diff --git a/.ci_script.sh b/.ci_script.sh deleted file mode 100755 index c915f2d47..000000000 --- a/.ci_script.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -e -set -x - -# Ensure it loads OK -python -m couchbase.connection - -# Set up our test directory.. -cp .tests.ini.travis tests.ini - -nosetests -v couchbase.tests.test_sync -nosetests -v gcouchbase.tests.test_api || echo "Gevent tests failed" -nosetests -v txcouchbase || echo "Twisted tests failed" diff --git a/.clang-format b/.clang-format new file mode 100644 index 000000000..a510e16bf --- /dev/null +++ b/.clang-format @@ -0,0 +1,258 @@ +# -*- mode: yaml; -*- + +--- +# Generated with: clang-format 18.1.6 +# clang-format --style=Mozilla --dump-config +AccessModifierOffset: -2 +AlignAfterOpenBracket: Align +AlignArrayOfStructures: None +AlignConsecutiveAssignments: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: true +AlignConsecutiveBitFields: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveDeclarations: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveMacros: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCompound: false + AlignFunctionPointers: false + PadOperators: false +AlignConsecutiveShortCaseStatements: + Enabled: false + AcrossEmptyLines: false + AcrossComments: false + AlignCaseColons: false +AlignEscapedNewlines: Right +AlignOperands: Align +AlignTrailingComments: + Kind: Always + OverEmptyLines: 0 +AllowAllArgumentsOnNextLine: true +AllowAllParametersOfDeclarationOnNextLine: false +AllowBreakBeforeNoexceptSpecifier: Never +AllowShortBlocksOnASingleLine: Never +AllowShortCaseLabelsOnASingleLine: false +AllowShortCompoundRequirementOnASingleLine: true +AllowShortEnumsOnASingleLine: true +AllowShortFunctionsOnASingleLine: Inline +AllowShortIfStatementsOnASingleLine: Never +AllowShortLambdasOnASingleLine: All +AllowShortLoopsOnASingleLine: false +AlwaysBreakAfterDefinitionReturnType: TopLevel +AlwaysBreakAfterReturnType: TopLevel +AlwaysBreakBeforeMultilineStrings: false +AlwaysBreakTemplateDeclarations: Yes +AttributeMacros: + - __capability +BinPackArguments: false +BinPackParameters: false +BitFieldColonSpacing: Both +BraceWrapping: + AfterCaseLabel: false + AfterClass: true + AfterControlStatement: Never + AfterEnum: true + AfterExternBlock: true + AfterFunction: true + AfterNamespace: false + AfterObjCDeclaration: false + AfterStruct: true + AfterUnion: true + BeforeCatch: false + BeforeElse: false + BeforeLambdaBody: false + BeforeWhile: false + IndentBraces: false + SplitEmptyFunction: true + SplitEmptyRecord: false + SplitEmptyNamespace: true +BreakAdjacentStringLiterals: true +BreakAfterAttributes: Leave +BreakAfterJavaFieldAnnotations: false +BreakArrays: true +BreakBeforeBinaryOperators: None +BreakBeforeConceptDeclarations: Always +BreakBeforeBraces: Mozilla +BreakBeforeInlineASMColon: OnlyMultiline +BreakBeforeTernaryOperators: true +BreakConstructorInitializers: BeforeComma +BreakInheritanceList: BeforeComma +BreakStringLiterals: true +ColumnLimit: 80 +CommentPragmas: '^ IWYU pragma:' +CompactNamespaces: false +ConstructorInitializerIndentWidth: 2 +ContinuationIndentWidth: 2 +Cpp11BracedListStyle: false +DerivePointerAlignment: false +DisableFormat: false +EmptyLineAfterAccessModifier: Never +EmptyLineBeforeAccessModifier: LogicalBlock +ExperimentalAutoDetectBinPacking: false +FixNamespaceComments: false +ForEachMacros: + - foreach + - Q_FOREACH + - BOOST_FOREACH +IfMacros: + - KJ_IF_MAYBE +IncludeBlocks: Preserve +IncludeCategories: + - Regex: '^"(llvm|llvm-c|clang|clang-c)/' + Priority: 2 + SortPriority: 0 + CaseSensitive: false + - Regex: '^(<|"(gtest|gmock|isl|json)/)' + Priority: 3 + SortPriority: 0 + CaseSensitive: false + - Regex: '.*' + Priority: 1 + SortPriority: 0 + CaseSensitive: false +IncludeIsMainRegex: '(Test)?$' +IncludeIsMainSourceRegex: '' +IndentAccessModifiers: false +IndentCaseBlocks: false +IndentCaseLabels: true +IndentExternBlock: AfterExternBlock +IndentGotoLabels: true +IndentPPDirectives: None +IndentRequiresClause: true +IndentWidth: 2 +IndentWrappedFunctionNames: false +InsertBraces: false +InsertNewlineAtEOF: false +InsertTrailingCommas: None +IntegerLiteralSeparator: + Binary: 0 + BinaryMinDigits: 0 + Decimal: 0 + DecimalMinDigits: 0 + Hex: 0 + HexMinDigits: 0 +JavaScriptQuotes: Leave +JavaScriptWrapImports: true +KeepEmptyLinesAtTheStartOfBlocks: true +KeepEmptyLinesAtEOF: false +LambdaBodyIndentation: Signature +LineEnding: DeriveLF +MacroBlockBegin: '' +MacroBlockEnd: '' +MaxEmptyLinesToKeep: 1 +NamespaceIndentation: None +ObjCBinPackProtocolList: Auto +ObjCBlockIndentWidth: 2 +ObjCBreakBeforeNestedBlockParam: true +ObjCSpaceAfterProperty: true +ObjCSpaceBeforeProtocolList: false +PackConstructorInitializers: BinPack +PenaltyBreakAssignment: 2 +PenaltyBreakBeforeFirstCallParameter: 19 +PenaltyBreakComment: 300 +PenaltyBreakFirstLessLess: 120 +PenaltyBreakOpenParenthesis: 0 +PenaltyBreakScopeResolution: 500 +PenaltyBreakString: 1000 +PenaltyBreakTemplateDeclaration: 10 +PenaltyExcessCharacter: 1000000 +PenaltyIndentedWhitespace: 0 +PenaltyReturnTypeOnItsOwnLine: 200 +PointerAlignment: Left +PPIndentWidth: -1 +QualifierAlignment: Leave +ReferenceAlignment: Pointer +ReflowComments: true +RemoveBracesLLVM: false +RemoveParentheses: Leave +RemoveSemicolon: false +RequiresClausePosition: OwnLine +RequiresExpressionIndentation: OuterScope +SeparateDefinitionBlocks: Leave +ShortNamespaceLines: 1 +SkipMacroDefinitionBody: false +SortIncludes: CaseSensitive +SortJavaStaticImport: Before +SortUsingDeclarations: LexicographicNumeric +SpaceAfterCStyleCast: false +SpaceAfterLogicalNot: false +SpaceAfterTemplateKeyword: false +SpaceAroundPointerQualifiers: Default +SpaceBeforeAssignmentOperators: true +SpaceBeforeCaseColon: false +SpaceBeforeCpp11BracedList: false +SpaceBeforeCtorInitializerColon: true +SpaceBeforeInheritanceColon: true +SpaceBeforeJsonColon: false +SpaceBeforeParens: ControlStatements +SpaceBeforeParensOptions: + AfterControlStatements: true + AfterForeachMacros: true + AfterFunctionDefinitionName: false + AfterFunctionDeclarationName: false + AfterIfMacros: true + AfterOverloadedOperator: false + AfterPlacementOperator: true + AfterRequiresInClause: false + AfterRequiresInExpression: false + BeforeNonEmptyParentheses: false +SpaceBeforeRangeBasedForLoopColon: true +SpaceBeforeSquareBrackets: false +SpaceInEmptyBlock: false +SpacesBeforeTrailingComments: 1 +SpacesInAngles: Never +SpacesInContainerLiterals: true +SpacesInLineCommentPrefix: + Minimum: 1 + Maximum: -1 +SpacesInParens: Never +SpacesInParensOptions: + InCStyleCasts: false + InConditionalStatements: false + InEmptyParentheses: false + Other: false +SpacesInSquareBrackets: false +Standard: Latest +StatementAttributeLikeMacros: + - Q_EMIT +StatementMacros: + - Q_UNUSED + - QT_REQUIRE_VERSION +TabWidth: 8 +UseTab: Never +VerilogBreakBetweenInstancePorts: true +WhitespaceSensitiveMacros: + - BOOST_PP_STRINGIZE + - CF_SWIFT_NAME + - NS_SWIFT_NAME + - PP_STRINGIZE + - STRINGIZE + +--- +# Couchbase Adjustments +Language: Cpp +Standard: c++17 +IndentWidth: 2 +ColumnLimit: 100 +AllowShortFunctionsOnASingleLine: None +AllowShortLambdasOnASingleLine: None +AllowShortEnumsOnASingleLine: false +BreakBeforeBraces: Linux +FixNamespaceComments: true diff --git a/.clang-tidy b/.clang-tidy new file mode 100644 index 000000000..be5df9838 --- /dev/null +++ b/.clang-tidy @@ -0,0 +1,38 @@ +HeaderFilterRegex: 'ext/couchbase.*' +Checks: |- + *, + bugprone-*, + cert-*, + google-*, + hicpp-*, + llvm-*, + misc-*, + modernize-*, + performance-*, + portability-*, + readability-*, + boost-*, + cppcoreguidelines-*, + -abseil-*, + -altera-*, + -cppcoreguidelines-avoid-magic-numbers, + -cppcoreguidelines-avoid-non-const-global-variables, + -cppcoreguidelines-pro-bounds-array-to-pointer-decay, + -cppcoreguidelines-pro-bounds-pointer-arithmetic, + -cppcoreguidelines-pro-type-cstyle-cast, + -cppcoreguidelines-pro-type-union-access, + -cppcoreguidelines-pro-type-vararg, + -fuchsia-*, + -google-runtime-int, + -google-runtime-references, + -hicpp-no-array-decay, + -hicpp-signed-bitwise, + -hicpp-vararg, + -performance-no-int-to-ptr, + -llvm-header-guard, + -llvmlibc-*, + -misc-no-recursion, + -misc-non-private-member-variables-in-classes, + -modernize-use-trailing-return-type, + -readability-function-cognitive-complexity, + -readability-magic-numbers diff --git a/.cmake-format.yaml b/.cmake-format.yaml new file mode 100644 index 000000000..d3c036051 --- /dev/null +++ b/.cmake-format.yaml @@ -0,0 +1,9 @@ +bullet_char: '*' +dangle_parens: false +enum_char: . +line_ending: unix +line_width: 120 +separate_ctrl_name_with_space: false +separate_fn_name_with_space: false +tab_size: 2 +max_pargs_hwrap: 3 diff --git a/.flake8 b/.flake8 new file mode 100644 index 000000000..f541d30f5 --- /dev/null +++ b/.flake8 @@ -0,0 +1,11 @@ +[flake8] +max-line-length = 120 +exclude = + .git, + __pycache__, + docs/source/conf.py, + build, + dist, + src + deps +max-complexity = 10 diff --git a/.gitignore b/.gitignore index 9b154568f..b1e51049d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,31 +1,28 @@ -build -dist -MANIFEST -couchbase/libcouchbase*.so -couchbase/libcouchbase.c -couchbase/libcouchbase.h -couchbase/__pycache__ -*.pyc -*.pdb -*.swp -*.out -*.log *.so -*.dylib -*.pyd +*.dylib*.* *.dll -*.exe -tests.ini -*.out* -# My personal development stuff -p3 -p2 -p3_nose -p2_nose -build*.sh -BUILD* +*.idea +__pycache__/ +cmake-build-*/ +*.DS_Store + +# Distribution / packaging +build/ couchbase/_version.py -ci/ -buildall.py -*egg-info* -_trial* +couchbase/*.so +couchbase/*.dylib*.* +couchbase/*.dll +couchbase/*.pyd +deps/couchbase-cxx-cache/ + +# Sphinx +docs/_build/ + +# VS Code +.vscode/ + +# tests +tests/test_logs/ +CouchbaseMock*.jar +gocaves* +.pytest_cache/ diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..38f33d928 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "deps/couchbase-cxx-client"] + path = deps/couchbase-cxx-client + url = https://github.com/couchbaselabs/couchbase-cxx-client.git diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..fd70d15ec --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,82 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + args: [--allow-multiple-documents] + - id: check-added-large-files + - id: check-toml + - id: check-merge-conflict + - repo: https://github.com/pre-commit/mirrors-clang-format + rev: v18.1.6 + hooks: + - id: clang-format + files: src/.*\.(cxx|hxx)$ + exclude: ^deps/ + - repo: https://github.com/hhatto/autopep8 + rev: v2.2.0 + hooks: + - id: autopep8 + exclude: ^(deps/|src/) + - repo: https://github.com/pycqa/flake8 + rev: 5.0.4 + hooks: + - id: flake8 + exclude: | + (?x)^( + deps/| + src/| + simple_acouchbase.py| + simple_couchbase.py| + simple_txcouchbase.py| + couchbase_kv_tests.py| + examples/ + ) + args: + [ + --max-line-length=120, + --max-complexity=10 + ] + - repo: https://github.com/PyCQA/isort + rev: 5.13.2 + hooks: + - id: isort + exclude: | + (?x)^( + deps/| + src/| + examples/txcouchbase/ + ) + args: + [ + "--multi-line 1", + "--force-grid-wrap 3", + "--use-parentheses True", + "--ensure-newline-before-comments True", + "--line-length 120", + "--order-by-type True" + ] + - repo: https://github.com/PyCQA/bandit + rev: 1.7.9 + hooks: + - id: bandit + exclude: | + (?x)^( + deps/| + src/| + acouchbase/tests/| + txcouchbase/tests/| + couchbase/tests/| + tests/| + simple_acouchbase.py| + simple_couchbase.py| + simple_txcouchbase.py| + couchbase_kv_tests.py| + couchbase_version.py + ) + args: + [ + --quiet + ] diff --git a/.tests.ini.travis b/.tests.ini.travis deleted file mode 100644 index 4cdeb83b7..000000000 --- a/.tests.ini.travis +++ /dev/null @@ -1,25 +0,0 @@ -[realserver] -host = localhost -port = 8091 -admin_username = Administrator - -; The administrative password. This is the password used to -; log into the admin console -admin_password = password - -bucket_name = default -; If a SASL bucket is being used (i.e. buckets are set up -; per the script, then this is the *bucket* password -; bucket_password sasl_password -bucket_password = - -; Set this to true if there is a real cluster available -enabled = False - -[mock] -; Set this to enabled to use the mock -enabled = True -; Local path for the mock -path = CouchbaseMock.jar -; Where to download it, if not available -url = http://packages.couchbase.com/clients/c/mock/CouchbaseMock-LATEST.jar diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index e827df0dd..000000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: python - -python: - - "2.7" - - "2.6" - - "3.2" - - "3.3" - -before_install: - - sudo rm -rf /etc/apt/sources.list.d/* - - wget -O- http://packages.couchbase.com/ubuntu/couchbase.key | sudo apt-key add - - - echo deb http://packages.couchbase.com/ubuntu precise precise/main | sudo tee /etc/apt/sources.list.d/couchbase.list - - sudo apt-get update - - sudo apt-cache search libcouchbase - -install: - - sudo apt-get -y install libcouchbase-dev libcouchbase2-core libcouchbase2-libevent libevent-dev - - pip -q install gevent || echo "Couldn't find gevent" - - pip -q install twisted - - pip -q install testresources - - export CFLAGS='-Wextra -Wno-long-long -Wno-missing-field-initializers' - - CFLAGS="$CFLAGS -fno-strict-aliasing -Wno-strict-aliasing" - - python setup.py build_ext --inplace - - sudo $(which python) setup.py install - -script: - - ./.ci_script.sh - -notifications: - email: - - mark.nunberg@couchbase.com diff --git a/BUILDING.md b/BUILDING.md new file mode 100644 index 000000000..fdc71841c --- /dev/null +++ b/BUILDING.md @@ -0,0 +1,63 @@ +# Building Couchbase Python Client + +>**NOTE:** This is the documentation for the 4.x version of the client. This is mostly compatible with the older 3.x version. Please refer to the *release32* branch for the older 3.x version. + +>**NOTE:** We highly recommend using a Python virtual environment (see [venv](https://docs.python.org/3/library/venv.html) for details). + +## Prerequisites<a id="prerequisites"></a> +- Supported Python version (see [Python Version Compatibility](https://docs.couchbase.com/python-sdk/current/project-docs/compatibility.html#python-version-compatibility) for details) +- A C++ compiler supporting C++ 17 +- CMake (version >= 3.18) +- Git +- Optional: OpenSSL + - The Couchbase Python Client can statically link against BoringSSL as of v4.1.9. The examples below are also using BoringSSL. + +## Clone the repository +```console +git clone --depth 1 --branch <tag_name> --recurse-submodules https://github.com/couchbase/couchbase-python-client.git +``` + +If the ```--recurse-submodules``` option was not used when cloning the Python SDK repository, run (after moving into the cloned repository directory) ```git submodule update --init --recursive``` to recursively update and initialize the submodules. + +## Set CPM Cache +The C++ core utilizes the CMake Package Manager (CPM) to include depencies. These can be set to a cache directory and can be used for future builds. Periodically the dependencies should be updated. So, in general it is good practice to configure the build environment by setting the CPM cache. + +```console +PYCBC_SET_CPM_CACHE=ON PYCBC_USE_OPENSSL=OFF python setup.py configure_ext +``` + +## Build the SDK +```console +PYCBC_USE_OPENSSL=OFF python setup.py build_ext --inplace +``` + +## Available Build Options +>Note: Section under construction + +## Alternate build from PyPI source distribution + +Make sure [minimum requirements](prerequisites) have been installed. + +>**NOTE:** After the source distribution has been obtained from PyPI, the build should be able to successfully complete without an internet connection. + +### Build with BoringSSL +```console +export CB_VERSION=<desired Python SDK version> SOURCE_DIR=couchbase-python-client +python -m pip download --no-deps --no-binary couchbase --no-cache-dir couchbase==$CB_VERSION +tar -xvf couchbase-$CB_VERSION.tar.gz +mkdir $SOURCE_DIR +mv couchbase-$CB_VERSION/* $SOURCE_DIR +cd $SOURCE_DIR +PYCBC_USE_OPENSSL=OFF python setup.py build_ext --inplace +``` + +### Build with system OpenSSL +```console +export CB_VERSION=<desired Python SDK version> SOURCE_DIR=couchbase-python-client +python -m pip download --no-deps --no-binary couchbase --no-cache-dir couchbase==$CB_VERSION +tar -xvf couchbase-$CB_VERSION.tar.gz +mkdir $SOURCE_DIR +mv couchbase-$CB_VERSION/* $SOURCE_DIR +cd $SOURCE_DIR +python setup.py build_ext --inplace +``` diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 000000000..6498a09ec --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,287 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cmake_minimum_required(VERSION 3.19) +# needed for CMAKE_MSVC_RUNTIME_LIBRARY +cmake_policy(SET CMP0091 NEW) +include(FetchContent) + +set(CMAKE_CXX_STANDARD 17) + +function(get_python_version) + if(DEFINED ENV{PYCBC_PYTHON3_EXECUTABLE}) + message(STATUS "Using python3 executable from environment: $ENV{PYCBC_PYTHON3_EXECUTABLE}") + set(PYTHON3_EXE $ENV{PYCBC_PYTHON3_EXECUTABLE}) + else() + message(STATUS "Using default python3 executable") + set(PYTHON3_EXE "python3") + endif() + + execute_process(COMMAND ${PYTHON3_EXE} -c "import platform;print(platform.python_version())" + OUTPUT_VARIABLE local_python_version) + string(STRIP "${local_python_version}" PYTHON_VERSION) + set(LOCAL_PYTHON_VERSION "${PYTHON_VERSION}" PARENT_SCOPE) +endfunction() + +if(WIN32) + # cmake-format: off + # MultiThreaded$<$<CONFIG:Debug>:Debug>DLL for /MD compile flag + # MultiThreaded$<$<CONFIG:Debug>:Debug> for /MT compile flag + # cmake-format: on + set(CMAKE_MSVC_RUNTIME_LIBRARY "MultiThreaded$<$<CONFIG:Debug>:Debug>DLL") + add_definitions(/bigobj) + add_definitions(-D_WIN32_WINNT=0x0601) +endif() + +project(couchbase_client) +set(CMAKE_EXPORT_COMPILE_COMMANDS ON) + +get_python_version() +message(STATUS "LOCAL_PYTHON_VERSION=${LOCAL_PYTHON_VERSION}") + +if(LOCAL_PYTHON_VERSION) + set(Python3_FIND_VIRTUALENV FIRST) + message("finding python version ${LOCAL_PYTHON_VERSION}") + find_package(Python3 ${LOCAL_PYTHON_VERSION} EXACT COMPONENTS Interpreter Development.Module) +else() + find_package(Python3 COMPONENTS Interpreter Development.Module) +endif() + + +if(WIN32) + set(PYCBC_C_MOD_SUFFIX ".pyd") +else() + set(PYCBC_C_MOD_SUFFIX ".so") +endif() + +file(READ "${PROJECT_SOURCE_DIR}/couchbase/_version.py" PYCBC_VERSION_CONTENTS) +string(REGEX MATCH "__version__.*([0-9]+\\.[0-9]+\\.[0-9]+)(\\.?[+a-z0-9]*)" PYCBC_FOUND_VERSION ${PYCBC_VERSION_CONTENTS}) +set(PYCBC_VERSION "${CMAKE_MATCH_1}") +if(NOT CMAKE_MATCH_2 STREQUAL "") + set(PYCBC_VERSION "${PYCBC_VERSION}${CMAKE_MATCH_2}") +endif() +message(STATUS "PYCBC_VERSION=${PYCBC_VERSION}") + +option(USE_STATIC_BORINGSSL "Statically link BoringSSL instead of dynamically linking OpenSSL" FALSE) +message(STATUS "USE_STATIC_BORINGSSL=${USE_STATIC_BORINGSSL}") +if(NOT USE_STATIC_BORINGSSL) + set(COUCHBASE_CXX_CLIENT_POST_LINKED_OPENSSL + ON + CACHE BOOL "" FORCE) + if(OPENSSL_ROOT_DIR) + message(STATUS "OPENSSL_ROOT_DIR set to ${OPENSSL_ROOT_DIR}, calling finder...") + find_package(OpenSSL REQUIRED) + endif() + + if(OPENSSL_FOUND) + message(STATUS "OpenSSL found, OPENSSL_ROOT_DIR set to ${OPENSSL_ROOT_DIR}") + else() + if(WIN32) + if("${CMAKE_SIZEOF_VOID_P}" EQUAL "8") + message(STATUS "++ 64 bit architecture") + set(PKGARCH "amd64") + else() + message(STATUS "++ 32 bit architecture") + set(PKGARCH "win32") + endif() + if(NOT OPENSSL_VERSION) + message(STATUS "No OpenSSL version set...cannot attempt to download.") + else() + # default version is currently 1.1.1g (see setup.py) + FetchContent_Declare(openssl + URL https://github.com/python/cpython-bin-deps/archive/openssl-bin-${OPENSSL_VERSION}.zip) + message(STATUS "fetching OpenSSL version: ${OPENSSL_VERSION}") + FetchContent_Populate(openssl) + message(STATUS "Downloaded OpenSSL: ${openssl_SOURCE_DIR}/${PKGARCH}") + set(OPENSSL_ROOT_DIR ${openssl_SOURCE_DIR}/${PKGARCH}) + endif() + elseif(APPLE) + # we were not supplied an OPENSSL_ROOT_DIR, so for macos assume brew is how it is installed, if it is... + find_program(BREW_COMMAND brew) + if(BREW_COMMAND) + message(STATUS "brew command: ${BREW_COMMAND}") + execute_process( + COMMAND ${BREW_COMMAND} --prefix openssl@1.1 + OUTPUT_VARIABLE BREW_OPENSSL_PREFIX + RESULT_VARIABLE BREW_RESULT + OUTPUT_STRIP_TRAILING_WHITESPACE) + message(STATUS "brew result: ${BREW_RESULT}, prefix: ${BREW_OPENSSL_PREFIX}") + if(BREW_RESULT EQUAL 0) + set(OPENSSL_ROOT_DIR + ${BREW_OPENSSL_PREFIX} + CACHE INTERNAL "" FORCE) + message(STATUS "brew set OPENSSL_ROOT_DIR to ${OPENSSL_ROOT_DIR}, finding OpenSSL...") + endif() + endif() + else() + message(STATUS "Not mac or windows, so assuming OpenSSL v1.1 is installed and findable...") + endif() + find_package(OpenSSL REQUIRED) + endif() + + message(STATUS "Adding ${OPENSSL_INCLUDE_DIR} to include dirs...") + include_directories(${OPENSSL_INCLUDE_DIR}) +else() + set(COUCHBASE_CXX_CLIENT_POST_LINKED_OPENSSL + OFF + CACHE BOOL "" FORCE) + set(COUCHBASE_CXX_CLIENT_STATIC_BORINGSSL + ON + CACHE BOOL "" FORCE) +endif() + +set(COUCHBASE_CXX_CLIENT_WRAPPER_UNIFIED_ID + "python/${PYCBC_VERSION}" + CACHE STRING "" FORCE) +set(COUCHBASE_CXX_CLIENT_PYTHON_WARNINGS + ON + CACHE INTERNAL "") +set(COUCHBASE_CXX_CLIENT_BUILD_STATIC + ON + CACHE BOOL "" FORCE) +set(COUCHBASE_CXX_CLIENT_BUILD_SHARED + OFF + CACHE BOOL "" FORCE) +set(COUCHBASE_CXX_CLIENT_BUILD_INSTALL + OFF + CACHE BOOL "" FORCE) +set(COUCHBASE_CXX_CLIENT_BUILD_DOCS + OFF + CACHE BOOL "" FORCE) +set(COUCHBASE_CXX_CLIENT_BUILD_EXAMPLES + OFF + CACHE BOOL "" FORCE) +set(COUCHBASE_CXX_CLIENT_BUILD_TESTS + OFF + CACHE BOOL "" FORCE) +set(COUCHBASE_CXX_CLIENT_BUILD_TOOLS + OFF + CACHE BOOL "" FORCE) + +# cmake-format: off +# PYCBC-1374 + PYCBC-1495: Move to dynamically link against static stdlib to avoid issues with: +# - other packages that also link against stdlibs (grpc) +# - building SDK on RHEL >= RHEL8 as static stdlibs are not available. +# cmake-format: on +option(USE_STATIC_STDLIB "Statically link C++ standard library" FALSE) +if(USE_STATIC_STDLIB) + set(COUCHBASE_CXX_CLIENT_STATIC_STDLIB + ON + CACHE BOOL "" FORCE) +else() + set(COUCHBASE_CXX_CLIENT_STATIC_STDLIB + OFF + CACHE BOOL "" FORCE) +endif() +message(STATUS "USE_STATIC_STDLIB=${USE_STATIC_STDLIB}") + +option(DOWNLOAD_MOZILLA_CA_BUNDLE "Allow C++ client to download and embed Mozilla certificates" TRUE) +if(DOWNLOAD_MOZILLA_CA_BUNDLE) + set(COUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE + ON + CACHE BOOL "" FORCE) +else() + set(COUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE + OFF + CACHE BOOL "" FORCE) +endif() +message(STATUS "DOWNLOAD_MOZILLA_CA_BUNDLE=${DOWNLOAD_MOZILLA_CA_BUNDLE}") + +# handle CPM cache dir +if(DEFINED COUCHBASE_CXX_CPM_CACHE_DIR AND NOT COUCHBASE_CXX_CPM_CACHE_DIR STREQUAL "") + set(CPM_SOURCE_CACHE "${COUCHBASE_CXX_CPM_CACHE_DIR}") +endif() + +if(DEFINED CPM_SOURCE_CACHE) + message(STATUS "CPM_SOURCE_CACHE=${CPM_SOURCE_CACHE}") +endif() + +add_subdirectory(deps/couchbase-cxx-client) + +set(COUCHBASE_CXX_BINARY_DIR "${CMAKE_BINARY_DIR}/deps/couchbase-cxx-client") +set(COUCHBASE_CXX_SOURCE_DIR "${PROJECT_SOURCE_DIR}/deps/couchbase-cxx-client") +message(STATUS "COUCHBASE_CXX_BINARY_DIR=${COUCHBASE_CXX_BINARY_DIR}") +message(STATUS "COUCHBASE_CXX_SOURCE_DIR=${COUCHBASE_CXX_SOURCE_DIR}") +if(DEFINED COUCHBASE_CXX_CPM_CACHE_DIR AND NOT COUCHBASE_CXX_CPM_CACHE_DIR STREQUAL "") + file(COPY "${COUCHBASE_CXX_BINARY_DIR}/mozilla-ca-bundle.crt" "${COUCHBASE_CXX_BINARY_DIR}/mozilla-ca-bundle.sha256" + DESTINATION "${COUCHBASE_CXX_CPM_CACHE_DIR}") + message(STATUS "Copied Mozilla cert bundle to cache. COUCHBASE_CXX_CPM_CACHE_DIR=${COUCHBASE_CXX_CPM_CACHE_DIR}") +endif() + +if(Python3_FOUND) + message(STATUS "Python executable: ${Python3_EXECUTABLE}") + message(STATUS "Python include dir: ${Python3_INCLUDE_DIRS}") + message(STATUS "Python libs: ${Python3_LIBRARIES}") +else() + message(FATAL_ERROR "Python3 not found.") +endif() + +include_directories(SYSTEM ${Python3_INCLUDE_DIRS}) +file( + GLOB + SOURCE_FILES + "src/*.cxx" + "src/management/*.cxx" + "src/transactions/*.cxx") +add_library(pycbc_core SHARED ${SOURCE_FILES}) + +target_include_directories( + pycbc_core PRIVATE SYSTEM + "${COUCHBASE_CXX_BINARY_DIR}/generated" + "${COUCHBASE_CXX_SOURCE_DIR}" + "${COUCHBASE_CXX_SOURCE_DIR}/third_party/cxx_function" + "${COUCHBASE_CXX_SOURCE_DIR}/third_party/expected/include") + +set(COUCHBASE_CXX_CLIENT_TARGET couchbase_cxx_client::couchbase_cxx_client_static) + +if(WIN32) + target_link_libraries( + pycbc_core PRIVATE + ${COUCHBASE_CXX_CLIENT_TARGET} + ${Python3_LIBRARIES} + asio + Microsoft.GSL::GSL + taocpp::json + spdlog::spdlog) +else() + target_link_libraries( + pycbc_core PRIVATE + ${COUCHBASE_CXX_CLIENT_TARGET} + asio + Microsoft.GSL::GSL + taocpp::json + spdlog::spdlog) + if(APPLE) + target_link_options( + pycbc_core + PRIVATE + -undefined + dynamic_lookup) + endif() +endif() + +if(CMAKE_VERBOSE_MAKEFILE) + target_link_options(pycbc_core PRIVATE -v) +endif() + +if(NOT USE_STATIC_BORINGSSL) + target_link_libraries(pycbc_core PUBLIC ${OPENSSL_LIBRARIES}) +endif() + +set_target_properties( + pycbc_core + PROPERTIES PREFIX "" + OUTPUT_NAME pycbc_core + SUFFIX ${PYCBC_C_MOD_SUFFIX}) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8638fe2f2..e218177d5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,78 +1,75 @@ -We've decided to use "Gerrit" for our code review system, making it -easier for all of us to contribute with code and comments. +# Contributing - 1. Visit http://review.couchbase.org and "Register" for an account - 2. Review http://review.couchbase.org/static/individual_agreement.html - 3. Agree to agreement by visiting http://review.couchbase.org/#/settings/agreements - 4. If you do not receive an email, please contact us - 5. Check out the Python SDK area http://review.couchbase.org/#/q/status:open+project:couchbase-python-client,n,z - 6. Join us on IRC at #libcouchbase on Freenode :-) +In addition to filing bugs, you may contribute by submitting patches to fix bugs in the library. Contributions may be submitted to <http://review.couchbase.com>. We use Gerrit as our code review system - and thus submitting a change requires an account there. While Github pull requests are not ignored, Gerrit pull requests will be responded to more quickly (and most likely with more detail). -We normally don't go looking for stuff in gerrit, so you should add at -least me (volker.mische@gmail.com) as a reviewer for your patch (and -I'll know who else to add and add them for you). +For something to be accepted into the codebase, it must be formatted properly and have undergone proper testing. Please see the [README guidelines](https://github.com/couchbase/couchbase-python-client#contributing) for details on formatting and linting. +## Branches and Tags -## Contributing Using Repo Tool +* The `master` branch represents the mainline branch. The master branch typically consists of content going into the next release. +* For older series of the Couchbase Python SDK see the corresponding branches: 2.x = `release25` and 3.x = `release32`. -If you haven't done so already you should -download the repo from http://code.google.com/p/git-repo/downloads/list -and put it in your path. +## Contributing Patches -All you should need to set up your development environment should be: +If you wish to contribute a new feature or a bug fix to the library, try to follow the following guidelines to help ensure your change gets merged upstream. - ~$ mkdir sdk - ~$ cd sdk - ~/sdk$ repo init -u git://github.com/vmx/manifest.git -m sdks/python.xml - ~/sdk$ repo sync - ~/sdk$ repo start my-branch-name --all - ~/sdk$ cd python - ~/sdk/python$ python setup.py build_ext --inplace +### Before you begin -You can work in the branch just as in any other git branch. Once you -are happy with your changes commit them as usual. Every commit will -show up as separate change within Gerrit, so you might want to squash -your commits into a single one before you upload them to gerrit with -the following command: +For any code change, ensure the new code you write looks similar to the code surrounding it. We have no strict code style policies, but do request that your code stand out as little as possible from its surrounding neighborhood (unless of course your change is stylistic in nature). - ~/sdk/python$ repo upload +If your change is going to involve a substantial amount of time or effort, please attempt to discuss it with the project developers first who will provide assistance and direction where possible. -You might experience a problem trying to upload the patches if you've -selected a different login name at review.couchbase.org than your login -name. Don't worry, all you need to do is to add the following to your -~/.gitconfig file: +#### For new features - [review "review.couchbase.org"] - username = your-gerrit-username +Ensure the feature you are adding does not already exist, and think about how this feature may be useful for other users. In general less intrusive changes are more likely to be accepted. +#### For fixing bugs -## Contributing Using Plain Git +Ensure the bug you are fixing is actually a bug (and not a usage) error, and that it has not been fixed in a more recent version. Please read the [release notes](https://docs.couchbase.com/python-sdk/current/project-docs/sdk-release-notes.html) as well as the [issue tracker](https://issues.couchbase.com/projects/PYCBC/issues/) to see a list of open and resolved issues. -If you not so familiar with repo tool and its workflow there is an -alternative way to do the same job. Just complete the Gerrit -registration steps above and clone the source repository -(remember the repository on github.com is just a mirror): +### Code Review - ~/sdk$ git clone ssh://YOURNAME@review.couchbase.org:29418/couchbase-python-client.git +#### Signing up on Gerrit -Install [`commit-msg` hook][1]: +Everything that is merged into the library goes through a code review process. The code review process is done via [Gerrit](http://review.couchbase.org). - ~/sdk$ cd couchbase-python-client - ~/sdk/couchbase-python-client$ scp -p -P 29418 YOURNAME@review.couchbase.org:hooks/commit-msg .git/hooks/ +To sign up for a Gerrit account, go to http://review.couchbase.org and click on the _Register_ link at the top right. Once you've signed in you will need to agree to the CLA (Contributor License Agreement) by going you your Gerrit account page and selecting the _Agreements_ link on the left. When you've done that, everything should flow through just fine. Be sure that you have registered your email address at http://review.couchbase.org/#/settings/contact as many sign-up methods won't pass emails along. Note that your email address in your code commit and in the Gerrit settings must match. -Make your changes and upload them for review: +Add your public SSH key to Gerrit before submitting. - ~/sdk/couchbase-python-client$ git commit - ~/sdk/couchbase-python-client$ git push origin HEAD:refs/for/master +#### Setting up your fork with Gerrit -If you need to fix or add something to your patch, do it and re-upload -the changes (all you need is to keep `Change-Id:` line the same to -allow gerrit to track the patch. +Assuming you have a repository created like so: - ~/couchbase-python-client % git commit --amend - ~/couchbase-python-client % git push origin HEAD:refs/for/master +``` +$ git clone https://github.com/couchbase/couchbase-python-client.git +``` -Happy hacking! +you can simply perform two simple steps to get started with Gerrit: +``` +$ git remote add gerrit ssh://${USERNAME}@review.couchbase.org:29418/couchbase-python-client +$ scp -P 29418 ${USERNAME}@review.couchbase.org:hooks/commit-msg .git/hooks +$ chmod a+x .git/hooks/commit-msg +``` -[1]: http://review.couchbase.org/Documentation/user-changeid.html +The last change is required for annotating each commit message with a special header known as `Change-Id`. This allows Gerrit to group together different revisions of the same patch. + +#### Pushing a changeset + +Now that you have your change and a Gerrit account to push to, you need to upload the change for review. To do so, invoke the following incantation: + +``` +$ git push gerrit HEAD:refs/for/master +``` + +Where `gerrit` is the name of the _remote_ added earlier. You may encounter some errors when pushing. The most common are: + +* "You are not authorized to push to this repository". You will get this if your account has not yet been approved. Please reach out in the [forums](https://www.couchbase.com/forums/c/python-sdk/10) if blocked. +* "Missing Change-Id". You need to install the `commit-msg` hook as described above. Note that even once you do this, you will need to ensure that any prior commits already have this header - this may be done by doing an interactive rebase (e.g. `git rebase -i origin/master` and selecting `reword` for all the commits; which will automatically fill in the Change-Id). + +Once you've pushed your changeset you can add people to review. Currently these are: + +* Jared Casey +* Dimitris Christodoulou +* Sergey Avseyev diff --git a/LICENSE b/LICENSE index 11069edd7..95c2436da 100644 --- a/LICENSE +++ b/LICENSE @@ -199,3 +199,132 @@ distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + +This library also makes use of boringssl so the following licenses also apply: + OpenSSL License + --------------- +/* ==================================================================== + * Copyright (c) 1998-2011 The OpenSSL Project. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * 3. All advertising materials mentioning features or use of this + * software must display the following acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit. (http://www.openssl.org/)" + * + * 4. The names "OpenSSL Toolkit" and "OpenSSL Project" must not be used to + * endorse or promote products derived from this software without + * prior written permission. For written permission, please contact + * openssl-core@openssl.org. + * + * 5. Products derived from this software may not be called "OpenSSL" + * nor may "OpenSSL" appear in their names without prior written + * permission of the OpenSSL Project. + * + * 6. Redistributions of any form whatsoever must retain the following + * acknowledgment: + * "This product includes software developed by the OpenSSL Project + * for use in the OpenSSL Toolkit (http://www.openssl.org/)" + * + * THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY + * EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE OpenSSL PROJECT OR + * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * ==================================================================== + * + * This product includes cryptographic software written by Eric Young + * (eay@cryptsoft.com). This product includes software written by Tim + * Hudson (tjh@cryptsoft.com). + * + */ + Original SSLeay License + ----------------------- +/* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com) + * All rights reserved. + * + * This package is an SSL implementation written + * by Eric Young (eay@cryptsoft.com). + * The implementation was written so as to conform with Netscapes SSL. + * + * This library is free for commercial and non-commercial use as long as + * the following conditions are aheared to. The following conditions + * apply to all code found in this distribution, be it the RC4, RSA, + * lhash, DES, etc., code; not just the SSL code. The SSL documentation + * included with this distribution is covered by the same copyright terms + * except that the holder is Tim Hudson (tjh@cryptsoft.com). + * + * Copyright remains Eric Young's, and as such any Copyright notices in + * the code are not to be removed. + * If this package is used in a product, Eric Young should be given attribution + * as the author of the parts of the library used. + * This can be in the form of a textual message at program startup or + * in documentation (online or textual) provided with the package. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * "This product includes cryptographic software written by + * Eric Young (eay@cryptsoft.com)" + * The word 'cryptographic' can be left out if the rouines from the library + * being used are not cryptographic related :-). + * 4. If you include any Windows specific code (or a derivative thereof) from + * the apps directory (application code) you must include an acknowledgement: + * "This product includes software written by Tim Hudson (tjh@cryptsoft.com)" + * + * THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * The licence and distribution terms for any publically available version or + * derivative of this code cannot be changed. i.e. this code cannot simply be + * copied and put under another distribution licence + * [including the GNU Public Licence.] + */ +ISC license used for completely new code in BoringSSL: +/* Copyright (c) 2015, Google Inc. + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY + * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION + * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN + * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */ diff --git a/MANIFEST.in b/MANIFEST.in index 1cd827e13..40816271f 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,76 @@ -include *.txt *.rst LICENSE -include couchbase_version.py +include *.txt LICENSE CONTRIBUTING.md BUILDING.md CMakeLists.txt pyproject.toml couchbase_version.py pycbc_build_setup.py +include couchbase-sdk-python-black-duck-manifest.yaml recursive-include couchbase *.py -recursive-include src *.c *.h -recursive-include docs * -recursive-exclude docs/build * -recursive-include tests *.py *.sample -recursive-include examples *.py -include couchbase libcouchbase.dll -exclude MANIFEST.in +recursive-include acouchbase *.py +recursive-include txcouchbase *.py +recursive-include src *.[ch]xx +include deps/couchbase-cxx-client/CMakeLists.txt +include deps/couchbase-cxx-client/couchbase-sdk-cxx-black-duck-manifest.yaml +include deps/couchbase-cxx-client/LICENSE.txt +include deps/couchbase-cxx-client/bin/build_boringssl* +include deps/couchbase-cxx-client/cmake/* +graft deps/couchbase-cxx-client/core +graft deps/couchbase-cxx-client/couchbase +include deps/couchbase-cxx-cache/cpm/*.cmake +include deps/couchbase-cxx-cache/asio/*/asio/COPYING +include deps/couchbase-cxx-cache/asio/*/asio/LICENSE* +include deps/couchbase-cxx-cache/asio/*/asio/asio/include/*.hpp +include deps/couchbase-cxx-cache/asio/*/asio/asio/src/asio.cpp +include deps/couchbase-cxx-cache/asio/*/asio/asio/src/asio_ssl.cpp +recursive-include deps/couchbase-cxx-cache/asio/*/asio/asio/include/asio *.[hi]pp +recursive-include deps/couchbase-cxx-cache/boringssl/*/boringssl *.[hcS] +recursive-include deps/couchbase-cxx-cache/boringssl/*/boringssl *.cc +recursive-include deps/couchbase-cxx-cache/boringssl/*/boringssl *.asm +recursive-include deps/couchbase-cxx-cache/boringssl/*/boringssl CMakeLists.txt +exclude deps/couchbase-cxx-cache/boringssl/*/boringssl/crypto_test_data.cc +include deps/couchbase-cxx-cache/boringssl/*/boringssl/LICENSE +include deps/couchbase-cxx-cache/gsl/*/gsl/CMakeLists.txt +include deps/couchbase-cxx-cache/gsl/*/gsl/GSL.natvis +include deps/couchbase-cxx-cache/gsl/*/gsl/LICENSE* +include deps/couchbase-cxx-cache/gsl/*/gsl/ThirdPartyNotices.txt +include deps/couchbase-cxx-cache/gsl/*/gsl/cmake/* +graft deps/couchbase-cxx-cache/gsl/*/gsl/include +include deps/couchbase-cxx-cache/hdr_histogram/*/hdr_histogram/*.pc.in +include deps/couchbase-cxx-cache/hdr_histogram/*/hdr_histogram/CMakeLists.txt +include deps/couchbase-cxx-cache/hdr_histogram/*/hdr_histogram/COPYING.txt +include deps/couchbase-cxx-cache/hdr_histogram/*/hdr_histogram/LICENSE.txt +include deps/couchbase-cxx-cache/hdr_histogram/*/hdr_histogram/cmake/* +include deps/couchbase-cxx-cache/hdr_histogram/*/hdr_histogram/config.cmake.in +graft deps/couchbase-cxx-cache/hdr_histogram/*/hdr_histogram/include +graft deps/couchbase-cxx-cache/hdr_histogram/*/hdr_histogram/src +include deps/couchbase-cxx-cache/llhttp/*/llhttp/*.pc.in +include deps/couchbase-cxx-cache/llhttp/*/llhttp/CMakeLists.txt +include deps/couchbase-cxx-cache/llhttp/*/llhttp/LICENSE* +include deps/couchbase-cxx-cache/llhttp/*/llhttp/include/*.h +include deps/couchbase-cxx-cache/llhttp/*/llhttp/src/*.c +include deps/couchbase-cxx-cache/snappy/*/snappy/CMakeLists.txt +include deps/couchbase-cxx-cache/snappy/*/snappy/COPYING +include deps/couchbase-cxx-cache/snappy/*/snappy/cmake/* +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy-c.h +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy-c.cc +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy-internal.h +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy-sinksource.h +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy-sinksource.cc +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy-stubs-internal.h +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy-stubs-internal.cc +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy-stubs-public.h.in +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy.h +include deps/couchbase-cxx-cache/snappy/*/snappy/snappy.cc +include deps/couchbase-cxx-cache/spdlog/*/spdlog/CMakeLists.txt +include deps/couchbase-cxx-cache/spdlog/*/spdlog/LICENSE +include deps/couchbase-cxx-cache/spdlog/*/spdlog/cmake/* +graft deps/couchbase-cxx-cache/spdlog/*/spdlog/include +graft deps/couchbase-cxx-cache/spdlog/*/spdlog/src +include deps/couchbase-cxx-cache/mozilla-ca-bundle.* +include deps/couchbase-cxx-cache/json/*/json/CMakeLists.txt +include deps/couchbase-cxx-cache/json/*/json/LICENSE* +include deps/couchbase-cxx-cache/json/*/json/external/PEGTL/.cmake/* +include deps/couchbase-cxx-cache/json/*/json/external/PEGTL/CMakeLists.txt +include deps/couchbase-cxx-cache/json/*/json/external/PEGTL/LICENSE* +graft deps/couchbase-cxx-cache/json/*/json/external/PEGTL/include +graft deps/couchbase-cxx-cache/json/*/json/include +include deps/couchbase-cxx-client/third_party/jsonsl/* +include deps/couchbase-cxx-client/third_party/expected/COPYING +graft deps/couchbase-cxx-client/third_party/expected/include +global-exclude *.py[cod] *.DS_Store +exclude .git .gitignore .gitmodules gocaves* *.jar .clang* .cmake* .pre* .flake* MANIFEST.in diff --git a/README.md b/README.md new file mode 100644 index 000000000..eab493945 --- /dev/null +++ b/README.md @@ -0,0 +1,637 @@ +# Couchbase Python Client +Python client for [Couchbase](https://couchbase.com) +>**NOTE:** This is the documentation for the 4.x version of the client. This is mostly compatible with the older 3.x version. Please refer to the *release32* branch for the older 3.x version. + + +# Contents<a id="contents"></a> +- [Prerequisites](#prerequisites) +- [Installing](#installing) +- [Building](#building) +- [Using the SDK](#using-the-sdk) +- [Building Documentation](#building-documentation) +- [Testing](#testing) +- [Contributing](#contributing) +- [Support & Additional Resources](#support-additional-resources) +- [License](#license) +- [Appendix](#appendix) + +# Prerequisites<a id="prerequisites"></a> + +- [Couchbase Server](http://couchbase.com/download) +- You may need a C++ compiler supporting C++ 17 and Python development files, unless a + binary wheel is available for your platform. With the 4.0.2 release, wheels are available on Windows, MacOS and Linux (via manylinux) for Python 3.7 - 3.10. +- CMake (version >= 3.18), unless a binary wheel is available for your platform. +- Git, unless a binary wheel is available for your platform. +- OpenSSL is now required for the 4.x Python SDK. +- If using the Twisted Framework and the txcouchbase API, Twisted >= 21.7.0 is required. + +## Debian and Ubuntu<a id="pre-deb-ubuntu"></a> + +First-time setup: +```console +$ sudo apt install git-all python3-dev python3-pip python3-setuptools cmake build-essential libssl-dev +``` + +>**NOTE:** We have provided *Dockerfiles* to demonstrate steps to achieve a working setup for various linux platforms. See the [dockerfiles folder](https://github.com/couchbase/couchbase-python-client/tree/master/examples/dockerfiles) in the Python SDK examples folder for details. + +See [Debian and Ubuntu](#install-deb-ubuntu) install section to install SDK. + +## RHEL and CentOS<a id="pre-rhel-centos"></a> + +First-time setup: +```console +$ sudo yum install git-all gcc gcc-c++ python3-devel python3-pip python3-setuptools cmake openssl-devel +``` + +>:exclamation:**IMPORTANT**:exclamation:<br>Some of the defaults for older operating systems like Centos/RHEL 7 and 8 have defaults to do not meet the 4.x Python SDK [minimum requirements](prerequisites). Be sure to update to the minimum requirements prior to installing the SDK. Most notably be sure to check the following: +>- The default Python version might be less than 3.7. If so, the Python version _will_ need to be udpated. +>- The default OpenSSL version might be less than 1.1.1. If so, the OpenSSL version _will_ need to be updated. +>- The gcc version must provide C++17 support. If the installed gcc version does not support C++17, gcc _will_ need to be updated. +>- The installed CMake version might be less than 3.17. If so, the CMake version _will_ need to be updated. Check out the steps [here](https://idroot.us/install-cmake-centos-8/) to update CMake. + +>**NOTE:** We have provided *Dockerfiles* to demonstrate steps to achieve a working setup for various linux platforms. See the [dockerfiles folder](https://github.com/couchbase/couchbase-python-client/tree/master/examples/dockerfiles) in the Python SDK examples folder for details. + +See [RHEL and Centos](#install-rhel-centos) install section to install SDK. + +## Mac OS<a id="pre-macos"></a> + +It is not recommended to use the vendor-supplied Python that ships with OS X. Best practice is to use a Python virtual environment such as pyenv or venv (after another version of Python that is not vendor-supplied has been installed) to manage multiple versions of Python. + +>:exclamation:**IMPORTANT**:exclamation:<br>There can be a problem when using the Python (3.8.2) that ships with Xcode on Catalina. It is advised to install Python with one of the following: +>- [pyenv](#macos-pyenv) +>- [Homebrew](#macos-homebrew) +>- Install Python via [python.org](https://www.python.org/downloads) + +### pyenv<a id="macos-pyenv"></a> + +See detailed walk through in [Appendix](#appendix-pyenv). Also, see pyenv install [docs](https://github.com/pyenv/pyenv#homebrew-on-macos) for further details. + +>**NOTE:** If using pyenv, make sure the python interpreter is the pyenv default, or a virtual environment has been activiated. Otherwise cmake might not be able to find the correct version of Python3 to use when building. + +### Homebrew<a id="macos-homebrew"></a> + +See Homebrew install [docs](https://brew.sh/) for further details. + +Get the latest packages: +```console +$ brew update +``` + +Install Python: +```console +$ brew install python +``` + +Update path: + +- **zsh:** + ```console + $ echo 'export PATH="/usr/local/bin:"$PATH' >> ~/.zshrc + ``` +- **bash:** + ```console + $ echo 'export PATH="/usr/local/bin:"$PATH' >> ~/.bash_profile + ``` + +Install OpenSSL: +```console +$ brew install openssl@1.1 +``` + +To get OpenSSL to be found by cmake on macos, find where openssl was +installed via homebrew: + +``` +brew info openssl@1.1 +``` + +This will show you how to get it seen by pkg-config. To check that it +worked, do this: + +``` +pkg-config --modversion openssl +``` + +See [Mac OS](#install-macos) install section to install SDK. + +## Windows<a id="pre-windows"></a> + +Wheels are available on Windows for Python 3.7, 3.8, 3.9 and 3.10. + +Best practice is to use a Python virtual environment such as venv or pyenv (checkout the [pyenv-win](https://github.com/pyenv-win/pyenv-win) project) to manage multiple versions of Python. + +If wanting to install from source, see the [Windows building](#building-windows) section for details. + +See [Windows install](#install-windows) section to install SDK. + +# Installing<a id="installing"></a> +[Back to Contents](#contents) + +You can always get the latest supported release version from [pypi](https://pypi.org/project/couchbase/). + +>**NOTE:** If you have a recent version of *pip*, you may use the latest development version by issuing the following incantation: +>```console +>pip install git+https://github.com/couchbase/couchbase-python-client.git +>``` + +>**NOTE:** The Python Client installer relies on PEP517 which older versions of PIP do not support. If you experience issues installing it is advised to upgrade your PIP/setuptools installation as follows:<br> +>```console +>python3 -m pip install --upgrade pip setuptools wheel +>``` + + +## Debian and Ubuntu<a id="install-deb-ubuntu"></a> + +First, make sure the [prerequisites](#pre-deb-ubuntu) have been installed. + +Install the SDK: +```console +$ python3 -m pip install couchbase +``` + +## RHEL and CentOS<a id="install-rhel-centos"></a> + +First, make sure the [prerequisites](#pre-rhel-centos) have been installed. + +Install the SDK: +```console +$ python3 -m pip install couchbase +``` + +## Mac OS<a id="install-macos"></a> + +First, make sure the [prerequisites](#pre-macos) have been installed. + +Install the SDK: +```console +$ python -m pip install couchbase +``` + + +## Windows<a id="install-windows"></a> + +First, make sure the [prerequisites](#pre-windows) have been installed. + +>**NOTE:** Commands assume user is working within a virtual environment. For example, the following commands have been executed after downloading and installing Python from [python.org](https://www.python.org/downloads/):<br> +>-```C:\Users\Administrator\AppData\Local\Programs\Python\Python39\python -m venv C:\python\python39```<br> +>-```C:\python\python39\Scripts\activate``` + +Install the SDK (if using Python 3.7, 3.8, 3.9 or 3.10): +```console +python -m pip install couchbase +``` + +### Alternative Installation Methods<a id="install-windows-alt"></a> + +In order to successfully install with the following methods, ensure a proper build system is in place (see the [Windows building](#building-windows) section for details). + +#### Source Install (i.e. no wheel) + +First, ensure all the [requirements](#building-windows) for a build system are met. + +Install the SDK: +```console +python -m pip install couchbase --no-binary couchbase +``` + +#### Local Install + +First, ensure all the [requirements](#building-windows) for a build system are met. + +Clone this Python SDK repository: +```console +git clone --depth 1 --branch <tag_name> --recurse-submodules https://github.com/couchbase/couchbase-python-client.git +``` + +>Where tag_name is equal to the latest release.<br> +Example: ```git clone --depth 1 --branch 4.0.0 --recurse-submodules https://github.com/couchbase/couchbase-python-client.git``` + +Move into the directory created after cloning the Python SDK repository: +```console +cd couchbase-python-client +``` + +>**NOTE:** If the ```--recurse-submodules``` option was not used when cloning the Python SDK repository, run (after moving into the cloned repository directory) ```git submodule update --init --recursive``` to recursively update and initialize the submodules. + +Install the SDK from source: +```console +python -m pip install . +``` + +## Anaconda/Miniconda<a id="install-anaconda"></a> + +To use the SDK within the Anaconda/Miniconda platform, make sure the prerequisites for the desired Operating System are met: +- [Debian and Ubuntu](#pre-deb-ubuntu) +- [RHEL and Centos](#pre-rhel-centos) +- [Mac OS](#pre-macos) +- [Windows](#pre-windows) + +In the *Anaconda Prompt*, create a new environment: +```console +(base) C:\Users\user1>conda create -n test_env python=3.9 +``` + +Activate the environment +```console +(base) C:\Users\user1>conda activate test_env +``` + +Install the SDK: +```console +(test_env) C:\Users\user1>python -m pip install couchbase +``` + +>**NOTE:** If using Windows, and no wheel is available, see the [Alternative Install Methods Windows](#install-windows-alt) section. The same process should work within the Anaconda/Miniconda platform. + +# Building<a id="building"></a> +[Back to Contents](#contents) + +>**NOTE:** This section only applies to building from source. + +## Build System Setup +### Linux<a id="building-linux"></a> + +Make sure the prerequisites have been installed: +- [Debian and Ubuntu](#pre-deb-ubuntu) +- [RHEL and Centos](#pre-rhel-centos) + +### Mac OS<a id="building-macos"></a> +First, make sure the [prerequisites](#pre-macos) have been installed. + +Install cmake: +```console +$ brew install cmake +``` + +Install command line developer tools: +``` +$ xcode-select --install +``` + +>**NOTE:** It is possible that installing or updating to the the latest version of [Xcode](https://developer.apple.com/download) is needed. + +If setuptools is not installed: +```console +$ python -m pip install setuptools +``` + +### Windows<a id="building-windows"></a> +#### Requirements +- Download and install [Git](https://git-scm.com/downloads) +- Download and install [Visual Studio 2019](https://visualstudio.microsoft.com/downloads/) + + Check *Desktop development with C++* prior to installing +- Download and install [CMake](https://cmake.org/download/) >= v 3.18 +- Download and install [Python](https://www.python.org/downloads/) + +#### VS2019 Notes + +If seeing issues when trying to build (steps in [](#)), some things to check/try: +- Try running the build commands within the *Developer Command Prompt for VS2019* +- Make sure *MSBuild* can find the correct *VCTargetsPath* + + It is possible the *VCTargetsPath* environment variable needs to be set. The below example is based on a typical path, but the actual setting should match that of your current environment setup. + * ```set VCTargetsPath=C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\MSBuild\Microsoft\VC\v160``` +- Make sure *CMake* is picking up the correct generator + + It is possible the *CMAKE_GENERATOR* environment variable needs to be set + * ```set CMAKE_GENERATOR=Visual Studio 16 2019``` + +## Build the Python SDK +Clone this Python SDK repository: +```console +git clone --depth 1 --recurse-submodules https://github.com/couchbase/couchbase-python-client.git +``` + +>**NOTE:** If the ```--recurse-submodules``` option was not used when cloning the Python SDK repository, run (after moving into the cloned repository directory) ```git submodule update --init --recursive``` to recursively update and initialize the submodules. + +Move into the directory created after cloning the Python SDK repository: +```console +cd couchbase-python-client +``` + +The following will compile the module locally: +```console +python setup.py build_ext --inplace +``` + +You can also modify the environment ```CFLAGS``` and ```LDFLAGS``` variables. + +>:exclamation:**WARNING:** If you do not intend to install this module, ensure you set the ```PYTHONPATH``` environment variable to this directory before running any scripts depending on it. Failing to do so may result in your script running against an older version of this module (if installed), or throwing an exception stating that the ```couchbase``` module could not be found. + +## Install +```console +pip install . +``` + +>:exclamation:**WARNING:** If you are on Linux/Mac OS you may need to remove the build directory: ```rm -rf ./build``` before installing with pip: ```pip3 install .```. + +# Using the SDK<a id="using-the-sdk"></a> +[Back to Contents](#contents) + +## Connecting<a id="sdk-connecting"></a> + +See [official documentation](https://docs.couchbase.com/python-sdk/current/howtos/managing-connections.html) for further details on connecting. + +```python +# needed for any cluster connection +from couchbase.cluster import Cluster +from couchbase.auth import PasswordAuthenticator + +# options for a cluster and SQL++ (N1QL) queries +from couchbase.options import ClusterOptions, QueryOptions + +# get a reference to our cluster +cluster = Cluster.connect('couchbase://localhost', ClusterOptions( + PasswordAuthenticator('Administrator', 'password'))) +``` + +>**NOTE:** The authenticator is always required. + +## Basic Operations<a id="sdk-basic-ops"></a> + +See official documentation for further details on [Working with Data](https://docs.couchbase.com/python-sdk/current/howtos/kv-operations.html). + +Building upon the example code in the [Connecting](#sdk-connecting) section: + +```python +# get a reference to our bucket +cb = cluster.bucket('travel-sample') + +# get a reference to the default collection +cb_coll = cb.default_collection() + +# get a document +result = cb_coll.get('airline_10') +print(result.content_as[dict]) + +# using SQL++ (a.k.a N1QL) +call_sign = 'CBS' +sql_query = 'SELECT VALUE name FROM `travel-sample` WHERE type = "airline" AND callsign = $1' +query_res = cluster.query(sql_query, QueryOptions(positional_parameters=[call_sign])) +for row in query_res: + print(row) +``` + +## Async Operations<a id="sdk-async-ops"></a> +The Python Couchbase SDK supports asynchronous I/O through the use of the asyncio (Python standard library) or the Twisted async framework. + +### Asyncio + +To use asyncio, import ```acouchbase.cluster``` instead of ```couchbase.cluster```. The ```acouchbase``` API offers an API similar to the ```couchbase``` API. + +```python +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.options import ClusterOptions +from couchbase.auth import PasswordAuthenticator + + +async def write_and_read(key, value): + cluster = await Cluster.connect('couchbase://localhost', + ClusterOptions(PasswordAuthenticator('Administrator', 'password'))) + cb = cluster.bucket('default') + await cb.on_connect() + cb_coll = cb.default_collection() + await cb_coll.upsert(key, value) + result = await cb_coll.get(key) + return result + +loop = get_event_loop() +rv = loop.run_until_complete(write_and_read('foo', 'bar')) +print(rv.content_as[str]) +``` +### Twisted + +To use with Twisted, import ```txcouchbase.cluster``` instead of ```couchbase.cluster```. The ```txcouchbase``` API offers an API similar to the ```couchbase``` API. + +>**NOTE:** The minimum required Twisted version is 21.7.0. + +>:exclamation:**WARNING:** The 4.x SDK introduced a breaking change where the txcouchbase package must be imported _prior_ to importing the reactor (see example below). This is so that the asyncio reactor can be installed. + + +```python +# IMPORTANT -- the txcouchbase import must occur PRIOR to importing the reactor +import txcouchbase +from twisted.internet import reactor, defer + +from txcouchbase.cluster import TxCluster +from couchbase.options import ClusterOptions +from couchbase.auth import PasswordAuthenticator + + +def after_upsert(res, key, d): + print('Set key. Result CAS: ', res.cas) + # trigger get_document callback + d.callback(key) + +def upsert_document(key, doc): + d = defer.Deferred() + res = cb.upsert(key, doc) + res.addCallback(after_upsert, key, d) + return d + +def on_get(res, _type=str): + print('Got res: \n', res.content_as[_type]) + reactor.stop() + +def get_document(key): + res = cb.get(key) + res.addCallback(on_get) + + +# create a cluster object +cluster = TxCluster('couchbase://localhost', + ClusterOptions(PasswordAuthenticator('Administrator', 'password'))) + +# create a bucket object +bucket = cluster.bucket('default') +# create a collection object +cb = bucket.default_collection() + +d = upsert_document('testDoc_1', {'id': 1, 'type': 'testDoc', 'info': 'fake document'}) +d.addCallback(get_document) + +reactor.run() +``` +# Building Documentation<a id="building-documentation"></a> +[Back to Contents](#contents) + +The documentation is using Sphinx and a number of extensions. To build the documentation be sure to +``pip install`` the sphinx_requirements.txt. + +```console +python3 -m pip install -r sphinx_requirements.txt +``` + +To build the documentation, go into the docs directory and run: +```console +make html +``` +The HTML output can be found in docs/build/html/. + +Alternatively, you can also build the documentation from the top-level directory: +```console +python setup.py build_sphinx +``` + +Once built, the docs will be in in build/sphinx/html. You can open the index.html file with the following command: +```console +open docs/build/sphinx/html/index.html +``` + +# Testing<a id="testing"></a> +[Back to Contents](#contents) + +For running the tests, be sure to ``pip install`` the dev_requirements.txt. The Couchbase Python SDK uses pytest for the test suite. + +```console +python3 -m pip install -r dev_requirements.txt +``` + +The tests need a running Couchbase instance. For this, a test_config.ini file must be present, containing various connection parameters. The default test_config.ini file may be found in the tests directory. You may modify the values of the test_config.ini file as needed. + +To run the tests for the blocking API (i.e. couchbase API): +```console +python -m pytest -m pycbc_couchbase -p no:asyncio -v -p no:warnings +``` + +To run the tests for the asyncio API (i.e. acouchbase API): +```console +python -m pytest -m pycbc_acouchbase --asyncio-mode=strict -v -p no:warnings +``` + +# Contributing<a id="contributing"></a> +[Back to Contents](#contents) + +We welcome contributions from the community! Please see follow the steps outlined [here](https://github.com/couchbase/couchbase-python-client/blob/master/CONTRIBUTING.md) to get started. + +The Python SDK uses pre-commit in order to handle linting, formatting and verifying the code base. +pre-commit can be installed either by installing the development requirements: + +``` +python3 -m pip install -r dev_requirements.txt +``` + +Or by installing pre-commit separately +``` +python3 -m pip install pre-commit +``` + +To run pre-commit, use the following: +``` +pre-commit run --all-files +``` + +# License +[Back to Contents](#contents) + +The Couchbase Python SDK is licensed under the Apache License 2.0. + +See [LICENSE](https://github.com/couchbase/couchbase-python-client/blob/master/LICENSE) for further details. + +# Support & Additional Resources<a id="support-additional-resources"></a> +[Back to Contents](#contents) + +If you found an issue, please file it in our [JIRA](https://issues.couchbase.com/projects/PYCBC/issues/). + +The Couchbase Discord server is a place where you can collaborate about all things Couchbase. Connect with others from the community, learn tips and tricks, and ask questions. [Join Discord and contribute](https://discord.com/invite/sQ5qbPZuTh). + +You can ask questions in our [forums](https://forums.couchbase.com/). + +The [official documentation](https://docs.couchbase.com/python-sdk/current/hello-world/start-using-sdk.html) can be consulted as well for general Couchbase concepts and offers a more didactic approach to using the SDK. + +# Appendix<a id="appendix"></a> +[Back to Contents](#contents) + +### Mac OS pyenv Install<a id="appendix-pyenv"></a> +See pyenv install [docs](https://github.com/pyenv/pyenv#homebrew-on-macos) for further details. + +Get the latest packages: +```console +$ brew update +``` + +For TLS/SSL support: +```console +$ brew install openssl@1.1 +``` + +Install pyenv: +```console +$ brew install pyenv +``` + +>**NOTE:** It is possible that Xcode might need to be reinstalled. Try **one** of the following:<br> +>- Use command ```xcode-select --install``` +>- Install the latest version of [Xcode](https://developer.apple.com/download) + +For *Zsh*, run the following commands to update *.zprofile* and *.zshrc*. See pyenv install [docs](https://github.com/pyenv/pyenv#homebrew-on-macos) for further details on other shells. + +```console +$ echo 'eval "$(pyenv init --path)"' >> ~/.zprofile +``` + +```console +$ echo 'eval "$(pyenv init -)"' >> ~/.zshrc +``` + +>**NOTE:** You need to restart your login session for changes to take affect. On MacOS, restarting terminal windows should suffice. + +Install Python version: +```console +$ pyenv install 3.9.7 +``` + +Set local shell to installed Python version: +```console +$ pyenv local 3.9.7 +``` + +To use virtualenvwrapper with pyenv, install pyenv-virtualenvwrapper: +```console +$ brew install pyenv-virtualenvwrapper +``` + +To setup a virtualenvwrapper in your pyenv shell, run either ```pyenv virtualenvwrapper``` or ```pyenv virtualenvwrapper_lazy``` + +>**NOTE:** If issues with ```pyenv virtualenvwrapper```, using ```python -m pip install virtualenvwrapper``` should accomplish the same goal. + +Make a virtualenv: +```console +$ mkvirtualenv python-3.9.7-test +``` + +Install the SDK: +```console +$ python -m pip install couchbase +``` + +### Run individual pre-commit commands<a id="appendix-precommit"></a> +To run pre-commit hooks separately, use the following. + +#### autopep8 +``` +pre-commit run autopep8 --all-files +``` + +#### bandit +``` +pre-commit run bandit --all-files +``` + +#### clang-format +``` +pre-commit run clang-format --all-files +``` + +#### flake8 +``` +pre-commit run flake8 --all-files +``` + +#### isort +``` +pre-commit run isort --all-files +``` + +#### trailing whitespace +``` +pre-commit run trailing-whitespace --all-files +``` diff --git a/README.rst b/README.rst deleted file mode 100644 index e888ff382..000000000 --- a/README.rst +++ /dev/null @@ -1,254 +0,0 @@ -======================= -Couchbase Python Client -======================= - -Client for Couchbase_. - -.. image:: https://travis-ci.org/couchbase/couchbase-python-client.png - :target: https://travis-ci.org/couchbase/couchbase-python-client - - -.. note:: - - This is the documentation for the 2.x version of the client. This is - mostly compatible with the older version. Please refer to the - *release12* branch for the older version. - ------------------------ -Building and Installing ------------------------ - -This only applies to building from source. If you are using a Windows -installer then everything (other than the server) is already included. -See below for windows snapshot releases. - -Also note that these instructions apply to building from source. -You can always get the latest supported release version from pypi_. - - -If you have a recent version of *pip*, you may use the latest development -version by issuing the following incantation - -.. code-block:: sh - - pip install git+git://github.com/couchbase/couchbase-python-client - -~~~~~~~~~~~~~ -Prerequisites -~~~~~~~~~~~~~ - -- Couchbase Server (http://couchbase.com/download) -- libcouchbase_. version 2.4.7 or greater (Bundled in Windows installer) -- libcouchbase development files. -- Python development files -- A C compiler. - -~~~~~~~~ -Building -~~~~~~~~ - -The following will compile the module locally; you can then test basic -functionality including running the examples. - -.. code-block:: sh - - python setup.py build_ext --inplace - - -If your libcouchbase install is in an alternate location (for example, -`/opt/local/libcouchbase`), you may add extra directives, like so - -.. code-block:: sh - - python setup.py build_ext --inplace \ - --library-dir /opt/local/libcouchbase/lib \ - --include-dir /opt/local/libcouchbase/include - -Or you can modify the environment ``CFLAGS`` and ``LDFLAGS`` variables. - - -.. warning:: - - If you do not intend to install this module, ensure you set the - ``PYTHONPATH`` environment variable to this directory before running - any scripts depending on it. Failing to do so may result in your script - running against an older version of this module (if installed), or - throwing an exception stating that the ``couchbase`` module could not - be found. - - -^^^^^^^^^^ -Installing -^^^^^^^^^^ -.. code-block:: sh - - python setup.py install - ------ -Using ------ - -Here's an example code snippet which sets a key and then reads it - -.. code-block:: pycon - - >>> from couchbase.bucket import Bucket - >>> c = Bucket('couchbase://localhost/default') - >>> c - <couchbase.bucket.Bucket bucket=default, nodes=['localhost:8091'] at 0x105991cd0> - >>> c.upsert("key", "value") - OperationResult<RC=0x0, Key=key, CAS=0x31c0e3f3fc4b0000> - >>> res = c.get("key") - >>> res - ValueResult<RC=0x0, Key=key, Value=u'value', CAS=0x31c0e3f3fc4b0000, Flags=0x0> - >>> res.value - u'value' - >>> - -You can also use views - -.. code-block:: pycon - - >>> from couchbase.bucket import Bucket - >>> c = Bucket('couchbase://localhost/beer-sample') - >>> resultset = c.query("beer", "brewery_beers", limit=5) - >>> resultset - View<Design=beer, View=brewery_beers, Query=Query:'limit=5', Rows Fetched=0> - >>> for row in resultset: print row.key - ... - [u'21st_amendment_brewery_cafe'] - [u'21st_amendment_brewery_cafe', u'21st_amendment_brewery_cafe-21a_ipa'] - [u'21st_amendment_brewery_cafe', u'21st_amendment_brewery_cafe-563_stout'] - [u'21st_amendment_brewery_cafe', u'21st_amendment_brewery_cafe-amendment_pale_ale'] - [u'21st_amendment_brewery_cafe', u'21st_amendment_brewery_cafe-bitter_american'] - -~~~~~~~~~~~ -Twisted API -~~~~~~~~~~~ - -The Python client now has support for the Twisted async network framework. -To use with Twisted, simply import ``txcouchbase.connection`` instead of -``couchbase.bucket`` - -.. code-block:: python - - from twisted.internet import reactor - from txcouchbase.bucket import Bucket - - cb = Bucket('couchbase://localhost/default') - def on_upsert(ret): - print "Set key. Result", ret - - def on_get(ret): - print "Got key. Result", ret - reactor.stop() - - cb.upsert("key", "value").addCallback(on_upsert) - cb.get("key").addCallback(on_get) - reactor.run() - - # Output: - # Set key. Result OperationResult<RC=0x0, Key=key, CAS=0x9a78cf56c08c0500> - # Got key. Result ValueResult<RC=0x0, Key=key, Value=u'value', CAS=0x9a78cf56c08c0500, Flags=0x0> - - -The ``txcouchbase`` API is identical to the ``couchbase`` API, except that where -the synchronous API will block until it receives a result, the async API will -return a `Deferred` which will be called later with the result or an appropriate -error. - -~~~~~~~~~~ -GEvent API -~~~~~~~~~~ - -.. code-block:: python - - from gcouchbase.bucket import Bucket - - conn = Bucket('couchbase://localhost/default') - print conn.upsert("foo", "bar") - print conn.get("foo") - -The API functions exactly like the normal Bucket API, except that the -implementation is significantly different. - - -~~~~ -PyPy -~~~~ - -`PyPy <http://pypy.org>`_ is an alternative high performance Python -implementation. Since PyPy does not work well with C extension modules, -this module will not work directly. You may refer to the alternate -implementation based on the *cffi* module: https://github.com/couchbaselabs/couchbase-python-cffi - -~~~~~~~~~~~~~~ -Other Examples -~~~~~~~~~~~~~~ - -There are other examples in the `examples` directory. To run them from the -source tree, do something like - -.. code-block:: sh - - PYTHONPATH=$PWD ./examples/bench.py -U couchbase://localhost/default - ----------------------- -Building documentation ----------------------- - - -The documentation is using Sphinx and also needs the numpydoc Sphinx extension. -To build the documentation, go into the `docs` directory and run - -.. code-block:: sh - - make html - -The HTML output can be found in `docs/build/html/`. - -------- -Testing -------- - -For running the tests, you need the standard `unittest` module, shipped -with Python. Additionally, the `testresources` package is required. - -To run them, use either `py.test`, `unittest` or `trial`. - -The tests need a running Couchbase instance. For this, a `tests.ini` -file must be present, containing various connection parameters. -An example of this file may be found in `tests.ini.sample`. -You may copy this file to `tests.ini` and modify the values as needed. - -The simplest way to run the tests is to declare a `bucket_prefix` in -the `tests.ini` file and run the `setup_tests.py` script to create -them for you. - -.. code-block:: sh - - python setup_tests.py - -To run the tests:: - - nosetests - -------- -Support -------- - -If you found an issue, please file it in our JIRA_. You may also ask in the -`#libcouchbase` IRC channel at freenode_. (which is where the author(s) -of this module may be found). - -------- -License -------- - -The Couchbase Python SDK is licensed under the Apache License 2.0. - -.. _Couchbase: http://couchbase.com -.. _libcouchbase: http://couchbase.com/develop/c/current -.. _JIRA: http://couchbase.com/issues/browse/pycbc -.. _freenode: http://freenode.net/irc_servers.shtml -.. _pypi: http://pypy.python.org/pypi/couchbase diff --git a/acouchbase/__init__.py b/acouchbase/__init__.py index e69de29bb..cc1f628ba 100644 --- a/acouchbase/__init__.py +++ b/acouchbase/__init__.py @@ -0,0 +1,73 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import selectors + + +class LoopValidator: + REQUIRED_METHODS = {'add_reader', 'remove_reader', + 'add_writer', 'remove_writer'} + + @staticmethod + def _get_working_loop(): + evloop = asyncio.get_event_loop() + gen_new_loop = not LoopValidator._is_valid_loop(evloop) + if gen_new_loop: + evloop.close() + selector = selectors.SelectSelector() + new_loop = asyncio.SelectorEventLoop(selector) + asyncio.set_event_loop(new_loop) + return new_loop + + return evloop + + @staticmethod + def _is_valid_loop(evloop): + if not evloop: + return False + for meth in LoopValidator.REQUIRED_METHODS: + abs_meth, actual_meth = ( + getattr(asyncio.AbstractEventLoop, meth), getattr(evloop.__class__, meth)) + if abs_meth == actual_meth: + return False + return True + + @staticmethod + def get_event_loop(evloop): + if LoopValidator._is_valid_loop(evloop): + return evloop + return LoopValidator._get_working_loop() + + @staticmethod + def close_loop(): + evloop = asyncio.get_event_loop() + evloop.close() + + +def get_event_loop( + evloop=None, # type: asyncio.AbstractEventLoop +): + """ + Get an event loop compatible with acouchbase. + Some Event loops, such as ProactorEventLoop (the default asyncio event + loop for Python 3.8 on Windows) are not compatible with acouchbase as + they don't implement all members in the abstract base class. + + :param evloop: preferred event loop + :return: The preferred event loop, if compatible, otherwise, a compatible + alternative event loop. + """ + return LoopValidator.get_event_loop(evloop) diff --git a/acouchbase/analytics.py b/acouchbase/analytics.py new file mode 100644 index 000000000..dc79ff5e8 --- /dev/null +++ b/acouchbase/analytics.py @@ -0,0 +1,109 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from typing import Awaitable + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.analytics import AnalyticsQuery # noqa: F401 +from couchbase.logic.analytics import AnalyticsRequestLogic + + +class AsyncAnalyticsRequest(AnalyticsRequestLogic): + def __init__(self, + connection, + loop, + query_params, + row_factory=lambda x: x, + **kwargs + ): + num_workers = kwargs.pop('num_workers', 2) + super().__init__(connection, query_params, row_factory=row_factory, **kwargs) + self._loop = loop + self._tp_executor = ThreadPoolExecutor(num_workers) + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @classmethod + def generate_analytics_request(cls, connection, loop, query_params, row_factory=lambda x: x, **kwargs): + return cls(connection, loop, query_params, row_factory=row_factory, **kwargs) + + def _get_metadata(self): + try: + analytics_response = next(self._streaming_result) + self._set_metadata(analytics_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def execute(self) -> Awaitable[None]: + async def _execute(): + return [r async for r in self] + + return asyncio.create_task(_execute()) + + def __aiter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_query() + + return self + + def _get_next_row(self): + if self.done_streaming is True: + return + + row = next(self._streaming_result) + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopAsyncIteration + # this should allow the event loop to pick up something else + return self.serializer.deserialize(row) + + async def __anext__(self): + try: + return await self._loop.run_in_executor(self._tp_executor, self._get_next_row) + except asyncio.QueueEmpty: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls('Unexpected QueueEmpty exception caught when doing Analytics query.') + raise excptn + except StopAsyncIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/acouchbase/asyncio_iops.py b/acouchbase/asyncio_iops.py deleted file mode 100644 index 61cf81cb4..000000000 --- a/acouchbase/asyncio_iops.py +++ /dev/null @@ -1,63 +0,0 @@ -try: - import asyncio -except ImportError: - import trollius as asyncio - -from couchbase.iops.base import ( - IOEvent, TimerEvent, LCB_READ_EVENT, LCB_WRITE_EVENT, - LCB_RW_EVENT, PYCBC_EVACTION_WATCH, PYCBC_EVACTION_UNWATCH -) - - -class AsyncioTimer(TimerEvent): - def __init__(self): - super(AsyncioTimer, self).__init__() - self._ashandle = None - - def cancel(self): - if self._ashandle: - self._ashandle.cancel() - self._ashandle = None - - def schedule(self, loop, usec): - sec = float(usec) / 1000000.0 - self._ashandle = loop.call_later(sec, self.ready, 0) - - -class IOPS(object): - def __init__(self, evloop = None): - if evloop is None: - evloop = asyncio.get_event_loop() - self.loop = evloop - - def update_event(self, event, action, flags): - if action == PYCBC_EVACTION_WATCH: - if flags & LCB_READ_EVENT: - self.loop.add_reader(event.fd, event.ready_r) - else: - self.loop.remove_reader(event.fd) - - if flags & LCB_WRITE_EVENT: - self.loop.add_writer(event.fd, event.ready_w) - else: - self.loop.remove_writer(event.fd) - - elif action == PYCBC_EVACTION_UNWATCH: - if event.flags & LCB_READ_EVENT: - self.loop.remove_reader(event.fd) - if event.flags & LCB_WRITE_EVENT: - self.loop.remove_writer(event.fd) - - def update_timer(self, timer, action, usecs): - timer.cancel() - if action == PYCBC_EVACTION_UNWATCH: - return - elif action == PYCBC_EVACTION_WATCH: - timer.schedule(self.loop, usecs) - - def start_watching(self): - pass - def stop_watching(self): - pass - def timer_event_factory(self): - return AsyncioTimer() diff --git a/acouchbase/binary_collection.py b/acouchbase/binary_collection.py new file mode 100644 index 000000000..6f1cc4ec7 --- /dev/null +++ b/acouchbase/binary_collection.py @@ -0,0 +1,242 @@ + +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Union) + +from couchbase.result import CounterResult, MutationResult + +if TYPE_CHECKING: + from couchbase.options import (AppendOptions, + DecrementOptions, + IncrementOptions, + PrependOptions) + + +class BinaryCollection: + + def __init__(self, collection): + self._collection = collection + + def increment( + self, + key, # type: str + *opts, # type: IncrementOptions + **kwargs, # type: Any + ) -> Awaitable[CounterResult]: + """Increments the ASCII value of the document, specified by the key, by the amount indicated in the delta + option (defaults to 1). + + Args: + key (str): The key of the document to increment. + opts (:class:`~couchbase.options.IncrementOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.IncrementOptions` + + Returns: + Awaitable[:class:`~couchbase.result.CounterResult`]: A future that contains an instance + of :class:`~couchbase.result.CounterResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple increment operation:: + + from couchbase.options import IncrementOptions, SignedInt64 + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().increment('counter-doc', IncrementOptions(initial=SignedInt64(100)) + print(f'Counter value: {res.content}') + + Simple increment operation, change default delta:: + + from couchbase.options import IncrementOptions, DeltaValue + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().increment('counter-doc', IncrementOptions(delta=DeltaValue(5)) + print(f'Counter value: {res.content}') + + """ + return self._collection._increment(key, *opts, **kwargs) + + def decrement( + self, + key, # type: str + *opts, # type: DecrementOptions + **kwargs, # type: Any + ) -> Awaitable[CounterResult]: + """Decrements the ASCII value of the document, specified by the key, by the amount indicated in the delta + option (defaults to 1). + + Args: + key (str): The key of the document to decrement. + opts (:class:`~couchbase.options.DecrementOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.DecrementOptions` + + Returns: + Awaitable[:class:`~couchbase.result.CounterResult`]: A future that contains an instance + of :class:`~couchbase.result.CounterResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple decrement operation:: + + from couchbase.options import DecrementOptions, SignedInt64 + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().decrement('counter-doc', DecrementOptions(initial=SignedInt64(100)) + print(f'Counter value: {res.content}') + + Simple decrement operation, change default delta:: + + from couchbase.options import DecrementOptions, DeltaValue + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().decrement('counter-doc', DecrementOptions(delta=DeltaValue(5)) + print(f'Counter value: {res.content}') + + """ + return self._collection._decrement(key, *opts, **kwargs) + + def append( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: AppendOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """Appends the specified value to the beginning of document of the specified key. + + Args: + key (str): The key of the document to append to. + value (Union[str, bytes, bytearray]): The value to append to the document. + opts (:class:`~couchbase.options.AppendOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.AppendOptions` + + Returns: + Awaitable[:class:`~couchbase.result.MutationResult`]: A future that contains an instance + of :class:`~couchbase.result.MutationResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple append string operation:: + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().append('string-doc', 'XYZ') + + Simple append binary operation:: + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().append('binary-doc', b'XYZ') + + Simple append operation with options:: + + from datetime import timedelta + + from couchbase.options import AppendOptions + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().append('string-doc', + 'XYZ', + AppendOptions(timeout=timedelta(seconds=2))) + + """ + return self._collection._append(key, value, *opts, **kwargs) + + def prepend( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: PrependOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """Prepends the specified value to the beginning of document of the specified key. + + Args: + key (str): The key of the document to prepend to. + value (Union[str, bytes, bytearray]): The value to prepend to the document. + opts (:class:`~couchbase.options.PrependOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.PrependOptions` + + Returns: + Awaitable[:class:`~couchbase.result.MutationResult`]: A future that contains an instance + of :class:`~couchbase.result.MutationResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple prepend string operation:: + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().prepend('string-doc', 'ABC') + + Simple prepend binary operation:: + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().prepend('binary-doc', b'ABC') + + Simple prepend operation with options:: + + from datetime import timedelta + + from couchbase.options import PrependOptions + + # ... other code ... + + collection = bucket.default_collection() + res = await collection.binary().prepend('string-doc', + 'ABC', + PrependOptions(timeout=timedelta(seconds=2))) + + """ + return self._collection._prepend(key, value, *opts, **kwargs) diff --git a/acouchbase/bucket.py b/acouchbase/bucket.py index 533ad6d6b..f392f84ca 100644 --- a/acouchbase/bucket.py +++ b/acouchbase/bucket.py @@ -1,50 +1,261 @@ -try: - import asyncio -except ImportError: - import trollius as asyncio - -from acouchbase.asyncio_iops import IOPS -from couchbase.async.bucket import AsyncBucket -from couchbase.experimental import enabled_or_raise; enabled_or_raise() - - -class Bucket(AsyncBucket): - def __init__(self, *args, **kwargs): - loop = asyncio.get_event_loop() - super(Bucket, self).__init__(IOPS(loop), *args, **kwargs) - self._loop = loop - - cft = asyncio.Future(loop=loop) - def ftresult(err): - if err: - cft.set_exception(err) - else: - cft.set_result(True) - - self._cft = cft - self._conncb = ftresult - - def _meth_factory(meth, name): - def ret(self, *args, **kwargs): - rv = meth(self, *args, **kwargs) - ft = asyncio.Future() - def on_ok(res): - ft.set_result(res) - rv.clear_callbacks() - - def on_err(res, excls, excval, exctb): - err = excls(excval) - ft.set_exception(err) - rv.clear_callbacks() - - rv.set_callbacks(on_ok, on_err) - return ft - - return ret - - locals().update(AsyncBucket._gen_memd_wrappers(_meth_factory)) - - def connect(self): - if not self.connected: - self._connect() - return self._cft +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict) + +from acouchbase.collection import Collection +from acouchbase.logic import AsyncWrapper +from acouchbase.management.collections import CollectionManager +from acouchbase.management.views import ViewIndexManager +from acouchbase.scope import Scope +from acouchbase.views import AsyncViewRequest, ViewQuery +from couchbase.logic.bucket import BucketLogic +from couchbase.result import PingResult, ViewResult + +if TYPE_CHECKING: + from acouchbase.cluster import Cluster + from couchbase.options import PingOptions, ViewOptions + + +class AsyncBucket(BucketLogic): + """Create a Couchbase Bucket instance. + + Exposes the operations which are available to be performed against a bucket. Namely the ability to + access to Collections as well as performing management operations against the bucket. + + Args: + cluster (:class:`~acouchbase.cluster.Cluster`): A :class:`~acouchbase.cluster.Cluster` instance. + bucket_name (str): Name of the bucket. + + Raises: + :class:`~couchbase.exceptions.BucketNotFoundException`: If provided `bucket_name` cannot be found. + + """ + + def __init__(self, cluster, # type: Cluster + bucket_name # type: str + ): + super().__init__(cluster, bucket_name) + self._close_ftr = None + self._connect_ftr = self._open_bucket() + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._cluster.loop + + # def _connect_bucket(self): + # """ + # **INTERNAL** + # Used w/in wrappers if a collection op is called when not connected + # """ + # if self._connect_ftr is not None: + # return + + # if not self._cluster.connected: + # self._connect_ftr = self.loop.create_future() + # ft = self._cluster.on_connect() + # ft.add_done_callback(partial(AsyncWrapper.chain_connect_callbacks, self)) + # else: + # self._connect_ftr = super()._open_or_close_bucket(open_bucket=True) + + @AsyncWrapper.inject_bucket_open_callbacks() + def _open_bucket(self, **kwargs) -> Awaitable: + super()._open_or_close_bucket(open_bucket=True, **kwargs) + + @AsyncWrapper.inject_close_callbacks() + def _close_bucket(self, **kwargs): + super()._open_or_close_bucket(open_bucket=False, **kwargs) + + def on_connect(self) -> Awaitable: + """Returns an awaitable future that indicates connecting to the Couchbase bucket has completed. + + Returns: + Awaitable: An empty future. If a result is provided, connecting to the Couchbase bucket is complete. + Otherwise an exception is raised. + + Raises: + :class:`~couchbase.exceptions.UnAmbiguousTimeoutException`: If an error occured while trying to connect. + """ + if not (self._connect_ftr or self.connected): + self._connect_ftr = self._open_bucket() + self._close_ftr = None + + return self._connect_ftr + + async def close(self) -> None: + """Shuts down this bucket instance. Cleaning up all resources associated with it. + + .. warning:: + Use of this method is almost *always* unnecessary. Bucket resources should be cleaned + up once the bucket instance falls out of scope. However, in some applications tuning resources + is necessary and in those types of applications, this method might be beneficial. + + """ + if self.connected and not self._close_ftr: + self._close_ftr = self._close_bucket() + self._connect_ftr = None + + await self._close_ftr + super()._destroy_connection() + + def default_scope(self + ) -> Scope: + """Creates a :class:`~acouchbase.scope.Scope` instance of the default scope. + + Returns: + :class:`~acouchbase.scope.Scope`: A :class:`~acouchbase.scope.Scope` instance of the default scope. + + """ + return self.scope(Scope.default_name()) + + def scope(self, name # type: str + ) -> Scope: + """Creates a :class:`~acouchbase.scope.Scope` instance of the specified scope. + + Args: + name (str): Name of the scope to reference. + + Returns: + :class:`~acouchbase.scope.Scope`: A :class:`~couchbase.scope.Scope` instance of the specified scope. + + """ + return Scope(self, name) + + def collection(self, collection_name): + """Creates a :class:`~acouchbase.collection.Collection` instance of the specified collection. + + Args: + collection_name (str): Name of the collection to reference. + + Returns: + :class:`~acouchbase.collection.Collection`: A :class:`~acouchbase.collection.Collection` instance of the specified collection. + + """ # noqa: E501 + scope = self.default_scope() + return scope.collection(collection_name) + + def default_collection(self): + """Creates a :class:`~acouchbase.collection.Collection` instance of the default collection. + + Returns: + :class:`~acouchbase.collection.Collection`: A :class:`~acouchbase.collection.Collection` instance of the default collection. + """ # noqa: E501 + scope = self.default_scope() + return scope.collection(Collection.default_name()) + + @AsyncWrapper.inject_cluster_callbacks(PingResult) + def ping(self, + *opts, # type: PingOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[PingResult]: + """Performs a ping operation against the bucket. + + The ping operation pings the services which are specified + (or all services if none are specified). Returns a report which describes the outcome of + the ping operations which were performed. + + Args: + opts (:class:`~couchbase.options.PingOptions`): Optional parameters for this operation. + + Returns: + Awaitable[:class:`~couchbase.result.PingResult`]: A report which describes the outcome of the ping + operations which were performed. + + """ + return super().ping(*opts, **kwargs) + + def view_query(self, + design_doc, # type: str + view_name, # type: str + *view_options, # type: ViewOptions + **kwargs # type: Dict[str, Any] + ) -> ViewResult: + """Executes a View query against the bucket. + + .. note:: + + The query is executed lazily in that it is executed once iteration over the + :class:`~.result.ViewResult` begins. + + .. seealso:: + * :class:`~.management.ViewIndexManager`: for how to manage query indexes + + Args: + design_doc (str): The name of the design document containing the view to execute. + view_name (str): The name of the view to execute. + view_options (:class:`~.options.ViewOptions`): Optional parameters for the view query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~.options.ViewOptions` + + Returns: + :class:`~.result.ViewResult`: An instance of a :class:`~.result.ViewResult` which + provides access to iterate over the query results and access metadata about the query. + + Examples: + Simple view query:: + + from couchbase.management.views import DesignDocumentNamespace + + # ... other code ... + + view_result = bucket.view_query('ddoc-name', + 'view-name', + limit=10, + namespace=DesignDocumentNamespace.DEVELOPMENT) + + async for row in view_result.rows(): + print(f'Found row: {row}') + + """ + request_args = dict(default_serialize=self.default_serializer, + streaming_timeout=self.streaming_timeouts.get('view_timeout', None)) + num_workers = kwargs.pop('num_workers', None) + if num_workers: + request_args['num_workers'] = num_workers + + query = ViewQuery.create_view_query_object(self.name, design_doc, view_name, *view_options, **kwargs) + return ViewResult(AsyncViewRequest.generate_view_request(self.connection, + self.loop, + query.as_encodable(), + **request_args)) + + def collections(self) -> CollectionManager: + """ + Get a :class:`~acouchbase.management.collections.CollectionManager` which can be used to manage the scopes and collections + of this bucket. + + Returns: + :class:`~acouchbase.management.collections.CollectionManager`: A :class:`~couchbase.management.collections.CollectionManager` instance. + """ # noqa: E501 + return CollectionManager(self.connection, self.loop, self.name) + + def view_indexes(self) -> ViewIndexManager: + """ + Get a :class:`~acouchbase.management.views.ViewIndexManager` which can be used to manage the view design documents + and views of this bucket. + + Returns: + :class:`~acouchbase.management.views.ViewIndexManager`: A :class:`~couchbase.management.views.ViewIndexManager` instance. + """ # noqa: E501 + return ViewIndexManager(self.connection, self.loop, self.name) + + +Bucket = AsyncBucket diff --git a/acouchbase/cluster.py b/acouchbase/cluster.py new file mode 100644 index 000000000..c9bac44fc --- /dev/null +++ b/acouchbase/cluster.py @@ -0,0 +1,873 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from asyncio import AbstractEventLoop +from datetime import timedelta +from time import perf_counter +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict) + +from acouchbase import get_event_loop +from acouchbase.analytics import AnalyticsQuery, AsyncAnalyticsRequest +from acouchbase.bucket import AsyncBucket +from acouchbase.logic import AsyncWrapper +from acouchbase.management.analytics import AnalyticsIndexManager +from acouchbase.management.buckets import BucketManager +from acouchbase.management.eventing import EventingFunctionManager +from acouchbase.management.queries import QueryIndexManager +from acouchbase.management.search import SearchIndexManager +from acouchbase.management.users import UserManager +from acouchbase.n1ql import AsyncN1QLRequest, N1QLQuery +from acouchbase.search import AsyncFullTextSearchRequest, SearchQueryBuilder +from acouchbase.transactions import Transactions +from couchbase.diagnostics import ClusterState, ServiceType +from couchbase.exceptions import UnAmbiguousTimeoutException +from couchbase.logic.cluster import ClusterLogic +from couchbase.options import PingOptions, forward_args +from couchbase.result import (AnalyticsResult, + ClusterInfoResult, + DiagnosticsResult, + PingResult, + QueryResult, + SearchResult) + +if TYPE_CHECKING: + from couchbase.options import (AnalyticsOptions, + ClusterOptions, + DiagnosticsOptions, + QueryOptions, + SearchOptions, + WaitUntilReadyOptions) + from couchbase.search import SearchQuery, SearchRequest + + +class AsyncCluster(ClusterLogic): + """Create a Couchbase Cluster instance. + + The cluster instance exposes the operations which are available to be performed against a cluster. + + .. note:: + Although creating an instance of :class:`.AsyncCluster` is allowed, it is recommended to + use the AsyncCluster's static :meth:`.AsyncCluster.connect` method. See :meth:`.AsyncCluster.connect` for + connect for examples. + + Args: + connstr (str): + The connection string to use for connecting to the cluster. + This is a URI-like string allowing specifying multiple hosts. + + The format of the connection string is the *scheme* + (``couchbase`` for normal connections, ``couchbases`` for + SSL enabled connections); a list of one or more *hostnames* + delimited by commas + options (:class:`~couchbase.options.ClusterOptions`): Global options to set for the cluster. + Some operations allow the global options to be overriden by passing in options to the + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + overrride provided :class:`~couchbase.options.ClusterOptions` + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If no :class:`~couchbase.auth.Authenticator` + is provided. Also raised if an invalid `ClusterOption` is provided. + :class:`~couchbase.exceptions.AuthenticationException`: If provided :class:`~couchbase.auth.Authenticator` + has incorrect credentials. + + """ + + def __init__(self, + connstr, # type: str + *options, # type: ClusterOptions + **kwargs, # type: Dict[str, Any] + ) -> AsyncCluster: + + self._loop = self._get_loop(kwargs.pop("loop", None)) + super().__init__(connstr, *options, **kwargs) + + self._close_ftr = None + self._connect_ftr = self._connect() + + @property + def loop(self) -> AbstractEventLoop: + """ + **INTERNAL** + """ + return self._loop + + def _get_loop(self, loop=None) -> AbstractEventLoop: + if not loop: + loop = get_event_loop() + + if not loop.is_running(): + raise RuntimeError("Event loop is not running.") + + return loop + + @property + def transactions(self) -> Transactions: + """ + :class:`~acouchbase.transactions.Transactions`: A Transactions instance which can be used to + perform transactions on this cluster. + """ + if not self._transactions: + self._transactions = Transactions(self, self._transaction_config) + return self._transactions + + @AsyncWrapper.inject_connection_callbacks() + def _connect(self, **kwargs) -> Awaitable: + """ + **INTERNAL** + """ + super()._connect_cluster(**kwargs) + + def on_connect(self) -> Awaitable: + """Returns an awaitable future that indicates connecting to the Couchbase cluster has completed. + + .. note:: + It is recommended to use the AsyncCluster's static :meth:`.AsyncCluster.connect` method. + See :meth:`.AsyncCluster.connect` for connect for examples. + + Returns: + Awaitable: An empty future. If a result is provided, connecting to the Couchbase cluster is complete. + Otherwise an exception is raised. + + Raises: + :class:`~couchbase.exceptions.UnAmbiguousTimeoutException`: If an error occured while trying to connect. + """ + if not (self._connect_ftr or self.connected): + self._connect_ftr = self._connect() + self._close_ftr = None + + return self._connect_ftr + + @AsyncWrapper.inject_close_callbacks() + def _close(self, **kwargs) -> Awaitable: + """ + **INTERNAL** + """ + super()._close_cluster(**kwargs) + + async def close(self) -> None: + """Shuts down this cluster instance. Cleaning up all resources associated with it. + + .. warning:: + Use of this method is almost *always* unnecessary. Cluster resources should be cleaned + up once the cluster instance falls out of scope. However, in some applications tuning resources + is necessary and in those types of applications, this method might be beneficial. + + """ + if self.connected and not self._close_ftr: + self._close_ftr = self._close() + self._connect_ftr = None + + await self._close_ftr + super()._destroy_connection() + + def bucket(self, bucket_name) -> AsyncBucket: + """Creates a Bucket instance to a specific bucket. + + .. seealso:: + :class:`.bucket.AsyncBucket` + + Args: + bucket_name (str): Name of the bucket to reference + + Returns: + :class:`~acouchbase.bucket.AsyncBucket`: A bucket instance + + Raises: + :class:`~couchbase.exceptions.BucketNotFoundException`: If provided `bucket_name` cannot + be found. + + """ + return AsyncBucket(self, bucket_name) + + def cluster_info(self) -> Awaitable[ClusterInfoResult]: + """Retrieve the Couchbase cluster information + + .. note:: + If using Couchbase Server version < 6.6, a bucket *must* be opened prior to calling + `cluster.cluster_info()`. If a bucket is not opened a + :class:`~couchbase.exceptions.ServiceUnavailableException` will be raised. + + + Returns: + Awaitable[:class:`~couchbase.result.ClusterInfoResult`]: Information about the connected cluster. + + Raises: + RuntimeError: If called prior to the cluster being connected. + :class:`~couchbase.exceptions.ServiceUnavailableException`: If called prior to connecting + to a bucket if using server version < 6.6. + + """ + if not self.connected: + # @TODO(jc): chain?? + raise RuntimeError( + "Cluster is not connected, cannot get cluster info. " + "Use await cluster.on_connect() to connect a cluster.") + + return self._get_cluster_info() + + @AsyncWrapper.inject_cluster_callbacks(ClusterInfoResult, set_cluster_info=True) + def _get_cluster_info(self, **kwargs) -> Awaitable[ClusterInfoResult]: + """**INTERNAL** + + use cluster_info() + + Returns: + Awaitable: _description_ + """ + super()._get_cluster_info(**kwargs) + + @AsyncWrapper.inject_cluster_callbacks(PingResult, chain_connection=True) + def ping(self, + *opts, # type: PingOptions + **kwargs # type: Any + ) -> Awaitable[PingResult]: + """Performs a ping operation against the cluster. + + The ping operation pings the services which are specified + (or all services if none are specified). Returns a report which describes the outcome of + the ping operations which were performed. + + Args: + opts (:class:`~couchbase.options.PingOptions`): Optional parameters for this operation. + + Returns: + Awaitable[:class:`~couchbase.result.PingResult`]: A report which describes the outcome of the + ping operations which were performed. + + """ + return super().ping(*opts, **kwargs) + + @AsyncWrapper.inject_cluster_callbacks(DiagnosticsResult, chain_connection=True) + def diagnostics(self, + *opts, # type: DiagnosticsOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[DiagnosticsResult]: + """Performs a diagnostic operation against the cluster. + + The diagnostic operations returns a report about the current active connections with the cluster. + Includes information about remote and local addresses, last activity, and other diagnostics information. + + Args: + opts (:class:`~couchbase.options.DiagnosticsOptions`): Optional parameters for this operation. + + Returns: + Awaitable[:class:`~couchbase.result.DiagnosticsResult`]: A report which describes current active + connections with the cluster. + + """ + return super().diagnostics(*opts, **kwargs) + + async def wait_until_ready(self, + timeout, # type: timedelta + *opts, # type: WaitUntilReadyOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + """Wait until the cluster is ready for use. + + Check the current connections to see if the desired state has been reached. If not, + perform a ping against the specified services. The ping operation will be performed + repeatedly with a slight delay in between until the specified timeout has been reached + or the cluster is ready for use, whichever comes first. + + .. seealso:: + * :class:`~couchbase.diagnostics.ServiceType` + * :class:`~couchbase.diagnostics.ClusterState` + + Args: + timeout (timedelta): Amount of time to wait for cluster to be ready before a + :class:`~couchbase.exceptions.UnAmbiguousTimeoutException` is raised. + opts (:class:`~couchbase.options.WaitUntilReadyOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.WaitUntilReadyOptions` + + Raises: + :class:`~couchbase.exceptions.UnAmbiguousTimeoutException`: If the specified timeout is reached prior to + the cluster being ready for use. + + Example: + + Wait until the cluster is ready to use KV and query services:: + + from acouchbase.cluster import Cluster + from couchbase.auth import PasswordAuthenticator + from couchbase.diagnostics import ServiceType + from couchbase.options import WaitUntilReadyOptions + + auth = PasswordAuthenticator('username', 'password') + cluster = Cluster.connect('couchbase://localhost', ClusterOptions(auth)) + + await cluster.wait_until_ready(timedelta(seconds=3), + WaitUntilReadyOptions(service_types=[ServiceType.KeyValue, ServiceType.Query])) + + """ + final_args = forward_args(kwargs, *opts) + service_types = final_args.get("service_types", None) + if not service_types: + service_types = [ServiceType(st.value) for st in ServiceType] + + desired_state = final_args.get("desired_state", ClusterState.Online) + service_types_set = set(map(lambda st: st.value if isinstance(st, ServiceType) else st, service_types)) + + # @TODO: handle units + timeout_millis = timeout.total_seconds() * 1000 + + interval_millis = float(50) + start = perf_counter() + time_left = timeout_millis + while True: + + diag_res = await self.diagnostics() + endpoint_svc_types = set(map(lambda st: st.value, diag_res.endpoints.keys())) + if not endpoint_svc_types.issuperset(service_types_set): + await self.ping(PingOptions(service_types=service_types)) + diag_res = await self.diagnostics() + + if diag_res.state == desired_state: + break + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise UnAmbiguousTimeoutException(message="Desired state not found.") + + await asyncio.sleep(interval_millis / 1000) + + def query(self, + statement, # type: str + *options, # type: QueryOptions + **kwargs # type: Dict[str, Any] + ) -> QueryResult: + """Executes a N1QL query against the cluster. + + .. note:: + The query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.QueryResult` begins. + + .. seealso:: + * :class:`~acouchbase.management.queries.QueryIndexManager`: for how to manage query indexes + * :meth:`~acouchbase.scope.AsyncScope.query`: For how to execute scope-level queries. + + Args: + statement (str): The N1QL statement to execute. + options (:class:`~couchbase.options.QueryOptions`): Optional parameters for the query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.QueryOptions` + + Returns: + :class:`~couchbase.result.QueryResult`: An instance of a :class:`~couchbase.result.QueryResult` which + provides access to iterate over the query results and access metadata and metrics about the query. + + Examples: + Simple query:: + + q_res = cluster.query('SELECT * FROM `travel-sample` WHERE country LIKE 'United%' LIMIT 2;') + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple query with positional parameters:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = cluster.query(q_str, QueryOptions(positional_parameters=['United%', 5])) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple query with named parameters:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = cluster.query(q_str, QueryOptions(named_parameters={'country': 'United%', 'lim':2})) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Retrieve metadata and/or metrics from query:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = cluster.query(q_str, QueryOptions(metrics=True)) + async for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'Query metadata: {q_res.metadata()}') + print(f'Query metrics: {q_res.metadata().metrics()}') + + """ + + request_args = dict(default_serialize=self.default_serializer, + streaming_timeout=self.streaming_timeouts.get('query_timeout', None)) + num_workers = kwargs.pop('num_workers', None) + if num_workers: + request_args['num_workers'] = num_workers + query = N1QLQuery.create_query_object(statement, *options, **kwargs) + return QueryResult(AsyncN1QLRequest.generate_n1ql_request(self.connection, + self.loop, + query.params, + **request_args)) + + def analytics_query(self, # type: Cluster + statement, # type: str + *options, # type: AnalyticsOptions + **kwargs, # type: Dict[str, Any] + ) -> AnalyticsResult: + """Executes an analaytics query against the cluster. + + .. note:: + The analytics query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.AnalyticsResult` begins. + + .. seealso:: + * :class:`~acouchbase.management.analytics.AnalyticsIndexManager`: for how to manage analytics dataverses, datasets, indexes and links. + * :meth:`~acouchbase.scope.AsyncScope.analytics_query`: for how to execute scope-level analytics queries + + Args: + statement (str): The analytics SQL++ statement to execute. + options (:class:`~couchbase.options.AnalyticsOptions`): Optional parameters for the analytics query + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.AnalyticsOptions` + + Returns: + :class:`~couchbase.result.AnalyticsResult`: An instance of a + :class:`~couchbase.result.AnalyticsResult` which provides access to iterate over the analytics + query results and access metadata and metrics about the analytics query. + + Examples: + .. note:: + Be sure to setup the necessary dataverse(s), dataset(s) for your analytics queries. + See :analytics_intro:`Analytics Introduction <>` in Couchbase Server docs. + + Simple analytics query:: + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = cluster.analytics_query(q_str) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple analytics query with positional parameters:: + + from couchbase.options import AnalyticsOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = cluster.analytics_query(q_str, AnalyticsOptions(positional_parameters=['United%', 5])) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple analytics query with named parameters:: + + from couchbase.options import AnalyticsOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = cluster.analytics_query(q_str, + AnalyticsOptions(named_parameters={'country': 'United%', 'lim':2})) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Retrieve metadata and/or metrics from analytics query:: + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = cluster.analytics_query(q_str) + async for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'Analytics query metadata: {q_res.metadata()}') + print(f'Analytics query metrics: {q_res.metadata().metrics()}') + + """ # noqa: E501 + + request_args = dict(default_serialize=self.default_serializer, + streaming_timeout=self.streaming_timeouts.get('analytics_timeout', None)) + num_workers = kwargs.pop('num_workers', None) + if num_workers: + request_args['num_workers'] = num_workers + query = AnalyticsQuery.create_query_object(statement, *options, **kwargs) + return AnalyticsResult(AsyncAnalyticsRequest.generate_analytics_request(self.connection, + self.loop, + query.params, + **request_args)) + + def search_query(self, + index, # type: str + query, # type: SearchQuery + *options, # type: SearchOptions + **kwargs # type: Dict[str, Any] + ) -> SearchResult: + """Executes an search query against the cluster. + + .. note:: + The search query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.SearchResult` begins. + + .. seealso:: + * :class:`~acouchbase.management.search.SearchIndexManager`: for how to manage search indexes. + * :meth:`~acouchbase.scope.AsyncScope.search_query`: for how to execute scope-level search queries + + Args: + index (str): Name of the search query to use. + query (:class:`~couchbase.search.SearchQuery`): Type of search query to perform. + options (:class:`~couchbase.options.SearchOptions`): Optional parameters for the search query + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.SearchOptions` + + Returns: + :class:`~couchbase.result.SearchResult`: An instance of a + :class:`~couchbase.result.SearchResult` which provides access to iterate over the search + query results and access metadata and metrics about the search query. + + Examples: + + .. note:: + Be sure to create a search index prior to executing search queries. Also, if an application + desires to utilize search row locations, highlighting, etc. make sure the search index is + setup appropriately. See :search_create_idx:`Creating Indexes <>` in Couchbase Server docs. + + Simple search query:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + query = search.TermQuery('home') + q_res = cluster.search_query('travel-sample-index', + query, + SearchOptions(limit=10)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + + + Simple search query with facets:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + facet_name = 'activity' + facet = search.TermFacet('activity') + query = search.TermQuery('home') + q_res = cluster.search_query('travel-sample-index', + query, + SearchOptions(limit=10, facets={facet_name: facet})) + + async for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'facets: {q_res.facets()}') + + + Simple search query with fields and locations:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + search_fields = ['name', 'activity'] + query = search.TermQuery('home') + q_res = cluster.search_query('travel-sample-index', + query, + SearchOptions(limit=10, + include_locations=True, + fields=search_fields)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + print(f'Fields: {row.fields}') + print(f'Locations: {row.locations}') + + """ + request_args = dict(default_serialize=self.default_serializer, + streaming_timeout=self.streaming_timeouts.get('search_timeout', None)) + num_workers = kwargs.pop('num_workers', None) + if num_workers: + request_args['num_workers'] = num_workers + query = SearchQueryBuilder.create_search_query_object(index, query, *options, **kwargs) + return SearchResult(AsyncFullTextSearchRequest.generate_search_request(self.connection, + self.loop, + query.as_encodable(), + **request_args)) + + def search(self, + index, # type: str + request, # type: SearchRequest + *options, # type: SearchOptions + **kwargs, # type: Dict[str, Any] + ) -> SearchResult: + """Executes an search against the cluster. + + .. note:: + The search is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.SearchResult` begins. + + .. seealso:: + * :class:`~couchbase.management.search.SearchIndexManager`: for how to manage search indexes. + * :meth:`~acouchbase.scope.AsyncScope.search`: for how to execute scope-level search + + Args: + index (str): Name of the search index to use. + request (:class:`~couchbase.search.SearchRequest`): Type of search request to perform. + options (:class:`~couchbase.options.SearchOptions`): Optional parameters for the search query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.SearchOptions` + + Returns: + :class:`~couchbase.result.SearchResult`: An instance of a + :class:`~couchbase.result.SearchResult` which provides access to iterate over the search + query results and access metadata and metrics about the search query. + + Examples: + + .. note:: + Be sure to create a search index prior to executing a search. Also, if an application + desires to utilize search row locations, highlighting, etc. make sure the search index is + setup appropriately. See :search_create_idx:`Creating Indexes <>` in Couchbase Server docs. + + Simple search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + request = search.SearchRequest.create(search.TermQuery('home')) + q_res = cluster.search('travel-sample-index', + request, + SearchOptions(limit=10)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple vector search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + from couchbase.vector_search import VectorQuery, VectorSearch + + # ... other code ... + + # NOTE: the vector is expected to be type List[float], set the vector to the appropriate value, this is an example. + vector = [-0.014653487130999565, -0.008658270351588726, 0.017129190266132355, -0.015563474968075752] + request = search.SearchRequest.create(VectorSearch.from_vector_query(VectorQuery('vector_field', vector))) + q_res = cluster.search('travel-sample-vector-index', + request, + SearchOptions(limit=10)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + + Combine search and vector search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + from couchbase.vector_search import VectorQuery, VectorSearch + + # ... other code ... + + # NOTE: the vector is expected to be type List[float], set the vector to the appropriate value, this is an example. + vector_search = VectorSearch.from_vector_query(VectorQuery('vector_field', [-0.014653487130999565, + -0.008658270351588726, + 0.017129190266132355, + -0.015563474968075752])) + request = search.SearchRequest.create(search.MatchAllQuery()).with_vector_search(vector_search) + q_res = cluster.search('travel-sample-vector-index', + request, + SearchOptions(limit=10)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + """ # noqa: E501 + request_args = dict(default_serialize=self.default_serializer, + streaming_timeout=self.streaming_timeouts.get('search_timeout', None)) + num_workers = kwargs.pop('num_workers', None) + if num_workers: + request_args['num_workers'] = num_workers + query = SearchQueryBuilder.create_search_query_from_request(index, request, *options, **kwargs) + return SearchResult(AsyncFullTextSearchRequest.generate_search_request(self.connection, + self.loop, + query.as_encodable(), + **request_args)) + + def buckets(self) -> BucketManager: + """ + Get a :class:`~acouchbase.management.buckets.BucketManager` which can be used to manage the buckets + of this cluster. + + Returns: + :class:`~acouchbase.management.buckets.BucketManager`: A :class:`~acouchbase.management.buckets.BucketManager` instance. + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return BucketManager(self.connection, self.loop) + + def users(self) -> UserManager: + """ + Get a :class:`~acouchbase.management.users.UserManager` which can be used to manage the users + of this cluster. + + Returns: + :class:`~acouchbase.management.users.UserManager`: A :class:`~couchbase.management.users.UserManager` instance. + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return UserManager(self.connection, self.loop) + + def query_indexes(self) -> QueryIndexManager: + """ + Get a :class:`~acouchbase.management.queries.QueryIndexManager` which can be used to manage the query + indexes of this cluster. + + Returns: + :class:`~acouchbase.management.queries.QueryIndexManager`: A :class:`~acouchbase.management.queries.QueryIndexManager` instance. + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return QueryIndexManager(self.connection, self.loop) + + def analytics_indexes(self) -> AnalyticsIndexManager: + """ + Get a :class:`~acouchbase.management.analytics.AnalyticsIndexManager` which can be used to manage the analytics + dataverses, dataset, indexes and links of this cluster. + + Returns: + :class:`~acouchbase.management.analytics.AnalyticsIndexManager`: An :class:`~acouchbase.management.analytics.AnalyticsIndexManager` instance. + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return AnalyticsIndexManager(self.connection, self.loop) + + def search_indexes(self) -> SearchIndexManager: + """ + Get a :class:`~acouchbase.management.search.SearchIndexManager` which can be used to manage the search + indexes of this cluster. + + Returns: + :class:`~acouchbase.management.search.SearchIndexManager`: A :class:`~acouchbase.management.search.SearchIndexManager` instance. + + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return SearchIndexManager(self.connection, self.loop) + + def eventing_functions(self) -> EventingFunctionManager: + """ + Get a :class:`~acouchbase.management.eventing.EventingFunctionManager` which can be used to manage the + eventing functions of this cluster. + + .. note:: + Eventing function management is an **uncommitted** API that is unlikely to change, + but may still change as final consensus on its behavior has not yet been reached. + + Returns: + :class:`~acouchbase.management.eventing.EventingFunctionManager`: An :class:`~acouchbase.management.eventing.EventingFunctionManager` instance. + + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return EventingFunctionManager(self.connection, self.loop) + + @staticmethod + async def connect(connstr, # type: str + *options, # type: ClusterOptions + **kwargs, # type: Dict[str, Any] + ) -> AsyncCluster: + """Create a Couchbase Cluster and connect + + Args: + connstr (str): + The connection string to use for connecting to the cluster. + This is a URI-like string allowing specifying multiple hosts. + + The format of the connection string is the *scheme* + (``couchbase`` for normal connections, ``couchbases`` for + SSL enabled connections); a list of one or more *hostnames* + delimited by commas + options (:class:`~couchbase.options.ClusterOptions`): Global options to set for the cluster. + Some operations allow the global options to be overriden by passing in options to the + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + overrride provided :class:`~couchbase.options.ClusterOptions` + + Returns: + :class:`.AsyncCluster`: If successful, a connect Couchbase Cluster instance. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If no :class:`~couchbase.auth.Authenticator` + is provided. Also raised if an invalid `ClusterOption` is provided. + :class:`~couchbase.exceptions.AuthenticationException`: If provided :class:`~couchbase.auth.Authenticator` + has incorrect credentials. + + + Examples: + Initialize cluster using default options:: + + from acouchbase.cluster import Cluster + from couchbase.auth import PasswordAuthenticator + from couchbase.options import ClusterOptions + + auth = PasswordAuthenticator('username', 'password') + cluster = await Cluster.connect('couchbase://localhost', ClusterOptions(auth)) + + Connect using SSL:: + + from acouchbase.cluster import Cluster + from couchbase.auth import PasswordAuthenticator + from couchbase.options import ClusterOptions + + auth = PasswordAuthenticator('username', 'password', cert_path='/path/to/cert') + cluster = await Cluster.connect('couchbases://localhost', ClusterOptions(auth)) + + Initialize cluster using with global timeout options:: + + from datetime import timedelta + + from acouchbase.cluster import Cluster + from couchbase.auth import PasswordAuthenticator + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + + auth = PasswordAuthenticator('username', 'password') + timeout_opts = ClusterTimeoutOptions(kv_timeout=timedelta(seconds=10), + query_timeout=timedelta(seconds=120)) + cluster = await Cluster.connect('couchbase://localhost', + ClusterOptions(auth, timeout_options=timeout_opts)) + + """ + cluster = AsyncCluster(connstr, *options, **kwargs) + await cluster.on_connect() + return cluster + + +Cluster = AsyncCluster diff --git a/acouchbase/collection.py b/acouchbase/collection.py new file mode 100644 index 000000000..c5d1869d2 --- /dev/null +++ b/acouchbase/collection.py @@ -0,0 +1,1344 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict, + Iterable, + Union) + +from acouchbase.binary_collection import BinaryCollection +from acouchbase.datastructures import (CouchbaseList, + CouchbaseMap, + CouchbaseQueue, + CouchbaseSet) +from acouchbase.kv_range_scan import AsyncRangeScanRequest +from acouchbase.logic import AsyncWrapper +from acouchbase.management.queries import CollectionQueryIndexManager +from couchbase.logic.collection import CollectionLogic +from couchbase.options import forward_args +from couchbase.result import (CounterResult, + ExistsResult, + GetReplicaResult, + GetResult, + LookupInReplicaResult, + LookupInResult, + MutateInResult, + MutationResult, + ScanResultIterable) + +if TYPE_CHECKING: + from datetime import timedelta + + from couchbase._utils import JSONType + from couchbase.kv_range_scan import ScanType + from couchbase.options import (AppendOptions, + DecrementOptions, + ExistsOptions, + GetAllReplicasOptions, + GetAndLockOptions, + GetAndTouchOptions, + GetAnyReplicaOptions, + GetOptions, + IncrementOptions, + InsertOptions, + LookupInAllReplicasOptions, + LookupInAnyReplicaOptions, + LookupInOptions, + MutateInOptions, + PrependOptions, + RemoveOptions, + ReplaceOptions, + ScanOptions, + TouchOptions, + UnlockOptions, + UpsertOptions) + from couchbase.subdocument import Spec + + +class AsyncCollection(CollectionLogic): + + def __init__(self, scope, name): + super().__init__(scope, name) + self._loop = scope.loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + def get(self, + key, # type: str + *opts, # type: GetOptions + **kwargs, # type: Any + ) -> Awaitable[GetResult]: + """Retrieves the value of a document from the collection. + + Args: + key (str): The key for the document to retrieve. + opts (:class:`~couchbase.options.GetOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetOptions` + + Returns: + Awaitable[:class:`~couchbase.result.GetResult`]: A future that contains an instance + of :class:`~couchbase.result.GetResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple get operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + res = await collection.get('airline_10') + print(f'Document value: {res.content_as[dict]}') + + + Simple get operation with options:: + + from datetime import timedelta + from couchbase.options import GetOptions + + # ... other code ... + + res = await collection.get('airline_10', GetOptions(timeout=timedelta(seconds=2))) + print(f'Document value: {res.content_as[dict]}') + + """ + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_internal(key, **final_args) + + @AsyncWrapper.inject_callbacks_and_decode(GetResult) + def _get_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Awaitable[GetResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`AsyncCollection.get` instead. + """ + super().get(key, **kwargs) + + def get_any_replica(self, + key, # type: str + *opts, # type: GetAnyReplicaOptions + **kwargs, # type: Dict[str, Any] + ) -> Awaitable[GetReplicaResult]: + """Retrieves the value of a document from the collection leveraging both active and all available replicas returning + the first available. + + Args: + key (str): The key for the document to retrieve. + opts (:class:`~couchbase.options.GetAnyReplicaOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAnyReplicaOptions` + + Returns: + :class:`~couchbase.result.GetReplicaResult`: An instance of :class:`~couchbase.result.GetReplicaResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentUnretrievableException`: If the key provided does not exist + on the server. + + Examples: + + Simple get_any_replica operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + res = await collection.get_any_replica('airline_10') + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + + + Simple get_any_replica operation with options:: + + from datetime import timedelta + from couchbase.options import GetAnyReplicaOptions + + # ... other code ... + + res = await collection.get_any_replica('airline_10', + GetAnyReplicaOptions(timeout=timedelta(seconds=5))) + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + + """ # noqa: E501 + + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_any_replica_internal(key, **final_args) + + @AsyncWrapper.inject_callbacks_and_decode(GetReplicaResult) + def _get_any_replica_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Awaitable[GetReplicaResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`AsyncCollection.get_any_replica` instead. + """ + super().get_any_replica(key, **kwargs) + + def get_all_replicas(self, + key, # type: str + *opts, # type: GetAllReplicasOptions + **kwargs, # type: Dict[str, Any] + ) -> Awaitable[Iterable[GetReplicaResult]]: + """Retrieves the value of a document from the collection returning both active and all available replicas. + + Args: + key (str): The key for the document to retrieve. + opts (:class:`~couchbase.options.GetAllReplicasOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAllReplicasOptions` + + Returns: + Iterable[:class:`~couchbase.result.GetReplicaResult`]: A stream of + :class:`~couchbase.result.GetReplicaResult` representing both active and replicas of the document retrieved. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple get_all_replicas operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + result = await collection.get_all_replicas('airline_10') + for res in results: + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + + + Simple get_all_replicas operation with options:: + + from datetime import timedelta + from couchbase.options import GetAllReplicasOptions + + # ... other code ... + + result = await collection.get_all_replicas('airline_10', + GetAllReplicasOptions(timeout=timedelta(seconds=10))) + for res in result: + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + + Stream get_all_replicas results:: + + from datetime import timedelta + from couchbase.options import GetAllReplicasOptions + + # ... other code ... + + result = await collection.get_all_replicas('airline_10', + GetAllReplicasOptions(timeout=timedelta(seconds=10))) + while True: + try: + res = next(result) + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + except StopIteration: + print('Done streaming replicas.') + break + + """ + + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_all_replicas_internal(key, **final_args) + + @AsyncWrapper.inject_callbacks_and_decode(GetReplicaResult) + def _get_all_replicas_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Awaitable[Iterable[GetReplicaResult]]: + """ **Internal Operation** + + Internal use only. Use :meth:`AsyncCollection.get_all_replicas` instead. + """ + # return super().get_all_replicas(key, **kwargs) + super().get_all_replicas(key, **kwargs) + + @AsyncWrapper.inject_callbacks(ExistsResult) + def exists( + self, + key, # type: str + *opts, # type: ExistsOptions + **kwargs, # type: Any + ) -> Awaitable[ExistsResult]: + """Checks whether a specific document exists or not. + + Args: + key (str): The key for the document to check existence. + opts (:class:`~couchbase.options.ExistsOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.ExistsOptions` + + Returns: + Awaitable[:class:`~couchbase.result.ExistsResult`]: A future that contains an instance + of :class:`~couchbase.result.ExistsResult` if successful. + + Examples: + + Simple exists operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_10' + res = await collection.exists(key) + print(f'Document w/ key - {key} {"exists" if res.exists else "does not exist"}') + + + Simple exists operation with options:: + + from datetime import timedelta + from couchbase.options import ExistsOptions + + # ... other code ... + + key = 'airline_10' + res = await collection.exists(key, ExistsOptions(timeout=timedelta(seconds=2))) + print(f'Document w/ key - {key} {"exists" if res.exists else "does not exist"}') + + """ + super().exists(key, *opts, **kwargs) + + @AsyncWrapper.inject_callbacks(MutationResult) + def insert( + self, # type: "Collection" + key, # type: str + value, # type: JSONType + *opts, # type: InsertOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """Inserts a new document to the collection, failing if the document already exists. + + Args: + key (str): Document key to insert. + value (JSONType): The value of the document to insert. + opts (:class:`~couchbase.options.InsertOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.InsertOptions` + + Returns: + Awaitable[:class:`~couchbase.result.MutationResult`]: A future that contains an instance + of :class:`~couchbase.result.MutationResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentExistsException`: If the document already exists on the + server. + + Examples: + + Simple insert operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_8091' + airline = { + "type": "airline", + "id": 8091, + "callsign": "CBS", + "iata": None, + "icao": None, + "name": "Couchbase Airways", + } + res = await collection.insert(key, doc) + + + Simple insert operation with options:: + + from couchbase.durability import DurabilityLevel, ServerDurability + from couchbase.options import InsertOptions + + # ... other code ... + + key = 'airline_8091' + airline = { + "type": "airline", + "id": 8091, + "callsign": "CBS", + "iata": None, + "icao": None, + "name": "Couchbase Airways", + } + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = await collection.insert(key, doc, InsertOptions(durability=durability)) + + """ + super().insert(key, value, *opts, **kwargs) + + @AsyncWrapper.inject_callbacks(MutationResult) + def upsert( + self, + key, # type: str + value, # type: JSONType + *opts, # type: UpsertOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """Upserts a document to the collection. This operation succeeds whether or not the document already exists. + + Args: + key (str): Document key to upsert. + value (JSONType): The value of the document to upsert. + opts (:class:`~couchbase.options.UpsertOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.UpsertOptions` + + Returns: + Awaitable[:class:`~couchbase.result.MutationResult`]: A future that contains an instance + of :class:`~couchbase.result.MutationResult` if successful. + + Examples: + + Simple upsert operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_8091' + airline = { + "type": "airline", + "id": 8091, + "callsign": "CBS", + "iata": None, + "icao": None, + "name": "Couchbase Airways", + } + res = await collection.upsert(key, doc) + + + Simple upsert operation with options:: + + from couchbase.durability import DurabilityLevel, ServerDurability + from couchbase.options import UpsertOptions + + # ... other code ... + + key = 'airline_8091' + airline = { + "type": "airline", + "id": 8091, + "callsign": "CBS", + "iata": None, + "icao": None, + "name": "Couchbase Airways", + } + durability = ServerDurability(level=DurabilityLevel.MAJORITY) + res = await collection.upsert(key, doc, InsertOptions(durability=durability)) + + """ + super().upsert(key, value, *opts, **kwargs) + + @AsyncWrapper.inject_callbacks(MutationResult) + def replace(self, + key, # type: str + value, # type: JSONType + *opts, # type: ReplaceOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """Replaces the value of an existing document. Failing if the document does not exist. + + Args: + key (str): Document key to replace. + value (JSONType): The value of the document to replace. + opts (:class:`~couchbase.options.ReplaceOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.ReplaceOptions` + + Returns: + Awaitable[:class:`~couchbase.result.MutationResult`]: A future that contains an instance + of :class:`~couchbase.result.MutationResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the document does not exist on the + server. + + Examples: + + Simple replace operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_8091' + res = await collection.get(key) + content = res.content_as[dict] + airline["name"] = "Couchbase Airways!!" + res = await collection.replace(key, doc) + + + Simple replace operation with options:: + + from couchbase.durability import DurabilityLevel, ServerDurability + from couchbase.options import ReplaceOptions + + # ... other code ... + + key = 'airline_8091' + res = await collection.get(key) + content = res.content_as[dict] + airline["name"] = "Couchbase Airways!!" + durability = ServerDurability(level=DurabilityLevel.MAJORITY) + res = await collection.replace(key, doc, InsertOptions(durability=durability)) + + """ + super().replace(key, value, *opts, **kwargs) + + @AsyncWrapper.inject_callbacks(MutationResult) + def remove(self, + key, # type: str + *opts, # type: RemoveOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """Removes an existing document. Failing if the document does not exist. + + Args: + key (str): Key for the document to remove. + opts (:class:`~couchbase.options.RemoveOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.RemoveOptions` + + Returns: + Awaitable[:class:`~couchbase.result.MutationResult`]: A future that contains an instance + of :class:`~couchbase.result.MutationResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the document does not exist on the + server. + + Examples: + + Simple remove operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + res = collection.remove('airline_10') + + + Simple remove operation with options:: + + from couchbase.durability import DurabilityLevel, ServerDurability + from couchbase.options import RemoveOptions + + # ... other code ... + + durability = ServerDurability(level=DurabilityLevel.MAJORITY) + res = collection.remove('airline_10', RemoveOptions(durability=durability)) + + """ + super().remove(key, *opts, **kwargs) + + @AsyncWrapper.inject_callbacks(MutationResult) + def touch(self, + key, # type: str + expiry, # type: timedelta + *opts, # type: TouchOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """Updates the expiry on an existing document. + + Args: + key (str): Key for the document to touch. + expiry (timedelta): The new expiry for the document. + opts (:class:`~couchbase.options.TouchOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TouchOptions` + + Returns: + Awaitable[:class:`~couchbase.result.MutationResult`]: A future that contains an instance + of :class:`~couchbase.result.MutationResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the document does not exist on the + server. + + Examples: + + Simple touch operation:: + + from datetime import timedelta + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + res = await collection.touch('airline_10', timedelta(seconds=300)) + + + Simple touch operation with options:: + + from datetime import timedelta + + from couchbase.options import TouchOptions + + # ... other code ... + + res = await collection.touch('airline_10', + timedelta(seconds=300), + TouchOptions(timeout=timedelta(seconds=2))) + + """ + super().touch(key, expiry, *opts, **kwargs) + + def get_and_touch(self, + key, # type: str + expiry, # type: timedelta + *opts, # type: GetAndTouchOptions + **kwargs, # type: Any + ) -> Awaitable[GetResult]: + """Retrieves the value of the document and simultanously updates the expiry time for the same document. + + Args: + key (str): The key for the document retrieve and set expiry time. + expiry (timedelta): The new expiry to apply to the document. + opts (:class:`~couchbase.options.GetAndTouchOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAndTouchOptions` + + Returns: + Awaitable[:class:`~couchbase.result.GetResult`]: A future that contains an instance + of :class:`~couchbase.result.GetResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple get and touch operation:: + + from datetime import timedelta + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_10' + res = await collection.get_and_touch(key, timedelta(seconds=20)) + print(f'Document w/ updated expiry: {res.content_as[dict]}') + + + Simple get and touch operation with options:: + + from datetime import timedelta + from couchbase.options import GetAndTouchOptions + + # ... other code ... + + key = 'airline_10' + res = await collection.get_and_touch(key, + timedelta(seconds=20), + GetAndTouchOptions(timeout=timedelta(seconds=2))) + print(f'Document w/ updated expiry: {res.content_as[dict]}') + + """ + # add to kwargs for conversion to int + kwargs["expiry"] = expiry + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_and_touch_internal(key, **final_args) + + @AsyncWrapper.inject_callbacks_and_decode(GetResult) + def _get_and_touch_internal(self, + key, # type: str + **kwargs, # type: Any + ) -> Awaitable[GetResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`AsyncCollection.get_and_touch` instead. + """ + super().get_and_touch(key, **kwargs) + + def get_and_lock( + self, + key, # type: str + lock_time, # type: timedelta + *opts, # type: GetAndLockOptions + **kwargs, # type: Any + ) -> Awaitable[GetResult]: + """Locks a document and retrieves the value of that document at the time it is locked. + + Args: + key (str): The key for the document to lock and retrieve. + lock_time (timedelta): The amount of time to lock the document. + opts (:class:`~couchbase.options.GetAndLockOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAndLockOptions` + + Returns: + Awaitable[:class:`~couchbase.result.GetResult`]: A future that contains an instance + of :class:`~couchbase.result.GetResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple get and lock operation:: + + from datetime import timedelta + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_10' + res = await collection.get_and_lock(key, timedelta(seconds=20)) + print(f'Locked document: {res.content_as[dict]}') + + + Simple get and lock operation with options:: + + from datetime import timedelta + from couchbase.options import GetAndLockOptions + + # ... other code ... + + key = 'airline_10' + res = await collection.get_and_lock(key, + timedelta(seconds=20), + GetAndLockOptions(timeout=timedelta(seconds=2))) + print(f'Locked document: {res.content_as[dict]}') + + """ + # add to kwargs for conversion to int + kwargs["lock_time"] = lock_time + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_and_lock_internal(key, **final_args) + + @AsyncWrapper.inject_callbacks_and_decode(GetResult) + def _get_and_lock_internal(self, + key, # type: str + **kwargs, # type: Any + ) -> Awaitable[GetResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`AsyncCollection.get_and_lock` instead. + """ + super().get_and_lock(key, **kwargs) + + @AsyncWrapper.inject_callbacks(None) + def unlock(self, + key, # type: str + cas, # type: int + *opts, # type: UnlockOptions + **kwargs, # type: Any + ) -> Awaitable[None]: + """Unlocks a previously locked document. + + Args: + key (str): The key for the document to unlock. + cas (int): The CAS of the document, used to validate lock ownership. + opts (:class:`couchbaseoptions.UnlockOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.UnlockOptions` + + Returns: + Awaitable[None]: A future that contains an empty result if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + :class:`~couchbase.exceptions.DocumentLockedException`: If the provided cas is invalid. + + Examples: + + Simple unlock operation:: + + from datetime import timedelta + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_10' + res = await collection.get_and_lock(key, timedelta(seconds=5)) + await collection.unlock(key, res.cas) + # this should be okay once document is unlocked + await collection.upsert(key, res.content_as[dict]) + + """ + super().unlock(key, cas, *opts, **kwargs) + + def lookup_in( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInOptions + **kwargs, # type: Any + ) -> Awaitable[LookupInResult]: + """Performs a lookup-in operation against a document, fetching individual fields or information + about specific fields inside the document value. + + Args: + key (str): The key for the document look in. + spec (Iterable[:class:`~couchbase.subdocument.Spec`]): A list of specs describing the data to fetch + from the document. + opts (:class:`~couchbase.options.LookupInOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.LookupInOptions` + + Returns: + Awaitable[:class:`~couchbase.result.LookupInResult`]: A future that contains an instance + of :class:`~couchbase.result.LookupInResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple look-up in operation:: + + import couchbase.subdocument as SD + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('hotel') + + key = 'hotel_10025' + res = await collection.lookup_in(key, (SD.get("geo"),)) + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + + Simple look-up in operation with options:: + + from datetime import timedelta + + import couchbase.subdocument as SD + from couchbase.options import LookupInOptions + + # ... other code ... + + key = 'hotel_10025' + res = await collection.lookup_in(key, + (SD.get("geo"),), + LookupInOptions(timeout=timedelta(seconds=2))) + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + """ + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_internal(key, spec, **final_args) + + @AsyncWrapper.inject_callbacks_and_decode(LookupInResult) + def _lookup_in_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Any + ) -> Awaitable[LookupInResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`AsyncCollection.lookup_in` instead. + + """ + super().lookup_in(key, spec, **kwargs) + + def lookup_in_any_replica( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInAnyReplicaOptions + **kwargs, # type: Any + ) -> Awaitable[LookupInReplicaResult]: + """Performs a lookup-in operation against a document, fetching individual fields or information + about specific fields inside the document value. It leverages both active and all available replicas + returning the first available + + Args: + key (str): The key for the document look in. + spec (Iterable[:class:`~couchbase.subdocument.Spec`]): A list of specs describing the data to fetch + from the document. + opts (:class:`~couchbase.options.LookupInAnyReplicaOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.LookupInAnyReplicaOptions` + + Returns: + Awaitable[:class:`~couchbase.result.LookupInReplicaResult`]: A future that contains an instance + of :class:`~couchbase.result.LookupInReplicaResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentUnretrievableException`: If the key provided does not exist + on the server. + + Examples: + + Simple lookup_in_any_replica operation:: + + import couchbase.subdocument as SD + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('hotel') + + key = 'hotel_10025' + res = await collection.lookup_in_any_replica(key, (SD.get("geo"),)) + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + + Simple lookup_in_any_replica operation with options:: + + from datetime import timedelta + + import couchbase.subdocument as SD + from couchbase.options import LookupInAnyReplicaOptions + + # ... other code ... + + key = 'hotel_10025' + res = await collection.lookup_in_any_replica(key, + (SD.get("geo"),), + LookupInAnyReplicaOptions(timeout=timedelta(seconds=2))) + print(f'Document is replica: {res.is_replica}') + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + """ + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_any_replica_internal(key, spec, **final_args) + + @AsyncWrapper.inject_callbacks_and_decode(LookupInReplicaResult) + def _lookup_in_any_replica_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Any + ) -> Awaitable[LookupInReplicaResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`AsyncCollection.lookup_in` instead. + + """ + super().lookup_in_any_replica(key, spec, **kwargs) + + def lookup_in_all_replicas( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInAllReplicasOptions + **kwargs, # type: Any + ) -> Awaitable[Iterable[LookupInReplicaResult]]: + """Performs a lookup-in operation against a document, fetching individual fields or information + about specific fields inside the document value, returning results from both active and all available replicas + + Args: + key (str): The key for the document look in. + spec (Iterable[:class:`~couchbase.subdocument.Spec`]): A list of specs describing the data to fetch + from the document. + opts (:class:`~couchbase.options.LookupInAllReplicasOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.LookupInAllReplicasOptions` + + Returns: + Iterable[:class:`~couchbase.result.LookupInReplicaResult`]: A stream of + :class:`~couchbase.result.LookupInReplicaResult` representing both active and replicas of the sub-document + retrieved. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple lookup_in_all_replicas operation:: + + import couchbase.subdocument as SD + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('hotel') + + key = 'hotel_10025' + results = await collection.lookup_in_all_replicas(key, (SD.get("geo"),)) + for res in results: + print(f'Document is replica: {res.is_replica}') + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + + Simple lookup_in_all_replicas operation with options:: + + import couchbase.subdocument as SD + from datetime import timedelta + from couchbase.options import LookupInAllReplicasOptions + + # ... other code ... + + key = 'hotel_10025' + results = await collection.lookup_in_all_replicas(key, + (SD.get("geo"),), + LookupInAllReplicasOptions(timeout=timedelta(seconds=2))) + + for res in results: + print(f'Document is replica: {res.is_replica}') + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + Stream lookup_in_all_replicas results:: + + from datetime import timedelta + from couchbase.options import GetAllReplicasOptions + + # ... other code ... + + key = 'hotel_10025' + results = await collection.lookup_in_all_replicas(key, + (SD.get("geo"),), + LookupInAllReplicasOptions(timeout=timedelta(seconds=2))) + while True: + try: + res = next(results) + print(f'Document is replica: {res.is_replica}') + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + except StopIteration: + print('Done streaming replicas.') + break + + """ # noqa: E501 + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_all_replicas_internal(key, spec, **final_args) + + @AsyncWrapper.inject_callbacks_and_decode(LookupInReplicaResult) + def _lookup_in_all_replicas_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Any + ) -> Awaitable[Iterable[LookupInReplicaResult]]: + """ **Internal Operation** + + Internal use only. Use :meth:`AsyncCollection.lookup_in_all_replicas` instead. + + """ + super().lookup_in_all_replicas(key, spec, **kwargs) + + @AsyncWrapper.inject_callbacks(MutateInResult) + def mutate_in( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: MutateInOptions + **kwargs, # type: Any + ) -> Awaitable[MutateInResult]: + """Performs a mutate-in operation against a document. Allowing atomic modification of specific fields + within a document. Also enables access to document extended-attributes (i.e. xattrs). + + Args: + key (str): The key for the document look in. + spec (Iterable[:class:`~couchbase.subdocument.Spec`]): A list of specs describing the operations to + perform on the document. + opts (:class:`~couchbase.options.MutateInOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.MutateInOptions` + + Returns: + Awaitable[:class:`~couchbase.result.MutateInResult`]: A future that contains an instance + of :class:`~couchbase.result.MutateInResult` if successful. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple mutate-in operation:: + + import couchbase.subdocument as SD + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('hotel') + + key = 'hotel_10025' + res = await collection.mutate_in(key, (SD.replace("city", "New City"),)) + + + Simple mutate-in operation with options:: + + from datetime import timedelta + + import couchbase.subdocument as SD + from couchbase.options import MutateInOptions + + # ... other code ... + + key = 'hotel_10025' + res = await collection.mutate_in(key, + (SD.replace("city", "New City"),), + MutateInOptions(timeout=timedelta(seconds=2))) + + """ + super().mutate_in(key, spec, *opts, **kwargs) + + def scan(self, scan_type, # type: ScanType + *opts, # type: ScanOptions + **kwargs, # type: Dict[str, Any] + ) -> ScanResultIterable: + """Execute a key-value range scan operation from the collection. + + .. note:: + Use this API for low concurrency batch queries where latency is not a critical as the system may have to scan a lot of documents to find the matching documents. + For low latency range queries, it is recommended that you use SQL++ with the necessary indexes. + + Args: + scan_type (:class:`~couchbase.kv_range_scan.ScanType`): Either a :class:`~couchbase.kv_range_scan.RangeScan`, + :class:`~couchbase.kv_range_scan.PrefixScan` or + :class:`~couchbase.kv_range_scan.SamplingScan` instance. + opts (:class:`~couchbase.options.ScanOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.ScanOptions` + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If scan_type is not either a RangeScan or SamplingScan instance. + :class:`~couchbase.exceptions.InvalidArgumentException`: If sort option is provided and is incorrect type. + :class:`~couchbase.exceptions.InvalidArgumentException`: If consistent_with option is provided and is not a + + Returns: + :class:`~couchbase.result.ScanResultIterable`: An instance of :class:`~couchbase.result.ScanResultIterable`. + + Examples: + + Simple range scan operation:: + + from couchbase.kv_range_scan import RangeScan + from couchbase.options import ScanOptions + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + scan_type = RangeScan(ScanTerm('airline-00'), ScanTerm('airline-99')) + scan_iter = collection.scan(scan_type, ScanOptions(ids_only=True)) + + async for res in scan_iter: + print(res) + + + """ # noqa: E501 + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + final_args['transcoder'] = self.default_transcoder + scan_args = super().build_scan_args(scan_type, **final_args) + range_scan_request = AsyncRangeScanRequest(self.loop, **scan_args) + return ScanResultIterable(range_scan_request) + + def binary(self) -> BinaryCollection: + """Creates a BinaryCollection instance, allowing access to various binary operations + possible against a collection. + + .. seealso:: + :class:`~acouchbase.binary_collection.BinaryCollection` + + Returns: + :class:`~acouchbase.binary_collection.BinaryCollection`: A BinaryCollection instance. + + """ + return BinaryCollection(self) + + @AsyncWrapper.inject_callbacks(MutationResult) + def _append( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: AppendOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`acouchbase.BinaryCollection.append` instead. + + """ + super().append(key, value, *opts, **kwargs) + + @AsyncWrapper.inject_callbacks(MutationResult) + def _prepend( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: PrependOptions + **kwargs, # type: Any + ) -> Awaitable[MutationResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`acouchbase.BinaryCollection.prepend` instead. + + """ + super().prepend(key, value, *opts, **kwargs) + + @AsyncWrapper.inject_callbacks(CounterResult) + def _increment( + self, + key, # type: str + *opts, # type: IncrementOptions + **kwargs, # type: Any + ) -> Awaitable[CounterResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`acouchbase.BinaryCollection.increment` instead. + + """ + super().increment(key, *opts, **kwargs) + + @AsyncWrapper.inject_callbacks(CounterResult) + def _decrement( + self, + key, # type: str + *opts, # type: DecrementOptions + **kwargs, # type: Any + ) -> Awaitable[CounterResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`acouchbase.BinaryCollection.decrement` instead. + + """ + super().decrement(key, *opts, **kwargs) + + def couchbase_list(self, key # type: str + ) -> CouchbaseList: + """Returns a CouchbaseList permitting simple list storage in a document. + + .. seealso:: + :class:`~acouchbase.datastructures.CouchbaseList` + + Returns: + :class:`~acouchbase.datastructures.CouchbaseList`: A CouchbaseList instance. + + """ + return CouchbaseList(key, self) + + def couchbase_map(self, key # type: str + ) -> CouchbaseMap: + """Returns a CouchbaseMap permitting simple map storage in a document. + + .. seealso:: + :class:`~acouchbase.datastructures.CouchbaseMap` + + Returns: + :class:`~acouchbase.datastructures.CouchbaseMap`: A CouchbaseMap instance. + + """ + return CouchbaseMap(key, self) + + def couchbase_set(self, key # type: str + ) -> CouchbaseSet: + """Returns a CouchbaseSet permitting simple map storage in a document. + + .. seealso:: + :class:`~acouchbase.datastructures.CouchbaseSet` + + Returns: + :class:`~acouchbase.datastructures.CouchbaseSet`: A CouchbaseSet instance. + + """ + return CouchbaseSet(key, self) + + def couchbase_queue(self, key # type: str + ) -> CouchbaseQueue: + """Returns a CouchbaseQueue permitting simple map storage in a document. + + .. seealso:: + :class:`~acouchbase.datastructures.CouchbaseQueue` + + Returns: + :class:`~acouchbase.datastructures.CouchbaseQueue`: A CouchbaseQueue instance. + + """ + return CouchbaseQueue(key, self) + + def query_indexes(self) -> CollectionQueryIndexManager: + """ + Get a :class:`~acouchbase.management.queries.CollectionQueryIndexManager` which can be used to manage the query + indexes of this cluster. + + Returns: + :class:`~acouchbase.management.queries.CollectionQueryIndexManager`: A :class:`~acouchbase.management.queries.CollectionQueryIndexManager` instance. + """ # noqa: E501 + return CollectionQueryIndexManager(self.connection, + self.loop, + self._scope.bucket_name, + self._scope.name, + self.name) + + @staticmethod + def default_name(): + return "_default" + + +Collection = AsyncCollection diff --git a/acouchbase/datastructures.py b/acouchbase/datastructures.py new file mode 100644 index 000000000..6cdffcad9 --- /dev/null +++ b/acouchbase/datastructures.py @@ -0,0 +1,665 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import time +from datetime import timedelta +from typing import (TYPE_CHECKING, + Any, + Dict, + Generator, + List, + Optional) + +from acouchbase.logic.wrappers import AsyncWrapper +from couchbase.exceptions import (CasMismatchException, + DocumentNotFoundException, + InvalidArgumentException, + PathExistsException, + PathNotFoundException, + QueueEmpty, + UnAmbiguousTimeoutException) +from couchbase.options import MutateInOptions +from couchbase.subdocument import (array_addunique, + array_append, + array_prepend, + count) +from couchbase.subdocument import exists as subdoc_exists +from couchbase.subdocument import get as subdoc_get +from couchbase.subdocument import (remove, + replace, + upsert) + +if TYPE_CHECKING: + from acouchbase.collection import Collection + from couchbase._utils import JSONType + + +class CouchbaseList: + def __init__(self, key, # type: str + collection # type: Collection + ) -> None: + self._key = key + self._collection = collection + self._iter = False + self._full_list = None + + @AsyncWrapper.datastructure_op(create_type=list) + async def _get(self) -> List: + """ + Get the entire list. + """ + + return await self._collection.get(self._key) + + @AsyncWrapper.datastructure_op(create_type=list) + async def append(self, value # type: JSONType + ) -> None: + """ + Add an item to the end of a list. + + :param value: The value to append + :return: None + :raise: :cb_exc:`DocumentNotFoundException` if the document does not exist. + and `create` was not specified. + + example:: + + cb.list_append('a_list', 'hello') + cb.list_append('a_list', 'world') + + .. seealso:: :meth:`map_add` + """ + op = array_append('', value) + await self._collection.mutate_in(self._key, (op,)) + + @AsyncWrapper.datastructure_op(create_type=list) + async def prepend(self, value # type: JSONType + ) -> None: + """ + Add an item to the beginning of a list. + + :param value: Value to prepend + :return: :class:`OperationResult`. + :raise: :cb_exc:`DocumentNotFoundException` if the document does not exist. + and `create` was not specified. + + This function is identical to :meth:`list_append`, except for prepending + rather than appending the item + + .. seealso:: :meth:`list_append`, :meth:`map_add` + """ + op = array_prepend('', value) + await self._collection.mutate_in(self._key, (op,)) + + async def set_at(self, index, # type: int + value # type: JSONType + ) -> None: + """ + Sets an item within a list at a given position. + + :param index: The position to replace + :param value: The value to be inserted + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + :raise: :exc:`IndexError` if the index is out of bounds + + example:: + + cb.upsert('a_list', ['hello', 'world']) + cb.list_set('a_list', 1, 'good') + cb.get('a_list').value # => ['hello', 'good'] + + .. seealso:: :meth:`map_add`, :meth:`list_append` + """ + try: + op = replace(f'[{index}]', value) + await self._collection.mutate_in(self._key, (op,)) + except PathNotFoundException: + raise InvalidArgumentException(message=f'Index: {index} is out of range.') from None + + @AsyncWrapper.datastructure_op(create_type=list) + async def get_at(self, index # type: int + ) -> Any: + """ + Get a specific element within a list. + + :param index: The index to retrieve + :return: value for the element + :raise: :exc:`IndexError` if the index does not exist + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + try: + op = subdoc_get(f'[{index}]') + sdres = await self._collection.lookup_in(self._key, (op,)) + return sdres.value[0].get("value", None) + except PathNotFoundException: + raise InvalidArgumentException(message=f'Index: {index} is out of range.') from None + + async def remove_at(self, index # type: int + ) -> None: + """ + Remove the element at a specific index from a list. + + :param index: The index to remove + :param kwargs: Arguments to :meth:`mutate_in` + :return: :class:`OperationResult` + :raise: :exc:`IndexError` if the index does not exist + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + try: + op = remove(f'[{index}]') + await self._collection.mutate_in(self._key, (op,)) + except PathNotFoundException: + raise InvalidArgumentException(message=f'Index: {index} is out of range.') from None + + @AsyncWrapper.datastructure_op(create_type=list) + async def size(self) -> int: + """ + Retrieve the number of elements in the list. + + :return: The number of elements within the list + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + op = count('') + sdres = await self._collection.lookup_in(self._key, (op,)) + return sdres.value[0].get("value", None) + + @AsyncWrapper.datastructure_op(create_type=list) + async def index_of(self, value # type: Any + ) -> int: + """ + Retrieve the index of the specified value in the list. + + :param value: the value to look-up + :return: The index of the specified value, -1 if not found + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + + list_ = await self._get() + for idx, val in enumerate(list_.content_as[list]): + if val == value: + return idx + + return -1 + + async def get_all(self) -> List[Any]: + """ + Retrieves the entire list. + + :return: The entire CouchbaseList + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + + list_ = await self._get() + return list_.content_as[list] + + async def clear(self) -> None: + """ + Clears the list. + + :return: clears the CouchbaseList + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + try: + await self._collection.remove(self._key) + except DocumentNotFoundException: + pass + + def __aiter__(self): + return self + + async def __anext__(self): + if self._iter is False: + list_ = await self._get() + self._full_list = (v for v in list_.content_as[list]) + self._iter = True + + try: + val = next(self._full_list) + # yield to the event loop + await asyncio.sleep(0) + return val + except StopIteration: + self._iter = False + raise StopAsyncIteration + + +class CouchbaseMap: + def __init__(self, key, # type: str + collection # type: Collection + ) -> None: + self._key = key + self._collection = collection + self._full_map = None + + @AsyncWrapper.datastructure_op(create_type=dict) + async def _get(self) -> Dict: + """ + Get the entire map. + """ + return await self._collection.get(self._key) + + @AsyncWrapper.datastructure_op(create_type=dict) + async def add(self, mapkey, # type: str + value # type: Any + ) -> None: + """ + Set a value for a key in a map. + + These functions are all wrappers around the :meth:`mutate_in` or + :meth:`lookup_in` methods. + + :param mapkey: The key in the map to set + :param value: The value to use (anything serializable to JSON) + :raise: :cb_exc:`Document.DocumentNotFoundException` if the document does not exist. + and `create` was not specified + + .. Initialize a map and add a value + + cb.upsert('a_map', {}) + cb.map_add('a_map', 'some_key', 'some_value') + cb.map_get('a_map', 'some_key').value # => 'some_value' + cb.get('a_map').value # => {'some_key': 'some_value'} + + """ + op = upsert(mapkey, value) + await self._collection.mutate_in(self._key, (op,)) + + @AsyncWrapper.datastructure_op(create_type=dict) + async def get(self, mapkey, # type: str + ) -> Any: + """ + Retrieve a value from a map. + + :param key: The document ID + :param mapkey: Key within the map to retrieve + :return: :class:`~.ValueResult` + :raise: :exc:`IndexError` if the mapkey does not exist + :raise: :cb_exc:`DocumentNotFoundException` if the document does not exist. + + .. seealso:: :meth:`map_add` for an example + """ + op = subdoc_get(mapkey) + sd_res = await self._collection.lookup_in(self._key, (op,)) + return sd_res.value[0].get("value", None) + + async def remove(self, mapkey # type: str + ) -> None: + """ + Remove an item from a map. + + :param key: The document ID + :param mapkey: The key in the map + :param See:meth:`mutate_in` for options + :raise: :exc:`IndexError` if the mapkey does not exist + :raise: :cb_exc:`DocumentNotFoundException` if the document does not exist. + + .. Remove a map key-value pair: + + cb.map_remove('a_map', 'some_key') + + .. seealso:: :meth:`map_add` + """ + try: + op = remove(mapkey) + await self._collection.mutate_in(self._key, (op,)) + except PathNotFoundException: + raise InvalidArgumentException(message=f'Key: {mapkey} is not in the map.') from None + + @AsyncWrapper.datastructure_op(create_type=dict) + async def size(self) -> int: + """ + Get the number of items in the map. + + :param key: The document ID of the map + :return int: The number of items in the map + :raise: :cb_exc:`DocumentNotFoundException` if the document does not exist. + + .. seealso:: :meth:`map_add` + """ + op = count('') + sd_res = await self._collection.lookup_in(self._key, (op,)) + return sd_res.value[0].get("value", None) + + @AsyncWrapper.datastructure_op(create_type=dict) + async def exists(self, key # type: Any + ) -> bool: + """ + hecks whether a specific key exists in the map. + + :param key: The key to check + :return bool: If the key exists in the map or not + :raise: :cb_exc:`DocumentNotFoundException` if the document does not exist. + + .. seealso:: :meth:`map_add` + """ + op = subdoc_exists(key) + sd_res = await self._collection.lookup_in(self._key, (op,)) + return sd_res.exists(0) + + async def keys(self) -> List[str]: + """ + Returns a list of all the keys which exist in the map. + + :return: The keys in CouchbaseMap + :raise: :cb_exc:`DocumentNotFoundException` if the map does not exist + """ + + map_ = await self._get() + return list(map_.content_as[dict].keys()) + + async def values(self) -> List[str]: + """ + Returns a list of all the values which exist in the map. + + :return: The keys in CouchbaseMap + :raise: :cb_exc:`DocumentNotFoundException` if the map does not exist + """ + + map_ = await self._get() + return list(map_.content_as[dict].values()) + + async def get_all(self) -> List[Any]: + """ + Retrieves the entire map. + + :return: The entire CouchbaseMap + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + + map_ = await self._get() + return map_.content_as[dict] + + async def clear(self) -> None: + """ + Clears the map. + + :return: clears the CouchbaseMap + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + try: + await self._collection.remove(self._key) + except DocumentNotFoundException: + pass + + async def items(self) -> Generator: + """ + Provide mechanism to loop over the entire map. + + :return: Generator expression for CouchbaseMap + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + + map_ = await self._get() + return ((k, v) for k, v in map_.content_as[dict].items()) + + +class CouchbaseSet: + def __init__(self, key, # type: str + collection # type: Collection + ) -> None: + self._key = key + self._collection = collection + + @AsyncWrapper.datastructure_op(create_type=list) + async def _get(self) -> List: + """ + Get the entire set. + """ + return await self._collection.get(self._key) + + @AsyncWrapper.datastructure_op(create_type=list) + async def add(self, value # type: Any + ) -> None: + """ + Add an item to a set if the item does not yet exist. + + :param value: Value to add + .. seealso:: :meth:`map_add` + """ + try: + op = array_addunique('', value) + await self._collection.mutate_in(self._key, (op,)) + return True + except PathExistsException: + return False + + async def remove(self, value, # type: Any # noqa: C901 + timeout=None # type: Optional[timedelta] + ) -> None: + """ + Remove an item from a set. + + :param value: Value to remove + :param kwargs: Arguments to :meth:`mutate_in` + :raise: :cb_exc:`DocumentNotFoundException` if the set does not exist. + + .. seealso:: :meth:`set_add`, :meth:`map_add` + """ + + if timeout is None: + timeout = timedelta(seconds=10) + + timeout_millis = timeout.total_seconds() * 1000 + + interval_millis = float(50) + start = time.perf_counter() + time_left = timeout_millis + while True: + sd_res = await self._get() + list_ = sd_res.content_as[list] + val_idx = -1 + for idx, v in enumerate(list_): + if v == value: + val_idx = idx + break + + if val_idx >= 0: + try: + op = remove(f'[{val_idx}]') + await self._collection.mutate_in(self._key, (op,), MutateInOptions(cas=sd_res.cas)) + break + except CasMismatchException: + pass + else: + break + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((time.perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise UnAmbiguousTimeoutException(message=f"Unable to remove {value} from the CouchbaseSet.") + + await asyncio.sleep(interval_millis / 1000) + + @AsyncWrapper.datastructure_op(create_type=list) + async def contains(self, value # type: Any + ) -> None: + """ + Check whether or not the CouchbaseSet contains a value + + :param value: Value to remove + :return: True if `value` exists in the set, False otherwise + :raise: :cb_exc:`DocumentNotFoundException` if the set does not exist. + + .. seealso:: :meth:`set_add`, :meth:`map_add` + """ + sd_res = await self._get() + list_ = sd_res.content_as[list] + return value in list_ + + @AsyncWrapper.datastructure_op(create_type=list) + async def size(self) -> int: + """ + Get the number of items in the set. + + :return int: The number of items in the map + :raise: :cb_exc:`DocumentNotFoundException` if the document does not exist. + + .. seealso:: :meth:`map_add` + """ + op = count('') + sd_res = await self._collection.lookup_in(self._key, (op,)) + return sd_res.value[0].get("value", None) + + async def clear(self) -> None: + """ + Clears the set. + + :return: clears the CouchbaseSet + """ + try: + await self._collection.remove(self._key) + except DocumentNotFoundException: + pass + + @AsyncWrapper.datastructure_op(create_type=list) + async def values(self) -> List[Any]: + """ + Returns a list of all the values which exist in the set. + + :return: The keys in CouchbaseSet + :raise: :cb_exc:`DocumentNotFoundException` if the map does not exist + """ + + list_ = await self._get() + return list_.content_as[list] + + +class CouchbaseQueue: + def __init__(self, key, # type: str + collection # type: Collection + ) -> None: + self._key = key + self._collection = collection + self._full_queue = None + self._iter = False + + @AsyncWrapper.datastructure_op(create_type=list) + async def _get(self) -> List: + """ + Get the entire queuee. + """ + return await self._collection.get(self._key) + + @AsyncWrapper.datastructure_op(create_type=list) + async def push(self, value # type: JSONType + ) -> None: + """ + Add an item to the queue. + + :param value: Value to push onto queue + """ + op = array_prepend('', value) + await self._collection.mutate_in(self._key, (op,)) + + async def pop(self, timeout=None # type: Optional[timedelta] + ) -> None: + """ + Pop an item from the queue. + + :param value: Value to remove + :raise: :cb_exc:`DocumentNotFoundException` if the set does not exist. + + .. seealso:: :meth:`set_add`, :meth:`map_add` + """ + + if timeout is None: + timeout = timedelta(seconds=10) + + timeout_millis = timeout.total_seconds() * 1000 + + interval_millis = float(50) + start = time.perf_counter() + time_left = timeout_millis + while True: + try: + op = subdoc_get('[-1]') + sd_res = await self._collection.lookup_in(self._key, (op,)) + val = sd_res.value[0].get("value", None) + + try: + op = remove('[-1]') + await self._collection.mutate_in(self._key, (op,), MutateInOptions(cas=sd_res.cas)) + return val + except CasMismatchException: + pass + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((time.perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise UnAmbiguousTimeoutException(message="Unable to pop from the CouchbaseQueue.") + + await asyncio.sleep(interval_millis / 1000) + except PathNotFoundException: + raise QueueEmpty('No items to remove from the queue') + + @AsyncWrapper.datastructure_op(create_type=list) + async def size(self) -> int: + """ + Get the number of items in the queue. + + :return int: The number of items in the queue + :raise: :cb_exc:`DocumentNotFoundException` if the document does not exist. + + .. seealso:: :meth:`map_add` + """ + op = count('') + sd_res = await self._collection.lookup_in(self._key, (op,)) + return sd_res.value[0].get("value", None) + + async def clear(self) -> None: + """ + Clears the queue. + + :return: clears the CouchbaseQueue + :raise: :cb_exc:`DocumentNotFoundException` if the list does not exist + """ + try: + await self._collection.remove(self._key) + except DocumentNotFoundException: + pass + + def __iter__(self): + raise TypeError('CouchbaseQueue is not iterable. Try using `async for`.') + + def __aiter__(self): + return self + + async def __anext__(self): + if self._iter is False: + list_ = await self._get() + self._full_queue = (v for v in list_.content_as[list]) + self._iter = True + + try: + val = next(self._full_queue) + # yield to the event loop + await asyncio.sleep(0) + return val + except StopIteration: + self._iter = False + raise StopAsyncIteration diff --git a/acouchbase/kv_range_scan.py b/acouchbase/kv_range_scan.py new file mode 100644 index 000000000..fecf89aed --- /dev/null +++ b/acouchbase/kv_range_scan.py @@ -0,0 +1,75 @@ +# Copyright 2021, Couchbase, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from typing import Any, Dict + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ExceptionMap, + RangeScanCompletedException) +from couchbase.logic.kv_range_scan import RangeScanRequestLogic + + +class AsyncRangeScanRequest(RangeScanRequestLogic): + def __init__(self, + loop, + **kwargs, # type: Dict[str, Any] + ): + num_workers = kwargs.pop('num_workers', 2) + super().__init__(**kwargs) + self._loop = loop + self._result_ftr = None + self._tp_executor = ThreadPoolExecutor(num_workers) + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + def __aiter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_scan() + + return self + + async def __anext__(self): + try: + return await self._loop.run_in_executor(self._tp_executor, self._get_next_row) + # We can stop iterator when we receive RangeScanCompletedException + except asyncio.QueueEmpty: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls('Unexpected QueueEmpty exception caught when doing N1QL query.') + raise excptn + except RangeScanCompletedException: + self._done_streaming = True + raise StopAsyncIteration + except StopAsyncIteration: + self._done_streaming = True + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/acouchbase/logic/__init__.py b/acouchbase/logic/__init__.py new file mode 100644 index 000000000..366cea612 --- /dev/null +++ b/acouchbase/logic/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .wrappers import AsyncWrapper # noqa: F401 +from .wrappers import call_async_fn # noqa: F401 diff --git a/acouchbase/logic/wrappers.py b/acouchbase/logic/wrappers.py new file mode 100644 index 000000000..b95fc8afe --- /dev/null +++ b/acouchbase/logic/wrappers.py @@ -0,0 +1,352 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from functools import partial, wraps + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + CouchbaseException, + DocumentExistsException, + DocumentNotFoundException, + ErrorMapper, + ExceptionMap, + MissingConnectionException, + ServiceUnavailableException) +from couchbase.logic import decode_replicas, decode_value + + +def call_async_fn(ft, self, fn, *args, **kwargs): + try: + fn(self, *args, **kwargs) + except CouchbaseException as e: + ft.set_exception(e) + except Exception as e: + if isinstance(e, (TypeError, ValueError)): + ft.set_exception(e) + else: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(e)) + ft.set_exception(excptn) + + +class AsyncWrapper: + + @classmethod + def inject_connection_callbacks(cls): + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(res): + self._set_connection(res) + self.loop.call_soon_threadsafe(ft.set_result, True) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + call_async_fn(ft, self, fn, *args, **kwargs) + return ft + + return wrapped_fn + + return decorator + + @classmethod + def inject_close_callbacks(cls): + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(_): + self.loop.call_soon_threadsafe(ft.set_result, True) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + call_async_fn(ft, self, fn, *args, **kwargs) + return ft + + return wrapped_fn + + return decorator + + @classmethod + def chain_connect_futures(cls, ft, self, fn, cft, **kwargs): + """ + **INTERNAL** + """ + if cft.cancelled(): + ft.cancel() + return + + exc = cft.exception() + if exc is not None: + ft.set_exception(exc) + else: + self._connection = self._cluster.connection + args = kwargs.pop("args", None) + call_async_fn(ft, self, fn, *args, **kwargs) + + @classmethod + def inject_bucket_open_callbacks(cls): + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(_): + self._set_connected(True) + self.loop.call_soon_threadsafe(ft.set_result, True) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + cluster_conn_ft = self._cluster.on_connect() + kwargs["args"] = args + cluster_conn_ft.add_done_callback( + partial(cls.chain_connect_futures, ft, self, fn, **kwargs)) + else: + call_async_fn(ft, self, fn, *args, **kwargs) + + return ft + + return wrapped_fn + + return decorator + + @classmethod + def chain_futures(cls, ft, self, fn, cft, set_connection=False, **kwargs): + """ + **INTERNAL** + """ + if cft.cancelled(): + ft.cancel() + return + + exc = cft.exception() + if exc is not None: + ft.set_exception(exc) + else: + if set_connection is True: + # the bucket will set it's connection, need to make sure + # the connection is set w/ the scope and collection as well + self._scope._set_connection() + self._set_connection() + args = kwargs.pop("args", None) + call_async_fn(ft, self, fn, *args, **kwargs) + + @classmethod # noqa: C901 + def inject_cluster_callbacks(cls, return_cls, chain_connection=False, set_cluster_info=False): # noqa: C901 + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(res): + if return_cls is None: + retval = None + elif return_cls is True: + retval = res + else: + retval = return_cls(res) + + if set_cluster_info is True: + self._cluster_info = retval + + self.loop.call_soon_threadsafe(ft.set_result, retval) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + if isinstance(excptn, ServiceUnavailableException) and fn.__name__ == '_get_cluster_info': + excptn._message = ('If using Couchbase Server < 6.6, ' + 'a bucket needs to be opened prior to cluster level operations') + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + if chain_connection is True: + c_ft = self.on_connect() + # in order to keep arg passing simple, add operation args to kwargs + # this will keep the positional args passed to the callback only ones + # in the scope of handling logic w.r.t. to handling logic between the futures + # (the cluster connect future and the operation future) + kwargs["args"] = args + c_ft.add_done_callback( + partial(cls.chain_futures, ft, self, fn, **kwargs)) + else: + exc = MissingConnectionException('Not connected. Cannot perform operation.') + ft.set_exception(exc) + else: + call_async_fn(ft, self, fn, *args, **kwargs) + + return ft + + return wrapped_fn + + return decorator + + @classmethod # noqa: C901 + def inject_callbacks(cls, return_cls): # noqa: C901 + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(res): + if return_cls is None: + retval = None + elif return_cls is True: + retval = res + else: + retval = return_cls(res) + + self.loop.call_soon_threadsafe(ft.set_result, retval) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + bucket_conn_ft = self._scope._connect_bucket() + # in order to keep arg passing simple, add operation args to kwargs + # This allows the chain_futures callback to only worry about positional args + # outside the scope of the operation. + # Since, the kwargs passed to chain_futures only apply to the operation, chain_futures + # can easily determine what it needs to pass to the original wrapped fn + kwargs["args"] = args + bucket_conn_ft.add_done_callback( + partial(cls.chain_futures, ft, self, fn, set_connection=True, **kwargs)) + else: + call_async_fn(ft, self, fn, *args, **kwargs) + + return ft + + return wrapped_fn + + return decorator + + @classmethod # noqa: C901 + def inject_callbacks_and_decode(cls, return_cls): # noqa: C901 + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + transcoder = kwargs.pop('transcoder') + + def on_ok(res): + try: + is_subdoc = fn.__name__ in [ + '_lookup_in_internal', '_lookup_in_any_replica_internal', '_lookup_in_all_replicas_internal' + ] + + # special case for get_all_replicas + if fn.__name__ in ['_get_all_replicas_internal', '_lookup_in_all_replicas_internal']: + self.loop.call_soon_threadsafe( + ft.set_result, + decode_replicas(transcoder, res, return_cls, is_subdoc=is_subdoc) + ) + return + + value = res.raw_result.get('value', None) + flags = res.raw_result.get('flags', None) + + res.raw_result['value'] = decode_value(transcoder, value, flags, is_subdoc=is_subdoc) + + if return_cls is None: + retval = None + elif return_cls is True: + retval = res + else: + retval = return_cls(res) + self.loop.call_soon_threadsafe(ft.set_result, retval) + except CouchbaseException as e: + self.loop.call_soon_threadsafe(ft.set_exception, e) + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(message=str(ex)) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + bucket_conn_ft = self._scope._connect_bucket() + # in order to keep arg passing simple, add operation args to kwargs + # This allows the chain_futures callback to only worry about positional args + # outside the scope of the operation. + # Since, the kwargs passed to chain_futures only apply to the operation, chain_futures + # can easily determine what it needs to pass to the original wrapped fn + kwargs["args"] = args + bucket_conn_ft.add_done_callback( + partial(cls.chain_futures, ft, self, fn, set_connection=True, **kwargs)) + else: + call_async_fn(ft, self, fn, *args, **kwargs) + + return ft + + return wrapped_fn + + return decorator + + @classmethod + def datastructure_op(cls, create_type=None): + def decorator(fn): + @wraps(fn) + async def wrapped_fn(self, *args, **kwargs): + try: + return await fn(self, *args, **kwargs) + except DocumentNotFoundException: + if create_type is not None: + try: + await self._collection.insert(self._key, create_type()) + except DocumentExistsException: + pass + return await fn(self, *args, **kwargs) + else: + raise + + return wrapped_fn + return decorator diff --git a/acouchbase/management/__init__.py b/acouchbase/management/__init__.py new file mode 100644 index 000000000..fc6346764 --- /dev/null +++ b/acouchbase/management/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/acouchbase/management/analytics.py b/acouchbase/management/analytics.py new file mode 100644 index 000000000..853c538dd --- /dev/null +++ b/acouchbase/management/analytics.py @@ -0,0 +1,244 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict, + Iterable, + Optional) + +from acouchbase.management.logic.wrappers import AsyncMgmtWrapper +from couchbase.exceptions import InvalidArgumentException +from couchbase.management.logic import ManagementType +from couchbase.management.logic.analytics_logic import (AnalyticsDataset, + AnalyticsDataType, + AnalyticsIndex, + AnalyticsLink, + AnalyticsManagerLogic, + AzureBlobExternalAnalyticsLink, + CouchbaseRemoteAnalyticsLink, + S3ExternalAnalyticsLink) + +if TYPE_CHECKING: + from couchbase.management.options import (ConnectLinkOptions, + CreateAnalyticsIndexOptions, + CreateDatasetOptions, + CreateDataverseOptions, + CreateLinkAnalyticsOptions, + DisconnectLinkOptions, + DropAnalyticsIndexOptions, + DropDatasetOptions, + DropDataverseOptions, + DropLinkAnalyticsOptions, + GetAllAnalyticsIndexesOptions, + GetAllDatasetOptions, + GetLinksAnalyticsOptions, + GetPendingMutationsOptions, + ReplaceLinkAnalyticsOptions) + + +class AnalyticsIndexManager(AnalyticsManagerLogic): + + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def create_dataverse(self, + dataverse_name, # type: str + options=None, # type: Optional[CreateDataverseOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + if not isinstance(dataverse_name, str): + raise InvalidArgumentException("dataverse_name must be provided when creating an analytics dataverse.") + + super().create_dataverse(dataverse_name, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def drop_dataverse(self, + dataverse_name, # type: str + options=None, # type: Optional[DropDataverseOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + if not isinstance(dataverse_name, str): + raise InvalidArgumentException("dataverse_name must be provided when dropping an analytics dataverse.") + + super().drop_dataverse(dataverse_name, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def create_dataset(self, + dataset_name, # type: str + bucket_name, # type: str + options=None, # type: Optional[CreateDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + if not isinstance(dataset_name, str): + raise InvalidArgumentException("dataset_name must be provided when creating an analytics dataset.") + + if not isinstance(bucket_name, str): + raise InvalidArgumentException("bucket_name must be provided when creating an analytics dataset.") + + super().create_dataset(dataset_name, bucket_name, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def drop_dataset(self, + dataset_name, # type: str + options=None, # type: Optional[DropDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + if not isinstance(dataset_name, str): + raise InvalidArgumentException("dataset_name must be provided when dropping an analytics dataset.") + + super().drop_dataset(dataset_name, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(AnalyticsDataset, ManagementType.AnalyticsIndexMgmt, + AnalyticsManagerLogic._ERROR_MAPPING) + def get_all_datasets(self, + options=None, # type: Optional[GetAllDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Iterable[AnalyticsDataset]]: + + super().get_all_datasets(options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def create_index(self, + index_name, # type: str + dataset_name, # type: str + fields, # type: Dict[str, AnalyticsDataType] + options=None, # type: Optional[CreateAnalyticsIndexOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + if not isinstance(index_name, str): + raise InvalidArgumentException("index_name must be provided when creating an analytics index.") + + if not isinstance(dataset_name, str): + raise InvalidArgumentException("dataset_name must be provided when creating an analytics index.") + + if fields is not None: + if not isinstance(fields, dict): + raise InvalidArgumentException("fields must be provided when creating an analytics index.") + + if not all(map(lambda v: isinstance(v, AnalyticsDataType), fields.values())): + raise InvalidArgumentException("fields must all be an AnalyticsDataType.") + + super().create_index(index_name, dataset_name, fields, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + dataset_name, # type: str + options=None, # type: Optional[DropAnalyticsIndexOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + if not isinstance(index_name, str): + raise InvalidArgumentException("index_name must be provided when dropping an analytics index.") + + if not isinstance(dataset_name, str): + raise InvalidArgumentException("dataset_name must be provided when dropping an analytics index.") + + super().drop_index(index_name, dataset_name, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(AnalyticsIndex, ManagementType.AnalyticsIndexMgmt, + AnalyticsManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + options=None, # type: Optional[GetAllAnalyticsIndexesOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Iterable[AnalyticsIndex]]: + + super().get_all_indexes(options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def connect_link(self, + options=None, # type: Optional[ConnectLinkOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().connect_link(options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def disconnect_link(self, + options=None, # type: Optional[DisconnectLinkOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().disconnect_link(options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(dict, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def get_pending_mutations(self, + options=None, # type: Optional[GetPendingMutationsOptions] + **kwargs # type: Dict[str, Any] + ) -> Dict[str, int]: + + super().get_pending_mutations(options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def create_link( + self, + link, # type: AnalyticsLink + options=None, # type: Optional[CreateLinkAnalyticsOptions] + **kwargs + ) -> Awaitable[None]: + super().create_link(link, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def replace_link( + self, + link, # type: AnalyticsLink + options=None, # type: Optional[ReplaceLinkAnalyticsOptions] + **kwargs + ) -> Awaitable[None]: + super().replace_link(link, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def drop_link( + self, + link_name, # type: str + dataverse_name, # type: str + options=None, # type: Optional[DropLinkAnalyticsOptions] + **kwargs + ) -> Awaitable[None]: + + if not isinstance(link_name, str): + raise InvalidArgumentException("link_name must be provided when dropping an analytics link.") + + if not isinstance(dataverse_name, str): + raise InvalidArgumentException("dataverse_name must be provided when dropping an analytics link.") + + super().drop_link(link_name, dataverse_name, options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks((CouchbaseRemoteAnalyticsLink, + S3ExternalAnalyticsLink, + AzureBlobExternalAnalyticsLink), + ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def get_links( + self, + options=None, # type: Optional[GetLinksAnalyticsOptions] + **kwargs + ) -> Awaitable[Iterable[AnalyticsLink]]: + super().get_links(options, **kwargs) diff --git a/acouchbase/management/buckets.py b/acouchbase/management/buckets.py new file mode 100644 index 000000000..c48752b88 --- /dev/null +++ b/acouchbase/management/buckets.py @@ -0,0 +1,136 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict, + List) + +from acouchbase.management.logic.wrappers import AsyncMgmtWrapper +from couchbase.management.logic import ManagementType +from couchbase.management.logic.buckets_logic import (BucketDescribeResult, + BucketManagerLogic, + BucketSettings, + CreateBucketSettings) + +if TYPE_CHECKING: + from couchbase.management.options import (BucketDescribeOptions, + CreateBucketOptions, + DropBucketOptions, + FlushBucketOptions, + GetAllBucketOptions, + GetBucketOptions, + UpdateBucketOptions) + + +class BucketManager(BucketManagerLogic): + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def create_bucket(self, + settings, # type: CreateBucketSettings + *options, # type: CreateBucketOptions + **kwargs # type: Any + ) -> Awaitable[None]: + """ + Creates a new bucket. + + :param: CreateBucketSettings settings: settings for the bucket. + :param: CreateBucketOptions options: options for setting the bucket. + :param: Any kwargs: override corresponding values in the options. + + :raises: BucketAlreadyExistsException + :raises: InvalidArgumentsException + """ + super().create_bucket(settings, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def update_bucket(self, + settings, # type: BucketSettings + *options, # type: UpdateBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + super().update_bucket(settings, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def drop_bucket(self, + bucket_name, # type: str + *options, # type: DropBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + super().drop_bucket(bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(BucketSettings, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def get_bucket(self, + bucket_name, # type: str + *options, # type: GetBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[BucketSettings]: + + super().get_bucket(bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(BucketSettings, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def get_all_buckets(self, + *options, # type: GetAllBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[List[BucketSettings]]: + + super().get_all_buckets(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def flush_bucket(self, + bucket_name, # type: str + *options, # type: FlushBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + super().flush_bucket(bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(BucketDescribeResult, + ManagementType.BucketMgmt, + BucketManagerLogic._ERROR_MAPPING) + def bucket_describe(self, + bucket_name, # type: str + *options, # type: BucketDescribeOptions + **kwargs # type: Dict[str, Any] + ) -> BucketDescribeResult: + """Provides details on provided the bucket. + + Args: + bucket_name (str): The name of the bucket to flush. + options (:class:`~couchbase.management.options.BucketDescribeOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`.BucketDescribeResult`: Key-value pair details describing the bucket. + + Raises: + :class:`~couchbase.exceptions.BucketDoesNotExistException`: If the bucket does not exist. + """ + super().bucket_describe(bucket_name, *options, **kwargs) diff --git a/acouchbase/management/collections.py b/acouchbase/management/collections.py new file mode 100644 index 000000000..e06618c7b --- /dev/null +++ b/acouchbase/management/collections.py @@ -0,0 +1,179 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from inspect import Parameter, Signature +from typing import (Any, + Awaitable, + Dict, + Iterable, + Optional) + +from acouchbase.management.logic.wrappers import AsyncMgmtWrapper +from couchbase._utils import OverloadType +from couchbase.logic.supportability import Supportability +from couchbase.management.logic import ManagementType +from couchbase.management.logic.collections_logic import (CollectionManagerLogic, + CollectionSpec, + CreateCollectionSettings, + ScopeSpec, + UpdateCollectionSettings) +from couchbase.management.options import (CreateCollectionOptions, + CreateScopeOptions, + DropCollectionOptions, + DropScopeOptions, + GetAllScopesOptions, + UpdateCollectionOptions) + + +class CollectionManager(CollectionManagerLogic): + + def __init__(self, connection, loop, bucket_name): + super().__init__(connection, bucket_name) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def create_scope(self, + scope_name: str, + *options: CreateScopeOptions, + **kwargs: Dict[str, Any] + ) -> Awaitable[None]: + super().create_scope(scope_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def drop_scope(self, + scope_name: str, + *options: DropScopeOptions, + **kwargs: Dict[str, Any] + ) -> Awaitable[None]: + super().drop_scope(scope_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks((ScopeSpec, CollectionSpec), ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING) + def get_all_scopes(self, + *options: GetAllScopesOptions, + **kwargs: Dict[str, Any] + ) -> Awaitable[Iterable[ScopeSpec]]: + super().get_all_scopes(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.DEFAULT) + def create_collection(self, + scope_name: str, + collection_name: str, + settings: Optional[CreateCollectionSettings] = None, + *options: CreateCollectionOptions, + **kwargs: Dict[str, Any] + ): + super().create_collection(scope_name, collection_name, settings, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.SECONDARY) + def create_collection(self, # noqa: F811 + collection: CollectionSpec, + *options: CreateCollectionOptions, + **kwargs: Dict[str, Any], + ) -> Awaitable[None]: + Supportability.method_signature_deprecated( + 'create_collection', + Signature( + parameters=[ + Parameter('collection', Parameter.POSITIONAL_OR_KEYWORD, annotation=CollectionSpec), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=CreateCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=Awaitable[None] + ), + Signature( + parameters=[ + Parameter('scope_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('collection_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('settings', Parameter.POSITIONAL_OR_KEYWORD, + annotation=Optional[CreateCollectionSettings]), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=CreateCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=Awaitable[None] + ) + ) + settings = None + if collection.max_expiry is not None: + settings = CreateCollectionSettings(max_expiry=collection.max_expiry) + super().create_collection(collection.scope_name, collection.name, settings, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def update_collection(self, + scope_name: str, + collection_name: str, + settings: UpdateCollectionSettings, + *options: UpdateCollectionOptions, + **kwargs: Dict[str, Any] + ) -> Awaitable[None]: + super().update_collection(scope_name, collection_name, settings, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.DEFAULT) + def drop_collection(self, + scope_name: str, + collection_name: str, + *options: DropCollectionOptions, + **kwargs: Dict[str, Any] + ) -> Awaitable[None]: + super().drop_collection(scope_name, collection_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.SECONDARY) + def drop_collection(self, # noqa: F811 + collection: CollectionSpec, + *options: DropCollectionOptions, + **kwargs: Dict[str, Any] + ) -> Awaitable[None]: + Supportability.method_signature_deprecated( + 'drop_collection', + Signature( + parameters=[ + Parameter('self', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('collection', Parameter.POSITIONAL_OR_KEYWORD, annotation=CollectionSpec), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=DropCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=Awaitable[None], + ), + Signature( + parameters=[ + Parameter('self', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('scope_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('collection_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=DropCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=Awaitable[None] + ) + ) + super().drop_collection(collection.scope_name, collection.name, *options, **kwargs) diff --git a/acouchbase/management/eventing.py b/acouchbase/management/eventing.py new file mode 100644 index 000000000..50f914549 --- /dev/null +++ b/acouchbase/management/eventing.py @@ -0,0 +1,272 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import (TYPE_CHECKING, + Any, + Dict, + List) + +from acouchbase.management.logic.wrappers import AsyncMgmtWrapper +from couchbase.management.logic import ManagementType +from couchbase.management.logic.eventing_logic import (EventingFunction, + EventingFunctionManagerLogic, + EventingFunctionsStatus, + EventingFunctionStatus) + +if TYPE_CHECKING: + from couchbase.management.options import (DeployFunctionOptions, + DropFunctionOptions, + FunctionsStatusOptions, + GetAllFunctionOptions, + GetFunctionOptions, + PauseFunctionOptions, + ResumeFunctionOptions, + UndeployFunctionOptions, + UpsertFunctionOptions) + + +class EventingFunctionManager(EventingFunctionManagerLogic): + + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def upsert_function( + self, + function, # type: EventingFunction + *options, # type: UpsertFunctionOptions + **kwargs # type: Dict[str, Any] + ) -> None: + super().upsert_function(function, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def drop_function( + self, + name, # type: str + *options, # type: DropFunctionOptions + **kwargs # type: Any + ) -> None: + super().drop_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def deploy_function( + self, + name, # type: str + *options, # type: DeployFunctionOptions + **kwargs # type: Any + ) -> None: + super().deploy_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(EventingFunction, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def get_all_functions( + self, + *options, # type: GetAllFunctionOptions + **kwargs # type: Any + ) -> List[EventingFunction]: + super().get_all_functions(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(EventingFunction, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def get_function( + self, + name, # type: str + *options, # type: GetFunctionOptions + **kwargs # type: Any + ) -> EventingFunction: + super().get_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def pause_function( + self, + name, # type: str + *options, # type: PauseFunctionOptions + **kwargs # type: Any + ) -> None: + super().pause_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def resume_function( + self, + name, # type: str + *options, # type: ResumeFunctionOptions + **kwargs # type: Any + ) -> None: + super().resume_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def undeploy_function( + self, + name, # type: str + *options, # type: UndeployFunctionOptions + **kwargs # type: Any + ) -> None: + super().undeploy_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(EventingFunctionsStatus, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def functions_status( + self, + *options, # type: FunctionsStatusOptions + **kwargs # type: Any + ) -> EventingFunctionsStatus: + super().functions_status(*options, **kwargs) + + async def _get_status( + self, + name, # type: str + ) -> EventingFunctionStatus: + + statuses = await self.functions_status() + + if statuses.functions: + return next((f for f in statuses.functions if f.name == name), None) + + return None + + +class ScopeEventingFunctionManager(EventingFunctionManagerLogic): + + def __init__(self, + connection, + loop, + bucket_name, # type: str + scope_name, # type: str + ): + super().__init__(connection, bucket_name=bucket_name, scope_name=scope_name) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def upsert_function( + self, + function, # type: EventingFunction + *options, # type: UpsertFunctionOptions + **kwargs # type: Dict[str, Any] + ) -> None: + super().upsert_function(function, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def drop_function( + self, + name, # type: str + *options, # type: DropFunctionOptions + **kwargs # type: Any + ) -> None: + super().drop_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def deploy_function( + self, + name, # type: str + *options, # type: DeployFunctionOptions + **kwargs # type: Any + ) -> None: + super().deploy_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(EventingFunction, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def get_all_functions( + self, + *options, # type: GetAllFunctionOptions + **kwargs # type: Any + ) -> List[EventingFunction]: + super().get_all_functions(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(EventingFunction, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def get_function( + self, + name, # type: str + *options, # type: GetFunctionOptions + **kwargs # type: Any + ) -> EventingFunction: + super().get_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def pause_function( + self, + name, # type: str + *options, # type: PauseFunctionOptions + **kwargs # type: Any + ) -> None: + super().pause_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def resume_function( + self, + name, # type: str + *options, # type: ResumeFunctionOptions + **kwargs # type: Any + ) -> None: + super().resume_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def undeploy_function( + self, + name, # type: str + *options, # type: UndeployFunctionOptions + **kwargs # type: Any + ) -> None: + super().undeploy_function(name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(EventingFunctionsStatus, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def functions_status( + self, + *options, # type: FunctionsStatusOptions + **kwargs # type: Any + ) -> EventingFunctionsStatus: + super().functions_status(*options, **kwargs) + + async def _get_status( + self, + name, # type: str + ) -> EventingFunctionStatus: + + statuses = await self.functions_status() + + if statuses.functions: + return next((f for f in statuses.functions if f.name == name), None) + + return None diff --git a/acouchbase/management/logic/__init__.py b/acouchbase/management/logic/__init__.py new file mode 100644 index 000000000..fc6346764 --- /dev/null +++ b/acouchbase/management/logic/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/acouchbase/management/logic/wrappers.py b/acouchbase/management/logic/wrappers.py new file mode 100644 index 000000000..dd5b056e2 --- /dev/null +++ b/acouchbase/management/logic/wrappers.py @@ -0,0 +1,116 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from functools import wraps + +from acouchbase.logic import call_async_fn +from couchbase._utils import Overload, OverloadType +from couchbase.exceptions import ErrorMapper, MissingConnectionException +from couchbase.logic.supportability import Supportability +from couchbase.management.logic import (ManagementType, + handle_analytics_index_mgmt_response, + handle_bucket_mgmt_response, + handle_collection_mgmt_response, + handle_eventing_function_mgmt_response, + handle_query_index_mgmt_response, + handle_search_index_mgmt_response, + handle_user_mgmt_response, + handle_view_index_mgmt_response) + + +def build_mgmt_exception(exc, mgmt_type, error_map): + return ErrorMapper.build_exception(exc, mapping=error_map) + + +mgmt_overload_registry = {} + + +class AsyncMgmtWrapper: + + @classmethod # noqa: C901 + def inject_callbacks(cls, return_cls, mgmt_type, error_map, overload_type=None): # noqa: C901 + + def decorator(fn): + if overload_type is not None: + mgmt_overload = mgmt_overload_registry.get(fn.__qualname__) + if mgmt_overload is None: + mgmt_overload = mgmt_overload_registry[fn.__qualname__] = Overload(fn.__qualname__) + if overload_type is OverloadType.DEFAULT: + mgmt_overload.register_default(fn) + else: + mgmt_overload.register(fn) + + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(ret): + if return_cls is None: + retval = None + elif return_cls is True: + retval = ret + else: + if mgmt_type == ManagementType.BucketMgmt: + retval = handle_bucket_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.CollectionMgmt: + retval = handle_collection_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.UserMgmt: + retval = handle_user_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.QueryIndexMgmt: + retval = handle_query_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.AnalyticsIndexMgmt: + retval = handle_analytics_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.SearchIndexMgmt: + retval = handle_search_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.ViewIndexMgmt: + retval = handle_view_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.EventingFunctionMgmt: + retval = handle_eventing_function_mgmt_response(ret, fn.__name__, return_cls) + else: + retval = None + + if not ft.done(): + self.loop.call_soon_threadsafe(ft.set_result, retval) + + def on_err(exc, exc_info=None, error_msg=None): + excptn = build_mgmt_exception(exc, mgmt_type, error_map) + if not ft.done(): + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + exc = MissingConnectionException('Not connected. Cannot perform bucket management operation.') + ft.set_exception(exc) + else: + func = mgmt_overload_registry.get(fn.__qualname__, fn) + # work-around for PYCBC-1375, I doubt users are calling the index mgmt method + # using fields=[...], but in the event they do (as we do in the tests) this corrects + # the kwarg name. + if ('QueryIndexManager' in fn.__qualname__ + and fn.__qualname__.endswith('create_index') + and 'fields' in kwargs): + kwargs['keys'] = kwargs.pop('fields') + Supportability.method_kwarg_deprecated('fields', 'keys') + call_async_fn(ft, self, func, *args, **kwargs) + + return ft + + return wrapped_fn + + return decorator diff --git a/acouchbase/management/queries.py b/acouchbase/management/queries.py new file mode 100644 index 000000000..820784f5d --- /dev/null +++ b/acouchbase/management/queries.py @@ -0,0 +1,586 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from datetime import timedelta +from time import perf_counter +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict, + Iterable) + +from acouchbase.management.logic.wrappers import AsyncMgmtWrapper +from couchbase.exceptions import (AmbiguousTimeoutException, + InvalidArgumentException, + QueryIndexNotFoundException, + WatchQueryIndexTimeoutException) +from couchbase.management.logic import ManagementType +from couchbase.management.logic.query_index_logic import QueryIndex, QueryIndexManagerLogic +from couchbase.management.options import GetAllQueryIndexOptions +from couchbase.options import forward_args + +if TYPE_CHECKING: + from couchbase.management.options import (BuildDeferredQueryIndexOptions, + CreatePrimaryQueryIndexOptions, + CreateQueryIndexOptions, + DropPrimaryQueryIndexOptions, + DropQueryIndexOptions, + WatchQueryIndexOptions) + + +class QueryIndexManager(QueryIndexManagerLogic): + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def create_index(self, + bucket_name, # type: str + index_name, # type: str + keys, # type: Iterable[str] + *options, # type: CreateQueryIndexOptions + **kwargs + ) -> Awaitable[None]: + """Creates a new query index. + + Args: + bucket_name (str): The name of the bucket this index is for. + index_name (str): The name of the index. + keys (Iterable[str]): The keys which this index should cover. + options (:class:`~couchbase.management.options.CreateQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name, index_name or keys + are invalid types. + :class:`~couchbase.exceptions.QueryIndexAlreadyExistsException`: If the index already exists. + """ + + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when creating a secondary index.') + if not isinstance(index_name, str): + raise InvalidArgumentException('The index_name must be provided when creating a secondary index.') + if not isinstance(keys, (list, tuple)): + raise InvalidArgumentException('Index keys must be provided when creating a secondary index.') + + super().create_index(bucket_name, index_name, keys, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def create_primary_index(self, + bucket_name, # type: str + *options, # type: CreatePrimaryQueryIndexOptions + **kwargs + ) -> Awaitable[None]: + """Creates a new primary query index. + + Args: + bucket_name (str): The name of the bucket this index is for. + options (:class:`~couchbase.management.options.CreatePrimaryQueryIndexOptions`): Optional parameters for + this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name is an invalid type. + :class:`~couchbase.exceptions.QueryIndexAlreadyExistsException`: If the index already exists. + """ + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when creating a primary index.') + + super().create_primary_index(bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + bucket_name, # type: str + index_name, # type: str + *options, # type: DropQueryIndexOptions + **kwargs) -> Awaitable[None]: + """Drops an existing query index. + + Args: + bucket_name (str): The name of the bucket containing the index to drop. + index_name (str): The name of the index to drop. + options (:class:`~couchbase.management.options.DropQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name or index_name are + invalid types. + :class:`~couchbase.exceptions.QueryIndexNotFoundException`: If the index does not exists. + """ + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when dropping a secondary index.') + if not isinstance(index_name, str): + raise InvalidArgumentException('The index_name must be provided when dropping a secondary index.') + + super().drop_index(bucket_name, index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def drop_primary_index(self, + bucket_name, # type: str + *options, # type: DropPrimaryQueryIndexOptions + **kwargs) -> Awaitable[None]: + """Drops an existing primary query index. + + Args: + bucket_name (str): The name of the bucket this index to drop. + options (:class:`~couchbase.management.options.DropPrimaryQueryIndexOptions`): Optional parameters for + this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name is an invalid type. + :class:`~couchbase.exceptions.QueryIndexNotFoundException`: If the index does not exists. + """ + + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when dropping a primary index.') + + super().drop_primary_index(bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(QueryIndex, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + bucket_name, # type: str + *options, # type: GetAllQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> Awaitable[Iterable[QueryIndex]]: + """Returns a list of indexes for a specific bucket. + + Args: + bucket_name (str): The name of the bucket to fetch indexes for. + options (:class:`~couchbase.management.options.GetAllQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Awaitable[Iterable[:class:`.QueryIndex`]]: A list of indexes for a specific bucket. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name is an invalid type. + """ + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when retrieving all indexes.') + + super().get_all_indexes(bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def build_deferred_indexes(self, + bucket_name, # type: str + *options, # type: BuildDeferredQueryIndexOptions + **kwargs + ) -> Awaitable[None]: + """Starts building any indexes which were previously created with ``deferred=True``. + + Args: + bucket_name (str): The name of the bucket to perform build on. + options (:class:`~couchbase.management.options.BuildDeferredQueryIndexOptions`): Optional parameters + for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name is an invalid type. + """ + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when building deferred indexes.') + + super().build_deferred_indexes(bucket_name, *options, **kwargs) + + async def watch_indexes(self, # noqa: C901 + bucket_name, # type: str + index_names, # type: Iterable[str] + *options, # type: WatchQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> Awaitable[None]: + """Waits for a number of indexes to finish creation and be ready to use. + + Args: + bucket_name (str): The name of the bucket to watch for indexes on. + index_names (Iterable[str]): The names of the indexes to watch. + options (:class:`~couchbase.management.options.WatchQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name or index_names are + invalid types. + :class:`~couchbase.exceptions.WatchQueryIndexTimeoutException`: If the specified timeout is reached + before all the specified indexes are ready to use. + """ + + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when watching indexes.') + if not isinstance(index_names, (list, tuple)): + raise InvalidArgumentException('One or more index_names must be provided when watching indexes.') + + final_args = forward_args(kwargs, *options) + + scope_name = final_args.get('scope_name', None) + collection_name = final_args.get('collection_name', None) + + if final_args.get('watch_primary', False): + index_names.append('#primary') + + timeout = final_args.get('timeout', None) + if not timeout: + raise InvalidArgumentException('Must specify a timeout condition for watch indexes') + + def check_indexes(index_names, indexes): + for idx_name in index_names: + match = next((i for i in indexes if i.name == idx_name), None) + if not match: + raise QueryIndexNotFoundException(f'Cannot find index with name: {idx_name}') + + return all(map(lambda i: i.state == 'online', indexes)) + + # timeout is converted to microsecs via final_args() + timeout_millis = timeout / 1000 + + interval_millis = float(50) + start = perf_counter() + time_left = timeout_millis + while True: + + opts = GetAllQueryIndexOptions( + timeout=timedelta(milliseconds=time_left)) + if scope_name: + opts['scope_name'] = scope_name + opts['collection_name'] = collection_name + + try: + indexes = await self.get_all_indexes(bucket_name, opts) + except AmbiguousTimeoutException: + pass # go ahead and move on, raise WatchQueryIndexTimeoutException later if needed + + all_online = check_indexes(index_names, indexes) + if all_online: + break + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise WatchQueryIndexTimeoutException('Failed to find all indexes online within the alloted time.') + + await asyncio.sleep(interval_millis / 1000) + + +class CollectionQueryIndexManager(QueryIndexManagerLogic): + def __init__(self, connection, loop, bucket_name, scope_name, collection_name): + super().__init__(connection) + self._bucket_name = bucket_name + self._scope_name = scope_name + self._collection_name = collection_name + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def create_index(self, + index_name, # type: str + keys, # type: Iterable[str] + *options, # type: CreateQueryIndexOptions + **kwargs + ) -> Awaitable[None]: + """Creates a new query index. + + Args: + index_name (str): The name of the index. + keys (Iterable[str]): The keys which this index should cover. + options (:class:`~couchbase.management.options.CreateQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the index_name or keys are invalid types. + :class:`~couchbase.exceptions.QueryIndexAlreadyExistsException`: If the index already exists. + """ + if not isinstance(index_name, str): + raise InvalidArgumentException('The index_name must be provided when creating a secondary index.') + if not isinstance(keys, (list, tuple)): + raise InvalidArgumentException('Index keys must be provided when creating a secondary index.') + + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + super().create_index(self._bucket_name, index_name, keys, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def create_primary_index(self, + *options, # type: CreatePrimaryQueryIndexOptions + **kwargs + ) -> Awaitable[None]: + """Creates a new primary query index. + + Args: + options (:class:`~couchbase.management.options.CreatePrimaryQueryIndexOptions`): Optional parameters for + this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.QueryIndexAlreadyExistsException`: If the index already exists. + """ + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + super().create_primary_index(self._bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + *options, # type: DropQueryIndexOptions + **kwargs) -> Awaitable[None]: + """Drops an existing query index. + + Args: + index_name (str): The name of the index to drop. + options (:class:`~couchbase.management.options.DropQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the index_name is an invalid type. + :class:`~couchbase.exceptions.QueryIndexNotFoundException`: If the index does not exists. + """ + if not isinstance(index_name, str): + raise InvalidArgumentException('The index_name must be provided when dropping a secondary index.') + + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + super().drop_index(self._bucket_name, index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def drop_primary_index(self, + *options, # type: DropPrimaryQueryIndexOptions + **kwargs) -> Awaitable[None]: + """Drops an existing primary query index. + + Args: + options (:class:`~couchbase.management.options.DropPrimaryQueryIndexOptions`): Optional parameters for + this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.QueryIndexNotFoundException`: If the index does not exists. + """ + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + super().drop_primary_index(self._bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(QueryIndex, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + *options, # type: GetAllQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> Awaitable[Iterable[QueryIndex]]: + """Returns a list of indexes for a specific collection. + + Args: + bucket_name (str): The name of the bucket to fetch indexes for. + options (:class:`~couchbase.management.options.GetAllQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Awaitable[Iterable[:class:`.QueryIndex`]]: A list of indexes. + """ + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + super().get_all_indexes(self._bucket_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def build_deferred_indexes(self, + *options, # type: BuildDeferredQueryIndexOptions + **kwargs + ) -> Awaitable[None]: + """Starts building any indexes which were previously created with ``deferred=True``. + + Args: + options (:class:`~couchbase.management.options.BuildDeferredQueryIndexOptions`): Optional parameters + for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + """ + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + super().build_deferred_indexes(self._bucket_name, *options, **kwargs) + + async def watch_indexes(self, # noqa: C901 + index_names, # type: Iterable[str] + *options, # type: WatchQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> Awaitable[None]: + """Waits for a number of indexes to finish creation and be ready to use. + + Args: + index_names (Iterable[str]): The names of the indexes to watch. + options (:class:`~couchbase.management.options.WatchQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name or index_names are + invalid types. + :class:`~couchbase.exceptions.WatchQueryIndexTimeoutException`: If the specified timeout is reached + before all the specified indexes are ready to use. + """ + + if not isinstance(index_names, (list, tuple)): + raise InvalidArgumentException('One or more index_names must be provided when watching indexes.') + + final_args = forward_args(kwargs, *options) + + if final_args.get('watch_primary', False): + index_names.append('#primary') + + if final_args.get('scope_name'): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if final_args.get('collection_name'): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + timeout = final_args.get('timeout', None) + if not timeout: + raise InvalidArgumentException('Must specify a timeout condition for watch indexes') + + def check_indexes(index_names, indexes): + for idx_name in index_names: + match = next((i for i in indexes if i.name == idx_name), None) + if not match: + raise QueryIndexNotFoundException(f'Cannot find index with name: {idx_name}') + + return all(map(lambda i: i.state == 'online', indexes)) + + # timeout is converted to microsecs via final_args() + timeout_millis = timeout / 1000 + + interval_millis = float(50) + start = perf_counter() + time_left = timeout_millis + while True: + opts = GetAllQueryIndexOptions( + timeout=timedelta(milliseconds=time_left)) + + try: + indexes = await self.get_all_indexes(opts) + except AmbiguousTimeoutException: + pass # go ahead and move on, raise WatchQueryIndexTimeoutException later if needed + + all_online = check_indexes(index_names, indexes) + if all_online: + break + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise WatchQueryIndexTimeoutException('Failed to find all indexes online within the alloted time.') + + await asyncio.sleep(interval_millis / 1000) diff --git a/acouchbase/management/search.py b/acouchbase/management/search.py new file mode 100644 index 000000000..894cea051 --- /dev/null +++ b/acouchbase/management/search.py @@ -0,0 +1,307 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict, + Iterable) + +from acouchbase.management.logic.wrappers import AsyncMgmtWrapper +from couchbase.management.logic import ManagementType +from couchbase.management.logic.search_index_logic import SearchIndex, SearchIndexManagerLogic + +if TYPE_CHECKING: + from couchbase.management.options import (AllowQueryingSearchIndexOptions, + AnalyzeDocumentSearchIndexOptions, + DisallowQueryingSearchIndexOptions, + DropSearchIndexOptions, + FreezePlanSearchIndexOptions, + GetAllSearchIndexesOptions, + GetAllSearchIndexStatsOptions, + GetSearchIndexedDocumentsCountOptions, + GetSearchIndexOptions, + GetSearchIndexStatsOptions, + PauseIngestSearchIndexOptions, + ResumeIngestSearchIndexOptions, + UnfreezePlanSearchIndexOptions, + UpsertSearchIndexOptions) + + +class SearchIndexManager(SearchIndexManagerLogic): + def __init__(self, + connection, + loop + ): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def upsert_index(self, + index, # type: SearchIndex + *options, # type: UpsertSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + super().upsert_index(index, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + *options, # type: DropSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + super().drop_index(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(SearchIndex, ManagementType.SearchIndexMgmt, + SearchIndexManagerLogic._ERROR_MAPPING) + def get_index(self, + index_name, # type: str + *options, # type: GetSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[SearchIndex]: + + super().get_index(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(SearchIndex, ManagementType.SearchIndexMgmt, + SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + *options, # type: GetAllSearchIndexesOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Iterable[SearchIndex]]: + super().get_all_indexes(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(int, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_indexed_documents_count(self, + index_name, # type: str + *options, # type: GetSearchIndexedDocumentsCountOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[int]: + super().get_indexed_documents_count(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def pause_ingest(self, + index_name, # type: str + *options, # type: PauseIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().pause_ingest(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def resume_ingest(self, + index_name, # type: str + *options, # type: ResumeIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().resume_ingest(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def allow_querying(self, + index_name, # type: str + *options, # type: AllowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().allow_querying(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def disallow_querying(self, + index_name, # type: str + *options, # type: DisallowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().disallow_querying(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def freeze_plan(self, + index_name, # type: str + *options, # type: FreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().freeze_plan(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def unfreeze_plan(self, + index_name, # type: str + *options, # type: UnfreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().unfreeze_plan(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def analyze_document(self, + index_name, # type: str + document, # type: Any + *options, # type: AnalyzeDocumentSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Dict[str, Any]]: + super().analyze_document(index_name, document, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index_stats(self, + index_name, # type: str + *options, # type: GetSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Dict[str, Any]]: + super().get_index_stats(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_index_stats(self, + *options, # type: GetAllSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Dict[str, Any]]: + super().get_all_index_stats(*options, **kwargs) + + +class ScopeSearchIndexManager(SearchIndexManagerLogic): + + def __init__(self, + connection, + loop, + bucket_name, # type: str + scope_name # type: str + ): + super().__init__(connection, bucket_name=bucket_name, scope_name=scope_name) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def upsert_index(self, + index, # type: SearchIndex + *options, # type: UpsertSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + super().upsert_index(index, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + *options, # type: DropSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + + super().drop_index(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(SearchIndex, ManagementType.SearchIndexMgmt, + SearchIndexManagerLogic._ERROR_MAPPING) + def get_index(self, + index_name, # type: str + *options, # type: GetSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[SearchIndex]: + + super().get_index(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(SearchIndex, ManagementType.SearchIndexMgmt, + SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + *options, # type: GetAllSearchIndexesOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Iterable[SearchIndex]]: + super().get_all_indexes(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(int, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_indexed_documents_count(self, + index_name, # type: str + *options, # type: GetSearchIndexedDocumentsCountOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[int]: + super().get_indexed_documents_count(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def pause_ingest(self, + index_name, # type: str + *options, # type: PauseIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().pause_ingest(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def resume_ingest(self, + index_name, # type: str + *options, # type: ResumeIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().resume_ingest(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def allow_querying(self, + index_name, # type: str + *options, # type: AllowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().allow_querying(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def disallow_querying(self, + index_name, # type: str + *options, # type: DisallowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().disallow_querying(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def freeze_plan(self, + index_name, # type: str + *options, # type: FreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().freeze_plan(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def unfreeze_plan(self, + index_name, # type: str + *options, # type: UnfreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().unfreeze_plan(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def analyze_document(self, + index_name, # type: str + document, # type: Any + *options, # type: AnalyzeDocumentSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Dict[str, Any]]: + super().analyze_document(index_name, document, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index_stats(self, + index_name, # type: str + *options, # type: GetSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Dict[str, Any]]: + super().get_index_stats(index_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_index_stats(self, + *options, # type: GetAllSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Dict[str, Any]]: + super().get_all_index_stats(*options, **kwargs) diff --git a/acouchbase/management/users.py b/acouchbase/management/users.py new file mode 100644 index 000000000..98fc0bb87 --- /dev/null +++ b/acouchbase/management/users.py @@ -0,0 +1,131 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Iterable) + +from acouchbase.management.logic.wrappers import AsyncMgmtWrapper +from couchbase.management.logic import ManagementType +from couchbase.management.logic.users_logic import (Group, + RoleAndDescription, + User, + UserAndMetadata, + UserManagerLogic) + +if TYPE_CHECKING: + from couchbase.management.options import (ChangePasswordOptions, + DropGroupOptions, + DropUserOptions, + GetAllGroupsOptions, + GetAllUsersOptions, + GetGroupOptions, + GetRolesOptions, + GetUserOptions, + UpsertGroupOptions, + UpsertUserOptions) + + +class UserManager(UserManagerLogic): + + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(UserAndMetadata, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_user(self, + username, # type: str + *options, # type: GetUserOptions + **kwargs # type: Any + ) -> Awaitable[UserAndMetadata]: + super().get_user(username, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(UserAndMetadata, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_all_users(self, + *options, # type: GetAllUsersOptions + **kwargs # type: Any + ) -> Awaitable[Iterable[UserAndMetadata]]: + super().get_all_users(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def upsert_user(self, + user, # type: User + *options, # type: UpsertUserOptions + **kwargs # type: Any + ) -> Awaitable[None]: + + super().upsert_user(user, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def drop_user(self, + username, # type: str + *options, # type: DropUserOptions + **kwargs # type: Any + ) -> Awaitable[None]: + super().drop_user(username, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def change_password(self, + new_password, # type: str + *options, # type: ChangePasswordOptions + **kwargs # type: Any + ) -> Awaitable[None]: + super().change_password(new_password, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(RoleAndDescription, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_roles(self, + *options, # type: GetRolesOptions + **kwargs # type: Any + ) -> Awaitable[Iterable[RoleAndDescription]]: + super().get_roles(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(Group, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_group(self, + group_name, # type: str + *options, # type: GetGroupOptions + **kwargs # type: Any + ) -> Awaitable[Group]: + super().get_group(group_name, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(Group, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_all_groups(self, + *options, # type: GetAllGroupsOptions + **kwargs # type: Any + ) -> Awaitable[Iterable[Group]]: + super().get_all_groups(*options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def upsert_group(self, + group, # type: Group + *options, # type: UpsertGroupOptions + **kwargs # type: Any + ) -> Awaitable[None]: + super().upsert_group(group, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def drop_group(self, + group_name, # type: str + *options, # type: DropGroupOptions + **kwargs # type: Any + ) -> Awaitable[None]: + super().drop_group(group_name, *options, **kwargs) diff --git a/acouchbase/management/views.py b/acouchbase/management/views.py new file mode 100644 index 000000000..7f54c87c6 --- /dev/null +++ b/acouchbase/management/views.py @@ -0,0 +1,93 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict, + Iterable) + +from acouchbase.management.logic.wrappers import AsyncMgmtWrapper +from couchbase.management.logic import ManagementType +from couchbase.management.logic.view_index_logic import (DesignDocument, + DesignDocumentNamespace, + ViewIndexManagerLogic) + +if TYPE_CHECKING: + from couchbase.management.options import (DropDesignDocumentOptions, + GetAllDesignDocumentsOptions, + GetDesignDocumentOptions, + PublishDesignDocumentOptions, + UpsertDesignDocumentOptions) + + +class ViewIndexManager(ViewIndexManagerLogic): + def __init__(self, connection, loop, bucket_name): + super().__init__(connection, bucket_name) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @AsyncMgmtWrapper.inject_callbacks(DesignDocument, ManagementType.ViewIndexMgmt, + ViewIndexManagerLogic._ERROR_MAPPING) + def get_design_document(self, + design_doc_name, # type: str + namespace, # type: DesignDocumentNamespace + *options, # type: GetDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[DesignDocument]: + super().get_design_document(design_doc_name, namespace, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(DesignDocument, ManagementType.ViewIndexMgmt, + ViewIndexManagerLogic._ERROR_MAPPING) + def get_all_design_documents(self, + namespace, # type: DesignDocumentNamespace + *options, # type: GetAllDesignDocumentsOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[Iterable[DesignDocument]]: + super().get_all_design_documents(namespace, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.ViewIndexMgmt, ViewIndexManagerLogic._ERROR_MAPPING) + def upsert_design_document(self, + design_doc_data, # type: DesignDocument + namespace, # type: DesignDocumentNamespace + *options, # type: UpsertDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().upsert_design_document(design_doc_data, namespace, *options, **kwargs) + + @AsyncMgmtWrapper.inject_callbacks(None, ManagementType.ViewIndexMgmt, ViewIndexManagerLogic._ERROR_MAPPING) + def drop_design_document(self, + design_doc_name, # type: str + namespace, # type: DesignDocumentNamespace + *options, # type: DropDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + super().drop_design_document(design_doc_name, namespace, *options, **kwargs) + + async def publish_design_document(self, + design_doc_name, # type: str + *options, # type: PublishDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + doc = await self.get_design_document( + design_doc_name, DesignDocumentNamespace.DEVELOPMENT, *options, **kwargs) + await self.upsert_design_document( + doc, DesignDocumentNamespace.PRODUCTION, *options, **kwargs) diff --git a/acouchbase/n1ql.py b/acouchbase/n1ql.py new file mode 100644 index 000000000..b1ed6b2ff --- /dev/null +++ b/acouchbase/n1ql.py @@ -0,0 +1,112 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging +from concurrent.futures import ThreadPoolExecutor +from typing import Awaitable + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.n1ql import N1QLQuery # noqa: F401 +from couchbase.logic.n1ql import QueryRequestLogic + +logger = logging.getLogger(__name__) + + +class AsyncN1QLRequest(QueryRequestLogic): + def __init__(self, + connection, + loop, + query_params, + row_factory=lambda x: x, + **kwargs + ): + num_workers = kwargs.pop('num_workers', 2) + super().__init__(connection, query_params, row_factory=row_factory, **kwargs) + self._loop = loop + self._tp_executor = ThreadPoolExecutor(num_workers) + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @classmethod + def generate_n1ql_request(cls, connection, loop, query_params, row_factory=lambda x: x, **kwargs): + return cls(connection, loop, query_params, row_factory=row_factory, **kwargs) + + def _get_metadata(self): + try: + analytics_response = next(self._streaming_result) + self._set_metadata(analytics_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def execute(self) -> Awaitable[None]: + async def _execute(): + return [r async for r in self] + + return asyncio.create_task(_execute()) + + def __aiter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_query() + + return self + + def _get_next_row(self): + if self.done_streaming is True: + return + + # this is a blocking operation + row = next(self._streaming_result) + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None once query request is complete and _no_ errors found + if row is None: + raise StopAsyncIteration + return self.serializer.deserialize(row) + + async def __anext__(self): + try: + return await self._loop.run_in_executor(self._tp_executor, self._get_next_row) + except asyncio.QueueEmpty: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls('Unexpected QueueEmpty exception caught when doing N1QL query.') + raise excptn + except StopAsyncIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/acouchbase/scope.py b/acouchbase/scope.py new file mode 100644 index 000000000..e9b559f35 --- /dev/null +++ b/acouchbase/scope.py @@ -0,0 +1,553 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Dict, + Optional) + +from acouchbase.analytics import AnalyticsQuery, AsyncAnalyticsRequest +from acouchbase.collection import Collection +from acouchbase.management.eventing import ScopeEventingFunctionManager +from acouchbase.management.search import ScopeSearchIndexManager +from acouchbase.n1ql import AsyncN1QLRequest, N1QLQuery +from acouchbase.search import AsyncFullTextSearchRequest, SearchQueryBuilder +from couchbase.options import (AnalyticsOptions, + QueryOptions, + SearchOptions) +from couchbase.result import (AnalyticsResult, + QueryResult, + SearchResult) +from couchbase.serializer import Serializer +from couchbase.transcoder import Transcoder + +if TYPE_CHECKING: + from couchbase.search import SearchQuery, SearchRequest + + +class AsyncScope: + """Create a Couchbase Scope instance. + + Exposes the operations which are available to be performed against a scope. Namely the ability to access + to Collections for performing operations. + + Args: + bucket (:class:`~acouchbase.bucket.Bucket`): A :class:`~acouchbase.bucket.Bucket` instance. + scope_name (str): Name of the scope. + + """ + + def __init__(self, bucket, scope_name): + self._bucket = bucket + self._set_connection() + self._scope_name = scope_name + + @property + def connection(self): + """ + **INTERNAL** + """ + return self._connection + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._bucket.loop + + @property + def default_serializer(self) -> Optional[Serializer]: + return self._bucket.default_serializer + + @property + def streaming_timeouts(self): + """ + **INTERNAL** + """ + return self._bucket.streaming_timeouts + + @property + def default_transcoder(self) -> Optional[Transcoder]: + return self._bucket.default_transcoder + + @property + def name(self): + """ + str: The name of this :class:`~acouchbase.scope.Scope` instance. + """ + return self._scope_name + + @property + def bucket_name(self): + """ + str: The name of the bucket in which this :class:`~acouchbase.scope.Scope` instance belongs. + """ + return self._bucket.name + + def collection(self, name # type: str + ) -> Collection: + """Creates a :class:`~acouchbase.collection.Collection` instance of the specified collection. + + Args: + name (str): Name of the collection to reference. + + Returns: + :class:`~acouchbase.collection.Collection`: A :class:`~acouchbase.collection.Collection` instance of the specified collection. + """ # noqa: E501 + return Collection(self, name) + + def _connect_bucket(self) -> Awaitable: + """ + **INTERNAL** + """ + return self._bucket.on_connect() + + def _set_connection(self): + """ + **INTERNAL** + """ + self._connection = self._bucket.connection + + def query(self, + statement, # type: str + *options, # type: QueryOptions + **kwargs # type: Dict[str, Any] + ) -> QueryResult: + """Executes a N1QL query against the scope. + + .. note:: + The query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.QueryResult` begins. + + .. note:: + Scope-level queries are only supported on Couchbase Server versions that support scopes and collections. + + .. seealso:: + * :class:`~acouchbase.management.queries.QueryIndexManager`: For how to manage query indexes. + * :meth:`~acouchbase.cluster.AsyncCluster.query`: For how to execute cluster-level queries. + + Args: + statement (str): The N1QL statement to execute. + options (:class:`~couchbase.options.QueryOptions`): Optional parameters for the query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.QueryOptions` + + Returns: + :class:`~couchbase.result.QueryResult`: An instance of a :class:`~couchbase.result.QueryResult` which + provides access to iterate over the query results and access metadata and metrics about the query. + + Examples: + Simple query:: + + q_res = scope.query('SELECT * FROM `inventory` WHERE country LIKE 'United%' LIMIT 2;') + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple query with positional parameters:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `inventory` WHERE country LIKE $1 LIMIT $2;' + q_res = scope.query(q_str, QueryOptions(positional_parameters=['United%', 5])) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple query with named parameters:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = scope.query(q_str, QueryOptions(named_parameters={'country': 'United%', 'lim':2})) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Retrieve metadata and/or metrics from query:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = scope.query(q_str, QueryOptions(metrics=True)) + async for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'Query metadata: {q_res.metadata()}') + print(f'Query metrics: {q_res.metadata().metrics()}') + + """ + opt = QueryOptions() + opts = list(options) + for o in opts: + if isinstance(o, QueryOptions): + opt = o + opts.remove(o) + + request_args = dict() + num_workers = kwargs.pop('num_workers', None) + if num_workers: + request_args['num_workers'] = num_workers + + # set the query context as this bucket and scope if not provided + if not ('query_context' in opt or 'query_context' in kwargs): + kwargs['query_context'] = '`{}`.`{}`'.format(self.bucket_name, self.name) + + query = N1QLQuery.create_query_object( + statement, opt, **kwargs) + return QueryResult(AsyncN1QLRequest.generate_n1ql_request(self.connection, + self.loop, + query.params, + **request_args)) + + def analytics_query(self, + statement, # type: str + *options, # type: AnalyticsOptions + **kwargs # type: Dict[str, Any] + ) -> AnalyticsResult: + """Executes an analaytics query against the scope. + + .. note:: + The analytics query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.AnalyticsResult` begins. + + .. seealso:: + * :class:`~acouchbase.management.analytics.AnalyticsIndexManager`: for how to manage analytics dataverses, datasets, indexes and links. + * :meth:`~acouchbase.cluster.AsyncCluster.analytics_query`: for how to execute cluster-level analytics queries + + Args: + statement (str): The analytics SQL++ statement to execute. + options (:class:`~couchbase.options.AnalyticsOptions`): Optional parameters for the analytics query + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.AnalyticsOptions` + + Returns: + :class:`~couchbase.result.AnalyticsResult`: An instance of a :class:`~couchbase.result.AnalyticsResult` which provides access to iterate over the analytics + query results and access metadata and metrics about the analytics query. + + Examples: + .. note:: + Be sure to setup the necessary dataverse(s), dataset(s) for your analytics queries. + See :analytics_intro:`Analytics Introduction <>` in Couchbase Server docs. + + Simple analytics query:: + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = scope.analytics_query(q_str) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple analytics query with positional parameters:: + + from couchbase.options import AnalyticsOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = scope.analytics_query(q_str, AnalyticsOptions(positional_parameters=['United%', 5])) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple analytics query with named parameters:: + + from couchbase.options import AnalyticsOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = scope.analytics_query(q_str, + AnalyticsOptions(named_parameters={'country': 'United%', 'lim':2})) + async for row in q_res.rows(): + print(f'Found row: {row}') + + Retrieve metadata and/or metrics from analytics query:: + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = scope.analytics_query(q_str) + async for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'Analytics query metadata: {q_res.metadata()}') + print(f'Analytics query metrics: {q_res.metadata().metrics()}') + + """ # noqa: E501 + opt = AnalyticsOptions() + opts = list(options) + for o in opts: + if isinstance(o, AnalyticsOptions): + opt = o + opts.remove(o) + + request_args = dict() + num_workers = kwargs.pop('num_workers', None) + if num_workers: + request_args['num_workers'] = num_workers + + # set the query context as this bucket and scope if not provided + if not ('query_context' in opt or 'query_context' in kwargs): + kwargs['query_context'] = 'default:`{}`.`{}`'.format(self.bucket_name, self.name) + + query = AnalyticsQuery.create_query_object( + statement, *options, **kwargs) + return AnalyticsResult(AsyncAnalyticsRequest.generate_analytics_request(self.connection, + self.loop, + query.params, + **request_args)) + + def search_query(self, + index, # type: str + query, # type: SearchQuery + *options, # type: SearchOptions + **kwargs # type: Dict[str, Any] + ) -> SearchResult: + """Executes an search query against the scope. + + .. warning:: + This method is **DEPRECATED** and will be removed in a future release. + Use :meth:`~acouchbase.scope.AsyncScope.search`: instead. + + .. note:: + The search query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.SearchResult` begins. + + Args: + index (str): Name of the search query to use. + query (:class:`~couchbase.search.SearchQuery`): Type of search query to perform. + options (:class:`~couchbase.options.SearchOptions`): Optional parameters for the search query + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.SearchOptions` + + Returns: + :class:`~couchbase.result.SearchResult`: An instance of a + :class:`~couchbase.result.SearchResult` which provides access to iterate over the search + query results and access metadata and metrics about the search query. + + Examples: + + .. note:: + Be sure to create a search index prior to executing search queries. Also, if an application + desires to utilize search row locations, highlighting, etc. make sure the search index is + setup appropriately. See :search_create_idx:`Creating Indexes <>` in Couchbase Server docs. + + Simple search query:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + query = search.TermQuery('home') + q_res = scope.search_query('travel-sample-index', + query, + SearchOptions(limit=10)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + + + Simple search query with facets:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + facet_name = 'activity' + facet = search.TermFacet('activity') + query = search.TermQuery('home') + q_res = scope.search_query('travel-sample-index', + query, + SearchOptions(limit=10, facets={facet_name: facet})) + + async for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'facets: {q_res.facets()}') + + + Simple search query with fields and locations:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + search_fields = ['name', 'activity'] + query = search.TermQuery('home') + q_res = scope.search_query('travel-sample-index', + query, + SearchOptions(limit=10, + include_locations=True, + fields=search_fields)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + print(f'Fields: {row.fields}') + print(f'Locations: {row.locations}') + + """ + opt = SearchOptions() + opts = list(options) + for o in opts: + if isinstance(o, SearchOptions): + opt = o + opts.remove(o) + + request_args = dict() + num_workers = kwargs.pop('num_workers', None) + if num_workers: + request_args['num_workers'] = num_workers + + # set the scope_name as this scope if not provided + if not ('scope_name' in opt or 'scope_name' in kwargs): + kwargs['scope_name'] = f'{self.name}' + + query = SearchQueryBuilder.create_search_query_object(index, query, *options, **kwargs) + return SearchResult(AsyncFullTextSearchRequest.generate_search_request(self.connection, + self.loop, + query.as_encodable(), + **request_args)) + + def search(self, + index, # type: str + request, # type: SearchRequest + *options, # type: SearchOptions + **kwargs, # type: Dict[str, Any] + ) -> SearchResult: + """Executes an search against the scope. + + .. note:: + The search is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.SearchResult` begins. + + .. seealso:: + * :class:`~couchbase.management.search.ScopeSearchIndexManager`: for how to manage search indexes. + * :meth:`acouchbase.cluster.AsyncCluster.search`: for how to execute cluster-level search + + Args: + index (str): Name of the search index to use. + request (:class:`~couchbase.search.SearchRequest`): Type of search request to perform. + options (:class:`~couchbase.options.SearchOptions`): Optional parameters for the search query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.SearchOptions` + + Returns: + :class:`~couchbase.result.SearchResult`: An instance of a + :class:`~couchbase.result.SearchResult` which provides access to iterate over the search + query results and access metadata and metrics about the search query. + + Examples: + + .. note:: + Be sure to create a search index prior to executing a search. Also, if an application + desires to utilize search row locations, highlighting, etc. make sure the search index is + setup appropriately. See :search_create_idx:`Creating Indexes <>` in Couchbase Server docs. + + Simple search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + request = search.SearchRequest.create(search.TermQuery('home')) + q_res = scope.search('travel-sample-index', request, SearchOptions(limit=10)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + + Simple vector search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + from couchbase.vector_search import VectorQuery, VectorSearch + + # ... other code ... + + # NOTE: the vector is expected to be type List[float], set the vector to the appropriate value, this is an example. + vector = [-0.014653487130999565, -0.008658270351588726, 0.017129190266132355, -0.015563474968075752] + request = search.SearchRequest.create(VectorSearch.from_vector_query(VectorQuery('vector_field', vector))) + q_res = scope.search('travel-sample-vector-index', request, SearchOptions(limit=10)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + + Combine search and vector search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + from couchbase.vector_search import VectorQuery, VectorSearch + + # ... other code ... + + # NOTE: the vector is expected to be type List[float], set the vector to the appropriate value, this is an example. + vector_search = VectorSearch.from_vector_query(VectorQuery('vector_field', [-0.014653487130999565, + -0.008658270351588726, + 0.017129190266132355, + -0.015563474968075752])) + request = search.SearchRequest.create(search.MatchAllQuery()).with_vector_search(vector_search) + q_res = scope.search('travel-sample-vector-index', request, SearchOptions(limit=10)) + + async for row in q_res.rows(): + print(f'Found row: {row}') + """ # noqa: E501 + # See couchbase.cluster.search() for note on streaming timeout + streaming_timeout = self.streaming_timeouts.get('search_timeout', None) + query = SearchQueryBuilder.create_search_query_from_request(index, request, *options, **kwargs) + req = AsyncFullTextSearchRequest.generate_search_request(self.connection, + self.loop, + query.as_encodable(), + default_serializer=self.default_serializer, + streaming_timeout=streaming_timeout, + bucket_name=self.bucket_name, + scope_name=self.name) + return SearchResult(req) + + def search_indexes(self) -> ScopeSearchIndexManager: + """ + Get a :class:`~acouchbase.management.search.ScopeSearchIndexManager` which can be used to manage the search + indexes of this scope. + + Returns: + :class:`~acouchbase.management.search.ScopeSearchIndexManager`: A :class:`~acouchbase.management.search.ScopeSearchIndexManager` instance. + + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return ScopeSearchIndexManager(self.connection, self.loop, self.bucket_name, self.name) + + def eventing_functions(self) -> ScopeEventingFunctionManager: + """ + Get a :class:`~acouchbase.management.search.ScopeEventingFunctionManager` which can be used to manage the eventing + functions of this scope. + + Returns: + :class:`~acouchbase.management.search.ScopeEventingFunctionManager`: A :class:`~acouchbase.management.search.ScopeSearchIndexManager` instance. + + """ # noqa: E501 + return ScopeEventingFunctionManager(self.connection, self.loop, self.bucket_name, self.name) + + @staticmethod + def default_name(): + return "_default" + + +Scope = AsyncScope diff --git a/acouchbase/search.py b/acouchbase/search.py new file mode 100644 index 000000000..0b9c4a38e --- /dev/null +++ b/acouchbase/search.py @@ -0,0 +1,108 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from typing import Awaitable + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.search import SearchQueryBuilder # noqa: F401 +from couchbase.logic.search import FullTextSearchRequestLogic + + +class AsyncFullTextSearchRequest(FullTextSearchRequestLogic): + def __init__(self, + connection, + loop, + encoded_query, + **kwargs + ): + num_workers = kwargs.pop('num_workers', 2) + super().__init__(connection, encoded_query, **kwargs) + self._loop = loop + self._tp_executor = ThreadPoolExecutor(num_workers) + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @classmethod + def generate_search_request(cls, connection, loop, encoded_query, **kwargs): + return cls(connection, loop, encoded_query, **kwargs) + + def _get_metadata(self): + try: + query_response = next(self._streaming_result) + self._set_metadata(query_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def execute(self) -> Awaitable[None]: + async def _execute(): + return [r async for r in self] + + return asyncio.create_task(_execute()) + + def __aiter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_query() + + return self + + def _get_next_row(self): + if self.done_streaming is True: + return + + row = next(self._streaming_result) + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopAsyncIteration + + return self._deserialize_row(row) + + async def __anext__(self): + try: + return await self._loop.run_in_executor(self._tp_executor, self._get_next_row) + except asyncio.QueueEmpty: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls('Unexpected QueueEmpty exception caught when doing Search query.') + raise excptn + except StopAsyncIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/acouchbase/tests/__init__.py b/acouchbase/tests/__init__.py new file mode 100644 index 000000000..fc6346764 --- /dev/null +++ b/acouchbase/tests/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/acouchbase/tests/_test_utils.py b/acouchbase/tests/_test_utils.py new file mode 100644 index 000000000..a6c82361e --- /dev/null +++ b/acouchbase/tests/_test_utils.py @@ -0,0 +1,598 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import asyncio +from typing import (Any, + Callable, + Dict, + Optional, + Tuple, + Type, + Union) +from urllib.parse import urlparse + +import pytest + +from acouchbase.bucket import Bucket +from acouchbase.cluster import Cluster +from acouchbase.collection import Collection +from acouchbase.management.analytics import AnalyticsIndexManager +from acouchbase.management.buckets import BucketManager +from acouchbase.management.collections import CollectionManager +from acouchbase.management.eventing import EventingFunctionManager +from acouchbase.management.queries import QueryIndexManager +from acouchbase.management.search import SearchIndexManager +from acouchbase.management.users import UserManager +from acouchbase.management.views import ViewIndexManager +from acouchbase.scope import Scope +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import (AmbiguousTimeoutException, + BucketAlreadyExistsException, + BucketDoesNotExistException, + CollectionAlreadyExistsException, + CouchbaseException, + ScopeAlreadyExistsException, + ScopeNotFoundException, + UnAmbiguousTimeoutException) +from couchbase.management.buckets import (BucketType, + CreateBucketSettings, + StorageBackend) +from couchbase.management.collections import CollectionSpec +from couchbase.options import ClusterOptions +from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder +from tests.helpers import CollectionType # noqa: F401 +from tests.helpers import EventingFunctionManagementTestStatusException # noqa: F401 +from tests.helpers import FakeTestObj # noqa: F401 +from tests.helpers import KVPair # noqa: F401 +from tests.helpers import (CouchbaseTestEnvironment, + CouchbaseTestEnvironmentException, + RateLimitData) + + +class TestEnvironment(CouchbaseTestEnvironment): + + TEST_SCOPE = "test-scope" + TEST_COLLECTION = "test-collection" + + def __init__(self, cluster, bucket, collection, cluster_config, **kwargs): + super().__init__(cluster, bucket, collection, cluster_config) + if kwargs.get("manage_buckets", False) is True: + self.check_if_feature_supported('basic_bucket_mgmt') + self._bm = self.cluster.buckets() + + if kwargs.get("manage_collections", False) is True: + self.check_if_feature_supported('collections') + self._cm = self.bucket.collections() + + if kwargs.get("manage_query_indexes", False) is True: + self.check_if_feature_supported('query_index_mgmt') + self._ixm = self.cluster.query_indexes() + + if kwargs.get("manage_search_indexes", False) is True: + self.check_if_feature_supported('search_index_mgmt') + self._sixm = self.cluster.search_indexes() + + if kwargs.get("manage_view_indexes", False) is True: + self.check_if_feature_supported('view_index_mgmt') + self._vixm = self.bucket.view_indexes() + + if kwargs.get("manage_users", False) is True: + self.check_if_feature_supported('user_mgmt') + self._um = self.cluster.users() + + if kwargs.get("manage_analytics", False) is True: + self.check_if_feature_supported('analytics') + self._am = self.cluster.analytics_indexes() + + if kwargs.get('manage_eventing_functions', False) is True: + self.check_if_feature_supported('eventing_function_mgmt') + self._efm = self.cluster.eventing_functions() + + if kwargs.get('manage_rate_limit', False) is True: + self.check_if_feature_supported('rate_limiting') + parsed_conn = urlparse(cluster_config.get_connection_string()) + url = f'http://{parsed_conn.netloc}:8091' + u, p = cluster_config.get_username_and_pw() + self._rate_limit_params = RateLimitData(url, u, p) + + self._test_bucket = None + self._test_bucket_cm = None + self._collection_spec = None + self._scope = None + self._named_collection = None + self._use_scope_search_mgmt = False + self._use_scope_eventing_mgmt = False + + @property + def collection(self) -> Collection: + if self._named_collection is None: + return super().collection + return self._named_collection + + @property + def scope(self) -> Optional[Scope]: + return self._scope + + @property + def fqdn(self) -> Optional[str]: + return f'`{self.bucket.name}`.`{self.scope.name}`.`{self.collection.name}`' + + @property + def bm(self) -> Optional[BucketManager]: + """Returns the default bucket's BucketManager""" + return self._bm if hasattr(self, '_bm') else None + + @property + def cm(self) -> Optional[CollectionManager]: + """Returns the default CollectionManager""" + return self._cm if hasattr(self, '_cm') else None + + @property + def ixm(self) -> Optional[QueryIndexManager]: + """Returns the default QueryIndexManager""" + return self._ixm if hasattr(self, '_ixm') else None + + @property + def sixm(self) -> Optional[SearchIndexManager]: + """Returns the default SearchIndexManager""" + return self._sixm if hasattr(self, '_sixm') else None + + @property + def vixm(self) -> Optional[ViewIndexManager]: + """Returns the default ViewIndexManager""" + return self._vixm if hasattr(self, '_vixm') else None + + @property + def um(self) -> Optional[UserManager]: + """Returns the default UserManager""" + return self._um if hasattr(self, '_um') else None + + @property + def am(self) -> Optional[AnalyticsIndexManager]: + """Returns the default AnalyticsIndexManager""" + return self._am if hasattr(self, '_am') else None + + @property + def efm(self) -> Optional[EventingFunctionManager]: + """Returns the default EventingFunctionManager""" + return self._efm if hasattr(self, '_efm') else None + + @property + def test_bucket(self) -> Optional[Bucket]: + """Returns the test bucket object""" + return self._test_bucket if hasattr(self, '_test_bucket') else None + + @property + def test_bucket_cm(self) -> Optional[CollectionManager]: + """Returns the test bucket's CollectionManager""" + return self._test_bucket_cm if hasattr(self, '_test_bucket_cm') else None + + @property + def rate_limit_params(self) -> Optional[RateLimitData]: + """Returns the rate limit testing data""" + return self._rate_limit_params if hasattr(self, '_rate_limit_params') else None + + @property + def use_scope_search_mgmt(self) -> bool: + return self._use_scope_search_mgmt + + @property + def use_scope_eventing_mgmt(self) -> bool: + return self._use_scope_eventing_mgmt + + @classmethod # noqa: C901 + async def get_environment(cls, # noqa: C901 + test_suite, + couchbase_config, + coll_type=CollectionType.DEFAULT, + **kwargs): + + # this will only return False _if_ using the mock server + mock_supports = CouchbaseTestEnvironment.mock_supports_feature(test_suite.split('.')[-1], + couchbase_config.is_mock_server) + if not mock_supports: + pytest.skip(f'Mock server does not support feature(s) required for test suite: {test_suite}') + + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + opts = ClusterOptions(PasswordAuthenticator(username, pw)) + transcoder = kwargs.pop('transcoder', None) + if transcoder: + opts['transcoder'] = transcoder + okay = False + cluster = None + bucket = None + for _ in range(3): + try: + cluster = await Cluster.connect(conn_string, opts) + bucket = cluster.bucket(f"{couchbase_config.bucket_name}") + await bucket.on_connect() + await cluster.cluster_info() + okay = True + break + except (UnAmbiguousTimeoutException, AmbiguousTimeoutException): + continue + + if not okay: + if couchbase_config.is_mock_server: + pytest.skip(('CAVES does not seem to be happy. Skipping tests as failure is not' + ' an accurate representation of the state of the test, but rather' + ' there is an environment issue.')) + elif not cluster or not bucket: + pytest.skip(('Skipping tests as failure is not' + ' an accurate representation of the state of the test, but rather' + ' there is an environment issue.')) + + coll = bucket.default_collection() + if coll_type == CollectionType.DEFAULT: + cb_env = cls(cluster, bucket, coll, couchbase_config, **kwargs) + elif coll_type == CollectionType.NAMED: + if 'manage_collections' not in kwargs: + kwargs['manage_collections'] = True + cb_env = cls(cluster, + bucket, + coll, + couchbase_config, + **kwargs) + + return cb_env + + def disable_scope_search_mgmt(self) -> TestEnvironment: + self._use_scope_search_mgmt = False + return self + + def disable_search_mgmt(self) -> TestEnvironment: + self.check_if_feature_supported('search_index_mgmt') + if hasattr(self, '_sixm'): + del self._sixm + + return self + + def disable_scope_eventing_mgmt(self) -> TestEnvironment: + self._use_scope_eventing_mgmt = False + return self + + def disable_eventing_mgmt(self) -> TestEnvironment: + self.check_if_feature_supported('eventing_function_mgmt') + if hasattr(self, '_efm'): + del self._efm + + return self + + def enable_scope_search_mgmt(self) -> TestEnvironment: + self.check_if_feature_supported('scope_search_index_mgmt') + + self._use_scope_search_mgmt = True + return self + + def enable_search_mgmt(self) -> TestEnvironment: + self.check_if_feature_supported('search_index_mgmt') + if self.use_scope_search_mgmt: + if not hasattr(self.scope, 'search_indexes'): + pytest.skip('Search index management not available on scope.') + self._sixm = self.scope.search_indexes() + else: + if not hasattr(self.cluster, 'search_indexes'): + pytest.skip('Search index management not available on cluster.') + self._sixm = self.cluster.search_indexes() + return self + + def enable_scope_eventing_mgmt(self) -> TestEnvironment: + self.check_if_feature_supported('scope_eventing_function_mgmt') + self._use_scope_eventing_mgmt = True + return self + + def enable_eventing_mgmt(self) -> TestEnvironment: + self.check_if_feature_supported('eventing_function_mgmt') + if self.use_scope_eventing_mgmt: + if not hasattr(self.scope, 'eventing_functions'): + pytest.skip('Eventing function management not available on scope.') + self._efm = self.scope.eventing_functions() + else: + if not hasattr(self.cluster, 'eventing_functions'): + pytest.skip('Eventing function management not available on cluster.') + self._efm = self.cluster.eventing_functions() + return self + + async def get_new_key_value(self, reset=True): + if reset is True: + try: + await self.collection.remove(self.NEW_KEY) + except BaseException: + pass + return self.NEW_KEY, self.NEW_CONTENT + + # standard data load/purge + + async def load_data(self): + data_types, sample_json = self.load_data_from_file() + for dt in data_types: + data = sample_json.get(dt, None) + if data and "results" in data: + await asyncio.gather(*[self.collection.upsert(f"{r['type']}_{r['id']}", r) for r in data["results"]]) + # for r in data["results"]: + # key = f"{r['type']}_{r['id']}" + # await self.collection.upsert(key, r) + self._loaded_keys.extend( + [f"{r['type']}_{r['id']}" for r in data["results"]]) + + def get_json_data_by_type(self, json_type): + _, sample_json = self.load_data_from_file() + data = sample_json.get(json_type, None) + if data and 'results' in data: + return data['results'] + + return None + + async def purge_data(self): + await asyncio.gather(*[self.collection.remove(key) for key in self._loaded_keys]) + + # binary data load/purge + + async def load_utf8_binary_data(self, start_value=None): + utf8_key = self.get_binary_key('UTF8') + value = start_value or '' + tc = RawStringTranscoder() + await self.collection.upsert(utf8_key, value, transcoder=tc) + await self.try_n_times(10, 1, self.collection.get, utf8_key, transcoder=tc) + return utf8_key, value + + async def load_bytes_binary_data(self, start_value=None): + bytes_key = self.get_binary_key('BYTES') + value = start_value or b'' + tc = RawBinaryTranscoder() + await self.collection.upsert(bytes_key, value, transcoder=tc) + await self.try_n_times(10, 1, self.collection.get, bytes_key, transcoder=tc) + return bytes_key, value + + async def load_counter_binary_data(self, start_value=None): + counter_key = self.get_binary_key('COUNTER') + if start_value: + await self.collection.upsert(counter_key, start_value) + await self.try_n_times(10, 1, self.collection.get, counter_key) + + return counter_key, start_value + + async def purge_binary_data(self): + for k in self.get_binary_keys(): + try: + await self.collection.remove(k) + except CouchbaseException: + pass + + # Collection MGMT + + async def setup_collection_mgmt(self, bucket_name): + await self.create_bucket(bucket_name) + self._test_bucket = self.cluster.bucket(bucket_name) + await self.try_n_times(3, 5, self._test_bucket.on_connect) + self._test_bucket_cm = self._test_bucket.collections() + + async def setup_named_collections(self): + try: + await self.cm.create_scope(self.TEST_SCOPE) + except ScopeAlreadyExistsException: + await self.cm.drop_scope(self.TEST_SCOPE) + await self.cm.create_scope(self.TEST_SCOPE) + + self._collection_spec = CollectionSpec(self.TEST_COLLECTION, self.TEST_SCOPE) + try: + await self.cm.create_collection(self._collection_spec) + except CollectionAlreadyExistsException: + await self.cm.drop_collection(self._collection_spec) + await self.cm.create_collection(self._collection_spec) + + c = await self.get_collection(self.TEST_SCOPE, self.TEST_COLLECTION, bucket_name=self.bucket.name) + if c is None: + raise CouchbaseTestEnvironmentException("Unabled to create collection for name collection testing") + + self._scope = self.bucket.scope(self.TEST_SCOPE) + self._named_collection = self._scope.collection(self.TEST_COLLECTION) + + async def teardown_named_collections(self): + await self.cm.drop_scope(self.TEST_SCOPE) + await self.try_n_times_till_exception(10, + 1, + self.cm.drop_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeNotFoundException,) + ) + self._collection_spec = None + self._scope = None + self._named_collection = None + + async def get_scope(self, scope_name, bucket_name=None): + if bucket_name is None and self._test_bucket is None: + raise CouchbaseTestEnvironmentException("Must provide a bucket name or have a valid test_bucket available") + + bucket_names = ['default'] + if self._test_bucket is not None: + bucket_names.append(self._test_bucket.name) + + bucket = bucket_name or self._test_bucket.name + if bucket not in bucket_names + [bucket_name]: + raise CouchbaseTestEnvironmentException( + f"{bucket} is an invalid bucket name.") + + scopes = [] + if bucket_name is not None: + scopes = await self.cluster.bucket(bucket_name).collections().get_all_scopes() + elif bucket == self.bucket.name: + scopes = await self.cm.get_all_scopes() + else: + scopes = await self.test_bucket_cm.get_all_scopes() + return next((s for s in scopes if s.name == scope_name), None) + + async def get_collection(self, scope_name, coll_name, bucket_name=None): + scope = await self.get_scope(scope_name, bucket_name=bucket_name) + if scope: + return next( + (c for c in scope.collections if c.name == coll_name), None) + + return None + + # Bucket MGMT + + async def create_bucket(self, bucket_name, storage_backend=None): + try: + settings_kwargs = { + 'name': bucket_name, + 'bucket_type': BucketType.COUCHBASE, + 'ram_quota_mb': 100, + } + if storage_backend is not None: + settings_kwargs['storage_backend'] = storage_backend + if storage_backend is StorageBackend.MAGMA: + settings_kwargs['ram_quota_mb'] = 1024 + + await self.bm.create_bucket(CreateBucketSettings(**settings_kwargs)) + except BucketAlreadyExistsException: + pass + await self.try_n_times(10, 1, self.bm.get_bucket, bucket_name) + + async def purge_buckets(self, buckets): + for bucket in buckets: + try: + await self.bm.drop_bucket(bucket) + except BucketDoesNotExistException: + continue + except Exception: + raise + + # now be sure it is really gone + await self.try_n_times_till_exception(10, + 3, + self.bm.get_bucket, + bucket, + expected_exceptions=(BucketDoesNotExistException)) + + # helper methods + + async def _try_n_times(self, + num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + **kwargs # type: Dict[str,Any] + ) -> Any: + + for _ in range(num_times): + try: + return await func(*args, **kwargs) + except Exception: + await asyncio.sleep(float(seconds_between)) + + async def try_n_times(self, + num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + reset_on_timeout=False, # type: Optional[bool] + reset_num_times=None, # type: Optional[int] + **kwargs # type: Dict[str,Any] + ) -> Any: + + if reset_on_timeout: + reset_times = reset_num_times or num_times + for _ in range(reset_times): + try: + return await self._try_n_times(num_times, + seconds_between, + func, + *args, + **kwargs) + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + continue + except Exception: + raise + else: + return await self._try_n_times(num_times, + seconds_between, + func, + *args, + **kwargs) + + raise CouchbaseTestEnvironmentException( + f"Unsuccessful execution of {func.__name__} after {num_times} times, " + f"waiting {seconds_between} seconds between calls.") + + async def _try_n_times_until_exception(self, + num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + expected_exceptions=(Exception,), # type: Tuple[Type[Exception],...] + raise_exception=False, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ): + # type: (...) -> Any + for _ in range(num_times): + try: + await func(*args, **kwargs) + await asyncio.sleep(float(seconds_between)) + except expected_exceptions: + if raise_exception: + raise + # helpful to have this print statement when tests fail + return + except Exception: + raise + + async def try_n_times_till_exception(self, + num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + expected_exceptions=(Exception,), # type: Tuple[Type[Exception],...] + raise_exception=False, # type: Optional[bool] + raise_if_no_exception=True, # type: Optional[bool] + reset_on_timeout=False, # type: Optional[bool] + reset_num_times=None, # type: Optional[int] + **kwargs # type: Dict[str, Any] + ) -> None: + if reset_on_timeout: + reset_times = reset_num_times or num_times + for _ in range(reset_times): + try: + await self._try_n_times_until_exception(num_times, + seconds_between, + func, + *args, + expected_exceptions=expected_exceptions, + raise_exception=raise_exception, + **kwargs) + return + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + continue + except Exception: + raise + else: + await self._try_n_times_until_exception(num_times, + seconds_between, + func, + *args, + expected_exceptions=expected_exceptions, + raise_exception=raise_exception, + **kwargs) + return # success -- return now + + # TODO: option to restart mock server? + if raise_if_no_exception is False: + return + + raise CouchbaseTestEnvironmentException((f"Exception not raised calling {func.__name__} {num_times} times " + f"waiting {seconds_between} seconds between calls.")) diff --git a/acouchbase/tests/analytics_t.py b/acouchbase/tests/analytics_t.py new file mode 100644 index 000000000..117f6f4bb --- /dev/null +++ b/acouchbase/tests/analytics_t.py @@ -0,0 +1,357 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from datetime import datetime, timedelta + +import pytest +import pytest_asyncio + +from couchbase.analytics import (AnalyticsMetaData, + AnalyticsMetrics, + AnalyticsStatus, + AnalyticsWarning) +from couchbase.exceptions import (AmbiguousTimeoutException, + DatasetNotFoundException, + DataverseNotFoundException) +from couchbase.options import AnalyticsOptions, UnsignedInt64 +from tests.environments import CollectionType +from tests.environments.analytics_environment import AsyncAnalyticsTestEnvironment + + +class AnalyticsCollectionTestSuite: + TEST_MANIFEST = [ + 'test_analytics_metadata', + 'test_analytics_with_metrics', + 'test_bad_query_context', + 'test_bad_scope_query', + 'test_cluster_query_context', + 'test_query_fully_qualified', + 'test_scope_query', + 'test_scope_query_fqdn', + 'test_scope_query_with_named_params_in_options', + 'test_scope_query_with_positional_params_in_options', + ] + + @pytest.mark.asyncio + async def test_analytics_metadata(self, cb_env): + result = cb_env.scope.analytics_query(f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2') + await cb_env.assert_rows(result, 2) + metadata = result.metadata() # type: AnalyticsMetaData + assert isinstance(metadata, AnalyticsMetaData) + for id_meth in (metadata.client_context_id, metadata.request_id): + id_res = id_meth() + fail_msg = "{} failed".format(id_meth) + assert isinstance(id_res, str), fail_msg + assert metadata.status() == AnalyticsStatus.SUCCESS + assert isinstance(metadata.signature(), (str, dict)) + assert isinstance(metadata.warnings(), (list)) + for warning in metadata.warnings(): + assert isinstance(warning, AnalyticsWarning) + assert isinstance(warning.message(), str) + assert isinstance(warning.code(), int) + + @pytest.mark.asyncio + async def test_analytics_with_metrics(self, cb_env): + initial = datetime.now() + result = cb_env.scope.analytics_query(f'SELECT * FROM `{cb_env.collection.name}` LIMIT 1') + await cb_env.assert_rows(result, 1) + taken = datetime.now() - initial + metadata = result.metadata() # type: AnalyticsMetaData + metrics = metadata.metrics() + assert isinstance(metrics, AnalyticsMetrics) + assert isinstance(metrics.elapsed_time(), timedelta) + assert metrics.elapsed_time() < taken + assert metrics.elapsed_time() > timedelta(milliseconds=0) + assert isinstance(metrics.execution_time(), timedelta) + assert metrics.execution_time() < taken + assert metrics.execution_time() > timedelta(milliseconds=0) + + expected_counts = {metrics.error_count: 0, + metrics.result_count: 1, + metrics.warning_count: 0} + for method, expected in expected_counts.items(): + count_result = method() + fail_msg = "{} failed".format(method) + assert isinstance(count_result, UnsignedInt64), fail_msg + assert count_result == UnsignedInt64(expected), fail_msg + assert metrics.result_size() > UnsignedInt64(0) + assert isinstance(metrics.processed_objects(), UnsignedInt64) + assert metrics.error_count() == UnsignedInt64(0) + + @pytest.mark.asyncio + async def test_bad_query_context(self, cb_env): + # test w/ no context + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.collection.name} `LIMIT 2') + # @TODO: DatasetNotFoundException + with pytest.raises(DatasetNotFoundException): + await cb_env.assert_rows(result, 2) + + # test w/ bad scope + q_context = f'default:`{cb_env.bucket.name}`.`fake-scope`' + result = cb_env.cluster.analytics_query( + f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2', AnalyticsOptions(query_context=q_context)) + # @TODO: DataverseNotFoundException + with pytest.raises(DataverseNotFoundException): + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_bad_scope_query(self, cb_env): + q_context = f'default:`{cb_env.bucket.name}`.`fake-scope`' + result = cb_env.scope.analytics_query(f'SELECT * FROM {cb_env.fqdn} LIMIT 2', + AnalyticsOptions(query_context=q_context)) + with pytest.raises(DataverseNotFoundException): + await cb_env.assert_rows(result, 2) + + q_context = f'default:`fake-bucket`.`{cb_env.scope.name}`' + result = cb_env.scope.analytics_query(f'SELECT * FROM {cb_env.fqdn} LIMIT 2', + query_context=q_context) + with pytest.raises(DataverseNotFoundException): + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_cluster_query_context(self, cb_env): + q_context = f'default:`{cb_env.bucket.name}`.`{cb_env.scope.name}`' + # test with QueryOptions + a_opts = AnalyticsOptions(query_context=q_context) + result = cb_env.cluster.analytics_query( + f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2', a_opts) + await cb_env.assert_rows(result, 2) + + # test with kwargs + result = cb_env.cluster.analytics_query( + f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2', query_context=q_context) + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_query_fully_qualified(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM {cb_env.fqdn} LIMIT 2') + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_scope_query(self, cb_env): + result = cb_env.scope.analytics_query(f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2') + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_scope_query_fqdn(self, cb_env): + # @TODO: look into this... + if cb_env.server_version_short >= 7.1: + pytest.skip("Analytics scope query format not allowed on server versions >= 7.1") + + result = cb_env.scope.analytics_query(f'SELECT * FROM {cb_env.fqdn} LIMIT 2', query_context='') + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_scope_query_with_named_params_in_options(self, cb_env): + q_str = f'SELECT * FROM `{cb_env.collection.name}` WHERE batch LIKE $batch LIMIT 1' + result = cb_env.scope.analytics_query(q_str, + AnalyticsOptions(named_parameters={'batch': f'{cb_env.get_batch_id()}%'})) + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_scope_query_with_positional_params_in_options(self, cb_env): + result = cb_env.scope.analytics_query(f'SELECT * FROM `{cb_env.collection.name}` WHERE batch LIKE $1 LIMIT 1', + AnalyticsOptions(positional_parameters=[f'{cb_env.get_batch_id()}%'])) + await cb_env.assert_rows(result, 1) + + +class AnalyticsTestSuite: + TEST_MANIFEST = [ + 'test_analytics_metadata', + 'test_analytics_with_metrics', + 'test_query_named_parameters', + 'test_query_named_parameters_no_options', + 'test_query_named_parameters_override', + 'test_query_positional_params', + 'test_query_positional_params_no_option', + 'test_query_positional_params_override', + 'test_query_raw_options', + 'test_query_timeout', + 'test_simple_query', + ] + + @pytest.mark.asyncio + async def test_analytics_metadata(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` LIMIT 2') + await cb_env.assert_rows(result, 2) + metadata = result.metadata() # type: AnalyticsMetaData + isinstance(metadata, AnalyticsMetaData) + for id_meth in (metadata.client_context_id, metadata.request_id): + id_res = id_meth() + fail_msg = "{} failed".format(id_meth) + assert isinstance(id_res, str), fail_msg + assert metadata.status() == AnalyticsStatus.SUCCESS + assert isinstance(metadata.signature(), (str, dict)) + assert isinstance(metadata.warnings(), (list)) + for warning in metadata.warnings(): + assert isinstance(warning, AnalyticsWarning) + assert isinstance(warning.message(), str) + assert isinstance(warning.code(), int) + + @pytest.mark.asyncio + async def test_analytics_with_metrics(self, cb_env): + initial = datetime.now() + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` LIMIT 1') + await cb_env.assert_rows(result, 1) + taken = datetime.now() - initial + metadata = result.metadata() # type: AnalyticsMetaData + metrics = metadata.metrics() + assert isinstance(metrics, AnalyticsMetrics) + assert isinstance(metrics.elapsed_time(), timedelta) + assert metrics.elapsed_time() < taken + assert metrics.elapsed_time() > timedelta(milliseconds=0) + assert isinstance(metrics.execution_time(), timedelta) + assert metrics.execution_time() < taken + assert metrics.execution_time() > timedelta(milliseconds=0) + + expected_counts = {metrics.error_count: 0, + metrics.result_count: 1, + metrics.warning_count: 0} + for method, expected in expected_counts.items(): + count_result = method() + fail_msg = "{} failed".format(method) + assert isinstance(count_result, UnsignedInt64), fail_msg + assert count_result == UnsignedInt64(expected), fail_msg + assert metrics.result_size() > UnsignedInt64(0) + assert isinstance(metrics.processed_objects(), UnsignedInt64) + assert metrics.error_count() == UnsignedInt64(0) + + @pytest.mark.asyncio + async def test_query_named_parameters(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $atype LIMIT 1', + AnalyticsOptions(named_parameters={'atype': 'vehicle'})) + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_query_named_parameters_no_options(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $atype LIMIT 1', + atype='vehicle') + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_query_named_parameters_override(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $atype LIMIT 1', + AnalyticsOptions(named_parameters={'atype': 'abcdefg'}), + atype='vehicle') + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_query_positional_params(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $1 LIMIT 1', + AnalyticsOptions(positional_parameters=["vehicle"])) + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_query_positional_params_no_option(self, cb_env): + result = cb_env.cluster.analytics_query( + f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $1 LIMIT 1', 'vehicle') + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_query_positional_params_override(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $1 LIMIT 1', + AnalyticsOptions(positional_parameters=['abcdefg']), 'vehicle') + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_query_raw_options(self, cb_env): + # via raw, we should be able to pass any option + # if using named params, need to match full name param in query + # which is different for when we pass in name_parameters via their specific + # query option (i.e. include the $ when using raw) + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $atype LIMIT $1', + AnalyticsOptions(raw={'$atype': 'vehicle', 'args': [1]})) + await cb_env.assert_rows(result, 1) + + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $1 LIMIT 1', + AnalyticsOptions(raw={'args': ['vehicle']})) + await cb_env.assert_rows(result, 1) + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_query_timeout(self, cb_env): + from acouchbase.cluster import Cluster + from couchbase.auth import PasswordAuthenticator + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + # Prior to PYCBC-1521, this test would fail as each request would override the cluster level analytics_timeout. + # If a timeout was not provided in the request, the default 75s timeout would be used. PYCBC-1521 corrects + # this behavior so this test will pass as we are essentially forcing an AmbiguousTimeoutException because + # we are setting the cluster level analytics_timeout such a small value. + timeout_opts = ClusterTimeoutOptions(analytics_timeout=timedelta(milliseconds=1)) + cluster = await Cluster.connect(f'{conn_string}', ClusterOptions(auth, timeout_options=timeout_opts)) + # don't need to do this except for older server versions + _ = cluster.bucket(f'{cb_env.bucket.name}') + q_str = f'SELECT * FROM `{cb_env.DATASET_NAME}` LIMIT 1;' + with pytest.raises(AmbiguousTimeoutException): + res = cluster.analytics_query(q_str) + [r async for r in res.rows()] + # if we override the timeout w/in the request the query should succeed. + res = cluster.analytics_query(q_str, timeout=timedelta(seconds=10)) + rows = [r async for r in res.rows()] + assert len(rows) > 0 + + @pytest.mark.asyncio + async def test_simple_query(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` LIMIT 1') + await cb_env.assert_rows(result, 1) + + +class ClassicAnalyticsCollectionTests(AnalyticsCollectionTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicAnalyticsCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicAnalyticsCollectionTests) if valid_test_method(meth)] + compare = set(AnalyticsCollectionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.NAMED]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + acb_env = AsyncAnalyticsTestEnvironment.from_environment(acb_base_env) + acb_env.enable_analytics_mgmt() + await acb_env.setup(request.param) + yield acb_env + await acb_env.teardown(request.param) + + +class ClassicAnalyticsTests(AnalyticsTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicAnalyticsTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicAnalyticsTests) if valid_test_method(meth)] + compare = set(AnalyticsTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + acb_env = AsyncAnalyticsTestEnvironment.from_environment(acb_base_env) + acb_env.enable_analytics_mgmt() + await acb_env.setup(request.param) + yield acb_env + await acb_env.teardown(request.param) diff --git a/acouchbase/tests/analyticsmgmt_t.py b/acouchbase/tests/analyticsmgmt_t.py new file mode 100644 index 000000000..eac2e274f --- /dev/null +++ b/acouchbase/tests/analyticsmgmt_t.py @@ -0,0 +1,791 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import (AnalyticsLinkExistsException, + AnalyticsLinkNotFoundException, + CouchbaseException, + DatasetAlreadyExistsException, + DatasetNotFoundException, + DataverseAlreadyExistsException, + DataverseNotFoundException, + InternalServerFailureException, + InvalidArgumentException) +from couchbase.management.analytics import (AnalyticsDataType, + AnalyticsEncryptionLevel, + AnalyticsLinkType, + AzureBlobExternalAnalyticsLink, + CouchbaseAnalyticsEncryptionSettings, + CouchbaseRemoteAnalyticsLink, + S3ExternalAnalyticsLink) +from couchbase.management.options import (ConnectLinkOptions, + CreateAnalyticsIndexOptions, + CreateDatasetOptions, + CreateDataverseOptions, + DisconnectLinkOptions, + DropAnalyticsIndexOptions, + DropDatasetOptions, + DropDataverseOptions, + GetLinksAnalyticsOptions) + +from ._test_utils import TestEnvironment + + +class AnalyticsManagementTests: + DATASET_NAME = 'test-dataset' + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_analytics=True) + yield cb_env + + @pytest.fixture(scope="class") + def empty_dataverse_name(self, cb_env): + if cb_env.server_version_short >= 7.0: + name = 'empty/dataverse' + else: + name = 'empty_dataverse' + return name + + @pytest_asyncio.fixture() + async def create_empty_dataverse(self, cb_env, empty_dataverse_name): + await cb_env.am.create_dataverse(empty_dataverse_name, ignore_if_exists=True) + + @pytest_asyncio.fixture() + async def drop_empty_dataverse(self, cb_env, empty_dataverse_name): + yield + await cb_env.am.drop_dataverse(empty_dataverse_name, ignore_if_not_exists=True) + + @pytest_asyncio.fixture() + async def create_empty_dataset(self, cb_env): + await cb_env.am.create_dataset(self.DATASET_NAME, cb_env.bucket.name, ignore_if_exists=True) + + @pytest_asyncio.fixture() + async def drop_empty_dataset(self, cb_env): + yield + await cb_env.am.drop_dataset(self.DATASET_NAME, ignore_if_not_exists=True) + + @pytest_asyncio.fixture() + async def clean_drop(self, cb_env, empty_dataverse_name): + yield + await cb_env.am.drop_dataset(self.DATASET_NAME, ignore_if_not_exists=True) + await cb_env.am.drop_dataset(self.DATASET_NAME, + DropDatasetOptions(ignore_if_not_exists=True, + dataverse_name=empty_dataverse_name)) + await cb_env.am.drop_dataverse(empty_dataverse_name, ignore_if_not_exists=True) + + @pytest.mark.usefixtures("drop_empty_dataverse") + @pytest.mark.asyncio + async def test_create_dataverse(self, cb_env, empty_dataverse_name): + await cb_env.am.create_dataverse(empty_dataverse_name) + + @pytest.mark.usefixtures("drop_empty_dataverse") + @pytest.mark.asyncio + async def test_create_dataverse_ignore_exists(self, cb_env, empty_dataverse_name): + await cb_env.am.create_dataverse( + empty_dataverse_name, CreateDataverseOptions(ignore_if_exists=True)) + + with pytest.raises(DataverseAlreadyExistsException): + await cb_env.am.create_dataverse(empty_dataverse_name) + + @pytest.mark.usefixtures("create_empty_dataverse") + @pytest.mark.usefixtures("drop_empty_dataverse") + @pytest.mark.asyncio + async def test_drop_dataverse(self, cb_env, empty_dataverse_name): + await cb_env.am.drop_dataverse(empty_dataverse_name) + + @pytest.mark.usefixtures("create_empty_dataverse") + @pytest.mark.usefixtures("drop_empty_dataverse") + @pytest.mark.asyncio + async def test_drop_dataverse_ignore_not_exists(self, cb_env, empty_dataverse_name): + await cb_env.am.drop_dataverse(empty_dataverse_name) + with pytest.raises(DataverseNotFoundException): + await cb_env.am.drop_dataverse(empty_dataverse_name) + await cb_env.am.drop_dataverse(empty_dataverse_name, DropDataverseOptions(ignore_if_not_exists=True)) + + @pytest.mark.usefixtures("drop_empty_dataset") + @pytest.mark.asyncio + async def test_create_dataset(self, cb_env): + await cb_env.am.create_dataset(self.DATASET_NAME, cb_env.bucket.name) + + @pytest.mark.usefixtures("drop_empty_dataset") + @pytest.mark.asyncio + async def test_create_dataset_ignore_exists(self, cb_env): + await cb_env.am.create_dataset(self.DATASET_NAME, cb_env.bucket.name) + with pytest.raises(DatasetAlreadyExistsException): + await cb_env.am.create_dataset(self.DATASET_NAME, cb_env.bucket.name) + + await cb_env.am.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(ignore_if_exists=True)) + + @pytest.mark.usefixtures("create_empty_dataset") + @pytest.mark.usefixtures("drop_empty_dataset") + @pytest.mark.asyncio + async def test_drop_dataset(self, cb_env): + await cb_env.am.drop_dataset(self.DATASET_NAME) + + @pytest.mark.usefixtures("create_empty_dataset") + @pytest.mark.usefixtures("drop_empty_dataset") + @pytest.mark.asyncio + async def test_drop_dataset_ignore_not_exists(self, cb_env): + await cb_env.am.drop_dataset(self.DATASET_NAME) + with pytest.raises(DatasetNotFoundException): + await cb_env.am.drop_dataset(self.DATASET_NAME) + await cb_env.am.drop_dataset(self.DATASET_NAME, DropDatasetOptions(ignore_if_not_exists=True)) + + @pytest.mark.usefixtures("create_empty_dataverse") + @pytest.mark.usefixtures("clean_drop") + @pytest.mark.asyncio + async def test_get_all_datasets(self, cb_env, empty_dataverse_name): + await cb_env.am.create_dataset(self.DATASET_NAME, cb_env.bucket.name, ignore_if_exists=True) + await cb_env.am.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, + ignore_if_exists=True)) + + datasets = await cb_env.am.get_all_datasets() + local_ds = [ds for ds in datasets if ds.dataset_name == self.DATASET_NAME] + assert len(local_ds) == 2 + assert any(map(lambda ds: ds.dataverse_name == 'Default', local_ds)) is True + assert any(map(lambda ds: ds.dataverse_name == empty_dataverse_name, local_ds)) is True + + @pytest.mark.usefixtures("create_empty_dataverse") + @pytest.mark.usefixtures("clean_drop") + @pytest.mark.asyncio + async def test_create_index(self, cb_env, empty_dataverse_name): + await cb_env.am.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, ignore_if_exists=True)) + await cb_env.am.create_index("test_idx", self.DATASET_NAME, + {'name': AnalyticsDataType.STRING, + 'description': AnalyticsDataType.STRING}, + CreateAnalyticsIndexOptions(dataverse_name=empty_dataverse_name)) + + async def check_for_idx(idx): + indexes = await cb_env.am.get_all_indexes() + for index in indexes: + if index.name == idx: + return + raise Exception( + "unable to find 'test_idx' in list of all indexes") + + await cb_env.try_n_times(10, 3, check_for_idx, 'test_idx') + + @pytest.mark.usefixtures("create_empty_dataverse") + @pytest.mark.usefixtures("clean_drop") + @pytest.mark.asyncio + async def test_drop_index(self, cb_env, empty_dataverse_name): + # create one first, if not already there + await cb_env.am.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, ignore_if_exists=True)) + await cb_env.am.create_index("test_idx", self.DATASET_NAME, + {'name': AnalyticsDataType.STRING, + 'description': AnalyticsDataType.STRING}, + CreateAnalyticsIndexOptions(dataverse_name=empty_dataverse_name)) + + async def check_for_idx(idx): + indexes = await cb_env.am.get_all_indexes() + for index in indexes: + if index.name == idx: + return + raise Exception( + "unable to find 'test_idx' in list of all indexes") + + await cb_env.try_n_times(10, 3, check_for_idx, 'test_idx') + await cb_env.am.drop_index("test_idx", self.DATASET_NAME, + DropAnalyticsIndexOptions(dataverse_name=empty_dataverse_name)) + await cb_env.try_n_times_till_exception( + 10, 3, check_for_idx, 'test_idx') + + @pytest.mark.usefixtures("create_empty_dataverse") + @pytest.mark.usefixtures("clean_drop") + @pytest.mark.asyncio + async def test_connect_disconnect_link(self, cb_env, empty_dataverse_name): + await cb_env.am.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, + ignore_if_exists=True)) + await cb_env.am.connect_link(ConnectLinkOptions(dataverse_name=empty_dataverse_name)) + + # # connect link should result in documents in the dataset, so... + # dataverse_name = self.mgr._scrub_dataverse_name(self.dataverse_name) + # self.assertRows( + # 'USE {}; SELECT * FROM `{}` LIMIT 1'.format(dataverse_name, self.dataset_name)) + # # manually stop it for now + # self.cluster.analytics_query( + # 'USE {}; DISCONNECT LINK Local'.format(dataverse_name, self.dataset_name)).metadata() + await cb_env.am.disconnect_link(DisconnectLinkOptions(dataverse_name=empty_dataverse_name)) + + @pytest.mark.usefixtures("create_empty_dataverse") + @pytest.mark.usefixtures("clean_drop") + @pytest.mark.asyncio + async def test_get_pending_mutations(self, cb_env, empty_dataverse_name): + cb_env.check_if_feature_supported('analytics_pending_mutations') + dv_name = empty_dataverse_name.replace('/', '.') + key = f'{dv_name}.{self.DATASET_NAME}' + result = await cb_env.am.get_pending_mutations() + assert key not in result.keys() + await cb_env.am.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, + ignore_if_exists=True)) + await cb_env.am.connect_link(ConnectLinkOptions(dataverse_name=empty_dataverse_name)) + await asyncio.sleep(1) + result = await cb_env.am.get_pending_mutations() + assert key in result.keys() + await cb_env.am.disconnect_link(DisconnectLinkOptions(dataverse_name=empty_dataverse_name)) + + @pytest.mark.asyncio + async def test_v6_dataverse_name_parsing(self, cb_env): + if cb_env.server_version_short >= 7.0: + pytest.skip("Test only for 6.x versions") + + # test.test_dataverse, valid format which is valid >= 6.0, but not on 6.6...weird + if cb_env.server_version_short >= 6.6: + with pytest.raises(CouchbaseException): + await cb_env.am.create_dataverse( + "test.test_dataverse", CreateDataverseOptions(ignore_if_exists=True)) + else: + await cb_env.am.create_dataverse( + "test.test_dataverse", CreateDataverseOptions(ignore_if_exists=True)) + + # test/test_dataverse, invalid format < 7.0 + with pytest.raises((InternalServerFailureException, CouchbaseException)): + await cb_env.am.create_dataverse( + "test/test_dataverse", CreateDataverseOptions(ignore_if_exists=True)) + + @pytest.mark.asyncio + async def test_v7_dataverse_name_parsing(self, cb_env): + if cb_env.server_version_short < 7.0: + pytest.skip("Test only for 7.x versions") + + # test.test_dataverse, valid format which is valid >= 6.6 + await cb_env.am.create_dataverse( + "test.test_dataverse", CreateDataverseOptions(ignore_if_exists=True)) + await cb_env.am.drop_dataverse("test.test_dataverse") + + # test/test_dataverse, valideformat which is valid >= 7.0 + await cb_env.am.create_dataverse( + "test/test_dataverse", CreateDataverseOptions(ignore_if_exists=True)) + await cb_env.am.drop_dataverse("test/test_dataverse") + + +class AnalyticsManagementLinksTests: + DATASET_NAME = 'test-dataset' + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_analytics=True) + + cb_env.check_if_feature_supported('analytics_link_mgmt') + yield cb_env + + @pytest.fixture(scope="class") + def empty_dataverse_name(self, cb_env): + if cb_env.server_version_short >= 7.0: + name = 'empty/dataverse' + else: + name = 'empty_dataverse' + return name + + @pytest_asyncio.fixture() + async def create_drop_empty_dataverse(self, cb_env, empty_dataverse_name): + await cb_env.am.create_dataverse(empty_dataverse_name, ignore_if_exists=True) + yield + await cb_env.am.drop_dataverse(empty_dataverse_name, ignore_if_not_exists=True) + + @pytest.mark.usefixtures('cb_env') + def test_couchbase_remote_link_encode(self): + link = CouchbaseRemoteAnalyticsLink("test_dataverse", + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username="Administrator", + password="password") + + encoded = link.as_dict() + assert isinstance(encoded, dict) + assert encoded.get('hostname') == 'localhost' + assert encoded.get('link_type') == AnalyticsLinkType.CouchbaseRemote.value + link_encryption = encoded.get('encryption', None) + assert isinstance(link_encryption, dict) + assert link_encryption.get('encryption_level') == AnalyticsEncryptionLevel.NONE.value + assert encoded.get('username') == 'Administrator' + assert encoded.get('password') == 'password' + + link = CouchbaseRemoteAnalyticsLink("test_dataverse", + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL, + certificate=bytes( + 'certificate', 'utf-8'), + client_certificate=bytes( + 'clientcertificate', 'utf-8'), + client_key=bytes('clientkey', 'utf-8')), + ) + + encoded = link.as_dict() + assert isinstance(encoded, dict) + assert encoded.get('hostname') == 'localhost' + assert encoded.get('link_type') == AnalyticsLinkType.CouchbaseRemote.value + link_encryption = encoded.get('encryption', None) + assert isinstance(link_encryption, dict) + assert link_encryption.get('encryption_level') == AnalyticsEncryptionLevel.FULL.value + assert link_encryption.get('certificate') == 'certificate' + assert link_encryption.get('client_certificate') == 'clientcertificate' + assert link_encryption.get('client_key') == 'clientkey' + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_create_s3_external_link(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + "s3link", + "accesskey", + "us-west-2", + secret_access_key="mysupersecretkey", + ) + + await cb_env.am.create_link(link) + + links = await cb_env.am.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 1 + assert links[0].dataverse_name() == link.dataverse_name() + assert links[0].name() == link.name() + assert links[0].link_type() == AnalyticsLinkType.S3External + assert links[0]._region == link._region + assert links[0]._access_key_id == link._access_key_id + + await cb_env.am.drop_link(link.name(), empty_dataverse_name) + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_replace_s3_external_link(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + "s3link", + "accesskey", + "us-west-2", + secret_access_key="mysupersecretkey", + ) + + await cb_env.am.create_link(link) + + links = await cb_env.am.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 1 + assert links[0].dataverse_name() == link.dataverse_name() + assert links[0].name() == link.name() + assert links[0].link_type() == AnalyticsLinkType.S3External + assert links[0]._region == link._region + assert links[0]._access_key_id == link._access_key_id + + new_link = S3ExternalAnalyticsLink(empty_dataverse_name, + "s3link", + "accesskey", + "eu-west-2", + secret_access_key="mysupersecretkey1", + ) + + await cb_env.am.replace_link(new_link) + + links = await cb_env.am.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 1 + assert links[0].dataverse_name() == new_link.dataverse_name() + assert links[0].name() == new_link.name() + assert links[0].link_type() == AnalyticsLinkType.S3External + assert links[0]._region == new_link._region + assert links[0]._access_key_id == new_link._access_key_id + + await cb_env.am.drop_link(link.name(), empty_dataverse_name) + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_drop_s3_external_link(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + "s3link", + "accesskey", + "us-west-2", + secret_access_key="mysupersecretkey", + ) + + await cb_env.am.create_link(link) + + links = await cb_env.am.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 1 + assert links[0].dataverse_name() == link.dataverse_name() + assert links[0].name() == link.name() + assert links[0].link_type() == AnalyticsLinkType.S3External + assert links[0]._region == link._region + assert links[0]._access_key_id == link._access_key_id + + await cb_env.am.drop_link(link.name(), empty_dataverse_name) + + links = await cb_env.am.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 0 + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_create_link_fail_link_exists(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + "s3link", + "accesskey", + "us-west-2", + secret_access_key="mysupersecretkey", + ) + + await cb_env.am.create_link(link) + + with pytest.raises(AnalyticsLinkExistsException): + await cb_env.am.create_link(link) + + await cb_env.am.drop_link(link.name(), empty_dataverse_name) + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_s3_link_fail_dataverse_not_found(self, cb_env): + + link = S3ExternalAnalyticsLink("notadataverse", + "s3link", + "accesskey", + "us-west-2", + secret_access_key="mysupersecretkey", + ) + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.create_link(link) + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.replace_link(link) + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_couchbase_link_fail_dataverse_not_found(self, cb_env): + + link = CouchbaseRemoteAnalyticsLink("notadataverse", + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username="Administrator", + password="password") + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.create_link(link) + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.replace_link(link) + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_azure_link_fail_dataverse_not_found(self, cb_env): + + link = AzureBlobExternalAnalyticsLink("notadataverse", + "azurebloblink", + account_name="myaccount", + account_key="myaccountkey") + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.create_link(link) + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.replace_link(link) + + with pytest.raises(DataverseNotFoundException): + await cb_env.am.drop_link(link.name(), link.dataverse_name()) + + @pytest.fixture() + def bad_couchbase_remote_links(self, empty_dataverse_name): + links = [] + links.append(CouchbaseRemoteAnalyticsLink("", + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username="Administrator", + password="password")) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username="Administrator", + password="password")) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username="Administrator", + password="password")) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + password="password")) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username="Administrator")) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.HALF), + password="password")) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.HALF), + username="Administrator")) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL) + )) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL, + certificate=bytes('certificate', 'utf-8')) + )) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL, + certificate=bytes( + 'certificate', 'utf-8'), + client_certificate=bytes('clientcert', 'utf-8')) + )) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL, + certificate=bytes( + 'certificate', 'utf-8'), + client_key=bytes('clientkey', 'utf-8')) + )) + + return links + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_create_couchbase_link_fail_invalid_argument(self, cb_env, bad_couchbase_remote_links): + for link in bad_couchbase_remote_links: + with pytest.raises(InvalidArgumentException): + await cb_env.am.create_link(link) + + @pytest.fixture() + def bad_s3_external_links(self, empty_dataverse_name): + links = [] + links.append(S3ExternalAnalyticsLink("", + "s3link", + "accesskey", + "us-west-2", + secret_access_key="mysupersecretkey", + )) + + links.append(S3ExternalAnalyticsLink(empty_dataverse_name, + "", + "accesskey", + "us-west-2", + secret_access_key="mysupersecretkey", + )) + + links.append(S3ExternalAnalyticsLink(empty_dataverse_name, + "s3link", + "", + "us-west-2", + secret_access_key="mysupersecretkey", + )) + + links.append(S3ExternalAnalyticsLink(empty_dataverse_name, + "s3link", + "accesskey", + "", + secret_access_key="mysupersecretkey", + )) + + links.append(S3ExternalAnalyticsLink("", + "s3link", + "accesskey", + "us-west-2", + )) + return links + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_create_s3_link_fail_invalid_argument(self, cb_env, bad_s3_external_links): + for link in bad_s3_external_links: + with pytest.raises(InvalidArgumentException): + await cb_env.am.create_link(link) + + @pytest.fixture() + def bad_azure_blob_external_links(self, empty_dataverse_name): + links = [] + links.append(AzureBlobExternalAnalyticsLink("", + "azurebloblink", + account_name="myaccount", + account_key="myaccountkey")) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + "", + account_name="myaccount", + account_key="myaccountkey")) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + "azurebloblink")) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + "azurebloblink", + account_name="myaccount")) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + "azurebloblink", + account_key="myaccountkey")) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + "azurebloblink", + shared_access_signature="sharedaccesssignature")) + return links + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_create_azure_block_link_fail_invalid_argument(self, cb_env, bad_azure_blob_external_links): + for link in bad_azure_blob_external_links: + with pytest.raises(InvalidArgumentException): + await cb_env.am.create_link(link) + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_s3_link_fail_link_not_found(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + "notalink", + "accesskey", + "us-west-2", + secret_access_key="mysupersecretkey", + ) + + with pytest.raises(AnalyticsLinkNotFoundException): + await cb_env.am.replace_link(link) + + with pytest.raises(AnalyticsLinkNotFoundException): + await cb_env.am.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_couchbase_link_fail_link_not_found(self, cb_env, empty_dataverse_name): + + link = CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + "cbremote", + "localhost", + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username="Administrator", + password="password") + + with pytest.raises(AnalyticsLinkNotFoundException): + await cb_env.am.replace_link(link) + + with pytest.raises(AnalyticsLinkNotFoundException): + await cb_env.am.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures("create_drop_empty_dataverse") + @pytest.mark.asyncio + async def test_azure_link_fail_link_not_found(self, cb_env, empty_dataverse_name): + + link = AzureBlobExternalAnalyticsLink(empty_dataverse_name, + "azurebloblink", + account_name="myaccount", + account_key="myaccountkey") + + with pytest.raises(AnalyticsLinkNotFoundException): + await cb_env.am.replace_link(link) + + with pytest.raises(AnalyticsLinkNotFoundException): + await cb_env.am.drop_link(link.name(), link.dataverse_name()) + + # @TODO: neither situation raises an exception... + # @pytest.mark.usefixtures("create_drop_empty_dataverse") + # @pytest.mark.asyncio + # async def test_get_links_fail(self, cb_env): + + # with pytest.raises(DataverseNotFoundException): + # await cb_env.am.get_links(GetLinksAnalyticsOptions( + # dataverse_name="notadataverse")) + + # with pytest.raises(InvalidArgumentException): + # await cb_env.am.get_links(GetLinksAnalyticsOptions(name="mylink")) diff --git a/acouchbase/tests/async_utilities_t.py b/acouchbase/tests/async_utilities_t.py new file mode 100644 index 000000000..28222a4ce --- /dev/null +++ b/acouchbase/tests/async_utilities_t.py @@ -0,0 +1,252 @@ +# Copyright 2016-2024. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from functools import wraps + +import pytest + +from acouchbase.logic.wrappers import call_async_fn +from acouchbase.transactions.transactions import AsyncWrapper as TxnAsyncWrapper +from couchbase.exceptions import (CouchbaseException, + ErrorMapper, + InternalSDKException) +from couchbase.exceptions import exception as BaseCouchbaseException + + +class AsyncTestWrapper: + + @classmethod + def inject_callbacks(cls): + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(res): + self.loop.call_soon_threadsafe(ft.set_result, res) + + def on_err(err): + self.loop.call_soon_threadsafe(ft.set_exception, err) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + call_async_fn(ft, self, fn, *args, **kwargs) + return ft + + return wrapped_fn + + return decorator + + +class AsyncTxnTestWrapper: + + @classmethod + def inject_callbacks(cls): + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(res): + self.loop.call_soon_threadsafe(ft.set_result, res) + + def on_err(err): + self.loop.call_soon_threadsafe(ft.set_exception, err) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + TxnAsyncWrapper.call_async_fn(ft, self, fn, *args, **kwargs) + return ft + + return wrapped_fn + + return decorator + + +class AsyncTester: + + def __init__(self, loop): + self._loop = loop + + @property + def loop(self): + return self._loop + + @AsyncTestWrapper.inject_callbacks() + def use_callback(self, result, **kwargs): + callback = kwargs.pop('callback') + callback(result) + + @AsyncTestWrapper.inject_callbacks() + def use_errback(self, error, **kwargs): + errback = kwargs.pop('errback') + errback(error) + + @AsyncTestWrapper.inject_callbacks() + def fn_failure(self, error, **kwargs): + cause = kwargs.pop('cause', None) + if cause is not None: + raise error from cause + raise error + + +class AsyncTxnTester: + + def __init__(self, loop): + self._loop = loop + + @property + def loop(self): + return self._loop + + @AsyncTxnTestWrapper.inject_callbacks() + def use_callback(self, result, **kwargs): + callback = kwargs.pop('callback') + callback(result) + + @AsyncTxnTestWrapper.inject_callbacks() + def use_errback(self, error, **kwargs): + errback = kwargs.pop('errback') + errback(error) + + @AsyncTxnTestWrapper.inject_callbacks() + def fn_failure(self, error, **kwargs): + cause = kwargs.pop('cause', None) + if cause is not None: + raise error from cause + raise error + + +class AsyncUtilityTestSuite: + TEST_MANIFEST = [ + 'test_call_async_fn_callback', + 'test_call_async_fn_errback', + 'test_call_async_fn_wrapped_fn_failure', + 'test_txn_call_async_fn_wrapped_fn_failure', + ] + + @pytest.mark.parametrize('tester_class', [AsyncTester, AsyncTxnTester]) + @pytest.mark.asyncio + async def test_call_async_fn_callback(self, tester_class): + expected = 'Success!' + tester = tester_class(asyncio.get_event_loop()) + res = await tester.use_callback(expected) + assert isinstance(res, str) + assert res == expected + + @pytest.mark.parametrize('err, tester_class', [(BaseCouchbaseException, AsyncTester), + (CouchbaseException, AsyncTester), + (SystemError, AsyncTester), + (BaseException, AsyncTester), + (KeyboardInterrupt, AsyncTester), + (SystemExit, AsyncTester), + (asyncio.CancelledError, AsyncTester), + (BaseCouchbaseException, AsyncTxnTester), + (CouchbaseException, AsyncTxnTester), + (SystemError, AsyncTxnTester), + (BaseException, AsyncTxnTester), + (KeyboardInterrupt, AsyncTxnTester), + (SystemExit, AsyncTxnTester), + (asyncio.CancelledError, AsyncTxnTester)]) + @pytest.mark.asyncio + async def test_call_async_fn_errback(self, err, tester_class): + tester = tester_class(asyncio.get_event_loop()) + + expected_error = err() + # need to convert the BaseCouchbaseException to a Python Exception + if err.__module__ == 'pycbc_core': + expected_error = ErrorMapper.build_exception(expected_error) + with pytest.raises(type(expected_error)): + await tester.use_errback(expected_error) + + @pytest.mark.parametrize('err', [BaseCouchbaseException, + CouchbaseException, + SystemError, + BaseException, + KeyboardInterrupt, + SystemExit, + asyncio.CancelledError]) + @pytest.mark.asyncio + async def test_call_async_fn_wrapped_fn_failure(self, err): + tester = AsyncTester(asyncio.get_event_loop()) + + raise_with_cause = False + # need to convert the BaseCouchbaseException to a Python Exception + if err.__module__ == 'pycbc_core': + expected_error = ErrorMapper.build_exception(err()) + else: + expected_error = err() + + class_name = expected_error.__class__.__name__ + if class_name == 'SystemError': + raise_with_cause = True + + if raise_with_cause is True: + # InternalSDKException b/c the SDK translates unknown to InternalSDKException + with pytest.raises(InternalSDKException): + await tester.fn_failure(expected_error) + cause = Exception('Fail!') + with pytest.raises(type(cause)): + await tester.fn_failure(expected_error, cause=cause) + else: + with pytest.raises(type(expected_error)): + await tester.fn_failure(expected_error) + + @pytest.mark.parametrize('err', [BaseCouchbaseException, + CouchbaseException, + SystemError, + BaseException, + KeyboardInterrupt, + SystemExit, + asyncio.CancelledError]) + @pytest.mark.asyncio + async def test_txn_call_async_fn_wrapped_fn_failure(self, err): + tester = AsyncTxnTester(asyncio.get_event_loop()) + + raise_with_cause = False + # need to convert the BaseCouchbaseException to a Python Exception + if err.__module__ == 'pycbc_core': + expected_error = ErrorMapper.build_exception(err()) + else: + expected_error = err() + + class_name = expected_error.__class__.__name__ + if class_name == 'SystemError': + raise_with_cause = True + + if raise_with_cause is True: + # TypeError b/c cause is None where a cause is expected + with pytest.raises(TypeError): + await tester.fn_failure(expected_error) + cause = Exception('Fail!') + with pytest.raises(type(cause)): + await tester.fn_failure(expected_error, cause=cause) + else: + with pytest.raises(type(expected_error)): + await tester.fn_failure(expected_error) + + +class AsyncUtilityTests(AsyncUtilityTestSuite): + + @pytest.fixture(scope='class', autouse=True) + def manifest_validated(self): + def valid_test_method(meth): + attr = getattr(AsyncUtilityTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(AsyncUtilityTests) if valid_test_method(meth)] + test_list = set(AsyncUtilityTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest not validated. Missing/extra tests: {test_list}.') diff --git a/acouchbase/tests/binary_collection_t.py b/acouchbase/tests/binary_collection_t.py new file mode 100644 index 000000000..f8ea2b180 --- /dev/null +++ b/acouchbase/tests/binary_collection_t.py @@ -0,0 +1,344 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import DocumentNotFoundException, InvalidArgumentException +from couchbase.options import (DecrementOptions, + DeltaValue, + IncrementOptions, + SignedInt64) +from couchbase.result import CounterResult, MutationResult +from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment) + + +class BinaryCollectionTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + + # teardown + await cb_env.try_n_times(5, 3, cb_env.purge_binary_data) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + # key/value fixtures + + @pytest_asyncio.fixture(name='utf8_empty_kvp') + async def utf8_key_and_empty_value(self, cb_env) -> KVPair: + key, value = await cb_env.try_n_times(5, 3, cb_env.load_utf8_binary_data) + yield KVPair(key, value) + await cb_env.collection.upsert(key, '', transcoder=RawStringTranscoder()) + + @pytest_asyncio.fixture(name='utf8_kvp') + async def utf8_key_and_value(self, cb_env) -> KVPair: + key, value = await cb_env.try_n_times(5, 3, cb_env.load_utf8_binary_data, start_value='XXXX') + yield KVPair(key, value) + await cb_env.collection.upsert(key, '', transcoder=RawStringTranscoder()) + + @pytest_asyncio.fixture(name='bytes_empty_kvp') + async def bytes_key_and_empty_value(self, cb_env) -> KVPair: + key, value = await cb_env.try_n_times(5, 3, cb_env.load_bytes_binary_data) + yield KVPair(key, value) + await cb_env.collection.upsert(key, b'', transcoder=RawBinaryTranscoder()) + + @pytest_asyncio.fixture(name='bytes_kvp') + async def bytes_key_and_value(self, cb_env) -> KVPair: + key, value = await cb_env.try_n_times(5, 3, cb_env.load_bytes_binary_data, start_value=b'XXXX') + yield KVPair(key, value) + await cb_env.collection.upsert(key, b'', transcoder=RawBinaryTranscoder()) + + @pytest_asyncio.fixture(name='counter_empty_kvp') + async def counter_key_and_empty_value(self, cb_env) -> KVPair: + key, value = await cb_env.try_n_times(5, 3, cb_env.load_counter_binary_data) + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,)) + + @pytest_asyncio.fixture(name='counter_kvp') + async def counter_key_and_value(self, cb_env) -> KVPair: + key, value = await cb_env.try_n_times(5, 3, cb_env.load_counter_binary_data, start_value=100) + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,)) + + # tests + + @pytest.mark.asyncio + async def test_append_string(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + result = await cb.binary().append(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = await cb.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.asyncio + async def test_append_string_not_empty(self, cb_env, utf8_kvp): + cb = cb_env.collection + key = utf8_kvp.key + value = utf8_kvp.value + result = await cb.binary().append(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = await cb.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == value + 'foo' + + @pytest.mark.asyncio + async def test_append_string_nokey(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + await cb.remove(key) + await cb_env.try_n_times_till_exception(10, + 1, + cb.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + # @TODO(jc): 3.2.x SDK tests for NotStoredException + with pytest.raises(DocumentNotFoundException): + await cb.binary().append(key, 'foo') + + @pytest.mark.asyncio + async def test_append_bytes(self, cb_env, bytes_empty_kvp): + cb = cb_env.collection + key = bytes_empty_kvp.key + result = await cb.binary().append(key, b'XXX') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = await cb.get(key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'XXX' + + @pytest.mark.asyncio + async def test_append_bytes_not_empty(self, cb_env, bytes_kvp): + cb = cb_env.collection + key = bytes_kvp.key + value = bytes_kvp.value + + result = await cb.binary().append(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = await cb.get(key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == value + b'foo' + + @pytest.mark.asyncio + async def test_prepend_string(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + result = await cb.binary().prepend(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = await cb.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.asyncio + async def test_prepend_string_not_empty(self, cb_env, utf8_kvp): + cb = cb_env.collection + key = utf8_kvp.key + value = utf8_kvp.value + + result = await cb.binary().prepend(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = await cb.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + value + + @pytest.mark.asyncio + async def test_prepend_string_nokey(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + await cb.remove(key) + await cb_env.try_n_times_till_exception(10, + 1, + cb.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + # @TODO(jc): 3.2.x SDK tests for NotStoredException + with pytest.raises(DocumentNotFoundException): + await cb.binary().prepend(key, 'foo') + + @pytest.mark.asyncio + async def test_prepend_bytes(self, cb_env, bytes_empty_kvp): + cb = cb_env.collection + key = bytes_empty_kvp.key + result = await cb.binary().prepend(key, b'XXX') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = await cb.get(key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'XXX' + + @pytest.mark.asyncio + async def test_prepend_bytes_not_empty(self, cb_env, bytes_kvp): + cb = cb_env.collection + key = bytes_kvp.key + value = bytes_kvp.value + + result = await cb.binary().prepend(key, b'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = await cb.get(key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'foo' + value + + @pytest.mark.asyncio + async def test_counter_increment_initial_value(self, cb_env, counter_empty_kvp): + cb = cb_env.collection + key = counter_empty_kvp.key + + result = await cb.binary().increment(key, IncrementOptions(initial=SignedInt64(100))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == 100 + + @pytest.mark.asyncio + async def test_counter_decrement_initial_value(self, cb_env, counter_empty_kvp): + cb = cb_env.collection + key = counter_empty_kvp.key + + result = await cb.binary().decrement(key, DecrementOptions(initial=SignedInt64(100))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == 100 + + @pytest.mark.asyncio + async def test_counter_increment(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + result = await cb.binary().increment(key) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value + 1 + + @pytest.mark.asyncio + async def test_counter_decrement(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + result = await cb.binary().decrement(key) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value - 1 + + @pytest.mark.asyncio + async def test_counter_increment_non_default(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + result = await cb.binary().increment(key, IncrementOptions(delta=DeltaValue(3))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value + 3 + + @pytest.mark.asyncio + async def test_counter_decrement_non_default(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + result = await cb.binary().decrement(key, DecrementOptions(delta=DeltaValue(3))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value - 3 + + @pytest.mark.asyncio + async def test_counter_increment_no_initial_value(self, cb_env): + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.binary().increment('non-existent-doc', IncrementOptions(initial=SignedInt64(-1))) + + @pytest.mark.asyncio + async def test_counter_decrement_no_initial_value(self, cb_env): + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.binary().decrement('non-existent-doc', DecrementOptions(initial=SignedInt64(-1))) + + @pytest.mark.asyncio + async def test_counter_bad_initial_value(self, cb_env, counter_empty_kvp): + cb = cb_env.collection + key = counter_empty_kvp.key + + with pytest.raises(InvalidArgumentException): + await cb.binary().increment(key, initial=100) + + with pytest.raises(InvalidArgumentException): + await cb.binary().decrement(key, initial=100) + + @pytest.mark.asyncio + async def test_counter_bad_delta_value(self, cb_env, counter_empty_kvp): + cb = cb_env.collection + key = counter_empty_kvp.key + + with pytest.raises(InvalidArgumentException): + await cb.binary().increment(key, delta=5) + + with pytest.raises(InvalidArgumentException): + await cb.binary().decrement(key, delta=5) + + @pytest.mark.asyncio + async def test_unsigned_int(self): + with pytest.raises(InvalidArgumentException): + x = DeltaValue(-1) + with pytest.raises(InvalidArgumentException): + x = DeltaValue(0x7FFFFFFFFFFFFFFF + 1) + + x = DeltaValue(5) + assert 5 == x.value + + @pytest.mark.asyncio + async def test_signed_int_64(self): + with pytest.raises(InvalidArgumentException): + x = SignedInt64(-0x7FFFFFFFFFFFFFFF - 2) + + with pytest.raises(InvalidArgumentException): + x = SignedInt64(0x7FFFFFFFFFFFFFFF + 1) + + x = SignedInt64(0x7FFFFFFFFFFFFFFF) + assert 0x7FFFFFFFFFFFFFFF == x.value + x = SignedInt64(-0x7FFFFFFFFFFFFFFF - 1) + assert -0x7FFFFFFFFFFFFFFF - 1 == x.value diff --git a/acouchbase/tests/binary_durability_t.py b/acouchbase/tests/binary_durability_t.py new file mode 100644 index 000000000..f16bb3a5e --- /dev/null +++ b/acouchbase/tests/binary_durability_t.py @@ -0,0 +1,367 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.durability import (ClientDurability, + DurabilityLevel, + PersistTo, + PersistToExtended, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import DocumentNotFoundException, DurabilityImpossibleException +from couchbase.options import (AppendOptions, + DecrementOptions, + IncrementOptions, + PrependOptions) +from couchbase.transcoder import RawStringTranscoder + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment) + + +class DurabilityTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + + # teardown + await cb_env.try_n_times(5, 3, cb_env.purge_binary_data) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name='utf8_empty_kvp') + async def utf8_key_and_empty_value(self, cb_env) -> KVPair: + key, value = await cb_env.try_n_times(5, 3, cb_env.load_utf8_binary_data) + yield KVPair(key, value) + await cb_env.collection.upsert(key, '', transcoder=RawStringTranscoder()) + + @pytest_asyncio.fixture(name='counter_kvp') + async def counter_key_and_value(self, cb_env) -> KVPair: + key, value = await cb_env.try_n_times(5, 3, cb_env.load_counter_binary_data, start_value=100) + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,)) + + @pytest.fixture(scope="class") + def check_sync_durability_supported(self, cb_env): + cb_env.check_if_feature_supported('sync_durability') + + @pytest.fixture(scope="class") + def num_nodes(self, cb_env): + return len(cb_env.cluster._cluster_info.nodes) + + @pytest.fixture(scope="class") + def check_multi_node(self, num_nodes): + if num_nodes == 1: + pytest.skip("Test only for clusters with more than a single node.") + + @pytest.fixture(scope="class") + def check_single_node(self, num_nodes): + if num_nodes != 1: + pytest.skip("Test only for clusters with a single node.") + + @pytest_asyncio.fixture(scope="class") + async def num_replicas(self, cb_env): + bucket_settings = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + return num_replicas + + @pytest.fixture(scope="class") + def check_has_replicas(self, num_replicas): + if num_replicas == 0: + pytest.skip("No replicas to test durability.") + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_append(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + + await cb.binary().append(key, 'foo', AppendOptions(durability=durability)) + result = await cb.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_append_single_node(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().append(key, 'foo', AppendOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_prepend(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + + await cb.binary().append(key, 'foo', AppendOptions(durability=durability)) + result = await cb.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_prepend_single_node(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().prepend(key, 'foo', PrependOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_append(self, cb_env, utf8_empty_kvp, num_replicas): + cb = cb_env.collection + key = utf8_empty_kvp.key + + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + await cb.binary().append(key, 'foo', AppendOptions(durability=durability)) + result = await cb.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_append_fail(self, cb_env, utf8_empty_kvp, num_replicas): + cb = cb_env.collection + key = utf8_empty_kvp.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().append(key, 'foo', AppendOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_append_single_node(self, cb_env, utf8_empty_kvp, num_replicas): + cb = cb_env.collection + key = utf8_empty_kvp.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().append(key, 'foo', AppendOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_prepend(self, cb_env, utf8_empty_kvp, num_replicas): + cb = cb_env.collection + key = utf8_empty_kvp.key + + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + await cb.binary().prepend(key, 'foo', PrependOptions(durability=durability)) + result = await cb.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_prepend_fail(self, cb_env, utf8_empty_kvp, num_replicas): + cb = cb_env.collection + key = utf8_empty_kvp.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().prepend(key, 'foo', PrependOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_prepend_single_node(self, cb_env, utf8_empty_kvp, num_replicas): + cb = cb_env.collection + key = utf8_empty_kvp.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().prepend(key, 'foo', PrependOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_increment(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + result = await cb.binary().increment(key, IncrementOptions(durability=durability)) + assert result.content == value + 1 + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_increment_single_node(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + await cb.binary().increment(key, IncrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_decrement(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + + result = await cb.binary().decrement(key, DecrementOptions(durability=durability)) + assert result.content == value - 1 + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_decrement_single_node(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + await cb.binary().decrement(key, DecrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_increment(self, cb_env, counter_kvp, num_replicas): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + result = await cb.binary().increment(key, IncrementOptions(durability=durability)) + assert result.content == value + 1 + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_increment_fail(self, cb_env, counter_kvp, num_replicas): + cb = cb_env.collection + key = counter_kvp.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().increment(key, IncrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_increment_single_node(self, cb_env, counter_kvp, num_replicas): + cb = cb_env.collection + key = counter_kvp.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().increment(key, IncrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_decrement(self, cb_env, counter_kvp, num_replicas): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + result = await cb.binary().decrement(key, DecrementOptions(durability=durability)) + assert result.content == value - 1 + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_decrement_fail(self, cb_env, counter_kvp, num_replicas): + cb = cb_env.collection + key = counter_kvp.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().decrement(key, DecrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_decrement_single_node(self, cb_env, counter_kvp, num_replicas): + cb = cb_env.collection + key = counter_kvp.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + with pytest.raises(DurabilityImpossibleException): + await cb.binary().decrement(key, DecrementOptions(durability=durability)) diff --git a/acouchbase/tests/bucket_t.py b/acouchbase/tests/bucket_t.py new file mode 100644 index 000000000..64f0c0c4b --- /dev/null +++ b/acouchbase/tests/bucket_t.py @@ -0,0 +1,134 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from uuid import uuid4 + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.diagnostics import (EndpointPingReport, + PingState, + ServiceType) +from couchbase.exceptions import InvalidArgumentException +from couchbase.options import PingOptions +from couchbase.result import PingResult + +from ._test_utils import TestEnvironment + + +class BucketDiagnosticsTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True) + + yield cb_env + + @pytest.fixture(scope="class") + def check_diagnostics_supported(self, cb_env): + cb_env.check_if_feature_supported('diagnostics') + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping(self, cb_env): + bucket = cb_env.bucket + result = await bucket.ping() + assert isinstance(result, PingResult) + + assert result.sdk is not None + assert result.id is not None + assert result.version is not None + assert result.endpoints is not None + for ping_reports in result.endpoints.values(): + for report in ping_reports: + assert isinstance(report, EndpointPingReport) + if report.state == PingState.OK: + assert report.id is not None + assert report.latency is not None + assert report.remote is not None + assert report.local is not None + assert report.service_type is not None + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_report_id(self, cb_env): + bucket = cb_env.bucket + report_id = uuid4() + result = await bucket.ping(PingOptions(report_id=report_id)) + assert str(report_id) == result.id + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_restrict_services(self, cb_env): + bucket = cb_env.bucket + services = [ServiceType.KeyValue] + result = await bucket.ping(PingOptions(service_types=services)) + keys = list(result.endpoints.keys()) + assert len(keys) == 1 + assert keys[0] == ServiceType.KeyValue + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_str_services(self, cb_env): + bucket = cb_env.bucket + services = [ServiceType.KeyValue.value, ServiceType.Query.value] + result = await bucket.ping(PingOptions(service_types=services)) + assert len(result.endpoints) >= 1 + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_mixed_services(self, cb_env): + bucket = cb_env.bucket + services = [ServiceType.KeyValue, ServiceType.Query.value] + result = await bucket.ping(PingOptions(service_types=services)) + assert len(result.endpoints) >= 1 + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_invalid_services(self, cb_env): + bucket = cb_env.bucket + with pytest.raises(InvalidArgumentException): + await bucket.ping(PingOptions(service_types=ServiceType.KeyValue)) + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_as_json(self, cb_env): + bucket = cb_env.bucket + result = await bucket.ping() + assert isinstance(result, PingResult) + result_str = result.as_json() + assert isinstance(result_str, str) + result_json = json.loads(result_str) + assert result_json['version'] is not None + assert result_json['id'] is not None + assert result_json['sdk'] is not None + assert result_json['services'] is not None + for _, data in result_json['services'].items(): + if len(data): + assert data[0]['id'] is not None + assert data[0]['latency_us'] is not None + assert data[0]['remote'] is not None + assert data[0]['local'] is not None + assert data[0]['state'] is not None diff --git a/acouchbase/tests/bucketmgmt_t.py b/acouchbase/tests/bucketmgmt_t.py new file mode 100644 index 000000000..85b6099da --- /dev/null +++ b/acouchbase/tests/bucketmgmt_t.py @@ -0,0 +1,438 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from random import choice + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.durability import DurabilityLevel +from couchbase.exceptions import (BucketAlreadyExistsException, + BucketDoesNotExistException, + BucketNotFlushableException, + FeatureUnavailableException, + InvalidArgumentException) +from couchbase.management.buckets import (BucketSettings, + BucketType, + ConflictResolutionType, + CreateBucketSettings, + StorageBackend) + +from ._test_utils import TestEnvironment + + +@pytest.mark.flaky(reruns=5) +class BucketManagementTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest.fixture(scope="class") + def test_buckets(self): + return [f"test-bucket-{i}" for i in range(5)] + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config, test_buckets): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True) + + yield cb_env + if cb_env.is_feature_supported('bucket_mgmt'): + await cb_env.purge_buckets(test_buckets) + + @pytest.fixture(scope="class") + def check_bucket_mgmt_supported(self, cb_env): + cb_env.check_if_feature_supported('bucket_mgmt') + + @pytest.fixture(scope="class") + def check_bucket_min_durability_supported(self, cb_env): + cb_env.check_if_feature_supported('bucket_min_durability') + + @pytest.fixture(scope="class") + def check_bucket_storage_backend_supported(self, cb_env): + cb_env.check_if_feature_supported('bucket_storage_backend') + + @pytest.fixture(scope="class") + def check_custom_conflict_resolution_supported(self, cb_env): + cb_env.check_if_feature_supported('custom_conflict_resolution') + + @pytest.fixture(scope="class") + def check_non_deduped_history_supported(self, cb_env): + cb_env.check_if_feature_supported('non_deduped_history') + + @pytest.fixture() + def test_bucket(self, test_buckets): + return choice(test_buckets) + + # TODO: more efficient, but cannot seem to get consistent results + # multiple tests fail with BucketAlreadyExistsException + # @pytest_asyncio.fixture() + # async def purge_bucket(self, cb_env, # type: TestEnvironment + # test_bucket # type: str + # ): + # yield + # await cb_env.purge_buckets(test_bucket) + + @pytest_asyncio.fixture() + async def purge_buckets(self, cb_env, test_buckets): + yield + await cb_env.purge_buckets(test_buckets) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_create(self, cb_env, test_bucket): + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100)) + bucket = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + if cb_env.server_version_short >= 6.6: + assert bucket["minimum_durability_level"] == DurabilityLevel.NONE + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_create_replica_index_true(self, cb_env, test_bucket): + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + replica_index=True)) + bucket = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + assert bucket.replica_index is True + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_create_replica_index_false(self, cb_env, test_bucket): + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + replica_index=False)) + bucket = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + assert bucket.replica_index is False + + @pytest.mark.usefixtures("check_bucket_min_durability_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_create_durability(self, cb_env, test_bucket): + min_durability = DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE + await cb_env.bm.create_bucket(CreateBucketSettings(name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + minimum_durability_level=min_durability)) + bucket = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + assert bucket["minimum_durability_level"] == min_durability + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_create_fail(self, cb_env, test_bucket): + settings = CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100) + await cb_env.bm.create_bucket(settings) + with pytest.raises(BucketAlreadyExistsException): + await cb_env.bm.create_bucket(settings) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.asyncio + async def test_bucket_drop_fail(self, cb_env, test_bucket): + settings = CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100) + await cb_env.bm.create_bucket(settings) + await cb_env.try_n_times(10, 1, cb_env.bm.drop_bucket, test_bucket) + with pytest.raises(BucketDoesNotExistException): + await cb_env.bm.drop_bucket(test_bucket) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_list(self, cb_env, test_buckets): + for bucket in test_buckets: + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket, + ram_quota_mb=100)) + await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, bucket) + + buckets = await cb_env.bm.get_all_buckets() + assert set() == set(test_buckets).difference( + set(map(lambda b: b.name, buckets))) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_cluster_sees_bucket(self, cb_env, test_bucket): + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100)) + await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + # cluster should be able to return it (though, not right away) + b = cb_env.cluster.bucket(test_bucket) + await cb_env.try_n_times(10, 1, b.on_connect) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_change_expiry(self, cb_env, test_bucket): + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100)) + await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + + # change bucket TTL + await cb_env.try_n_times(10, 3, cb_env.bm.update_bucket, BucketSettings( + name=test_bucket, max_expiry=timedelta(seconds=500))) + + async def get_bucket_expiry_equal(name, expiry): + b = await cb_env.bm.get_bucket(name) + + if not expiry == b.max_expiry: + raise Exception("not equal") + + await cb_env.try_n_times(10, 3, get_bucket_expiry_equal, test_bucket, timedelta(seconds=500)) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_flush(self, cb_env, test_bucket): + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + flush_enabled=True)) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.flush_enabled is True + # flush the bucket + await cb_env.try_n_times(10, 3, cb_env.bm.flush_bucket, bucket.name) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_flush_fail(self, cb_env, test_bucket): + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + flush_enabled=False)) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.flush_enabled is False + + with pytest.raises(BucketNotFlushableException): + await cb_env.bm.flush_bucket(test_bucket) + + @pytest.mark.usefixtures("check_bucket_storage_backend_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_backend_default(self, cb_env, test_bucket): + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + flush_enabled=False)) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + if cb_env.supports_feature('magma_128_buckets'): + assert bucket.storage_backend == StorageBackend.MAGMA + else: + assert bucket.storage_backend == StorageBackend.COUCHSTORE + + @pytest.mark.usefixtures("check_bucket_storage_backend_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_backend_magma(self, cb_env, test_bucket): + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=1024, + flush_enabled=False, + storage_backend=StorageBackend.MAGMA)) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.storage_backend == StorageBackend.MAGMA + + @pytest.mark.usefixtures("check_bucket_storage_backend_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_backend_couchstore(self, cb_env, test_bucket): + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + flush_enabled=False, + storage_backend=StorageBackend.COUCHSTORE)) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.storage_backend == StorageBackend.COUCHSTORE + + @pytest.mark.usefixtures("check_bucket_storage_backend_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_backend_ephemeral(self, cb_env, test_bucket): + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + bucket_type=BucketType.EPHEMERAL, + flush_enabled=False)) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.storage_backend == StorageBackend.UNDEFINED + + @pytest.mark.usefixtures("check_custom_conflict_resolution_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_custom_conflict_resolution(self, cb_env, test_bucket): + if cb_env.is_developer_preview: + # Create the bucket + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + conflict_resolution_type=ConflictResolutionType.CUSTOM, + flush_enabled=False)) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.conflict_resolution_type == ConflictResolutionType.CUSTOM + else: + with pytest.raises(FeatureUnavailableException): + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + conflict_resolution_type=ConflictResolutionType.CUSTOM, + flush_enabled=False)) + + @pytest.mark.usefixtures("check_non_deduped_history_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_create_history_retention(self, cb_env, test_bucket): + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.MAGMA, + ram_quota_mb=1024, + history_retention_collection_default=True, + history_retention_bytes=2**31, + history_retention_duration=timedelta(days=1) + ) + ) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.history_retention_collection_default + assert bucket.history_retention_bytes == 2**31 + assert bucket.history_retention_duration == timedelta(days=1) + + @pytest.mark.usefixtures("check_non_deduped_history_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_create_history_retention_unsupported(self, cb_env, test_bucket): + with pytest.raises(InvalidArgumentException): + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.COUCHSTORE, + ram_quota_mb=1024, + history_retention_collection_default=True, + history_retention_bytes=2**31, + history_retention_duration=timedelta(days=1) + ) + ) + + @pytest.mark.usefixtures("check_non_deduped_history_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_update_history_retention(self, cb_env, test_bucket): + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.MAGMA, + ram_quota_mb=1024, + ) + ) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket is not None + assert bucket.history_retention_collection_default + assert bucket.history_retention_bytes == 0 + assert bucket.history_retention_duration == timedelta(0) + + await cb_env.bm.update_bucket( + BucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.MAGMA, + ram_quota_mb=1024, + history_retention_collection_default=True, + history_retention_bytes=2**31, + history_retention_duration=timedelta(minutes=10) + ) + ) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.history_retention_collection_default + assert bucket.history_retention_bytes == 2**31 + assert bucket.history_retention_duration == timedelta(minutes=10) + + @pytest.mark.usefixtures("check_non_deduped_history_supported") + @pytest.mark.usefixtures("purge_buckets") + @pytest.mark.asyncio + async def test_bucket_update_history_retention_unsupported(self, cb_env, test_bucket): + await cb_env.bm.create_bucket( + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.COUCHSTORE, + ram_quota_mb=1024, + ) + ) + bucket = await cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket is not None + assert bucket.history_retention_collection_default is None + assert bucket.history_retention_bytes is None + assert bucket.history_retention_duration is None + + with pytest.raises(InvalidArgumentException): + await cb_env.bm.update_bucket( + BucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.MAGMA, + ram_quota_mb=1024, + history_retention_collection_default=True, + history_retention_bytes=2**31, + history_retention_duration=timedelta(minutes=10) + ) + ) diff --git a/acouchbase/tests/chain_t.py b/acouchbase/tests/chain_t.py new file mode 100644 index 000000000..64805c105 --- /dev/null +++ b/acouchbase/tests/chain_t.py @@ -0,0 +1,85 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +import pytest +import pytest_asyncio + +from acouchbase.bucket import AsyncBucket +from acouchbase.cluster import (AsyncCluster, + Cluster, + get_event_loop) +from couchbase.auth import PasswordAuthenticator +from couchbase.options import ClusterOptions +from couchbase.result import GetResult + + +class ConnectionChainingTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest.mark.asyncio + async def test_bucket_create_chain(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster = Cluster(conn_string, ClusterOptions(auth)) + # validate the cluster, at this point, the connect future should be pending + assert isinstance(cluster, AsyncCluster) + assert cluster._connect_ftr.done() is False + + bucket = cluster.bucket(couchbase_config.bucket_name) + assert isinstance(bucket, AsyncBucket) + await bucket.on_connect() + # after connecting the bucket, the cluster connection should now exist b/c the connection + # futures are chained (i.e. cluster.connect -> bucket.connect) + assert cluster._connect_ftr is not None + assert cluster._connect_ftr.done() is True + assert cluster._connection is not None + assert bucket._connect_ftr.done() is True + + @pytest.mark.asyncio + async def test_kv_op_chain(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster = Cluster(conn_string, ClusterOptions(auth)) + bucket = cluster.bucket(couchbase_config.bucket_name) + coll = bucket.default_collection() + key = 'async-test-conn-key' + doc = {'id': key, 'what': 'this is an asyncio test.'} + await coll.upsert(key, doc) + res = None + for _ in range(3): + try: + res = await coll.get(key) + except Exception: + await asyncio.sleep(float(1.0)) + + # after executing the KV op, the cluster connection should now exist b/c the connection + # futures are chained (i.e. cluster.connect -> bucket.connect -> KV op) + assert cluster._connect_ftr.done() is True + assert bucket._connect_ftr.done() is True + assert isinstance(res, GetResult) + assert res.content_as[dict] == doc + + # @TODO(jc): PYCBC-1414 - once complete, validate cluster operations chaining diff --git a/acouchbase/tests/cluster_t.py b/acouchbase/tests/cluster_t.py new file mode 100644 index 000000000..98381344b --- /dev/null +++ b/acouchbase/tests/cluster_t.py @@ -0,0 +1,220 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta +from uuid import uuid4 + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.diagnostics import (ClusterState, + EndpointPingReport, + EndpointState, + PingState, + ServiceType) +from couchbase.exceptions import (InternalServerFailureException, + InvalidArgumentException, + ParsingFailedException, + QueryIndexNotFoundException) +from couchbase.options import DiagnosticsOptions, PingOptions +from couchbase.result import DiagnosticsResult, PingResult + +from ._test_utils import TestEnvironment + + +class ClusterDiagnosticsTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True) + + yield cb_env + + @pytest.fixture(scope="class") + def check_diagnostics_supported(self, cb_env): + cb_env.check_if_feature_supported('diagnostics') + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping(self, cb_env): + cluster = cb_env.cluster + result = await cluster.ping() + assert isinstance(result, PingResult) + + assert result.sdk is not None + assert result.id is not None + assert result.version is not None + assert result.endpoints is not None + for ping_reports in result.endpoints.values(): + for report in ping_reports: + assert isinstance(report, EndpointPingReport) + if report.state == PingState.OK: + assert report.id is not None + assert report.latency is not None + assert report.remote is not None + assert report.local is not None + assert report.service_type is not None + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_report_id(self, cb_env): + cluster = cb_env.cluster + report_id = uuid4() + result = await cluster.ping(PingOptions(report_id=report_id)) + assert str(report_id) == result.id + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_restrict_services(self, cb_env): + cluster = cb_env.cluster + services = [ServiceType.KeyValue] + result = await cluster.ping(PingOptions(service_types=services)) + keys = list(result.endpoints.keys()) + assert len(keys) == 1 + assert keys[0] == ServiceType.KeyValue + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_str_services(self, cb_env): + cluster = cb_env.cluster + services = [ServiceType.KeyValue.value, ServiceType.Query.value] + result = await cluster.ping(PingOptions(service_types=services)) + assert len(result.endpoints) >= 1 + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_mixed_services(self, cb_env): + cluster = cb_env.cluster + services = [ServiceType.KeyValue, ServiceType.Query.value] + result = await cluster.ping(PingOptions(service_types=services)) + assert len(result.endpoints) >= 1 + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_invalid_services(self, cb_env): + cluster = cb_env.cluster + with pytest.raises(InvalidArgumentException): + await cluster.ping(PingOptions(service_types=ServiceType.KeyValue)) + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_ping_as_json(self, cb_env): + cluster = cb_env.cluster + result = await cluster.ping() + assert isinstance(result, PingResult) + result_str = result.as_json() + assert isinstance(result_str, str) + result_json = json.loads(result_str) + assert result_json['version'] is not None + assert result_json['id'] is not None + assert result_json['sdk'] is not None + assert result_json['services'] is not None + for _, data in result_json['services'].items(): + if len(data): + assert data[0]['id'] is not None + assert data[0]['latency_us'] is not None + assert data[0]['remote'] is not None + assert data[0]['local'] is not None + assert data[0]['state'] is not None + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_diagnostics(self, cb_env): + cluster = cb_env.cluster + report_id = str(uuid4()) + result = await cluster.diagnostics( + DiagnosticsOptions(report_id=report_id)) + assert isinstance(result, DiagnosticsResult) + assert result.id == report_id + assert result.sdk is not None + assert result.version is not None + assert result.state == ClusterState.Online + + kv_endpoints = result.endpoints[ServiceType.KeyValue] + assert len(kv_endpoints) > 0 + assert kv_endpoints[0].id is not None + assert kv_endpoints[0].local is not None + assert kv_endpoints[0].remote is not None + assert kv_endpoints[0].last_activity_us is not None + assert kv_endpoints[0].state == EndpointState.Connected + assert kv_endpoints[0].service_type == ServiceType.KeyValue + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_diagnostics_after_query(self, cb_env): + cluster = cb_env.cluster + # lets make sure there is at least 1 row + key, value = cb_env.get_default_key_value() + await cb_env.collection.upsert(key, value) + bucket_name = cb_env.bucket.name + report_id = str(uuid4()) + # the mock will fail on query, but diagnostics should + # still return a query service type + try: + rows = await cluster.query(f'SELECT * FROM `{bucket_name}` LIMIT 1').execute() + assert len(rows) > 0 + except (InternalServerFailureException, ParsingFailedException, QueryIndexNotFoundException): + pass + except Exception as e: + print(f'exception: {e.__class__.__name__}, {e}') + raise e + + result = await cluster.diagnostics( + DiagnosticsOptions(report_id=report_id)) + assert result.id == report_id + + query_diag_result = result.endpoints[ServiceType.Query] + assert len(query_diag_result) >= 1 + for q in query_diag_result: + assert q.id is not None + assert q.local is not None + assert q.remote is not None + assert isinstance(q.last_activity_us, timedelta) + assert q.state == EndpointState.Connected + assert q.service_type == ServiceType.Query + + @pytest.mark.usefixtures("check_diagnostics_supported") + @pytest.mark.asyncio + async def test_diagnostics_as_json(self, cb_env): + cluster = cb_env.cluster + report_id = str(uuid4()) + result = await cluster.diagnostics( + DiagnosticsOptions(report_id=report_id)) + + assert isinstance(result, DiagnosticsResult) + result_str = result.as_json() + assert isinstance(result_str, str) + result_json = json.loads(result_str) + assert result_json['version'] is not None + assert result_json['id'] is not None + assert result_json['sdk'] is not None + assert result_json['services'] is not None + for _, data in result_json['services'].items(): + if len(data): + assert data[0]['id'] is not None + assert data[0]['last_activity_us'] is not None + assert data[0]['remote'] is not None + assert data[0]['local'] is not None + assert data[0]['state'] is not None diff --git a/acouchbase/tests/collection_t.py b/acouchbase/tests/collection_t.py new file mode 100644 index 000000000..e7e47a676 --- /dev/null +++ b/acouchbase/tests/collection_t.py @@ -0,0 +1,724 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from datetime import datetime, timedelta +from time import time + +import pytest +import pytest_asyncio + +import couchbase.subdocument as SD +from acouchbase.cluster import get_event_loop +from couchbase.diagnostics import ServiceType +from couchbase.exceptions import (CasMismatchException, + DocumentExistsException, + DocumentLockedException, + DocumentNotFoundException, + DocumentNotLockedException, + DocumentUnretrievableException, + InvalidArgumentException, + PathNotFoundException, + TemporaryFailException) +from couchbase.options import (GetAllReplicasOptions, + GetAnyReplicaOptions, + GetOptions, + InsertOptions, + ReplaceOptions, + UpsertOptions) +from couchbase.replica_reads import ReadPreference +from couchbase.result import (ExistsResult, + GetReplicaResult, + GetResult, + MutationResult) +from tests.mock_server import MockServerType + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment) + + +class CollectionTests: + + NO_KEY = "not-a-key" + FIFTY_YEARS = 50 * 365 * 24 * 60 * 60 + THIRTY_DAYS = 30 * 24 * 60 * 60 + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + manage_buckets=True) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + await cb_env.try_n_times(3, 5, cb_env.load_data) + yield cb_env + await cb_env.try_n_times_till_exception(3, 5, + cb_env.purge_data, + raise_if_no_exception=False) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest.fixture(scope="class") + def check_preserve_expiry_supported(self, cb_env): + cb_env.check_if_feature_supported('preserve_expiry') + + @pytest.fixture(scope="class") + def check_sync_durability_supported(self, cb_env): + cb_env.check_if_feature_supported('sync_durability') + + @pytest.fixture(scope="class") + def check_xattr_supported(self, cb_env): + cb_env.check_if_feature_supported('xattr') + + @pytest.fixture(scope="class") + def check_not_locked_supported(self, cb_env): + cb_env.check_if_feature_supported('kv_not_locked') + + @pytest.fixture(scope='class') + def check_server_groups_supported(self, cb_env): + cb_env.check_if_feature_supported('server_groups') + + @pytest_asyncio.fixture(name="new_kvp") + async def new_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = await cb_env.get_new_key_value() + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="default_kvp") + def default_key_and_value(self, cb_env) -> KVPair: + key, value = cb_env.get_default_key_value() + yield KVPair(key, value) + + @pytest_asyncio.fixture(name="default_kvp_and_reset") + async def default_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_default_key_value() + yield KVPair(key, value) + await cb_env.try_n_times(5, 3, cb_env.collection.upsert, key, value) + + @pytest_asyncio.fixture(scope="class") + async def num_replicas(self, cb_env): + bucket_settings = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + return num_replicas + + @pytest_asyncio.fixture(scope="class") + async def check_replicas(self, cb_env, num_replicas): + ping_res = await cb_env.bucket.ping() + kv_endpoints = ping_res.endpoints.get(ServiceType.KeyValue, None) + if kv_endpoints is None or len(kv_endpoints) < (num_replicas + 1): + pytest.skip("Not all replicas are online") + + @pytest.fixture(scope="class") + def num_nodes(self, cb_env): + return len(cb_env.cluster._cluster_info.nodes) + + @pytest.fixture(scope="class") + def check_multi_node(self, num_nodes): + if num_nodes == 1: + pytest.skip("Test only for clusters with more than a single node.") + + @pytest.fixture(scope="class") + def skip_if_go_caves(self, cb_env): + if cb_env.is_mock_server and cb_env.mock_server_type == MockServerType.GoCAVES: + pytest.skip("GoCAVES does not like this operation.") + + @pytest.mark.asyncio + async def test_exists(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + result = await cb.exists(key) + assert isinstance(result, ExistsResult) + assert result.exists is True + + @pytest.mark.asyncio + async def test_does_not_exists(self, cb_env): + cb = cb_env.collection + result = await cb.exists(self.NO_KEY) + assert isinstance(result, ExistsResult) + assert result.exists is False + + @pytest.mark.asyncio + async def test_get(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = await cb.get(key) + assert isinstance(result, GetResult) + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + assert result.content_as[dict] == value + + @pytest.mark.asyncio + async def test_get_options(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = await cb.get(key, GetOptions(timeout=timedelta(seconds=2), with_expiry=False)) + assert isinstance(result, GetResult) + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + assert result.content_as[dict] == value + + @pytest.mark.asyncio + async def test_get_fails(self, cb_env): + cb = cb_env.collection + with pytest.raises(DocumentNotFoundException): + await cb.get(self.NO_KEY) + + @pytest.mark.usefixtures("check_xattr_supported") + @pytest.mark.asyncio + async def test_get_with_expiry(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=1000))) + + expiry_path = "$document.exptime" + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry = res.content_as[int](0) + assert expiry is not None + assert expiry > 0 + expires_in = (datetime.fromtimestamp(expiry) - datetime.now()).total_seconds() + # when running local, this can be be up to 1050, so just make sure > 0 + assert expires_in > 0 + + @pytest.mark.asyncio + async def test_expiry_really_expires(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + result = await cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + assert result.cas != 0 + await asyncio.sleep(3) + with pytest.raises(DocumentNotFoundException): + await cb.get(key) + + @pytest.mark.asyncio + async def test_project(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = await cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + + async def cas_matches(cb, new_cas): + r = await cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + await cb_env.try_n_times(10, 3, cas_matches, cb, result.cas) + result = await cb.get(key, GetOptions(project=["faa"])) + assert {"faa": "ORD"} == result.content_as[dict] + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + + @pytest.mark.asyncio + async def test_project_bad_path(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + with pytest.raises(PathNotFoundException): + await cb.get(key, GetOptions(project=["some", "qzx"])) + + @pytest.mark.asyncio + async def test_project_project_not_list(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + # @TODO(jc): better exception + with pytest.raises(InvalidArgumentException): + await cb.get(key, GetOptions(project="thiswontwork")) + + @pytest.mark.usefixtures('skip_if_go_caves') + @pytest.mark.asyncio + async def test_project_too_many_projections(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = {f'f{i}': i for i in range(1, 21)} + result = await cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + + async def cas_matches(cb, new_cas): + r = await cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + await cb_env.try_n_times(10, 3, cas_matches, cb, result.cas) + + project = [f'f{i}' for i in range(1, 19)] + result = await cb.get(key, GetOptions(project=project)) + assert result.cas is not None + res_dict = result.content_as[dict] + assert res_dict != {} + assert res_dict.get('f20') is None + for field in project: + assert res_dict.get(field) is not None + + @pytest.mark.asyncio + async def test_upsert(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = await cb.upsert(key, value, UpsertOptions(timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = await cb_env.try_n_times(10, 3, cb.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + @pytest.mark.asyncio + async def test_upsert_preserve_expiry_not_used(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value1 = new_kvp.value + await cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = "$document.exptime" + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + await cb.upsert(key, value1) + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + await asyncio.sleep(3) + result = await cb.get(key) + assert isinstance(result, GetResult) + assert result.content_as[dict] == value1 + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + @pytest.mark.asyncio + async def test_upsert_preserve_expiry(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value1 = new_kvp.value + + await cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = "$document.exptime" + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + await cb.upsert(key, value1, UpsertOptions(preserve_expiry=True)) + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + await asyncio.sleep(3) + with pytest.raises(DocumentNotFoundException): + await cb.get(key) + + @pytest.mark.asyncio + async def test_insert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + result = await cb.insert(key, value, InsertOptions(timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = await cb_env.try_n_times(10, 3, cb.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + @pytest.mark.asyncio + async def test_insert_document_exists(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + with pytest.raises(DocumentExistsException): + await cb.insert(key, value) + + @pytest.mark.asyncio + async def test_replace(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = await cb.replace(key, value, ReplaceOptions(timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = await cb_env.try_n_times(10, 3, cb.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + @pytest.mark.asyncio + async def test_replace_with_cas(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value1 = new_kvp.value + result = await cb.get(key) + old_cas = result.cas + result = await cb.replace(key, value1, ReplaceOptions(cas=old_cas)) + assert isinstance(result, MutationResult) + assert result.cas != old_cas + + # try same cas again, must fail. + with pytest.raises(CasMismatchException): + await cb.replace(key, value1, ReplaceOptions(cas=old_cas)) + + @pytest.mark.asyncio + async def test_replace_fail(self, cb_env): + cb = cb_env.collection + with pytest.raises(DocumentNotFoundException): + await cb.replace(self.NO_KEY, {"some": "content"}) + + @pytest.mark.asyncio + async def test_remove(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + result = await cb.remove(key) + assert isinstance(result, MutationResult) + + with pytest.raises(DocumentNotFoundException): + await cb.get(key) + + @pytest.mark.asyncio + async def test_remove_fail(self, cb_env): + cb = cb_env.collection + with pytest.raises(DocumentNotFoundException): + await cb.remove(self.NO_KEY) + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + @pytest.mark.asyncio + async def test_replace_preserve_expiry_not_used(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value1 = new_kvp.value + + await cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = "$document.exptime" + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + await cb.replace(key, value1) + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + await asyncio.sleep(3) + result = await cb.get(key) + assert isinstance(result, GetResult) + assert result.content_as[dict] == value1 + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + @pytest.mark.asyncio + async def test_replace_preserve_expiry(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value1 = new_kvp.value + + await cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = "$document.exptime" + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + await cb.replace(key, value1, ReplaceOptions(preserve_expiry=True)) + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + await asyncio.sleep(3) + with pytest.raises(DocumentNotFoundException): + await cb.get(key) + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + @pytest.mark.asyncio + async def test_replace_preserve_expiry_fail(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + opts = ReplaceOptions( + expiry=timedelta( + seconds=5), + preserve_expiry=True) + with pytest.raises(InvalidArgumentException): + await cb.replace(key, value, opts) + + @pytest.mark.asyncio + async def test_touch(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb.upsert(key, value) + await cb_env.try_n_times(10, 1, cb.get, key) + result = await cb.touch(key, timedelta(seconds=2)) + assert isinstance(result, MutationResult) + await asyncio.sleep(3) + with pytest.raises(DocumentNotFoundException): + await cb.get(key) + + @pytest.mark.asyncio + async def test_touch_no_expire(self, cb_env, new_kvp): + # TODO: handle MOCK + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb.upsert(key, value) + await cb_env.try_n_times(10, 1, cb.get, key) + await cb.touch(key, timedelta(seconds=15)) + g_result = await cb.get(key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is not None + await cb.touch(key, timedelta(seconds=0)) + g_result = await cb.get(key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is None + + @pytest.mark.asyncio + async def test_get_and_touch(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb.upsert(key, value) + await cb_env.try_n_times(10, 1, cb.get, key) + result = await cb.get_and_touch(key, timedelta(seconds=2)) + assert isinstance(result, GetResult) + await asyncio.sleep(3) + with pytest.raises(DocumentNotFoundException): + await cb.get(key) + + @pytest.mark.asyncio + async def test_get_and_touch_no_expire(self, cb_env, new_kvp): + # TODO: handle MOCK + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb.upsert(key, value) + await cb_env.try_n_times(10, 1, cb.get, key) + await cb.get_and_touch(key, timedelta(seconds=15)) + g_result = await cb.get(key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is not None + await cb.get_and_touch(key, timedelta(seconds=0)) + g_result = await cb.get(key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is None + + @pytest.mark.asyncio + async def test_get_and_lock(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = await cb.get_and_lock(key, timedelta(seconds=3)) + assert isinstance(result, GetResult) + with pytest.raises(DocumentLockedException): + await cb.upsert(key, value) + + await cb_env.try_n_times(10, 1, cb.upsert, key, value) + + @pytest.mark.asyncio + async def test_get_after_lock(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + orig = await cb.get_and_lock(key, timedelta(seconds=5)) + assert isinstance(orig, GetResult) + result = await cb.get(key) + assert orig.content_as[dict] == result.content_as[dict] + assert orig.cas != result.cas + + # @TODO(jc): cxx client raises ambiguous timeout w/ retry reason: kv_temporary_failure + await cb_env.try_n_times_till_exception(10, 1, + cb.unlock, key, orig.cas, + expected_exceptions=(TemporaryFailException, + DocumentNotLockedException)) + + @pytest.mark.asyncio + async def test_get_and_lock_replace_with_cas(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + result = await cb.get_and_lock(key, timedelta(seconds=5)) + assert isinstance(result, GetResult) + cas = result.cas + with pytest.raises(DocumentLockedException): + await cb.upsert(key, value) + + await cb.replace(key, value, ReplaceOptions(cas=cas)) + # @TODO(jc): cxx client raises ambiguous timeout w/ retry reason: kv_temporary_failure + await cb_env.try_n_times_till_exception(10, 1, + cb.unlock, key, cas, + expected_exceptions=(TemporaryFailException, + DocumentNotLockedException)) + + @pytest.mark.asyncio + async def test_unlock(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + result = await cb.get_and_lock(key, timedelta(seconds=5)) + assert isinstance(result, GetResult) + await cb.unlock(key, result.cas) + await cb.upsert(key, value) + + @pytest.mark.asyncio + async def test_unlock_wrong_cas(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + result = await cb.get_and_lock(key, timedelta(seconds=5)) + cas = result.cas + # @TODO(jc): MOCK - TemporaryFailException + with pytest.raises(CasMismatchException): + await cb.unlock(key, 100) + + await cb_env.try_n_times_till_exception(10, 1, + cb.unlock, key, cas, + expected_exceptions=(TemporaryFailException, + DocumentNotLockedException)) + + @pytest.mark.usefixtures("check_not_locked_supported") + @pytest.mark.asyncio + async def test_unlock_not_locked(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + result = await cb.get_and_lock(key, timedelta(seconds=5)) + cas = result.cas + await cb.unlock(key, cas) + with pytest.raises(DocumentNotLockedException): + await cb.unlock(key, cas) + + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.asyncio + async def test_get_any_replica(self, cb_env, default_kvp): + result = await cb_env.try_n_times(10, 3, cb_env.collection.get_any_replica, default_kvp.key) + assert isinstance(result, GetReplicaResult) + assert isinstance(result.is_replica, bool) + assert default_kvp.value == result.content_as[dict] + + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.asyncio + async def test_get_any_replica_fail(self, cb_env): + with pytest.raises(DocumentUnretrievableException): + await cb_env.collection.get_any_replica('not-a-key') + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.usefixtures("check_server_groups_supported") + @pytest.mark.asyncio + async def test_get_any_replica_read_preference(self, cb_env, default_kvp): + # No preferred server group was specified in the cluster options so this should raise + # DocumentUnretrievableException + with pytest.raises(DocumentUnretrievableException): + await cb_env.collection.get_any_replica( + default_kvp.key, GetAnyReplicaOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.asyncio + async def test_get_all_replicas(self, cb_env, default_kvp): + result = await cb_env.try_n_times(10, 3, cb_env.collection.get_all_replicas, default_kvp.key) + # make sure we can iterate over results + while True: + try: + res = next(result) + assert isinstance(res, GetReplicaResult) + assert isinstance(res.is_replica, bool) + assert default_kvp.value == res.content_as[dict] + except StopIteration: + break + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.asyncio + async def test_get_all_replicas_fail(self, cb_env): + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.get_all_replicas('not-a-key') + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.asyncio + async def test_get_all_replicas_results(self, cb_env, default_kvp): + result = await cb_env.try_n_times(10, 3, cb_env.collection.get_all_replicas, default_kvp.key) + active_cnt = 0 + replica_cnt = 0 + for res in result: + assert isinstance(res, GetReplicaResult) + assert isinstance(res.is_replica, bool) + assert default_kvp.value == res.content_as[dict] + if res.is_replica: + replica_cnt += 1 + else: + active_cnt += 1 + + assert active_cnt == 1 + assert replica_cnt >= active_cnt + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.usefixtures("check_server_groups_supported") + @pytest.mark.asyncio + async def test_get_all_replicas_read_preference(self, cb_env, default_kvp): + # No preferred server group was specified in the cluster options so this should raise + # DocumentUnretrievableException + with pytest.raises(DocumentUnretrievableException): + await cb_env.collection.get_all_replicas( + default_kvp.key, GetAllReplicasOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP)) + + # @TODO(jc): - should an expiry of -1 raise an InvalidArgumentException? + @pytest.mark.usefixtures("check_xattr_supported") + @pytest.mark.asyncio + @pytest.mark.parametrize("expiry", [FIFTY_YEARS + 1, + FIFTY_YEARS, + THIRTY_DAYS - 1, + THIRTY_DAYS, + int(time() - 1.0), 60, -1]) + async def test_document_expiry_values(self, cb_env, new_kvp, expiry): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + if expiry == -1: + with pytest.raises(InvalidArgumentException): + await cb.upsert(key, value, expiry=timedelta(seconds=expiry)) + else: + before = int(time() - 1.0) + result = await cb.upsert(key, value, expiry=timedelta(seconds=expiry)) + assert result.cas is not None + + expiry_path = "$document.exptime" + res = await cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + res_expiry = res.content_as[int](0) + + after = int(time() + 1.0) + before + expiry <= res_expiry <= after + expiry diff --git a/acouchbase/tests/collectionmgmt_t.py b/acouchbase/tests/collectionmgmt_t.py new file mode 100644 index 000000000..a19fc7acc --- /dev/null +++ b/acouchbase/tests/collectionmgmt_t.py @@ -0,0 +1,591 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from datetime import timedelta +from uuid import uuid4 + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import (BucketDoesNotExistException, + CollectionAlreadyExistsException, + CollectionNotFoundException, + DocumentNotFoundException, + FeatureUnavailableException, + InvalidArgumentException, + ScopeAlreadyExistsException, + ScopeNotFoundException) +from couchbase.management.buckets import StorageBackend +from couchbase.management.collections import (CollectionSpec, + CreateCollectionSettings, + UpdateCollectionSettings) + +from ._test_utils import TestEnvironment + + +class CollectionManagementTests: + + TEST_BUCKET = "test-bucket" + TEST_SCOPE = "test-scope" + TEST_COLLECTION = "test-collection" + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest.fixture(scope="class") + def check_non_deduped_history_supported(self, cb_env): + cb_env.check_if_feature_supported('non_deduped_history') + + @pytest.fixture(scope="class") + def check_update_collection_supported(self, cb_env): + cb_env.check_if_feature_supported('update_collection') + + @pytest.fixture(scope="class") + def check_update_collection_max_expiry_supported(self, cb_env): + cb_env.check_if_feature_supported('update_collection_max_expiry') + + @pytest.fixture(scope="class") + def check_negative_collection_max_expiry_supported(self, cb_env): + cb_env.check_if_feature_supported('negative_collection_max_expiry') + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True, + manage_collections=True) + # will create a new bucket w/ name test-bucket + await cb_env.try_n_times(3, 5, cb_env.setup_collection_mgmt, self.TEST_BUCKET) + yield cb_env + if cb_env.is_feature_supported('bucket_mgmt'): + await cb_env.purge_buckets([self.TEST_BUCKET]) + + @pytest_asyncio.fixture() + async def cleanup_scope(self, cb_env): + await cb_env.try_n_times_till_exception(5, 1, + cb_env.test_bucket_cm.drop_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeNotFoundException,)) + yield + await cb_env.try_n_times_till_exception(5, 1, + cb_env.test_bucket_cm.drop_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeNotFoundException,)) + + @pytest_asyncio.fixture() + async def cleanup_collection(self, cb_env): + await cb_env.try_n_times_till_exception(5, 1, + cb_env.test_bucket_cm.drop_collection, + CollectionSpec(self.TEST_COLLECTION), + expected_exceptions=(CollectionNotFoundException,)) + yield + await cb_env.try_n_times_till_exception(5, 1, + cb_env.test_bucket_cm.drop_collection, + CollectionSpec(self.TEST_COLLECTION), + expected_exceptions=(CollectionNotFoundException,)) + + # temporary until we consoldate w/ new test env setup + async def _get_collection(self, cm, scope_name, coll_name): + scopes = await cm.get_all_scopes() + scope = next((s for s in scopes if s.name == scope_name), None) + if scope: + return next( + (c for c in scope.collections if c.name == coll_name), None) + + return None + + @pytest.mark.usefixtures("cleanup_scope") + @pytest.mark.asyncio + async def test_create_scope(self, cb_env): + await cb_env.test_bucket_cm.create_scope(self.TEST_SCOPE) + assert await cb_env.get_scope(self.TEST_SCOPE) is not None + + @pytest.mark.usefixtures("cleanup_scope") + @pytest.mark.asyncio + async def test_create_scope_already_exists(self, cb_env): + await cb_env.test_bucket_cm.create_scope(self.TEST_SCOPE) + assert await cb_env.get_scope(self.TEST_SCOPE) is not None + with pytest.raises(ScopeAlreadyExistsException): + await cb_env.test_bucket_cm.create_scope(self.TEST_SCOPE) + + @pytest.mark.asyncio + async def test_get_all_scopes(self, cb_env): + scope_names = [str(uuid4())[:8] for _ in range(4)] + for name in scope_names: + await cb_env.test_bucket_cm.create_scope(name) + for _ in range(2): + await cb_env.test_bucket_cm.create_collection(name, str(uuid4())[:8]) + + scopes = await cb_env.test_bucket_cm.get_all_scopes() + assert (sum(s.name[0] != '_' for s in scopes) == len(scope_names)) + # should have a _default scope + assert any(map(lambda s: s.name == '_default', scopes)) + for scope_name in scope_names: + assert any(map(lambda s: s.name == scope_name, scopes)) + for s in scopes: + for c in s.collections: + assert isinstance(c, CollectionSpec) + assert isinstance(c.name, str) + assert isinstance(c.scope_name, str) + assert isinstance(c.max_expiry, timedelta) + assert c.history is None or isinstance(c.history, bool) + + for name in scope_names: + await cb_env.test_bucket_cm.drop_scope(name) + + # deprecated + # @pytest.mark.asyncio + # async def test_get_scope(self, cb_env): + # await cb_env.test_bucket_cm.get_scope('_default') + + @pytest.mark.asyncio + async def test_drop_scope(self, cb_env): + await cb_env.test_bucket_cm.create_scope(self.TEST_SCOPE) + assert await cb_env.get_scope(self.TEST_SCOPE) is not None + await cb_env.test_bucket_cm.drop_scope(self.TEST_SCOPE) + with pytest.raises(ScopeNotFoundException): + await cb_env.test_bucket_cm.drop_scope(self.TEST_SCOPE) + + @pytest.mark.asyncio + async def test_drop_scope_not_found(self, cb_env): + with pytest.raises(ScopeNotFoundException): + await cb_env.test_bucket_cm.drop_scope("some-random-scope") + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_create_collection(self, cb_env): + # create a collection under default_ scope + await cb_env.test_bucket_cm.create_collection("_default", self.TEST_COLLECTION) + assert await cb_env.get_collection("_default", self.TEST_COLLECTION) is not None + + @pytest.mark.usefixtures("cleanup_scope") + @pytest.mark.asyncio + async def test_create_scope_and_collection(self, cb_env): + await cb_env.test_bucket_cm.create_scope(self.TEST_SCOPE) + assert await cb_env.get_scope(self.TEST_SCOPE) is not None + await cb_env.test_bucket_cm.create_collection(self.TEST_SCOPE, self.TEST_COLLECTION) + assert await cb_env.get_collection(self.TEST_SCOPE, self.TEST_COLLECTION) is not None + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_create_collection_max_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection TTL.") + + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=2)) + + await cb_env.test_bucket_cm.create_collection("_default", self.TEST_COLLECTION, settings) + coll_spec = await cb_env.get_collection("_default", self.TEST_COLLECTION) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=2) + + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(self.TEST_COLLECTION) + key = "test-coll-key0" + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + await cb_env.try_n_times(10, 1, coll.upsert, key, {"some": "thing"}) + await cb_env.try_n_times(10, 1, coll.get, key) + await cb_env.try_n_times_till_exception(4, 1, coll.get, key, expected_exceptions=(DocumentNotFoundException,)) + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + @pytest.mark.asyncio + async def test_create_collection_max_expiry_default(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + + await cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + # TODO: consistency + coll_spec = await cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + @pytest.mark.asyncio + async def test_create_collection_max_expiry_invalid(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + collection_name = self.TEST_COLLECTION + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=-20)) + + with pytest.raises(InvalidArgumentException): + await cb_env.test_bucket_cm.create_collection(scope_name, collection_name, settings) + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + @pytest.mark.asyncio + async def test_create_collection_max_expiry_no_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=-1)) + await cb_env.test_bucket_cm.create_collection(scope_name, collection_name, settings) + # TODO: consistency + coll_spec = await cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=-1) + + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_update_collection_max_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection TTL.") + + await cb_env.test_bucket_cm.create_collection("_default", self.TEST_COLLECTION) + + coll_spec = await cb_env.get_collection("_default", self.TEST_COLLECTION) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=2)) + await cb_env.test_bucket_cm.update_collection("_default", self.TEST_COLLECTION, settings) + + coll_spec = await cb_env.get_collection("_default", self.TEST_COLLECTION) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=2) + + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(self.TEST_COLLECTION) + key = "test-coll-key0" + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + await cb_env.try_n_times(10, 1, coll.upsert, key, {"some": "thing"}) + await cb_env.try_n_times(10, 1, coll.get, key) + await cb_env.try_n_times_till_exception(4, 1, coll.get, key, expected_exceptions=(DocumentNotFoundException,)) + + @pytest.mark.asyncio + async def test_create_collection_bad_scope(self, cb_env): + with pytest.raises(ScopeNotFoundException): + await cb_env.test_bucket_cm.create_collection("im-a-fake-scope", self.TEST_COLLECTION) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_create_collection_already_exists(self, cb_env): + await cb_env.test_bucket_cm.create_collection("_default", self.TEST_COLLECTION) + # verify the collection exists w/in other-bucket + assert await cb_env.get_collection("_default", self.TEST_COLLECTION) is not None + # now, it will fail if we try to create it again... + with pytest.raises(CollectionAlreadyExistsException): + await cb_env.test_bucket_cm.create_collection("_default", self.TEST_COLLECTION) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_collection_goes_in_correct_bucket(self, cb_env): + await cb_env.test_bucket_cm.create_collection("_default", self.TEST_COLLECTION) + # make sure it actually is in the other-bucket + assert await cb_env.get_collection("_default", self.TEST_COLLECTION) is not None + # also be sure this isn't in the default bucket + assert await cb_env.get_collection("_default", + self.TEST_COLLECTION, + bucket_name=cb_env.bucket.name) is None + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_drop_collection(self, cb_env): + await cb_env.test_bucket_cm.create_collection("_default", self.TEST_COLLECTION) + # verify the collection exists w/in other-bucket + assert await cb_env.get_collection("_default", self.TEST_COLLECTION) is not None + # attempt to drop it again will raise CollectionNotFoundException + await cb_env.test_bucket_cm.drop_collection("_default", self.TEST_COLLECTION) + with pytest.raises(CollectionNotFoundException): + await cb_env.test_bucket_cm.drop_collection("_default", self.TEST_COLLECTION) + + @pytest.mark.asyncio + async def test_drop_collection_not_found(self, cb_env): + with pytest.raises(CollectionNotFoundException): + await cb_env.test_bucket_cm.drop_collection("_default", "fake-collection") + + @pytest.mark.asyncio + async def test_drop_collection_scope_not_found(self, cb_env): + with pytest.raises(ScopeNotFoundException): + await cb_env.test_bucket_cm.drop_collection("fake-scope", "fake-collection") + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.asyncio + async def test_create_collection_history_retention(self, cb_env): + bucket_name = 'test-magma-bucket' + scope_name = '_default' + collection_name = self.TEST_COLLECTION + + await cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA) + bucket = cb_env.cluster.bucket(bucket_name) + await cb_env.try_n_times(10, 1, bucket.on_connect) + cm = bucket.collections() + + await cm.create_collection(scope_name, collection_name, CreateCollectionSettings(history=True)) + collection_spec = None + retry = 0 + while retry < 5 and collection_spec is None: + collection_spec = await self._get_collection(cm, scope_name, collection_name) + await asyncio.sleep(1) + retry += 1 + assert collection_spec is not None + assert collection_spec.history + + await cb_env.try_n_times_till_exception(10, + 3, + cb_env.bm.drop_bucket, + bucket_name, + expected_exceptions=(BucketDoesNotExistException,)) + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.asyncio + async def test_create_collection_history_retention_unsupported(self, cb_env): + scope_name = '_default' + collection_name = self.TEST_COLLECTION + + # Couchstore does not support history retention + with pytest.raises(FeatureUnavailableException): + await cb_env.test_bucket_cm.create_collection( + scope_name, collection_name, CreateCollectionSettings(history=True)) + + with pytest.raises(FeatureUnavailableException): + await cb_env.test_bucket_cm.create_collection( + scope_name, collection_name, CreateCollectionSettings(history=False)) + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.asyncio + async def test_update_collection_history_retention(self, cb_env): + bucket_name = 'test-magma-bucket' + scope_name = '_default' + collection_name = self.TEST_COLLECTION + + await cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA) + bucket = cb_env.cluster.bucket(bucket_name) + await cb_env.try_n_times(10, 1, bucket.on_connect) + cm = bucket.collections() + + await cm.create_collection(scope_name, collection_name, CreateCollectionSettings(history=False)) + collection_spec = None + retry = 0 + while retry < 5 and collection_spec is None: + collection_spec = await self._get_collection(cm, scope_name, collection_name) + await asyncio.sleep(1) + retry += 1 + assert collection_spec is not None + assert not collection_spec.history + + await cm.update_collection(scope_name, collection_name, UpdateCollectionSettings(history=True)) + collection_spec = None + retry = 0 + while retry < 5 and collection_spec is None: + collection_spec = await self._get_collection(cm, scope_name, collection_name) + await asyncio.sleep(1) + retry += 1 + assert collection_spec is not None + assert collection_spec.history + + await cb_env.try_n_times_till_exception(10, + 3, + cb_env.bm.drop_bucket, + bucket_name, + expected_exceptions=(BucketDoesNotExistException,)) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.asyncio + async def test_update_collection_history_retention_unsupported(self, cb_env): + scope_name = '_default' + collection_name = self.TEST_COLLECTION + + await cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + collection_spec = await cb_env.get_collection(scope_name, collection_name) + assert collection_spec is not None + assert collection_spec.history is False + + # Couchstore does not support history retention + with pytest.raises(FeatureUnavailableException): + await cb_env.test_bucket_cm.update_collection( + scope_name, collection_name, UpdateCollectionSettings(history=True)) + + # Collection history retention setting remains unchanged + collection_spec = await cb_env.get_collection(scope_name, collection_name) + assert collection_spec is not None + assert collection_spec.history is False + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + @pytest.mark.asyncio + async def test_update_collection_max_expiry_bucket_default(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=5)) + + await cb_env.test_bucket_cm.create_collection(scope_name, collection_name, settings) + coll_spec = await cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=5) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=0)) + await cb_env.test_bucket_cm.update_collection(scope_name, collection_name, settings) + coll_spec = await cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + @pytest.mark.asyncio + async def test_update_collection_max_expiry_invalid(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + collection_name = self.TEST_COLLECTION + scope_name = '_default' + + await cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + coll_spec = await cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=-20)) + with pytest.raises(InvalidArgumentException): + await cb_env.test_bucket_cm.update_collection(scope_name, collection_name, settings) + + @pytest.mark.usefixtures('cleanup_collection') + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + @pytest.mark.asyncio + async def test_update_collection_max_expiry_no_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + await cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + coll_spec = await cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=-1)) + await cb_env.test_bucket_cm.update_collection(scope_name, collection_name, settings) + coll_spec = await cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=-1) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_deprecated_create_collection(self, cb_env): + # create a collection under default_ scope + collection = CollectionSpec(self.TEST_COLLECTION) + await cb_env.test_bucket_cm.create_collection(collection) + assert await cb_env.get_collection(collection.scope_name, collection.name) is not None + + @pytest.mark.usefixtures("cleanup_scope") + @pytest.mark.asyncio + async def test_deprecated_create_scope_and_collection(self, cb_env): + await cb_env.test_bucket_cm.create_scope(self.TEST_SCOPE) + assert await cb_env.get_scope(self.TEST_SCOPE) is not None + collection = CollectionSpec(self.TEST_COLLECTION, self.TEST_SCOPE) + await cb_env.test_bucket_cm.create_collection(collection) + assert await cb_env.get_collection(collection.scope_name, collection.name) is not None + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_deprecated_create_collection_max_ttl(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection TTL.") + collection = CollectionSpec(self.TEST_COLLECTION, max_ttl=timedelta(seconds=2)) + + await cb_env.test_bucket_cm.create_collection(collection) + assert await cb_env.get_collection(collection.scope_name, collection.name) is not None + + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(collection.name) + key = "test-coll-key0" + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + await cb_env.try_n_times(10, 1, coll.upsert, key, {"some": "thing"}) + await cb_env.try_n_times(10, 1, coll.get, key) + await cb_env.try_n_times_till_exception(4, 1, coll.get, key, expected_exceptions=(DocumentNotFoundException,)) + + @pytest.mark.asyncio + async def test_deprecated_create_collection_bad_scope(self, cb_env): + collection = CollectionSpec(self.TEST_COLLECTION, "im-a-fake-scope") + with pytest.raises(ScopeNotFoundException): + await cb_env.test_bucket_cm.create_collection(collection) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_deprecated_create_collection_already_exists(self, cb_env): + collection = CollectionSpec(self.TEST_COLLECTION) + await cb_env.test_bucket_cm.create_collection(collection) + # verify the collection exists w/in other-bucket + assert await cb_env.get_collection(collection.scope_name, collection.name) is not None + # now, it will fail if we try to create it again... + with pytest.raises(CollectionAlreadyExistsException): + await cb_env.test_bucket_cm.create_collection(collection) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_deprecated_collection_goes_in_correct_bucket(self, cb_env): + collection = CollectionSpec(self.TEST_COLLECTION) + await cb_env.test_bucket_cm.create_collection(collection) + # make sure it actually is in the other-bucket + assert await cb_env.get_collection(collection.scope_name, collection.name) is not None + # also be sure this isn't in the default bucket + assert await cb_env.get_collection(collection.scope_name, + collection.name, + bucket_name=cb_env.bucket.name) is None + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.asyncio + async def test_deprecated_drop_collection(self, cb_env): + collection = CollectionSpec(self.TEST_COLLECTION) + await cb_env.test_bucket_cm.create_collection(collection) + # verify the collection exists w/in other-bucket + assert await cb_env.get_collection(collection.scope_name, collection.name) is not None + # attempt to drop it again will raise CollectionNotFoundException + await cb_env.test_bucket_cm.drop_collection(collection) + with pytest.raises(CollectionNotFoundException): + await cb_env.test_bucket_cm.drop_collection(collection) + + @pytest.mark.asyncio + async def test_deprecated_drop_collection_not_found(self, cb_env): + collection = CollectionSpec("fake-collection") + with pytest.raises(CollectionNotFoundException): + await cb_env.test_bucket_cm.drop_collection(collection) + + @pytest.mark.asyncio + async def test_deprecated_drop_collection_scope_not_found(self, cb_env): + collection = CollectionSpec("fake-collection", "fake-scope") + with pytest.raises(ScopeNotFoundException): + await cb_env.test_bucket_cm.drop_collection(collection) diff --git a/acouchbase/tests/datastructures_t.py b/acouchbase/tests/datastructures_t.py new file mode 100644 index 000000000..cff4cf7da --- /dev/null +++ b/acouchbase/tests/datastructures_t.py @@ -0,0 +1,216 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from acouchbase.datastructures import (CouchbaseList, + CouchbaseMap, + CouchbaseQueue, + CouchbaseSet) +from couchbase.exceptions import (DocumentNotFoundException, + InvalidArgumentException, + QueueEmpty) + +from ._test_utils import CollectionType, TestEnvironment + + +class DatastructuresTests: + + TEST_DS_KEY = 'ds-key' + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + manage_buckets=True) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture() + async def remove_ds(self, cb_env) -> None: + yield + try: + await cb_env.collection.remove(self.TEST_DS_KEY) + except DocumentNotFoundException: + pass + + @pytest.mark.usefixtures("remove_ds") + @pytest.mark.asyncio + async def test_list(self, cb_env): + + cb_list = cb_env.collection.couchbase_list(self.TEST_DS_KEY) + assert isinstance(cb_list, CouchbaseList) + + await cb_list.append('world') + rv = await cb_list.get_at(0) + assert str(rv) == 'world' + + await cb_list.prepend('hello') + rv = await cb_list.get_at(0) + assert str(rv) == 'hello' + + rv = await cb_list.get_at(1) + assert str(rv) == 'world' + assert 2 == await cb_list.size() + + await cb_list.remove_at(1) + res = await cb_env.collection.get(self.TEST_DS_KEY) + assert ['hello'] == res.content_as[list] + + await cb_list.append('world') + res = await cb_env.collection.get(self.TEST_DS_KEY) + assert ['hello', 'world'] == res.content_as[list] + + await cb_list.set_at(1, 'after') + res = await cb_env.collection.get(self.TEST_DS_KEY) + assert ['hello', 'after'] == res.content_as[list] + + res = await cb_list.get_all() + assert ['hello', 'after'] == res + + res = await cb_list.index_of('after') + assert res == 1 + + expected = ['hello', 'after'] + idx = 0 + async for v in cb_list: + assert expected[idx] == v + idx += 1 + + await cb_list.clear() + + assert 0 == await cb_list.size() + + @pytest.mark.usefixtures("remove_ds") + @pytest.mark.asyncio + async def test_map(self, cb_env): + + cb_map = cb_env.collection.couchbase_map(self.TEST_DS_KEY) + assert isinstance(cb_map, CouchbaseMap) + + await cb_map.add('key1', 'val1') + + rv = await cb_map.get('key1') + assert rv == 'val1' + + assert 1 == await cb_map.size() + + with pytest.raises(InvalidArgumentException): + await cb_map.remove('key2') + + await cb_map.add('key2', 'val2') + + keys = await cb_map.keys() + assert ['key1', 'key2'] == keys + + values = await cb_map.values() + assert ['val1', 'val2'] == values + + assert await cb_map.exists('key1') is True + assert await cb_map.exists('no-key') is False + + expected_keys = ['key1', 'key2'] + expected_values = ['val1', 'val2'] + items = await cb_map.items() + for k, v in items: + assert k in expected_keys + assert v in expected_values + + with pytest.raises(TypeError): + async for k, v in cb_map: + assert k in expected_keys + assert v in expected_values + + await cb_map.remove('key1') + assert 1 == await cb_map.size() + + await cb_map.clear() + assert 0 == await cb_map.size() + + @pytest.mark.usefixtures("remove_ds") + @pytest.mark.asyncio + async def test_sets(self, cb_env): + + cb_set = cb_env.collection.couchbase_set(self.TEST_DS_KEY) + assert isinstance(cb_set, CouchbaseSet) + + rv = await cb_set.add(123) + rv = await cb_set.add(123) + assert 1 == await cb_set.size() + assert await cb_set.contains(123) is True + + rv = await cb_set.remove(123) + assert 0 == await cb_set.size() + rv = await cb_set.remove(123) + assert rv is None + assert await cb_set.contains(123) is False + + await cb_set.add(1) + await cb_set.add(2) + await cb_set.add(3) + await cb_set.add(4) + + values = await cb_set.values() + assert values == [1, 2, 3, 4] + await cb_set.clear() + assert 0 == await cb_set.size() + + @pytest.mark.usefixtures("remove_ds") + @pytest.mark.asyncio + async def test_queue(self, cb_env): + cb_queue = cb_env.collection.couchbase_queue(self.TEST_DS_KEY) + assert isinstance(cb_queue, CouchbaseQueue) + + await cb_queue.push(1) + await cb_queue.push(2) + await cb_queue.push(3) + + # Pop the items now + assert await cb_queue.pop() == 1 + assert await cb_queue.pop() == 2 + assert await cb_queue.pop() == 3 + with pytest.raises(QueueEmpty): + await cb_queue.pop() + + await cb_queue.push(1) + await cb_queue.push(2) + await cb_queue.push(3) + + assert await cb_queue.size() == 3 + + expected = [3, 2, 1] + idx = 0 + async for v in cb_queue: + assert expected[idx] == v + idx += 1 + + await cb_queue.clear() + + assert 0 == await cb_queue.size() diff --git a/acouchbase/tests/durability_t.py b/acouchbase/tests/durability_t.py new file mode 100644 index 000000000..cf82be808 --- /dev/null +++ b/acouchbase/tests/durability_t.py @@ -0,0 +1,485 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest +import pytest_asyncio + +import couchbase.subdocument as SD +from acouchbase.cluster import get_event_loop +from couchbase.durability import (ClientDurability, + DurabilityLevel, + PersistTo, + PersistToExtended, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import DocumentNotFoundException, DurabilityImpossibleException +from couchbase.options import (InsertOptions, + MutateInOptions, + RemoveOptions, + ReplaceOptions, + UpsertOptions) + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment) + + +class DurabilityTests: + NO_KEY = "not-a-key" + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + manage_buckets=True) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + await cb_env.try_n_times(3, 5, cb_env.load_data) + yield cb_env + await cb_env.try_n_times_till_exception(3, 5, + cb_env.purge_data, + raise_if_no_exception=False) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name="new_kvp") + async def new_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = await cb_env.get_new_key_value() + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="default_kvp_and_reset") + async def default_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.default_durable_key_value() + yield KVPair(key, value) + await cb_env.try_n_times(5, 3, cb_env.collection.upsert, key, value) + + @pytest.fixture(scope="class") + def check_sync_durability_supported(self, cb_env): + cb_env.check_if_feature_supported('sync_durability') + + @pytest.fixture(scope="class") + def num_nodes(self, cb_env): + return len(cb_env.cluster._cluster_info.nodes) + + @pytest.fixture(scope="class") + def check_multi_node(self, num_nodes): + if num_nodes == 1: + pytest.skip("Test only for clusters with more than a single node.") + + @pytest.fixture(scope="class") + def check_single_node(self, num_nodes): + if num_nodes != 1: + pytest.skip("Test only for clusters with a single node.") + + @pytest_asyncio.fixture(scope="class") + async def num_replicas(self, cb_env): + bucket_settings = await cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + return num_replicas + + @pytest.fixture(scope="class") + def check_has_replicas(self, num_replicas): + if num_replicas == 0: + pytest.skip("No replicas to test durability.") + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_upsert(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + + await cb.upsert(key, value, UpsertOptions(durability=durability)) + result = await cb.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_upsert_single_node(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + await cb.upsert(key, value, UpsertOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_insert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + await cb.insert(key, value, InsertOptions(durability=durability)) + result = await cb.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_insert_single_node(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + await cb.insert(key, value, InsertOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_replace(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + await cb.replace(key, value, ReplaceOptions(durability=durability)) + result = await cb.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_replace_single_node(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + await cb.replace(key, value, ReplaceOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_remove(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + await cb.remove(key, RemoveOptions(durability=durability)) + with pytest.raises(DocumentNotFoundException): + await cb.get(key) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_remove_single_node(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + await cb.remove(key, RemoveOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_upsert(self, cb_env, default_kvp_and_reset, num_replicas): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + await cb.upsert(key, value, + UpsertOptions(durability=durability), timeout=timedelta(seconds=3)) + result = await cb.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_upsert_fail(self, cb_env, default_kvp_and_reset, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + await cb.upsert(key, value, UpsertOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_upsert_single_node(self, cb_env, default_kvp_and_reset, num_replicas): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + await cb.upsert(key, value, UpsertOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_insert(self, cb_env, new_kvp, num_replicas): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + await cb.insert(key, value, InsertOptions(durability=durability)) + result = await cb.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_insert_fail(self, cb_env, new_kvp, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + await cb.insert(key, value, InsertOptions(durability=durability)) + + # @TODO: why DurabilityImpossibleException not raised? + # @pytest.mark.usefixtures("check_single_node") + # @pytest.mark.usefixtures("check_has_replicas") + # @pytest.mark.asyncio + # async def test_client_durable_insert_single_node(self, cb_env, new_kvp, num_replicas): + # cb = cb_env.collection + # key = new_kvp.key + # value = new_kvp.value + + # durability = ClientDurability( + # persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + # with pytest.raises(DurabilityImpossibleException): + # await cb.insert(key, value, InsertOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_replace(self, cb_env, default_kvp_and_reset, num_replicas): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + await cb.replace(key, value, ReplaceOptions(durability=durability)) + result = await cb.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_replace_fail(self, cb_env, default_kvp_and_reset, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + await cb.replace(key, value, ReplaceOptions(durability=durability)) + + # @TODO: why DurabilityImpossibleException not raised? + # @pytest.mark.usefixtures("check_single_node") + # @pytest.mark.usefixtures("check_has_replicas") + # @pytest.mark.asyncio + # async def test_client_durable_replace_single_node(self, cb_env, default_kvp_and_reset, num_replicas): + # cb = cb_env.collection + # key = default_kvp_and_reset.key + # value = default_kvp_and_reset.value + + # durability = ClientDurability( + # persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + # with pytest.raises(DurabilityImpossibleException): + # await cb.replace(key, value, ReplaceOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_remove(self, cb_env, default_kvp_and_reset, num_replicas): + cb = cb_env.collection + key = default_kvp_and_reset.key + + durability = ClientDurability(persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + await cb.remove(key, RemoveOptions(durability=durability)) + with pytest.raises(DocumentNotFoundException): + await cb.get(key) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_remove_fail(self, cb_env, default_kvp_and_reset, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + await cb.remove(key, RemoveOptions(durability=durability)) + + # @TODO: why DurabilityImpossibleException not raised? + # @pytest.mark.usefixtures("check_single_node") + # @pytest.mark.usefixtures("check_has_replicas") + # @pytest.mark.asyncio + # async def test_client_durable_remove_single_node(self, cb_env, default_kvp_and_reset, num_replicas): + # cb = cb_env.collection + # key = default_kvp_and_reset.key + + # durability = ClientDurability( + # persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + # with pytest.raises(DurabilityImpossibleException): + # await cb.remove(key, RemoveOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.parametrize("persist_to", [PersistToExtended.NONE, PersistToExtended.ACTIVE, PersistToExtended.ONE]) + @pytest.mark.asyncio + async def test_client_persist_to_extended(self, cb_env, default_kvp_and_reset, persist_to): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ClientDurability( + persist_to=persist_to, replicate_to=ReplicateTo.ONE) + + await cb.upsert(key, value, UpsertOptions(durability=durability)) + result = await cb.get(key) + assert value == result.content_as[dict] + + # Sub-document durable operations + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_mutate_in(self, cb_env, default_kvp_and_reset): + if cb_env.is_mock_server: + pytest.skip("Mock will not return expiry in the xaddrs.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value['city'] = 'New City' + value['faa'] = 'CTY' + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + await cb.mutate_in(key, + (SD.upsert('city', 'New City'), SD.replace('faa', 'CTY')), + MutateInOptions(durability=durability)) + result = await cb.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_server_durable_mutate_in_single_node(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + await cb.mutate_in(key, (SD.upsert('city', 'New City'),), MutateInOptions(durability=durability)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_mutate_in(self, cb_env, default_kvp_and_reset, num_replicas): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value['city'] = 'New City' + value['faa'] = 'CTY' + + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + + await cb.mutate_in(key, + (SD.upsert('city', 'New City'), SD.replace('faa', 'CTY')), + MutateInOptions(durability=durability)) + result = await cb.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_has_replicas") + @pytest.mark.asyncio + async def test_client_durable_mutate_in_fail(self, cb_env, default_kvp_and_reset, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + await cb.mutate_in(key, (SD.upsert('city', 'New City'),), MutateInOptions(durability=durability)) diff --git a/acouchbase/tests/eventingmgmt_t.py b/acouchbase/tests/eventingmgmt_t.py new file mode 100644 index 000000000..00c67952d --- /dev/null +++ b/acouchbase/tests/eventingmgmt_t.py @@ -0,0 +1,1301 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from datetime import timedelta + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import (CollectionAlreadyExistsException, + EventingFunctionAlreadyDeployedException, + EventingFunctionCollectionNotFoundException, + EventingFunctionNotBootstrappedException, + EventingFunctionNotDeployedException, + EventingFunctionNotFoundException, + EventingFunctionNotUnDeployedException) +from couchbase.management.collections import CollectionSpec +from couchbase.management.eventing import (EventingFunction, + EventingFunctionBucketAccess, + EventingFunctionBucketBinding, + EventingFunctionConstantBinding, + EventingFunctionDcpBoundary, + EventingFunctionDeploymentStatus, + EventingFunctionKeyspace, + EventingFunctionLanguageCompatibility, + EventingFunctionProcessingStatus, + EventingFunctionSettings, + EventingFunctionsStatus, + EventingFunctionState, + EventingFunctionStatus, + EventingFunctionUrlAuthBasic, + EventingFunctionUrlAuthBearer, + EventingFunctionUrlAuthDigest, + EventingFunctionUrlBinding, + EventingFunctionUrlNoAuth) +from couchbase.management.options import GetFunctionOptions, UpsertFunctionOptions + +from ._test_utils import (CollectionType, + EventingFunctionManagementTestStatusException, + TestEnvironment) + + +@pytest.mark.flaky(reruns=5) +class EventingManagementTests: + + EVT_SRC_BUCKET_NAME = "beer-sample" + EVT_META_BUCKET_NAME = "default" + TEST_EVT_NAME = 'test-evt-func' + SIMPLE_EVT_CODE = ('function OnUpdate(doc, meta) {\n log("Doc created/updated", meta.id);\n}' + '\n\nfunction OnDelete(meta, options) {\n log("Doc deleted/expired", meta.id);\n}') + EVT_VERSION = None + BASIC_FUNC = EventingFunction( + TEST_EVT_NAME, + SIMPLE_EVT_CODE, + "evt-7.0.0-5302-ee", + metadata_keyspace=EventingFunctionKeyspace(EVT_META_BUCKET_NAME), + source_keyspace=EventingFunctionKeyspace(EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc", + name=EventingFunctionKeyspace(EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_eventing_functions=True) + + if cb_env.is_feature_supported('collections'): + cb_env._cm = cb_env.bucket.collections() + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if cb_env.is_feature_supported('collections'): + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest.fixture(scope="class", name='evt_version') + def get_eventing_function_version(self, cb_env): + version = "evt-{}".format( + cb_env.server_version.replace("enterprise", "ee").replace("community", "ce") + ) + return version + + @pytest_asyncio.fixture() + async def create_eventing_function(self, cb_env): + await cb_env.efm.upsert_function(self.BASIC_FUNC) + + @pytest_asyncio.fixture() + async def drop_eventing_function(self, cb_env): + yield + await cb_env.efm.drop_function(self.TEST_EVT_NAME) + + @pytest_asyncio.fixture() + async def undeploy_and_drop_eventing_function(self, cb_env): + yield + await cb_env.efm.undeploy_function(self.TEST_EVT_NAME) + await self._wait_until_status( + cb_env, 15, 2, EventingFunctionState.Undeployed, self.TEST_EVT_NAME + ) + await cb_env.efm.drop_function(self.TEST_EVT_NAME) + + @pytest_asyncio.fixture() + async def create_and_drop_eventing_function(self, cb_env): + await cb_env.efm.upsert_function(self.BASIC_FUNC) + yield + await cb_env.efm.drop_function(self.BASIC_FUNC.name) + + async def _wait_until_status(self, + cb_env, # type: TestEnvironment + num_times, # type: int + seconds_between, # type: int + state, # type: EventingFunctionState + name # type: str + ) -> None: + + func_status = None + for _ in range(num_times): + func_status = await cb_env.efm._get_status(name) + if func_status is None or func_status.state != state: + await asyncio.sleep(seconds_between) + else: + break + + if func_status is None: + raise EventingFunctionManagementTestStatusException( + "Unable to obtain function status for {}".format(name) + ) + if func_status.state != state: + raise EventingFunctionManagementTestStatusException( + "Function {} status is {} which does not match desired status of {}.".format( + name, func_status.state.value, state.value + ) + ) + + @pytest.mark.usefixtures("drop_eventing_function") + @pytest.mark.asyncio + async def test_upsert_function(self, cb_env, evt_version): + local_func = EventingFunction( + self.TEST_EVT_NAME, + self.SIMPLE_EVT_CODE, + evt_version, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + await cb_env.efm.upsert_function(local_func) + func = await cb_env.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func, shallow=True) + + @pytest.mark.asyncio + async def test_upsert_function_fail(self, cb_env, evt_version): + # bad appcode + local_func = EventingFunction( + self.TEST_EVT_NAME, + 'func OnUpdate(doc, meta) {\n log("Doc created/updated", meta.id);\n}\n\n', + evt_version, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + + # @TODO: + # with pytest.raises(EventingFunctionCompilationFailureException): + # await cb_env.efm.upsert_function(local_func) + + local_func.code = self.SIMPLE_EVT_CODE + local_func.source_keyspace = EventingFunctionKeyspace( + "beer-sample", "test-scope", "test-collection" + ) + with pytest.raises(EventingFunctionCollectionNotFoundException): + await cb_env.efm.upsert_function(local_func) + + @pytest.mark.usefixtures("create_eventing_function") + @pytest.mark.asyncio + async def test_drop_function(self, cb_env): + await cb_env.efm.drop_function(self.BASIC_FUNC.name) + await cb_env.try_n_times_till_exception( + 10, + 1, + cb_env.efm.get_function, + self.BASIC_FUNC.name, + EventingFunctionNotFoundException + ) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_drop_function_fail(self, cb_env): + with pytest.raises( + (EventingFunctionNotDeployedException, + EventingFunctionNotFoundException) + ): + await cb_env.efm.drop_function("not-a-function") + + # deploy function -- but first verify in undeployed state + await self._wait_until_status( + cb_env, 15, 2, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + # now, wait for it to be deployed + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + + with pytest.raises(EventingFunctionNotUnDeployedException): + await cb_env.efm.drop_function(self.BASIC_FUNC.name) + + @pytest.mark.usefixtures("create_and_drop_eventing_function") + @pytest.mark.asyncio + async def test_get_function(self, cb_env): + func = await cb_env.try_n_times( + 5, 3, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func) + + @pytest.mark.asyncio + async def test_get_function_fail(self, cb_env): + with pytest.raises(EventingFunctionNotFoundException): + await cb_env.efm.get_function("not-a-function") + + @pytest.mark.asyncio + async def test_get_all_functions(self, cb_env): + new_funcs = [ + EventingFunction( + "test-evt-func-1", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ), + EventingFunction( + "test-evt-func-2", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_5_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc2", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadOnly + ) + ] + ) + ] + for func in new_funcs: + await cb_env.efm.upsert_function(func) + await cb_env.try_n_times(5, 3, cb_env.efm.get_function, func.name) + + funcs = await cb_env.efm.get_all_functions() + for func in funcs: + cb_env.validate_eventing_function(func) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_deploy_function(self, cb_env): + # deploy function -- but first verify in undeployed state + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status has changed + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Deployed + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_deploy_function_fail(self, cb_env): + with pytest.raises(EventingFunctionNotFoundException): + await cb_env.efm.deploy_function("not-a-function") + + # deploy function -- but first verify in undeployed state + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + with pytest.raises(EventingFunctionAlreadyDeployedException): + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_undeploy_function(self, cb_env): + # deploy function -- but first verify in undeployed state + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Deployed + # now, undeploy function + await cb_env.efm.undeploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 15, 2, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status has changed + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Undeployed + + @pytest.mark.asyncio + async def test_undeploy_function_fail(self, cb_env): + with pytest.raises( + (EventingFunctionNotDeployedException, + EventingFunctionNotFoundException) + ): + await cb_env.efm.undeploy_function("not-a-function") + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_pause_function(self, cb_env): + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + await cb_env.efm.pause_function(self.BASIC_FUNC.name) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function processing status + assert func.settings.processing_status == EventingFunctionProcessingStatus.Paused + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_pause_function_fail(self, cb_env): + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + + with pytest.raises(EventingFunctionNotFoundException): + await cb_env.efm.pause_function("not-a-function") + + with pytest.raises(EventingFunctionNotBootstrappedException): + await cb_env.efm.pause_function(self.BASIC_FUNC.name) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_resume_function(self, cb_env): + # make sure function has been deployed + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + # pause function - verify status is paused + await cb_env.efm.pause_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 15, 2, EventingFunctionState.Paused, self.BASIC_FUNC.name + ) + # resume function + await cb_env.efm.resume_function(self.BASIC_FUNC.name) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function processing status + assert func.settings.processing_status == EventingFunctionProcessingStatus.Running + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_resume_function_fail(self, cb_env): + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + + with pytest.raises(EventingFunctionNotFoundException): + await cb_env.efm.pause_function("not-a-function") + + with pytest.raises(EventingFunctionNotBootstrappedException): + await cb_env.efm.pause_function(self.BASIC_FUNC.name) + + @pytest.mark.asyncio + async def test_constant_bindings(self, cb_env): + # TODO: look into why timeout occurs when providing > 1 constant + # binding + local_func = EventingFunction( + "test-evt-const-func", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace(self.EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc", + name=EventingFunctionKeyspace(self.EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ], + constant_bindings=[ + EventingFunctionConstantBinding( + alias="testConstant", literal="1234"), + EventingFunctionConstantBinding( + alias="testConstant1", literal="\"another test value\"") + ] + ) + + await cb_env.efm.upsert_function(local_func) + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, local_func.name + ) + func = await cb_env.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + @pytest.mark.asyncio + async def test_url_bindings(self, cb_env): + local_func = EventingFunction( + "test-evt-url-func", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace(self.EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc", + name=EventingFunctionKeyspace(self.EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ], + url_bindings=[ + EventingFunctionUrlBinding( + hostname="http://localhost:5000", + alias="urlBinding1", + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlNoAuth() + ), + EventingFunctionUrlBinding( + hostname="http://localhost:5001", + alias="urlBinding2", + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthBasic("username", "password") + ), + EventingFunctionUrlBinding( + hostname="http://localhost:5002", + alias="urlBinding3", + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthBearer( + "IThinkTheBearerTokenIsSupposedToGoHere" + ) + ), + EventingFunctionUrlBinding( + hostname="http://localhost:5003", + alias="urlBinding4", + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthDigest("username", "password") + ) + ] + ) + + await cb_env.efm.upsert_function(local_func) + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, local_func.name + ) + func = await cb_env.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + @pytest.mark.asyncio + async def test_functions_status(self, cb_env): + new_funcs = [ + EventingFunction( + "test-evt-func-1", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ), + EventingFunction( + "test-evt-func-2", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_5_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc2", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadOnly + ) + ] + ) + ] + for func in new_funcs: + await cb_env.efm.upsert_function(func) + await cb_env.try_n_times(5, 3, cb_env.efm.get_function, func.name) + + funcs = await cb_env.efm.functions_status() + assert isinstance(funcs, EventingFunctionsStatus) + for func in funcs.functions: + assert isinstance(func, EventingFunctionStatus) + + @pytest.mark.asyncio + async def test_options_simple(self, cb_env): + local_func = EventingFunction( + "test-evt-func-1", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + await cb_env.efm.upsert_function( + local_func, UpsertFunctionOptions(timeout=timedelta(seconds=20)) + ) + func = await cb_env.try_n_times( + 5, + 3, + cb_env.efm.get_function, + local_func.name, + GetFunctionOptions(timeout=timedelta(seconds=15)) + ) + cb_env.validate_eventing_function(func, shallow=True) + + @pytest.mark.asyncio + async def test_with_scope_and_collection(self, cb_env, evt_version): + if not cb_env.is_feature_supported('collections'): + pytest.skip('Server does not support scopes/collections.') + + coll_name = 'test-collection-1' + collection_spec = CollectionSpec(coll_name, cb_env.TEST_SCOPE) + try: + await cb_env.cm.create_collection(collection_spec) + except CollectionAlreadyExistsException: + await cb_env.cm.drop_collection(collection_spec) + await cb_env.cm.create_collection(collection_spec) + + await cb_env.try_n_times(5, 3, cb_env.get_collection, + cb_env.TEST_SCOPE, + coll_name, + bucket_name=cb_env.bucket.name) + + local_func = EventingFunction( + "test-evt-func-coll", + self.SIMPLE_EVT_CODE, + evt_version, + metadata_keyspace=EventingFunctionKeyspace( + cb_env.bucket.name, + cb_env.TEST_SCOPE, + cb_env.TEST_COLLECTION + ), + source_keyspace=EventingFunctionKeyspace( + cb_env.bucket.name, + cb_env.TEST_SCOPE, + coll_name + ), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc", + name=EventingFunctionKeyspace(cb_env.bucket.name), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + await cb_env.efm.upsert_function(local_func) + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, local_func.name + ) + func = await cb_env.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + await cb_env.efm.drop_function(local_func.name) + + +@pytest.mark.flaky(reruns=5) +class ScopeEventingManagementTests: + + EVT_SRC_BUCKET_NAME = "beer-sample" + EVT_META_BUCKET_NAME = "default" + TEST_EVT_NAME = 'test-evt-func' + SIMPLE_EVT_CODE = ('function OnUpdate(doc, meta) {\n log("Doc created/updated", meta.id);\n}' + '\n\nfunction OnDelete(meta, options) {\n log("Doc deleted/expired", meta.id);\n}') + EVT_VERSION = None + BASIC_FUNC = EventingFunction( + TEST_EVT_NAME, + SIMPLE_EVT_CODE, + "evt-7.0.0-5302-ee", + metadata_keyspace=EventingFunctionKeyspace(EVT_META_BUCKET_NAME), + source_keyspace=EventingFunctionKeyspace(EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc", + name=EventingFunctionKeyspace(EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + CollectionType.NAMED) + + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + cb_env.enable_scope_eventing_mgmt().enable_eventing_mgmt() + yield cb_env + cb_env.disable_eventing_mgmt().disable_scope_eventing_mgmt() + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest.fixture(scope="class", name='evt_version') + def get_eventing_function_version(self, cb_env): + version = "evt-{}".format( + cb_env.server_version.replace("enterprise", "ee").replace("community", "ce") + ) + return version + + @pytest_asyncio.fixture() + async def create_eventing_function(self, cb_env): + await cb_env.efm.upsert_function(self.BASIC_FUNC) + + @pytest_asyncio.fixture() + async def drop_eventing_function(self, cb_env): + yield + await cb_env.efm.drop_function(self.TEST_EVT_NAME) + + @pytest_asyncio.fixture() + async def undeploy_and_drop_eventing_function(self, cb_env): + yield + await cb_env.efm.undeploy_function(self.TEST_EVT_NAME) + await self._wait_until_status( + cb_env, 15, 2, EventingFunctionState.Undeployed, self.TEST_EVT_NAME + ) + await cb_env.efm.drop_function(self.TEST_EVT_NAME) + + @pytest_asyncio.fixture() + async def create_and_drop_eventing_function(self, cb_env): + await cb_env.efm.upsert_function(self.BASIC_FUNC) + yield + await cb_env.efm.drop_function(self.BASIC_FUNC.name) + + async def _wait_until_status(self, + cb_env, # type: TestEnvironment + num_times, # type: int + seconds_between, # type: int + state, # type: EventingFunctionState + name # type: str + ) -> None: + + func_status = None + for _ in range(num_times): + func_status = await cb_env.efm._get_status(name) + if func_status is None or func_status.state != state: + await asyncio.sleep(seconds_between) + else: + break + + if func_status is None: + raise EventingFunctionManagementTestStatusException( + "Unable to obtain function status for {}".format(name) + ) + if func_status.state != state: + raise EventingFunctionManagementTestStatusException( + "Function {} status is {} which does not match desired status of {}.".format( + name, func_status.state.value, state.value + ) + ) + + @pytest.mark.usefixtures("drop_eventing_function") + @pytest.mark.asyncio + async def test_upsert_function(self, cb_env, evt_version): + local_func = EventingFunction( + self.TEST_EVT_NAME, + self.SIMPLE_EVT_CODE, + evt_version, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + await cb_env.efm.upsert_function(local_func) + func = await cb_env.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func, shallow=True) + + @pytest.mark.asyncio + async def test_upsert_function_fail(self, cb_env, evt_version): + # bad appcode + local_func = EventingFunction( + self.TEST_EVT_NAME, + 'func OnUpdate(doc, meta) {\n log("Doc created/updated", meta.id);\n}\n\n', + evt_version, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + + # @TODO: + # with pytest.raises(EventingFunctionCompilationFailureException): + # await cb_env.efm.upsert_function(local_func) + + local_func.code = self.SIMPLE_EVT_CODE + local_func.source_keyspace = EventingFunctionKeyspace( + "beer-sample", "test-scope", "test-collection" + ) + with pytest.raises(EventingFunctionCollectionNotFoundException): + await cb_env.efm.upsert_function(local_func) + + @pytest.mark.usefixtures("create_eventing_function") + @pytest.mark.asyncio + async def test_drop_function(self, cb_env): + await cb_env.efm.drop_function(self.BASIC_FUNC.name) + await cb_env.try_n_times_till_exception( + 10, + 1, + cb_env.efm.get_function, + self.BASIC_FUNC.name, + EventingFunctionNotFoundException + ) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_drop_function_fail(self, cb_env): + with pytest.raises( + (EventingFunctionNotDeployedException, + EventingFunctionNotFoundException) + ): + await cb_env.efm.drop_function("not-a-function") + + # deploy function -- but first verify in undeployed state + await self._wait_until_status( + cb_env, 15, 2, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + # now, wait for it to be deployed + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + + with pytest.raises(EventingFunctionNotUnDeployedException): + await cb_env.efm.drop_function(self.BASIC_FUNC.name) + + @pytest.mark.usefixtures("create_and_drop_eventing_function") + @pytest.mark.asyncio + async def test_get_function(self, cb_env): + func = await cb_env.try_n_times( + 5, 3, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func) + + @pytest.mark.asyncio + async def test_get_function_fail(self, cb_env): + with pytest.raises(EventingFunctionNotFoundException): + await cb_env.efm.get_function("not-a-function") + + @pytest.mark.asyncio + async def test_get_all_functions(self, cb_env): + new_funcs = [ + EventingFunction( + "test-evt-func-1", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ), + EventingFunction( + "test-evt-func-2", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_5_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc2", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadOnly + ) + ] + ) + ] + for func in new_funcs: + await cb_env.efm.upsert_function(func) + await cb_env.try_n_times(5, 3, cb_env.efm.get_function, func.name) + + funcs = await cb_env.efm.get_all_functions() + for func in funcs: + cb_env.validate_eventing_function(func) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_deploy_function(self, cb_env): + # deploy function -- but first verify in undeployed state + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status has changed + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Deployed + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_deploy_function_fail(self, cb_env): + with pytest.raises(EventingFunctionNotFoundException): + await cb_env.efm.deploy_function("not-a-function") + + # deploy function -- but first verify in undeployed state + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + with pytest.raises(EventingFunctionAlreadyDeployedException): + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_undeploy_function(self, cb_env): + # deploy function -- but first verify in undeployed state + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Deployed + # now, undeploy function + await cb_env.efm.undeploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 15, 2, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status has changed + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Undeployed + + @pytest.mark.asyncio + async def test_undeploy_function_fail(self, cb_env): + with pytest.raises( + (EventingFunctionNotDeployedException, + EventingFunctionNotFoundException) + ): + await cb_env.efm.undeploy_function("not-a-function") + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_pause_function(self, cb_env): + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + await cb_env.efm.pause_function(self.BASIC_FUNC.name) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function processing status + assert func.settings.processing_status == EventingFunctionProcessingStatus.Paused + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_pause_function_fail(self, cb_env): + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + + with pytest.raises(EventingFunctionNotFoundException): + await cb_env.efm.pause_function("not-a-function") + + with pytest.raises(EventingFunctionNotBootstrappedException): + await cb_env.efm.pause_function(self.BASIC_FUNC.name) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_resume_function(self, cb_env): + # make sure function has been deployed + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + await cb_env.efm.deploy_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 20, 3, EventingFunctionState.Deployed, self.BASIC_FUNC.name + ) + # pause function - verify status is paused + await cb_env.efm.pause_function(self.BASIC_FUNC.name) + await self._wait_until_status( + cb_env, 15, 2, EventingFunctionState.Paused, self.BASIC_FUNC.name + ) + # resume function + await cb_env.efm.resume_function(self.BASIC_FUNC.name) + func = await cb_env.try_n_times( + 5, 1, cb_env.efm.get_function, self.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function processing status + assert func.settings.processing_status == EventingFunctionProcessingStatus.Running + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + @pytest.mark.asyncio + async def test_resume_function_fail(self, cb_env): + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, self.BASIC_FUNC.name + ) + + with pytest.raises(EventingFunctionNotFoundException): + await cb_env.efm.pause_function("not-a-function") + + with pytest.raises(EventingFunctionNotBootstrappedException): + await cb_env.efm.pause_function(self.BASIC_FUNC.name) + + @pytest.mark.asyncio + async def test_constant_bindings(self, cb_env): + # TODO: look into why timeout occurs when providing > 1 constant + # binding + local_func = EventingFunction( + "test-evt-const-func", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace(self.EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc", + name=EventingFunctionKeyspace(self.EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ], + constant_bindings=[ + EventingFunctionConstantBinding( + alias="testConstant", literal="1234"), + EventingFunctionConstantBinding( + alias="testConstant1", literal="\"another test value\"") + ] + ) + + await cb_env.efm.upsert_function(local_func) + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, local_func.name + ) + func = await cb_env.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + @pytest.mark.asyncio + async def test_url_bindings(self, cb_env): + local_func = EventingFunction( + "test-evt-url-func", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace(self.EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc", + name=EventingFunctionKeyspace(self.EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ], + url_bindings=[ + EventingFunctionUrlBinding( + hostname="http://localhost:5000", + alias="urlBinding1", + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlNoAuth() + ), + EventingFunctionUrlBinding( + hostname="http://localhost:5001", + alias="urlBinding2", + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthBasic("username", "password") + ), + EventingFunctionUrlBinding( + hostname="http://localhost:5002", + alias="urlBinding3", + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthBearer( + "IThinkTheBearerTokenIsSupposedToGoHere" + ) + ), + EventingFunctionUrlBinding( + hostname="http://localhost:5003", + alias="urlBinding4", + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthDigest("username", "password") + ) + ] + ) + + await cb_env.efm.upsert_function(local_func) + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, local_func.name + ) + func = await cb_env.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + @pytest.mark.asyncio + async def test_functions_status(self, cb_env): + new_funcs = [ + EventingFunction( + "test-evt-func-1", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ), + EventingFunction( + "test-evt-func-2", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_5_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc2", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadOnly + ) + ] + ) + ] + for func in new_funcs: + await cb_env.efm.upsert_function(func) + await cb_env.try_n_times(5, 3, cb_env.efm.get_function, func.name) + + funcs = await cb_env.efm.functions_status() + assert isinstance(funcs, EventingFunctionsStatus) + for func in funcs.functions: + assert isinstance(func, EventingFunctionStatus) + + @pytest.mark.asyncio + async def test_options_simple(self, cb_env): + local_func = EventingFunction( + "test-evt-func-1", + self.SIMPLE_EVT_CODE, + self.EVT_VERSION, + metadata_keyspace=EventingFunctionKeyspace("default"), + source_keyspace=EventingFunctionKeyspace("beer-sample"), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc1", + name=EventingFunctionKeyspace("beer-sample"), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + await cb_env.efm.upsert_function( + local_func, UpsertFunctionOptions(timeout=timedelta(seconds=20)) + ) + func = await cb_env.try_n_times( + 5, + 3, + cb_env.efm.get_function, + local_func.name, + GetFunctionOptions(timeout=timedelta(seconds=15)) + ) + cb_env.validate_eventing_function(func, shallow=True) + + @pytest.mark.asyncio + async def test_with_scope_and_collection(self, cb_env, evt_version): + if not cb_env.is_feature_supported('collections'): + pytest.skip('Server does not support scopes/collections.') + + coll_name = 'test-collection-1' + collection_spec = CollectionSpec(coll_name, cb_env.TEST_SCOPE) + try: + await cb_env.cm.create_collection(collection_spec) + except CollectionAlreadyExistsException: + await cb_env.cm.drop_collection(collection_spec) + await cb_env.cm.create_collection(collection_spec) + + await cb_env.try_n_times(5, 3, cb_env.get_collection, + cb_env.TEST_SCOPE, + coll_name, + bucket_name=cb_env.bucket.name) + + local_func = EventingFunction( + "test-evt-func-coll", + self.SIMPLE_EVT_CODE, + evt_version, + metadata_keyspace=EventingFunctionKeyspace( + cb_env.bucket.name, + cb_env.TEST_SCOPE, + cb_env.TEST_COLLECTION + ), + source_keyspace=EventingFunctionKeyspace( + cb_env.bucket.name, + cb_env.TEST_SCOPE, + coll_name + ), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias="evtFunc", + name=EventingFunctionKeyspace(cb_env.bucket.name), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + await cb_env.efm.upsert_function(local_func) + await self._wait_until_status( + cb_env, 10, 1, EventingFunctionState.Undeployed, local_func.name + ) + func = await cb_env.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + await cb_env.efm.drop_function(local_func.name) diff --git a/acouchbase/tests/kv_range_scan_t.py b/acouchbase/tests/kv_range_scan_t.py new file mode 100644 index 000000000..2fa1306dd --- /dev/null +++ b/acouchbase/tests/kv_range_scan_t.py @@ -0,0 +1,368 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from uuid import uuid4 + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import (DocumentNotFoundException, + FeatureUnavailableException, + InvalidArgumentException) +from couchbase.kv_range_scan import (PrefixScan, + RangeScan, + SamplingScan, + ScanTerm) +from couchbase.mutation_state import MutationState +from couchbase.options import ScanOptions +from couchbase.result import ScanResult, ScanResultIterable +from tests.environments import CollectionType +from tests.test_features import EnvironmentFeatures + + +class RangeScanTestSuite: + TEST_MANIFEST = [ + 'test_range_scan', + 'test_range_scan_exclusive', + 'test_range_scan_ids_only', + 'test_range_scan_default_terms', + 'test_prefix_scan', + 'test_sampling_scan', + 'test_sampling_scan_with_seed', + 'test_range_scan_with_batch_byte_limit', + 'test_range_scan_with_batch_item_limit', + 'test_range_scan_with_concurrency', + 'test_prefix_scan_with_batch_byte_limit', + 'test_prefix_scan_with_batch_item_limit', + 'test_prefix_scan_with_concurrency', + 'test_sampling_scan_with_batch_byte_limit', + 'test_sampling_scan_with_batch_item_limit', + 'test_sampling_scan_with_concurrency', + 'test_range_scan_with_zero_concurrency', + 'test_sampling_scan_with_zero_limit', + 'test_sampling_scan_with_negative_limit', + 'test_range_scan_feature_unavailable', + ] + + @pytest_asyncio.fixture(scope='session') + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest.fixture(scope='class') + def check_range_scan_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('kv_range_scan', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_range_scan_not_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_not_supported('kv_range_scan', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def test_id(self): + scan_uuid = str(uuid4()) + return scan_uuid + + @pytest.fixture(scope="class") + def test_ids(self, test_id): + scan_ids = [f'{test_id}-{i}' for i in range(100)] + return scan_ids + + @pytest_asyncio.fixture(scope="class") + async def test_mutation_state(self, test_ids, cb_env): + results = [] + for doc_id in test_ids: + res = await cb_env.collection.upsert(doc_id, {'id': doc_id}) + results.append(res) + + return MutationState(*results) + + async def _purge_temp_docs(self, cb_env, test_ids): + for doc_id in test_ids: + try: + await cb_env.collection.remove(doc_id) + except DocumentNotFoundException: + pass + + async def _validate_result(self, result, expected_count=0, ids_only=False, return_rows=False, from_sample=False): + assert isinstance(result, ScanResultIterable) + rows = [] + async for r in result: + assert isinstance(r, ScanResult) + if ids_only: + with pytest.raises(InvalidArgumentException): + r.expiry_time + with pytest.raises(InvalidArgumentException): + r.cas + with pytest.raises(InvalidArgumentException): + r.content_as[str] + else: + assert r.cas is not None + assert r.id is not None + content = r.content_as[dict] + assert content is not None + assert content == {'id': r.id} + rows.append(r) + + if from_sample is True: + assert len(rows) <= expected_count + else: + assert len(rows) >= expected_count + + if return_rows: + return rows + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_range_scan(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + await self._validate_result(res, 12) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_range_scan_exclusive(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1', True), ScanTerm(f'{test_id}-2', True)) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + rows = await self._validate_result(res, 10, return_rows=True) + ids = [r.id for r in rows] + assert f'{test_id}-1' not in ids + assert f'{test_id}-2' not in ids + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_range_scan_ids_only(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + await self._validate_result(res, 12, ids_only=True) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_range_scan_default_terms(self, cb_env, test_ids): + scan_type = RangeScan() + res = cb_env.collection.scan(scan_type, ids_only=True) + await self._validate_result(res, len(test_ids), ids_only=True) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_sampling_scan(self, cb_env, test_mutation_state): + limit = 10 + scan_type = SamplingScan(limit) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=False, + consistent_with=test_mutation_state)) + await self._validate_result(res, limit, ids_only=False, return_rows=False, from_sample=True) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_sampling_scan_with_seed(self, cb_env, test_ids, test_mutation_state): + limit = 10 + scan_type = SamplingScan(limit, 50) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + rows = await self._validate_result(res, limit, ids_only=True, return_rows=True, from_sample=True) + result_ids = [] + for r in rows: + result_ids.append(r.id) + + # w/ the seed, we should get the same results + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + rows = await self._validate_result(res, limit, ids_only=True, return_rows=True, from_sample=True) + compare_ids = list(map(lambda r: r.id, rows)) + assert result_ids == compare_ids + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_prefix_scan(self, cb_env, test_id, test_ids, test_mutation_state): + scan_type = PrefixScan(f'{test_id}') + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + rows = await self._validate_result(res, 100, ids_only=True, return_rows=True) + for r in rows: + assert r.id in test_ids + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_byte_limit', [0, 1, 25, 100]) + async def test_range_scan_with_batch_byte_limit(self, cb_env, test_id, test_mutation_state, batch_byte_limit): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_byte_limit=batch_byte_limit, + consistent_with=test_mutation_state)) + await self._validate_result(res, 12) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_item_limit', [0, 1, 25, 100]) + async def test_range_scan_with_batch_item_limit(self, cb_env, test_id, test_mutation_state, batch_item_limit): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_item_limit=batch_item_limit, + consistent_with=test_mutation_state)) + await self._validate_result(res, 12) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('concurrency', [1, 2, 4, 16, 64, 128]) + async def test_range_scan_with_concurrency(self, cb_env, test_id, test_mutation_state, concurrency): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + concurrency=concurrency, + consistent_with=test_mutation_state)) + await self._validate_result(res, 12) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_byte_limit', [0, 1, 25, 100]) + async def test_prefix_scan_with_batch_byte_limit(self, cb_env, test_id, test_mutation_state, batch_byte_limit): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_byte_limit=batch_byte_limit, + consistent_with=test_mutation_state)) + await self._validate_result(res, 100) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_item_limit', [0, 1, 25, 100]) + async def test_prefix_scan_with_batch_item_limit(self, cb_env, test_id, test_mutation_state, batch_item_limit): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_item_limit=batch_item_limit, + consistent_with=test_mutation_state)) + await self._validate_result(res, 100) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('concurrency', [1, 2, 4, 16, 64, 128]) + async def test_prefix_scan_with_concurrency(self, cb_env, test_id, test_mutation_state, concurrency): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + concurrency=concurrency, + consistent_with=test_mutation_state)) + await self._validate_result(res, 100) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_byte_limit', [0, 1, 25, 100]) + async def test_sampling_scan_with_batch_byte_limit(self, cb_env, test_id, test_mutation_state, batch_byte_limit): + scan_type = SamplingScan(50) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_byte_limit=batch_byte_limit, + consistent_with=test_mutation_state)) + await self._validate_result(res, 50, from_sample=True) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_item_limit', [0, 1, 25, 100]) + async def test_sampling_scan_with_batch_item_limit(self, cb_env, test_id, test_mutation_state, batch_item_limit): + scan_type = SamplingScan(50) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_item_limit=batch_item_limit, + consistent_with=test_mutation_state)) + await self._validate_result(res, 50, from_sample=True) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('concurrency', [1, 2, 4, 16, 64, 128]) + async def test_sampling_scan_with_concurrency(self, cb_env, test_id, test_mutation_state, concurrency): + scan_type = SamplingScan(50) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + concurrency=concurrency, + consistent_with=test_mutation_state)) + await self._validate_result(res, 50, from_sample=True) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_range_scan_with_zero_concurrency(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + with pytest.raises(InvalidArgumentException): + cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + concurrency=0, + consistent_with=test_mutation_state)) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_sampling_scan_with_zero_limit(self, cb_env, test_id, test_mutation_state): + scan_type = SamplingScan(0) + with pytest.raises(InvalidArgumentException): + cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_supported') + async def test_sampling_scan_with_negative_limit(self, cb_env, test_id, test_mutation_state): + scan_type = SamplingScan(-10) + with pytest.raises(InvalidArgumentException): + cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_range_scan_not_supported') + async def test_range_scan_feature_unavailable(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + with pytest.raises(FeatureUnavailableException): + await self._validate_result(res) + + +class ClassicRangeScanTests(RangeScanTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicRangeScanTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + + method_list = [meth for meth in dir(ClassicRangeScanTests) if valid_test_method(meth)] + compare = set(ClassicRangeScanTests.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request, test_ids): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + await acb_base_env.setup(collection_type=request.param, num_docs=0) + yield acb_base_env + await self._purge_temp_docs(acb_base_env, test_ids) + await acb_base_env.teardown(request.param) diff --git a/acouchbase/tests/mutation_tokens_t.py b/acouchbase/tests/mutation_tokens_t.py new file mode 100644 index 000000000..449933477 --- /dev/null +++ b/acouchbase/tests/mutation_tokens_t.py @@ -0,0 +1,138 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import pytest_asyncio + +import couchbase.subdocument as SD +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import DocumentNotFoundException + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment) + + +class MutationTokensEnabledTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + manage_buckets=True) + if request.param == CollectionType.NAMED: + if cb_env.is_mock_server: + pytest.skip('Jenkins + GoCAVES not playing nice...') + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name="new_kvp") + async def new_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = await cb_env.get_new_key_value() + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + def verify_mutation_tokens(self, bucket_name, result): + mutation_token = result.mutation_token() + assert mutation_token is not None + partition_id, partition_uuid, sequence_number, mt_bucket_name = mutation_token.as_tuple() + assert isinstance(partition_id, int) + assert isinstance(partition_uuid, int) + assert isinstance(sequence_number, int) + assert bucket_name == mt_bucket_name + + @pytest.mark.asyncio + async def test_mutation_tokens_upsert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + result = await cb_env.try_n_times(5, 3, cb.upsert, key, value) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + @pytest.mark.asyncio + async def test_mutation_tokens_insert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + result = await cb_env.try_n_times(5, 3, cb.insert, key, value) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + @pytest.mark.asyncio + async def test_mutation_tokens_replace(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb_env.try_n_times(5, 3, cb.upsert, key, value) + result = await cb_env.try_n_times(5, 3, cb.replace, key, value) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + @pytest.mark.asyncio + async def test_mutation_tokens_remove(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb_env.try_n_times(5, 3, cb.upsert, key, value) + result = await cb_env.try_n_times(5, 3, cb.remove, key) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + # @TODO: c++ client does not provide mutation token for touch + # @pytest.mark.asyncio + # async def test_mutation_tokens_touch(self, cb_env, new_kvp): + # cb = cb_env.collection + # key = new_kvp.key + # value = new_kvp.value + # await cb.upsert(key, value) + # result = await cb.touch(key, timedelta(seconds=3)) + # self.verify_mutation_tokens(cb_env.bucket.name, result) + + @pytest.mark.asyncio + async def test_mutation_tokens_mutate_in(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + + async def cas_matches(key, cas): + result = await cb.get(key) + if result.cas == cas: + return result + raise Exception("nope") + + res = await cb_env.try_n_times(5, 3, cb.upsert, key, {"a": "aaa", "b": {"c": {"d": "yo!"}}}) + cas = res.cas + await cb_env.try_n_times(10, 3, cas_matches, key, cas) + result = await cb_env.try_n_times(5, 3, cb.mutate_in, key, (SD.upsert("c", "ccc"), SD.replace("b", "XXX"),)) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + +# @TODO: need to update client settings first +class MutationTokensDisabledTests: + pass diff --git a/acouchbase/tests/query_t.py b/acouchbase/tests/query_t.py new file mode 100644 index 000000000..79c350d90 --- /dev/null +++ b/acouchbase/tests/query_t.py @@ -0,0 +1,540 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import asyncio +import time +from datetime import datetime, timedelta + +import pytest +import pytest_asyncio + +import couchbase.subdocument as SD +from couchbase.exceptions import (AmbiguousTimeoutException, + CouchbaseException, + KeyspaceNotFoundException, + ParsingFailedException, + QueryErrorContext, + ScopeNotFoundException) +from couchbase.mutation_state import MutationState +from couchbase.n1ql import (QueryMetaData, + QueryMetrics, + QueryProfile, + QueryStatus, + QueryWarning) +from couchbase.options import (QueryOptions, + UnsignedInt64, + UpsertOptions) +from tests.environments import CollectionType +from tests.environments.query_environment import AsyncQueryTestEnvironment +from tests.environments.test_environment import AsyncTestEnvironment +from tests.test_features import EnvironmentFeatures + + +class QueryCollectionTestSuite: + TEST_MANIFEST = [ + 'test_bad_query_context', + 'test_bad_scope_query', + 'test_cluster_query_context', + 'test_query_fully_qualified', + 'test_query_metadata', + 'test_query_ryow', + 'test_query_with_metrics', + 'test_scope_query', + 'test_scope_query_with_named_params_in_options', + 'test_scope_query_with_positional_params_in_options', + ] + + @pytest.mark.asyncio + async def test_bad_query_context(self, cb_env): + q_str = f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2" + # test w/ no context + with pytest.raises(KeyspaceNotFoundException): + await cb_env.cluster.query(q_str).execute() + + # test w/ bad scope + q_context = f'{cb_env.bucket.name}.`fake-scope`' + with pytest.raises(ScopeNotFoundException): + await cb_env.cluster.query(q_str, QueryOptions(query_context=q_context)).execute() + + @pytest.mark.asyncio + async def test_bad_scope_query(self, cb_env): + q_str = f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2" + q_context = f'{cb_env.bucket.name}.`fake-scope`' + with pytest.raises(ScopeNotFoundException): + await cb_env.scope.query(q_str, QueryOptions(query_context=q_context)).execute() + + q_context = f'`fake-bucket`.`{cb_env.scope.name}`' + with pytest.raises(KeyspaceNotFoundException): + await cb_env.scope.query(q_str, query_context=q_context).execute() + + @pytest.mark.asyncio + async def test_cluster_query_context(self, cb_env): + q_context = f'{cb_env.bucket.name}.{cb_env.scope.name}' + # test with QueryOptions + q_opts = QueryOptions(query_context=q_context, adhoc=True) + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2", q_opts) + await cb_env.assert_rows(result, 2) + + # test with kwargs + result = cb_env.cluster.query( + f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2", query_context=q_context) + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_query_fully_qualified(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM {cb_env.fqdn} LIMIT 2") + await cb_env.assert_rows(result, 2) + assert result.metadata() is not None + # if adhoc is not set, it should be None + assert result._request.params.get('adhoc', None) is None + + @pytest.mark.asyncio + async def test_query_metadata(self, cb_env): + result = cb_env.scope.query(f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2") + await cb_env.assert_rows(result, 2) + metadata = result.metadata() # type: QueryMetaData + for id_meth in (metadata.client_context_id, metadata.request_id): + id_res = id_meth() + fail_msg = "{} failed".format(id_meth) + assert isinstance(id_res, str), fail_msg + assert metadata.status() == QueryStatus.SUCCESS + assert isinstance(metadata.signature(), (str, dict)) + assert isinstance(metadata.warnings(), list) + + for warning in metadata.warnings(): + assert isinstance(warning, QueryWarning) + assert isinstance(warning.message(), str) + assert isinstance(warning.code(), int) + + @pytest.mark.asyncio + async def test_query_ryow(self, cb_env): + key, value = cb_env.get_new_doc() + result = cb_env.scope.query(f'SELECT * FROM `{cb_env.collection.name}` USE KEYS "{key}"') + await cb_env.assert_rows(result, 0) + res = await cb_env.collection.insert(key, value) + ms = MutationState().add_mutation_token(res.mutation_token()) + result = cb_env.scope.query(f'SELECT * FROM `{cb_env.collection.name}` USE KEYS "{key}"', + QueryOptions(consistent_with=ms)) + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_query_with_metrics(self, cb_env): + initial = datetime.now() + result = cb_env.scope.query( + f"SELECT * FROM `{cb_env.collection.name}` LIMIT 1", QueryOptions(metrics=True)) + await cb_env.assert_rows(result, 1) + taken = datetime.now() - initial + metadata = result.metadata() # type: QueryMetaData + assert isinstance(metadata, QueryMetaData) + metrics = metadata.metrics() + assert isinstance(metrics, QueryMetrics) + assert isinstance(metrics.elapsed_time(), timedelta) + assert metrics.elapsed_time() < taken + assert metrics.elapsed_time() > timedelta(milliseconds=0) + assert isinstance(metrics.execution_time(), timedelta) + assert metrics.execution_time() < taken + assert metrics.execution_time() > timedelta(milliseconds=0) + + expected_counts = {metrics.mutation_count: 0, + metrics.result_count: 1, + metrics.sort_count: 0, + metrics.warning_count: 0} + for method, expected in expected_counts.items(): + count_result = method() + fail_msg = "{} failed".format(method) + assert isinstance(count_result, UnsignedInt64), fail_msg + assert UnsignedInt64(expected) == count_result, fail_msg + assert metrics.result_size() > UnsignedInt64(0) + assert metrics.error_count() == UnsignedInt64(0) + assert metadata.profile() is None + + @pytest.mark.asyncio + async def test_scope_query(self, cb_env): + result = cb_env.scope.query(f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2") + await cb_env.assert_rows(result, 2) + result = cb_env.scope.query(f"SELECT * FROM {cb_env.fqdn} LIMIT 2") + + @pytest.mark.asyncio + async def test_scope_query_with_named_params_in_options(self, cb_env): + result = cb_env.scope.query(f"SELECT * FROM `{cb_env.collection.name}` WHERE batch LIKE $batch LIMIT 1", + QueryOptions(named_parameters={'batch': f'{cb_env.get_batch_id()}%'})) + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_scope_query_with_positional_params_in_options(self, cb_env): + result = cb_env.scope.query(f"SELECT * FROM `{cb_env.collection.name}` WHERE batch LIKE $1 LIMIT 1", + QueryOptions(positional_parameters=[f'{cb_env.get_batch_id()}%'])) + await cb_env.assert_rows(result, 1) + + +class QueryTestSuite: + TEST_MANIFEST = [ + 'test_bad_query', + 'test_mixed_named_parameters', + 'test_mixed_positional_parameters', + 'test_non_blocking', + 'test_preserve_expiry', + 'test_query_error_context', + 'test_query_metadata', + 'test_query_raw_options', + 'test_query_ryow', + 'test_query_timeout', + 'test_query_with_metrics', + 'test_query_with_profile', + 'test_simple_query', + 'test_simple_query_explain', + 'test_simple_query_prepared', + 'test_simple_query_with_named_params', + 'test_simple_query_with_named_params_in_options', + 'test_simple_query_with_positional_params', + 'test_simple_query_with_positional_params_in_options', + 'test_simple_query_without_options_with_kwargs_named_params', + 'test_simple_query_without_options_with_kwargs_positional_params', + ] + + @pytest_asyncio.fixture(name='setup_udf') + async def setup_teardown_udf(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('query_user_defined_functions', + cb_env.server_version_short, + cb_env.mock_server_type) + await AsyncTestEnvironment.try_n_times(3, + 1, + cb_env.load_udf) + yield + await AsyncTestEnvironment.try_n_times(3, + 1, + cb_env.drop_udf) + + @pytest.fixture(scope='class') + def check_preserve_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('preserve_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.mark.asyncio + async def test_bad_query(self, cb_env): + with pytest.raises(ParsingFailedException): + await cb_env.cluster.query("I'm not N1QL!").execute() + + @pytest.mark.asyncio + async def test_mixed_named_parameters(self, cb_env): + batch_id = cb_env.get_batch_id() + result = cb_env.cluster.query(f'SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT 1', + QueryOptions(named_parameters={'batch': 'xgfflq'}), batch=batch_id) + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_mixed_positional_parameters(self, cb_env): + # we assume that positional overrides one in the Options + result = cb_env.cluster.query(f'SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 1', + QueryOptions(positional_parameters=['xgfflq']), f'{cb_env.get_batch_id()}') + await cb_env.assert_rows(result, 1) + + @pytest.mark.usefixtures('setup_udf') + @pytest.mark.asyncio + async def test_non_blocking(self, cb_env): + async def run_query(cluster, idx): + result = cluster.query("EXECUTE FUNCTION loop(1000000000)") + rows = [] + async for r in result: + rows.append(r) + return {time.time(): idx} + + tasks = [asyncio.create_task(run_query(cb_env.cluster, idx)) for idx in range(10)] + results = await asyncio.gather(*tasks) + # order tasks via the finished timestamp, prior to PYCBC-1471, the queries + # would return in order b/c we were blocking when pulling rows from the binding's + # streaming result object. + ordered_results = dict(sorted({k: v for r in results for k, v in r.items()}.items())) + assert list(ordered_results.values()) != list(i for i in range(10)) + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + @pytest.mark.asyncio + async def test_preserve_expiry(self, cb_env): + key = "imakey" + content = {"a": "aaa", "b": "bbb"} + + await cb_env.collection.upsert(key, content, UpsertOptions(expiry=timedelta(days=1))) + + expiry_path = "$document.exptime" + res = await AsyncTestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + q_str = f"UPDATE `{cb_env.bucket.name}` AS content USE KEYS '{key}' SET content.a = 'aaaa'" + await cb_env.cluster.query(q_str, QueryOptions(preserve_expiry=True)).execute() + + res = await AsyncTestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + + @pytest.mark.asyncio + async def test_query_error_context(self, cb_env): + try: + await cb_env.cluster.query("SELECT * FROM no_such_bucket").execute() + except CouchbaseException as ex: + assert isinstance(ex.context, QueryErrorContext) + assert ex.context.statement is not None + assert ex.context.first_error_code is not None + assert ex.context.first_error_message is not None + assert ex.context.client_context_id is not None + assert ex.context.response_body is not None + # @TODO: these are diff from 3.x -> 4.x + # self.assertIsNotNone(ex.context.endpoint) + # self.assertIsNotNone(ex.context.error_response_body) + + @pytest.mark.asyncio + async def test_query_metadata(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 2") + await cb_env.assert_rows(result, 2) + metadata = result.metadata() # type: QueryMetaData + for id_meth in (metadata.client_context_id, metadata.request_id): + id_res = id_meth() + fail_msg = "{} failed".format(id_meth) + assert isinstance(id_res, str), fail_msg + assert metadata.status() == QueryStatus.SUCCESS + assert isinstance(metadata.signature(), (str, dict)) + assert isinstance(metadata.warnings(), list) + + for warning in metadata.warnings(): + assert isinstance(warning, QueryWarning) + assert isinstance(warning.message(), str) + assert isinstance(warning.code(), int) + + @pytest.mark.asyncio + async def test_query_raw_options(self, cb_env): + # via raw, we should be able to pass any option + # if using named params, need to match full name param in query + # which is different for when we pass in name_parameters via their specific + # query option (i.e. include the $ when using raw) + batch_id = cb_env.get_batch_id() + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT $1", + QueryOptions(raw={'$batch': f'{batch_id}%', 'args': [2]})) + await cb_env.assert_rows(result, 2) + + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 1", + QueryOptions(raw={'args': [f'{batch_id}%']})) + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_query_ryow(self, cb_env): + key, value = cb_env.get_new_doc() + q_str = f'SELECT * FROM `{cb_env.bucket.name}` USE KEYS "{key}"' + result = cb_env.cluster.query(q_str) + await cb_env.assert_rows(result, 0) + res = await cb_env.collection.insert(key, value) + ms = MutationState().add_mutation_token(res.mutation_token()) + result = cb_env.cluster.query(q_str, QueryOptions(consistent_with=ms)) + await cb_env.assert_rows(result, 1) + + # prior to PYCBC-1477 the SDK _could_ crash w/ this this sort of MS creation + key, value = cb_env.get_new_doc() + result = cb_env.cluster.query(q_str) + await cb_env.assert_rows(result, 0) + res = await cb_env.collection.insert(key, value) + ms = MutationState(res) + result = cb_env.cluster.query(q_str, QueryOptions(consistent_with=ms)) + await cb_env.assert_rows(result, 1) + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_query_timeout(self, cb_env): + from acouchbase.cluster import Cluster + from couchbase.auth import PasswordAuthenticator + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + # Prior to PYCBC-1521, this test would fail as each request would override the cluster level query_timeout. + # If a timeout was not provided in the request, the default 75s timeout would be used. PYCBC-1521 corrects + # this behavior so this test will pass as we are essentially forcing an AmbiguousTimeoutException because + # we are setting the cluster level query_timeout such a small value. + timeout_opts = ClusterTimeoutOptions(query_timeout=timedelta(milliseconds=1)) + cluster = await Cluster.connect(f'{conn_string}', ClusterOptions(auth, timeout_options=timeout_opts)) + # don't need to do this except for older server versions + _ = cluster.bucket(f'{cb_env.bucket.name}') + q_str = f'SELECT * FROM `{cb_env.bucket.name}` LIMIT 10;' + with pytest.raises(AmbiguousTimeoutException): + await cluster.query(q_str).execute() + # If we override the timeout w/in the request the query should succeed. + rows = await cluster.query(q_str, timeout=timedelta(seconds=10)).execute() + assert len(rows) > 0 + + @pytest.mark.asyncio + async def test_query_with_metrics(self, cb_env): + initial = datetime.now() + result = cb_env.cluster.query( + f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 1", QueryOptions(metrics=True)) + await cb_env.assert_rows(result, 1) + taken = datetime.now() - initial + metadata = result.metadata() # type: QueryMetaData + assert isinstance(metadata, QueryMetaData) + metrics = metadata.metrics() + assert isinstance(metrics, QueryMetrics) + assert isinstance(metrics.elapsed_time(), timedelta) + assert metrics.elapsed_time() < taken + assert metrics.elapsed_time() > timedelta(milliseconds=0) + assert isinstance(metrics.execution_time(), timedelta) + assert metrics.execution_time() < taken + assert metrics.execution_time() > timedelta(milliseconds=0) + + expected_counts = {metrics.mutation_count: 0, + metrics.result_count: 1, + metrics.sort_count: 0, + metrics.warning_count: 0} + for method, expected in expected_counts.items(): + count_result = method() + fail_msg = "{} failed".format(method) + assert isinstance(count_result, UnsignedInt64), fail_msg + assert UnsignedInt64(expected) == count_result, fail_msg + assert metrics.result_size() > UnsignedInt64(0) + assert metrics.error_count() == UnsignedInt64(0) + assert metadata.profile() is None + + @pytest.mark.asyncio + async def test_query_with_profile(self, cb_env): + result = cb_env.cluster.query( + f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 1", QueryOptions(profile=QueryProfile.TIMINGS)) + await cb_env.assert_rows(result, 1) + assert result.metadata().profile() is not None + + @pytest.mark.asyncio + async def test_simple_query(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 2") + await cb_env.assert_rows(result, 2) + assert result.metadata() is not None + # if adhoc is not set, it should be None + assert result._request.params.get('adhoc', None) is None + + @pytest.mark.asyncio + async def test_simple_query_explain(self, cb_env): + result = cb_env.cluster.query(f"EXPLAIN SELECT * FROM `{cb_env.bucket.name}` LIMIT 2", + QueryOptions(metrics=True)) + rows = [] + async for r in result.rows(): + rows.append(r) + + assert len(rows) == 1 + assert 'plan' in rows[0] + assert result.metadata() is not None + assert result.metadata().metrics() is not None + + @pytest.mark.asyncio + async def test_simple_query_prepared(self, cb_env): + # @TODO(CXXCBC-174) + if cb_env.server_version_short < 6.5: + pytest.skip(f'Skipped on server versions < 6.5 (using {cb_env.server_version_short}). Pending CXXCBC-174') + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 2", + QueryOptions(adhoc=False, metrics=True)) + await cb_env.assert_rows(result, 2) + assert result.metadata() is not None + assert result.metadata().metrics() is not None + assert result._request.params.get('adhoc', None) is False + + @pytest.mark.asyncio + async def test_simple_query_with_named_params(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT 2", + batch=f'{cb_env.get_batch_id()}%') + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_simple_query_with_named_params_in_options(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT 1", + QueryOptions(named_parameters={'batch': f'{cb_env.get_batch_id()}%'})) + await cb_env.assert_rows(result, 1) + + @pytest.mark.asyncio + async def test_simple_query_with_positional_params(self, cb_env): + result = cb_env.cluster.query( + f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 2", f'{cb_env.get_batch_id()}%') + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_simple_query_with_positional_params_in_options(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 2", + QueryOptions(positional_parameters=[f'{cb_env.get_batch_id()}%'])) + await cb_env.assert_rows(result, 2) + + @pytest.mark.asyncio + async def test_simple_query_without_options_with_kwargs_named_params(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT 1", + named_parameters={'batch': f'{cb_env.get_batch_id()}%'}) + await cb_env.assert_rows(result, 1) + + # NOTE: Ideally I'd notice a set of positional parameters in the query call, and assume they were the positional + # parameters for the query (once popping off the options if it is in there). But this seems a bit tricky so for + # now, kwargs override the corresponding value in the options, only. + + @pytest.mark.asyncio + async def test_simple_query_without_options_with_kwargs_positional_params(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 1", + positional_parameters=[f'{cb_env.get_batch_id()}%']) + await cb_env.assert_rows(result, 1) + + +class ClassicQueryCollectionTests(QueryCollectionTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicQueryCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicQueryCollectionTests) if valid_test_method(meth)] + compare = set(QueryCollectionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.NAMED]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + acb_env = AsyncQueryTestEnvironment.from_environment(acb_base_env) + acb_env.enable_query_mgmt() + await acb_env.setup(request.param) + yield acb_env + await acb_env.teardown(request.param) + + +class ClassicQueryTests(QueryTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicQueryTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicQueryTests) if valid_test_method(meth)] + compare = set(QueryTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + acb_env = AsyncQueryTestEnvironment.from_environment(acb_base_env) + acb_env.enable_query_mgmt() + await acb_env.setup(request.param) + yield acb_env + await acb_env.teardown(request.param) diff --git a/acouchbase/tests/querymgmt_t.py b/acouchbase/tests/querymgmt_t.py new file mode 100644 index 000000000..a34b4f44a --- /dev/null +++ b/acouchbase/tests/querymgmt_t.py @@ -0,0 +1,1250 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from datetime import timedelta + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import (ParsingFailedException, + QueryIndexAlreadyExistsException, + QueryIndexNotFoundException, + WatchQueryIndexTimeoutException) +from couchbase.management.options import (CreatePrimaryQueryIndexOptions, + CreateQueryIndexOptions, + DropPrimaryQueryIndexOptions, + DropQueryIndexOptions, + GetAllQueryIndexOptions, + WatchQueryIndexOptions) + +from ._test_utils import TestEnvironment + + +class QueryIndexManagementTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True, + manage_query_indexes=True) + + yield cb_env + await cb_env.try_n_times(5, 3, self._clear_all_indexes, cb_env, ignore_fail=True) + + async def _clear_all_indexes(self, cb_env, ignore_fail=False): + # Drop all indexes! + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + indexes = await ixm.get_all_indexes(bucket_name) + for index in indexes: + # @TODO: will need to update once named primary allowed + if index.is_primary: + await ixm.drop_primary_index(bucket_name) + else: + await ixm.drop_index(bucket_name, index.name) + for _ in range(10): + indexes = await ixm.get_all_indexes(bucket_name) + if 0 == len(indexes): + return + await asyncio.sleep(2) + if ignore_fail is True: + return + + pytest.xfail( + "Indexes were not dropped after {} waits of {} seconds each".format(10, 3)) + + @pytest.fixture(scope="class") + def check_query_index_mgmt_supported(self, cb_env): + cb_env.check_if_feature_supported('query_index_mgmt') + + @pytest_asyncio.fixture() + async def clear_all_indexes(self, cb_env): + await self._clear_all_indexes(cb_env) + + @staticmethod + def supports_query_without_index(cb_env): + return cb_env.is_feature_supported('query_without_index') + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + await ixm.create_primary_index( + bucket_name, timeout=timedelta(seconds=60)) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM `{bucket_name}` LIMIT 1' + + await cb_env.cluster.query(n1ql).execute() + # Drop the primary index + await ixm.drop_primary_index(bucket_name) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + await cb_env.cluster.query(n1ql).execute() + + # @TODO: couchbase++ does not handle named primary + # @pytest.mark.usefixtures("check_query_index_mgmt_supported") + # @pytest.mark.usefixtures("clear_all_indexes") + # @pytest.mark.asyncio + # async def test_create_named_primary(self, cb_env): + # bucket_name = cb_env.bucket.name + # ixname = 'namedPrimary' + # n1ql = f'SELECT * FROM {bucket_name} LIMIT 1' + # ixm = cb_env.ixm + # # Try to create a _named_ primary index + # await ixm.create_index(bucket_name, ixname, [], primary=True) + # await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary_ignore_if_exists(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + await ixm.create_primary_index(bucket_name) + await ixm.create_primary_index( + bucket_name, CreatePrimaryQueryIndexOptions(ignore_if_exists=True)) + + with pytest.raises(QueryIndexAlreadyExistsException): + await ixm.create_primary_index(bucket_name) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary_ignore_if_exists_kwargs(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + await ixm.create_primary_index(bucket_name) + await ixm.create_primary_index(bucket_name, ignore_if_exists=True) + + with pytest.raises(QueryIndexAlreadyExistsException): + await ixm.create_primary_index(bucket_name) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_primary(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + + # create an index so we can drop + await ixm.create_primary_index( + bucket_name, timeout=timedelta(seconds=60)) + + await ixm.drop_primary_index( + bucket_name, timeout=timedelta(seconds=60)) + # this should fail now + with pytest.raises(QueryIndexNotFoundException): + await ixm.drop_primary_index(bucket_name) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_primary_ignore_if_not_exists(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + await ixm.drop_primary_index(bucket_name, ignore_if_not_exists=True) + await ixm.drop_primary_index(bucket_name, DropPrimaryQueryIndexOptions(ignore_if_not_exists=True)) + with pytest.raises(QueryIndexNotFoundException): + await ixm.drop_primary_index(bucket_name) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + keys = ('fld1', 'fld2') + await ixm.create_index(bucket_name, ixname, + keys=keys, timeout=timedelta(seconds=120)) + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format( + bucket_name, *keys) + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_condition(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + keys = ('fld1', 'fld2') + + await cb_env.try_n_times_till_exception(10, 5, ixm.drop_index, bucket_name, ixname, + expected_exceptions=(QueryIndexNotFoundException,)) + condition = '((`fld1` = 1) and (`fld2` = 2))' + await ixm.create_index(bucket_name, ixname, keys, + CreateQueryIndexOptions(timeout=timedelta(days=1), condition=condition)) + + async def check_index(): + indexes = await ixm.get_all_indexes(bucket_name) + result = next((idx for idx in indexes if idx.name == ixname), None) + assert result is not None + return result + result = await cb_env.try_n_times(10, 5, check_index) + assert result.condition == condition + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_fields_kwarg(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + keys = ('fld1', 'fld2') + await ixm.create_index(bucket_name, ixname, + fields=keys, timeout=timedelta(seconds=120)) + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format( + bucket_name, *keys) + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_secondary_indexes(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + keys = ('fld1', 'fld2') + await ixm.create_index(bucket_name, ixname, keys, timeout=timedelta(seconds=120)) + + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format(bucket_name, *keys) + + # Drop the index + await ixm.drop_index(bucket_name, ixname) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises((QueryIndexNotFoundException, ParsingFailedException)): + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_index_no_fields(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # raises a TypeError b/c not providing fields means + # create_index() is missing a required positional param + with pytest.raises(TypeError): + await ixm.create_index(bucket_name, 'noFields') + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_ignore_if_exists(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + await ixm.create_index(bucket_name, ixname, ['hello']) + await ixm.create_index(bucket_name, ixname, ['hello'], ignore_if_exists=True) + await ixm.create_index(bucket_name, ixname, ['hello'], CreateQueryIndexOptions(ignore_if_exists=True)) + with pytest.raises(QueryIndexAlreadyExistsException): + await ixm.create_index(bucket_name, ixname, ['hello']) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_secondary_indexes_ignore_if_not_exists(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # Create it + ixname = 'ix2' + await ixm.create_index(bucket_name, ixname, ['hello']) + # Drop it + await ixm.drop_index(bucket_name, ixname) + await ixm.drop_index(bucket_name, ixname, ignore_if_not_exists=True) + await ixm.drop_index(bucket_name, ixname, DropQueryIndexOptions(ignore_if_not_exists=True)) + with pytest.raises(QueryIndexNotFoundException): + await ixm.drop_index(bucket_name, ixname) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_list_indexes(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # start with no indexes + ixs = await ixm.get_all_indexes(bucket_name) + assert len(ixs) == 0 + + # Create the primary index + await ixm.create_primary_index(bucket_name) + ixs = await ixm.get_all_indexes(bucket_name) + assert len(ixs) == 1 + assert ixs[0].is_primary is True + assert ixs[0].name == '#primary' + assert ixs[0].bucket_name == bucket_name + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_index_partition_info(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # use query to create index w/ partition, cannot do that via manager + # ATM + n1ql = 'CREATE INDEX idx_fld1 ON `{0}`(fld1) PARTITION BY HASH(fld1)'.format( + bucket_name) + await cb_env.cluster.query(n1ql).execute() + ixs = await ixm.get_all_indexes(bucket_name) + idx = next((ix for ix in ixs if ix.name == "idx_fld1"), None) + assert idx is not None + assert idx.partition is not None + assert idx.partition == 'HASH(`fld1`)' + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_watch(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # Create primary index + await ixm.create_primary_index(bucket_name, deferred=True) + ixs = await ixm.get_all_indexes(bucket_name) + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + defer = False + if n % 2 == 0: + defer = True + await ixm.create_index(bucket_name, + 'ix{0}'.format(n), ['fld{0}'.format(n)], deferred=defer) + + ixs = await ixm.get_all_indexes(bucket_name) + assert len(ixs) == 6 + # by not building deffered indexes, should timeout + with pytest.raises(WatchQueryIndexTimeoutException): + await ixm.watch_indexes(bucket_name, + [i.name for i in ixs], + WatchQueryIndexOptions(timeout=timedelta(seconds=5))) + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_deferred(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # Create primary index + await ixm.create_primary_index(bucket_name, deferred=True) + ixs = await ixm.get_all_indexes(bucket_name) + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + await ixm.create_index(bucket_name, + 'ix{0}'.format(n), ['fld{0}'.format(n)], CreateQueryIndexOptions(deferred=True)) + + ixs = await ixm.get_all_indexes(bucket_name) + assert len(ixs) == 6 + + ix_names = list(map(lambda i: i.name, ixs)) + + await ixm.build_deferred_indexes(bucket_name) + await ixm.watch_indexes(bucket_name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30))) # Should be OK + await ixm.watch_indexes(bucket_name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30), + watch_primary=True)) # Should be OK again + with pytest.raises(QueryIndexNotFoundException): + await ixm.watch_indexes(bucket_name, + ['idontexist'], + WatchQueryIndexOptions(timeout=timedelta(seconds=10))) + + # TODO: what is the purpose of this test? It is flaky in that timeout of dropping the index matters + # @pytest.mark.usefixtures("check_query_index_mgmt_supported") + # @pytest.mark.usefixtures("clear_all_indexes") + # @pytest.mark.asyncio + # async def test_deferred_fail(self, cb_env): + # # Create a bunch of indexes + # for n in range(10): + # await cb_env.ixm.create_index(cb_env.bucket.name, + # 'ix{0}'.format(n), + # ['fld{0}'.format(n)], + # CreateQueryIndexOptions(deferred=True)) + + # ixs = await cb_env.ixm.get_all_indexes(cb_env.bucket.name) + # assert len(ixs) == 10 + # ix_names = list(map(lambda i: i.name, ixs)) + + # # lets drop the last index while building all deferred indexes to trigger an exception + # results = await asyncio.gather(cb_env.ixm.drop_index(cb_env.bucket.name, ix_names[-1]), + # cb_env.ixm.build_deferred_indexes(cb_env.bucket.name), + # return_exceptions=True) + # assert isinstance(results[1], InternalServerFailureException) + # assert isinstance(results[1].context, ManagementErrorContext) + + +class QueryIndexCollectionManagementTests: + + TEST_SCOPE = "test-scope" + TEST_COLLECTION = "test-collection" + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True, + manage_collections=True, + manage_query_indexes=True) + + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + await cb_env.try_n_times(5, 3, self._clear_all_indexes, cb_env, ignore_fail=True) + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + async def _drop_all_indexes(self, cb_env, indexes, scope_name='_default', collection_name='_default'): + for index in indexes: + # @TODO: will need to update once named primary allowed + if index.is_primary: + await cb_env.ixm.drop_primary_index(cb_env.bucket.name, + DropPrimaryQueryIndexOptions(scope_name=scope_name, + collection_name=collection_name)) + else: + await cb_env.ixm.drop_index(cb_env.bucket.name, + index.name, + DropQueryIndexOptions(scope_name=scope_name, + collection_name=collection_name)) + for _ in range(10): + indexes = await cb_env.ixm.get_all_indexes(cb_env.bucket.name, + GetAllQueryIndexOptions(scope_name=scope_name, + collection_name=collection_name)) + if 0 == len(indexes): + return True + await asyncio.sleep(2) + + return False + + async def _clear_all_indexes(self, cb_env, ignore_fail=False): + # Drop all indexes! + for scope_col in [(self.TEST_SCOPE, self.TEST_COLLECTION), ('_default', '_default')]: + indexes = await cb_env.ixm.get_all_indexes(cb_env.bucket.name, + GetAllQueryIndexOptions(scope_name=scope_col[0], + collection_name=scope_col[1])) + + success = await self._drop_all_indexes(cb_env, + indexes, + scope_name=scope_col[0], + collection_name=scope_col[1]) + if success: + continue + elif ignore_fail is True: + continue + else: + pytest.xfail( + "Indexes were not dropped after {} waits of {} seconds each".format(10, 3)) + + @pytest.fixture(scope="class") + def check_query_index_mgmt_supported(self, cb_env): + cb_env.check_if_feature_supported('query_index_mgmt') + + @pytest.fixture(scope="class", name="fqdn") + def get_fqdn(self, cb_env): + return f'`{cb_env.bucket.name}`.`{self.TEST_SCOPE}`.`{self.TEST_COLLECTION}`' + + @pytest_asyncio.fixture() + async def clear_all_indexes(self, cb_env): + await self._clear_all_indexes(cb_env) + + @staticmethod + def supports_query_without_index(cb_env): + return cb_env.is_feature_supported('query_without_index') + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary(self, cb_env, fqdn): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + await ixm.create_primary_index(bucket_name, + CreatePrimaryQueryIndexOptions(scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION, + timeout=timedelta(seconds=60))) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM {fqdn} LIMIT 1' + + await cb_env.cluster.query(n1ql).execute() + # Drop the primary index + await ixm.drop_primary_index(bucket_name, + DropPrimaryQueryIndexOptions(scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary_ignore_if_exists(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + await ixm.create_primary_index(bucket_name, + CreatePrimaryQueryIndexOptions(scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + await ixm.create_primary_index(bucket_name, + CreatePrimaryQueryIndexOptions(ignore_if_exists=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + + with pytest.raises(QueryIndexAlreadyExistsException): + await ixm.create_primary_index(bucket_name, + CreatePrimaryQueryIndexOptions(scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary_ignore_if_exists_kwargs(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + await ixm.create_primary_index(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + await ixm.create_primary_index(bucket_name, + ignore_if_exists=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + with pytest.raises(QueryIndexAlreadyExistsException): + await ixm.create_primary_index(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_primary(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + + # create an index so we can drop + await ixm.create_primary_index(bucket_name, + timeout=timedelta(seconds=60), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + await ixm.drop_primary_index(bucket_name, + timeout=timedelta(seconds=60), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + # this should fail now + with pytest.raises(QueryIndexNotFoundException): + await ixm.drop_primary_index(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_primary_ignore_if_not_exists(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + await ixm.drop_primary_index(bucket_name, + ignore_if_not_exists=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + await ixm.drop_primary_index(bucket_name, + DropPrimaryQueryIndexOptions(ignore_if_not_exists=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + with pytest.raises(QueryIndexNotFoundException): + await ixm.drop_primary_index(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes(self, cb_env, fqdn): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + keys = ('fld1', 'fld2') + await ixm.create_index(bucket_name, ixname, + keys=keys, + timeout=timedelta(seconds=120), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + n1ql = "SELECT {1}, {2} FROM {0} WHERE {1}=1 AND {2}=2 LIMIT 1".format(fqdn, *keys) + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_condition(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + keys = ('fld1', 'fld2') + + await cb_env.try_n_times_till_exception(10, 5, ixm.drop_index, bucket_name, ixname, + expected_exceptions=(QueryIndexNotFoundException,), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + condition = '((`fld1` = 1) and (`fld2` = 2))' + await ixm.create_index(bucket_name, ixname, keys, + CreateQueryIndexOptions(timeout=timedelta(days=1), + condition=condition, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + + async def check_index(): + indexes = await ixm.get_all_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + result = next((idx for idx in indexes if idx.name == ixname), None) + assert result is not None + return result + result = await cb_env.try_n_times(10, 5, check_index) + assert result.condition == condition + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_fields_kwarg(self, cb_env, fqdn): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + keys = ('fld1', 'fld2') + await ixm.create_index(bucket_name, ixname, + fields=keys, + timeout=timedelta(seconds=120), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + n1ql = "SELECT {1}, {2} FROM {0} WHERE {1}=1 AND {2}=2 LIMIT 1".format(fqdn, *keys) + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_secondary_indexes(self, cb_env, fqdn): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + keys = ('fld1', 'fld2') + await ixm.create_index(bucket_name, ixname, + keys, + timeout=timedelta(seconds=120), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format(fqdn, *keys) + + # Drop the index + await ixm.drop_index(bucket_name, + ixname, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises((QueryIndexNotFoundException, ParsingFailedException)): + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_index_no_fields(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # raises a TypeError b/c not providing fields means + # create_index() is missing a required positional param + with pytest.raises(TypeError): + await ixm.create_index(bucket_name, + 'noFields', + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_ignore_if_exists(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + ixname = 'ix2' + await ixm.create_index(bucket_name, + ixname, + ['hello'], + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + await ixm.create_index(bucket_name, + ixname, ['hello'], + ignore_if_exists=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + await ixm.create_index(bucket_name, + ixname, + ['hello'], + CreateQueryIndexOptions(ignore_if_exists=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + with pytest.raises(QueryIndexAlreadyExistsException): + await ixm.create_index(bucket_name, + ixname, + ['hello'], + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_secondary_indexes_ignore_if_not_exists(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # Create it + ixname = 'ix2' + await ixm.create_index(bucket_name, + ixname, + ['hello'], + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + # Drop it + await ixm.drop_index(bucket_name, + ixname, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + await ixm.drop_index(bucket_name, + ixname, + ignore_if_not_exists=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + await ixm.drop_index(bucket_name, + ixname, + DropQueryIndexOptions(ignore_if_not_exists=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + with pytest.raises(QueryIndexNotFoundException): + await ixm.drop_index(bucket_name, + ixname, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_list_indexes(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # start with no indexes + ixs = await ixm.get_all_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + assert len(ixs) == 0 + + # Create the primary index + await ixm.create_primary_index(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + ixs = await ixm.get_all_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + assert len(ixs) == 1 + assert ixs[0].is_primary is True + assert ixs[0].name == '#primary' + assert ixs[0].bucket_name == bucket_name + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_index_partition_info(self, cb_env, fqdn): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # use query to create index w/ partition, cannot do that via manager + # ATM + n1ql = f'CREATE INDEX idx_fld1 ON {fqdn}(fld1) PARTITION BY HASH(fld1)' + await cb_env.cluster.query(n1ql).execute() + ixs = await ixm.get_all_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + idx = next((ix for ix in ixs if ix.name == "idx_fld1"), None) + assert idx is not None + assert idx.partition is not None + assert idx.partition == 'HASH(`fld1`)' + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_watch(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # Create primary index + await ixm.create_primary_index(bucket_name, + deferred=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + ixs = await ixm.get_all_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + defer = False + if n % 2 == 0: + defer = True + await ixm.create_index(bucket_name, + 'ix{0}'.format(n), + ['fld{0}'.format(n)], + deferred=defer, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + ixs = await ixm.get_all_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + assert len(ixs) == 6 + # by not building deffered indexes, should timeout + with pytest.raises(WatchQueryIndexTimeoutException): + await ixm.watch_indexes(bucket_name, + [i.name for i in ixs], + WatchQueryIndexOptions(timeout=timedelta(seconds=5), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_deferred(self, cb_env): + bucket_name = cb_env.bucket.name + ixm = cb_env.ixm + # Create primary index + await ixm.create_primary_index(bucket_name, + deferred=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + ixs = await ixm.get_all_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + await ixm.create_index(bucket_name, + 'ix{0}'.format(n), + ['fld{0}'.format(n)], + CreateQueryIndexOptions(deferred=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + + ixs = await ixm.get_all_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + assert len(ixs) == 6 + + ix_names = list(map(lambda i: i.name, ixs)) + + await ixm.build_deferred_indexes(bucket_name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + await ixm.watch_indexes(bucket_name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) # Should be OK + await ixm.watch_indexes(bucket_name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30), + watch_primary=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) # Should be OK again + with pytest.raises(QueryIndexNotFoundException): + await ixm.watch_indexes(bucket_name, + ['idontexist'], + WatchQueryIndexOptions(timeout=timedelta(seconds=10), + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_get_all_correct_collection(self, cb_env): + # create some indexes in the _default scope & collection + for i in range(2): + await cb_env.ixm.create_index(cb_env.bucket.name, + f'ix{i}', + [f'fld{i}'], + CreateQueryIndexOptions(deferred=True)) + + # create some indexes in the test scope & collection + for i in range(2): + await cb_env.ixm.create_index(cb_env.bucket.name, + f'ix{i}', + [f'fld{i}'], + CreateQueryIndexOptions(deferred=True, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION)) + + # all indexes in bucket (i.e. in test and _default scopes/collections) + all_ixs = await cb_env.ixm.get_all_indexes(cb_env.bucket.name) + + # _should_ be only indexes in test scope & collection + collection_ixs = await cb_env.ixm.get_all_indexes(cb_env.bucket.name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + + assert len(all_ixs) != len(collection_ixs) + assert len(all_ixs) == 4 + assert len(collection_ixs) == 2 + + +class CollectionQueryIndexManagerTests: + + TEST_SCOPE = "test-scope" + TEST_COLLECTION = "test-collection" + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True, + manage_collections=True, + manage_query_indexes=True) + + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + cb_env.cixm = cb_env.collection.query_indexes() + + yield cb_env + await cb_env.try_n_times(5, 3, self._clear_all_indexes, cb_env, ignore_fail=True) + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + del cb_env.cixm + + async def _drop_all_indexes(self, cb_env, indexes, scope_name='_default', collection_name='_default'): + for index in indexes: + # @TODO: will need to update once named primary allowed + if index.is_primary: + await cb_env.cixm.drop_primary_index() + else: + await cb_env.cixm.drop_index(index.name) + for _ in range(10): + indexes = await cb_env.cixm.get_all_indexes() + if 0 == len(indexes): + return True + await asyncio.sleep(2) + + return False + + async def _clear_all_indexes(self, cb_env, ignore_fail=False): + # Drop all indexes! + for scope_col in [(self.TEST_SCOPE, self.TEST_COLLECTION), ('_default', '_default')]: + indexes = await cb_env.cixm.get_all_indexes() + + success = await self._drop_all_indexes(cb_env, indexes) + if success: + continue + elif ignore_fail is True: + continue + else: + pytest.xfail( + "Indexes were not dropped after {} waits of {} seconds each".format(10, 3)) + + @pytest.fixture(scope="class") + def check_query_index_mgmt_supported(self, cb_env): + cb_env.check_if_feature_supported('query_index_mgmt') + + @pytest.fixture(scope="class", name="fqdn") + def get_fqdn(self, cb_env): + return f'`{cb_env.bucket.name}`.`{self.TEST_SCOPE}`.`{self.TEST_COLLECTION}`' + + @pytest_asyncio.fixture() + async def clear_all_indexes(self, cb_env): + await self._clear_all_indexes(cb_env) + + @staticmethod + def supports_query_without_index(cb_env): + return cb_env.is_feature_supported('query_without_index') + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary(self, cb_env, fqdn): + await cb_env.cixm.create_primary_index(CreatePrimaryQueryIndexOptions(timeout=timedelta(seconds=60))) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM {fqdn} LIMIT 1' + + await cb_env.cluster.query(n1ql).execute() + # Drop the primary index + await cb_env.cixm.drop_primary_index() + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary_ignore_if_exists(self, cb_env): + await cb_env.cixm.create_primary_index() + await cb_env.cixm.create_primary_index(CreatePrimaryQueryIndexOptions(ignore_if_exists=True)) + + with pytest.raises(QueryIndexAlreadyExistsException): + await cb_env.cixm.create_primary_index() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_primary_ignore_if_exists_kwargs(self, cb_env): + await cb_env.cixm.create_primary_index() + await cb_env.cixm.create_primary_index(ignore_if_exists=True) + + with pytest.raises(QueryIndexAlreadyExistsException): + await cb_env.cixm.create_primary_index() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_primary(self, cb_env): + # create an index so we can drop + await cb_env.cixm.create_primary_index(timeout=timedelta(seconds=60)) + + await cb_env.cixm.drop_primary_index(timeout=timedelta(seconds=60)) + # this should fail now + with pytest.raises(QueryIndexNotFoundException): + await cb_env.cixm.drop_primary_index() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_primary_ignore_if_not_exists(self, cb_env): + await cb_env.cixm.drop_primary_index(ignore_if_not_exists=True) + await cb_env.cixm.drop_primary_index(DropPrimaryQueryIndexOptions(ignore_if_not_exists=True)) + with pytest.raises(QueryIndexNotFoundException): + await cb_env.cixm.drop_primary_index() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes(self, cb_env, fqdn): + ixname = 'ix2' + keys = ('fld1', 'fld2') + await cb_env.cixm.create_index(ixname, keys=keys, timeout=timedelta(seconds=120)) + n1ql = "SELECT {1}, {2} FROM {0} WHERE {1}=1 AND {2}=2 LIMIT 1".format(fqdn, *keys) + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_condition(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + + await cb_env.try_n_times_till_exception(10, + 5, + cb_env.cixm.drop_index, + ixname, + expected_exceptions=(QueryIndexNotFoundException,),) + condition = '((`fld1` = 1) and (`fld2` = 2))' + await cb_env.cixm.create_index(ixname, + keys, + CreateQueryIndexOptions(timeout=timedelta(days=1), condition=condition)) + + async def check_index(): + indexes = await cb_env.cixm.get_all_indexes() + result = next((idx for idx in indexes if idx.name == ixname), None) + assert result is not None + return result + result = await cb_env.try_n_times(10, 5, check_index) + assert result.condition == condition + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_fields_kwarg(self, cb_env, fqdn): + ixname = 'ix2' + keys = ('fld1', 'fld2') + await cb_env.cixm.create_index(ixname, fields=keys, timeout=timedelta(seconds=120)) + n1ql = "SELECT {1}, {2} FROM {0} WHERE {1}=1 AND {2}=2 LIMIT 1".format(fqdn, *keys) + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_secondary_indexes(self, cb_env, fqdn): + ixname = 'ix2' + keys = ('fld1', 'fld2') + await cb_env.cixm.create_index(ixname, keys, timeout=timedelta(seconds=120)) + + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format(fqdn, *keys) + + # Drop the index + await cb_env.cixm.drop_index(ixname) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises((QueryIndexNotFoundException, ParsingFailedException)): + await cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_index_no_fields(self, cb_env): + # raises a TypeError b/c not providing fields means + # create_index() is missing a required positional param + with pytest.raises(TypeError): + await cb_env.cixm.create_index('noFields') + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_create_secondary_indexes_ignore_if_exists(self, cb_env): + ixname = 'ix2' + await cb_env.cixm.create_index(ixname, ['hello']) + await cb_env.cixm.create_index(ixname, ['hello'], ignore_if_exists=True) + await cb_env.cixm.create_index(ixname, ['hello'], CreateQueryIndexOptions(ignore_if_exists=True)) + with pytest.raises(QueryIndexAlreadyExistsException): + await cb_env.cixm.create_index(ixname, ['hello']) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_drop_secondary_indexes_ignore_if_not_exists(self, cb_env): + # Create it + ixname = 'ix2' + await cb_env.cixm.create_index(ixname, ['hello']) + # Drop it + await cb_env.cixm.drop_index(ixname) + await cb_env.cixm.drop_index(ixname, ignore_if_not_exists=True) + await cb_env.cixm.drop_index(ixname, DropQueryIndexOptions(ignore_if_not_exists=True)) + with pytest.raises(QueryIndexNotFoundException): + await cb_env.cixm.drop_index(ixname) + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_list_indexes(self, cb_env): + # start with no indexes + ixs = await cb_env.cixm.get_all_indexes() + assert len(ixs) == 0 + + # Create the primary index + await cb_env.cixm.create_primary_index() + ixs = await cb_env.cixm.get_all_indexes() + assert len(ixs) == 1 + assert ixs[0].is_primary is True + assert ixs[0].name == '#primary' + assert ixs[0].bucket_name == cb_env.bucket.name + + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_index_partition_info(self, cb_env, fqdn): + # use query to create index w/ partition, cannot do that via manager ATM + n1ql = f'CREATE INDEX idx_fld1 ON {fqdn}(fld1) PARTITION BY HASH(fld1)' + await cb_env.cluster.query(n1ql).execute() + ixs = await cb_env.cixm.get_all_indexes() + idx = next((ix for ix in ixs if ix.name == "idx_fld1"), None) + assert idx is not None + assert idx.partition is not None + assert idx.partition == 'HASH(`fld1`)' + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_watch(self, cb_env): + # Create primary index + await cb_env.cixm.create_primary_index(deferred=True) + ixs = await cb_env.cixm.get_all_indexes() + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + defer = False + if n % 2 == 0: + defer = True + await cb_env.cixm.create_index(f'ix{n}', [f'fld{n}'], deferred=defer) + + ixs = await cb_env.cixm.get_all_indexes() + assert len(ixs) == 6 + # by not building deffered indexes, should timeout + with pytest.raises(WatchQueryIndexTimeoutException): + await cb_env.cixm.watch_indexes([i.name for i in ixs], + WatchQueryIndexOptions(timeout=timedelta(seconds=5))) + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures("check_query_index_mgmt_supported") + @pytest.mark.usefixtures("clear_all_indexes") + @pytest.mark.asyncio + async def test_deferred(self, cb_env): + # Create primary index + await cb_env.cixm.create_primary_index(deferred=True) + ixs = await cb_env.cixm.get_all_indexes() + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + await cb_env.cixm.create_index(f'ix{n}', [f'fld{n}'], CreateQueryIndexOptions(deferred=True)) + + ixs = await cb_env.cixm.get_all_indexes() + assert len(ixs) == 6 + + ix_names = list(map(lambda i: i.name, ixs)) + + await cb_env.cixm.build_deferred_indexes() + await cb_env.cixm.watch_indexes(ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30))) # Should be OK + await cb_env.cixm.watch_indexes(ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30), + watch_primary=True)) # Should be OK again + with pytest.raises(QueryIndexNotFoundException): + await cb_env.cixm.watch_indexes(['idontexist'], WatchQueryIndexOptions(timeout=timedelta(seconds=10))) diff --git a/acouchbase/tests/rate_limit_t.py b/acouchbase/tests/rate_limit_t.py new file mode 100644 index 000000000..7c2d09696 --- /dev/null +++ b/acouchbase/tests/rate_limit_t.py @@ -0,0 +1,542 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import json +import random +import time +from datetime import timedelta + +import pytest +import pytest_asyncio +import requests + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import (CouchbaseException, + DocumentNotFoundException, + QuotaLimitedException, + RateLimitedException, + ScopeNotFoundException) +from couchbase.management.buckets import CreateBucketSettings +from couchbase.management.collections import CollectionSpec +from couchbase.management.options import GetUserOptions +from couchbase.management.users import Role, User +from couchbase.options import ClusterOptions, GetOptions +from tests.helpers import CouchbaseTestEnvironmentException + +from ._test_utils import TestEnvironment + + +@pytest.mark.flaky(reruns=5, reruns_delay=1) +class RateLimitTests: + """ + These tests should just test the collection interface, as simply + as possible. We have the Scenario tests for more complicated + stuff. + """ + CONTENT = {"some": "content"} + KEY = "imakey" + NOKEY = "somerandomkey" + USERNAME = "rate-limit-user" + RATE_LIMIT_SCOPE_NAME = 'rate-limit-scope' + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True, + manage_collections=True, + manage_users=True, + manage_rate_limit=True) + # if couchbase_config.is_mock_server: + # pytest.skip('Mock server does not support feature: rate limit testing.') + # conn_string = couchbase_config.get_connection_string() + # username, pw = couchbase_config.get_username_and_pw() + # opts = ClusterOptions(PasswordAuthenticator(username, pw)) + # cluster = await Cluster.connect(conn_string, opts) + # bucket = cluster.bucket(f"{couchbase_config.bucket_name}") + # await bucket.on_connect() + # await cluster.cluster_info() + + # coll = bucket.default_collection() + + # self._conn_str = conn_string + # parsed_conn = urlparse(conn_string) + # url = f'http://{parsed_conn.netloc}:8091' + + # cb_env = TestEnvironment(cluster, + # bucket, + # coll, + # couchbase_config, + # manage_buckets=True, + # manage_collections=True, + # manage_users=True, + # rate_limit_params=RateLimitData(url, username, pw)) + + self._fts_indexes = [] + self._enforce_rate_limits(cb_env, True) + + yield cb_env + await self.tear_down(cb_env) + + def _enforce_rate_limits(self, cb_env, enforce=True): + url = f'{cb_env.rate_limit_params.url}/internalSettings' + payload = {'enforceLimits': f'{"true" if enforce is True else "false"}'} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + r = requests.post(url, + headers=headers, + data=payload, + auth=(cb_env.rate_limit_params.username, + cb_env.rate_limit_params.pw)) + + if r.status_code != 200: + raise CouchbaseTestEnvironmentException('Unable to enforce rate limits.') + + async def tear_down(self, cb_env): + await self._drop_rate_limit_user(cb_env) + scopes = await cb_env.cm.get_all_scopes() + scope_names = ["concurrent-scope-{}".format(i) for i in range(10)] + scope_names.append(self.RATE_LIMIT_SCOPE_NAME) + for scope in scopes: + if scope.name in scope_names: + await cb_env.cm.drop_scope(scope.name) + + num_indexes = 0 + if cb_env.rate_limit_params.fts_indexes: + num_indexes = len(cb_env.rate_limit_params.fts_indexes) + if num_indexes > 0: + ixm = cb_env.cluster.search_indexes() + for idx in cb_env.rate_limit_params.fts_indexes: + await cb_env.try_n_times_till_exception(10, 3, ixm.drop_index, idx) + + qm = cb_env.cluster.query_indexes() + await qm.drop_primary_index(cb_env.bucket.name, ignore_if_not_exists=True) + + self._enforce_rate_limits(cb_env, False) + + @pytest_asyncio.fixture() + async def remove_docs(self, cb_env): + try: + await cb_env.collection.remove('ratelimit-ingress') + await cb_env.try_n_times_till_exception(10, 3, cb_env.collection.get, + 'ratelimit-ingress', (DocumentNotFoundException,)) + except CouchbaseException: + pass + + try: + await cb_env.collection.remove('ratelimit-egress') + await cb_env.try_n_times_till_exception(10, 3, cb_env.collection.get, + 'ratelimit-egress', (DocumentNotFoundException,)) + except CouchbaseException: + pass + + @pytest_asyncio.fixture() + async def cleanup_scope_and_collection(self, cb_env): + await cb_env.try_n_times_till_exception(5, 1, + cb_env.cm.drop_scope, + self.RATE_LIMIT_SCOPE_NAME, + expected_exceptions=(ScopeNotFoundException,)) + yield + await cb_env.try_n_times_till_exception(5, 1, + cb_env.cm.drop_scope, + self.RATE_LIMIT_SCOPE_NAME, + expected_exceptions=(ScopeNotFoundException,)) + + async def _create_rate_limit_user(self, cb_env, username, limits): + params = { + "password": "password", + "roles": "admin" + } + + user_limits = {} + kv_limits = limits.get("kv_limits", None) + if kv_limits: + user_limits["kv"] = { + "num_connections": kv_limits["num_connections"], + "num_ops_per_min": kv_limits["num_ops_per_min"], + "ingress_mib_per_min": kv_limits["ingress_mib_per_min"], + "egress_mib_per_min": kv_limits["egress_mib_per_min"] + } + + query_limits = limits.get("query_limits", None) + if query_limits: + user_limits["query"] = { + "num_queries_per_min": query_limits["num_queries_per_min"], + "num_concurrent_requests": query_limits["num_concurrent_requests"], + "ingress_mib_per_min": query_limits["ingress_mib_per_min"], + "egress_mib_per_min": query_limits["egress_mib_per_min"] + } + + fts_limits = limits.get("fts_limits", None) + if fts_limits: + user_limits["fts"] = { + "num_queries_per_min": fts_limits["num_queries_per_min"], + "num_concurrent_requests": fts_limits["num_concurrent_requests"], + "ingress_mib_per_min": fts_limits["ingress_mib_per_min"], + "egress_mib_per_min": fts_limits["egress_mib_per_min"] + } + + if user_limits: + params["limits"] = json.dumps(user_limits) + + path = f'/settings/rbac/users/local/{username}' + + url = f'{cb_env.rate_limit_params.url}/{path}' + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + r = requests.put(url, + headers=headers, + data=params, + auth=(cb_env.rate_limit_params.username, + cb_env.rate_limit_params.pw)) + + if r.status_code != 200: + raise CouchbaseTestEnvironmentException('Unable to create rate-limit-user.') + + # lets verify user exists + user_metadata = await cb_env.try_n_times(10, 1, cb_env.um.get_user, username, + GetUserOptions(domain_name="local")) + + assert user_metadata is not None + assert username == user_metadata.user.username + + async def _drop_rate_limit_user(self, cb_env): + await cb_env.try_n_times_till_exception(10, 3, cb_env.um.drop_user, self.USERNAME) + + async def _create_rate_limit_scope(self, cb_env, scope_name, limits): + params = { + "name": scope_name + } + + scope_limits = {} + kv_limits = limits.get("kv_limits", None) + if kv_limits: + scope_limits["kv"] = { + "data_size": kv_limits["data_size"] + } + + index_limits = limits.get("index_limits", None) + if index_limits: + scope_limits["index"] = { + "num_indexes": index_limits["num_indexes"] + } + + fts_limits = limits.get("fts_limits", None) + if fts_limits: + scope_limits["fts"] = { + "num_fts_indexes": fts_limits["num_fts_indexes"] + } + + cluster_mgr_limits = limits.get("cluster_mgr_limits", None) + if cluster_mgr_limits: + scope_limits["clusterManager"] = { + "num_collections": cluster_mgr_limits["num_collections"] + } + + if scope_limits: + params["limits"] = json.dumps(scope_limits) + + path = f'/pools/default/buckets/{cb_env.bucket.name}/scopes' + url = f'{cb_env.rate_limit_params.url}/{path}' + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + requests.post(url, + headers=headers, + data=params, + auth=(cb_env.rate_limit_params.username, + cb_env.rate_limit_params.pw)) + # verify the scope exists + all_scopes = await cb_env.cm.get_all_scopes() + assert scope_name in map(lambda s: s.name, all_scopes) + + def _random_doc_by_size(self, size): + doc = bytearray((random.getrandbits(8) for i in range(size))) + return doc.hex() + + async def _try_until_timeout(self, timeout, interval, func, *args, **kwargs): + """Execute provided func until specified timeout has been reached. + + :param timeout: timeout in seconds + :type timeout: int + :param interval: sleep interval in milliseconds + :type interval: int + :param func: function to execute periodically, sleeping interval milliseconds between each execution + :type func: function + """ + timeout_ms = timeout * 1000 + time_left = timeout_ms + interval_ms = float(interval / 1000) + start = time.perf_counter() + is_query = kwargs.pop("query", False) + is_fts = kwargs.pop("fts", False) + + while True: + if is_query is True: + await func(*args, **kwargs).execute() + elif is_fts is True: + res = func(*args, **kwargs) + [r async for r in res.rows()] + else: + await func(*args, **kwargs) + time_left = timeout_ms - ((time.perf_counter() - start) * 1000) + if time_left <= 0: + break + + await asyncio.sleep(interval_ms) + + @pytest.mark.asyncio + async def test_rate_limits(self, couchbase_config, cb_env): + await self._create_rate_limit_user(cb_env, + self.USERNAME, { + "kv_limits": { + "num_connections": 10, + "num_ops_per_min": 10, + "ingress_mib_per_min": 1, + "egress_mib_per_min": 10 + } + }) + conn_string = couchbase_config.get_connection_string() + cluster = None + try: + cluster = await Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(self.USERNAME, "password"))) + bucket = cluster.bucket('default') + await bucket.on_connect() + collection = bucket.default_collection() + + await self._try_until_timeout( + 5, 10, collection.upsert, "ratelimit", "test") + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + @pytest.mark.usefixtures('remove_docs') + @pytest.mark.asyncio + async def test_rate_limits_ingress(self, couchbase_config, cb_env): + await self._create_rate_limit_user(cb_env, + self.USERNAME, { + "kv_limits": { + "num_connections": 10, + "num_ops_per_min": 100, + "ingress_mib_per_min": 1, + "egress_mib_per_min": 10 + } + }) + conn_string = couchbase_config.get_connection_string() + cluster = None + try: + cluster = await Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(self.USERNAME, "password"))) + bucket = cluster.bucket("default") + await bucket.on_connect() + collection = bucket.default_collection() + + doc = self._random_doc_by_size(1024*512) + for _ in range(3): + await collection.upsert("ratelimit-ingress", doc) + + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + @pytest.mark.usefixtures('remove_docs') + @pytest.mark.asyncio + async def test_rate_limits_egress(self, couchbase_config, cb_env): + await self._create_rate_limit_user(cb_env, + self.USERNAME, {"kv_limits": { + "num_connections": 10, + "num_ops_per_min": 100, + "ingress_mib_per_min": 10, + "egress_mib_per_min": 2 + } + }) + conn_string = couchbase_config.get_connection_string() + cluster = None + try: + cluster = await Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(self.USERNAME, "password"))) + bucket = cluster.bucket("default") + await bucket.on_connect() + collection = bucket.default_collection() + + doc = self._random_doc_by_size(1024*512) + key = "ratelimit-egress" + await collection.upsert(key, doc) + for _ in range(3): + await collection.get(key, GetOptions(timeout=timedelta(seconds=10))) + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + @pytest.mark.asyncio + async def test_rate_limits_max_conns(self, couchbase_config, cb_env): + await self._create_rate_limit_user(cb_env, + self.USERNAME, { + "kv_limits": { + "num_connections": 1, + "num_ops_per_min": 100, + "ingress_mib_per_min": 10, + "egress_mib_per_min": 10 + } + }) + cluster = None + cluster1 = None + conn_string = couchbase_config.get_connection_string() + try: + cluster = await Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(self.USERNAME, "password"))) + bucket = cluster.bucket("default") + await bucket.on_connect() + collection = bucket.default_collection() + collection.exists("some-key") + + cluster1 = await Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(self.USERNAME, "password"))) + bucket1 = cluster1.bucket("default") + await bucket.on_connect() + collection1 = bucket1.default_collection() + await collection1.exists("some-key") + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + @pytest.mark.usefixtures('cleanup_scope_and_collection') + @pytest.mark.asyncio + async def test_rate_limits_kv_scopes_data_size(self, cb_env): + scope_name = self.RATE_LIMIT_SCOPE_NAME + await self._create_rate_limit_scope(cb_env, scope_name, { + "kv_limits": {"data_size": 1024*1024} + }) + + collection_spec = CollectionSpec( + 'rate-limit-collection', scope_name=scope_name) + await cb_env.cm.create_collection(collection_spec) + + # verify collection exists + created = await cb_env.try_n_times( + 10, 3, cb_env.get_collection, scope_name, collection_spec.name, cb_env.bucket.name) + assert created is not None + + scope = cb_env.bucket.scope(scope_name) + collection = scope.collection(collection_spec.name) + + doc = self._random_doc_by_size(1024*512) + with pytest.raises(QuotaLimitedException): + for _ in range(5): + await collection.upsert("ratelimit-datasize", doc) + + # await cb_env.cm.drop_collection(collection_spec) + # for _ in range(5): + # res = await cb_env.get_collection(scope_name, + # collection_spec.name, + # cb_env.bucket.name) + # if not res: + # break + # await cb_env.cm.drop_scope(scope_name) + # for _ in range(5): + # res = await cb_env.get_scope(scope_name, cb_env.bucket.name) + # if not res: + # break + + @pytest.mark.usefixtures('cleanup_scope_and_collection') + @pytest.mark.asyncio + async def test_rate_limits_collections_scopes_limits(self, cb_env): + scope_name = self.RATE_LIMIT_SCOPE_NAME + await self._create_rate_limit_scope(cb_env, scope_name, { + "cluster_mgr_limits": {"num_collections": 1} + }) + + collection_spec = CollectionSpec( + 'rate-limit-collection', scope_name=scope_name) + await cb_env.cm.create_collection(collection_spec) + + # verify collection exists + created = await cb_env.try_n_times(10, + 3, + cb_env.get_collection, + scope_name, + collection_spec.name, + cb_env.bucket.name) + assert created is not None + + with pytest.raises(QuotaLimitedException): + collection_spec = CollectionSpec( + 'rate-limit-collection-1', scope_name=scope_name) + await cb_env.cm.create_collection(collection_spec) + + @pytest.mark.asyncio + async def test_rate_limits_cluster_mgr_concurrency(self, couchbase_config, cb_env): + pytest.skip('Needs some work, not raising RateLimitedExceptions for some reason...') + await self._create_rate_limit_user(cb_env, + self.USERNAME, { + "cluster_mgr_limits": { + "num_concurrent_requests": 3, + "ingress_mib_per_min": 10, + "egress_mib_per_min": 10 + } + }) + + conn_string = couchbase_config.get_connection_string() + cluster = await Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(self.USERNAME, "password"))) + bucket = cluster.bucket("default") + await bucket.on_connect() + + cm = bucket.collections() + + async def create_collection(scope_name): + await cm.create_scope(scope_name) + + scope_names = ["concurrent-scope-{}".format(i) for i in range(10)] + + with pytest.raises(RateLimitedException): + await asyncio.gather(*[create_collection(s) for s in scope_names]) + + bm = cluster.buckets() + + async def create_bucket(bucket_settings): + await bm.create_bucket(bucket_settings) + + bucket_list = [CreateBucketSettings( + name="testBucket{}".format(i), + bucket_type="couchbase", + ram_quota_mb=100) for i in range(10)] + + with pytest.raises(RateLimitedException): + await asyncio.gather(*[create_bucket(b) for b in bucket_list]) + + um = cluster.users() + + async def create_user(user): + await um.upsert_user(user, domain_name="local") + + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + user_list = [User(username="user{}".format( + i), roles=roles, password="password{}".format(i)) for i in range(10)] + + with pytest.raises(RateLimitedException): + await asyncio.gather(*[create_user(u) for u in user_list]) diff --git a/acouchbase/tests/search_t.py b/acouchbase/tests/search_t.py new file mode 100644 index 000000000..3d3cfbafd --- /dev/null +++ b/acouchbase/tests/search_t.py @@ -0,0 +1,811 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import uuid +from copy import copy +from datetime import datetime, timedelta + +import pytest +import pytest_asyncio + +import couchbase.search as search +from couchbase.exceptions import (AmbiguousTimeoutException, + InvalidArgumentException, + QueryIndexNotFoundException) +from couchbase.mutation_state import MutationState +from couchbase.options import SearchOptions +from couchbase.search import (HighlightStyle, + SearchDateRangeFacet, + SearchFacetResult, + SearchNumericRangeFacet, + SearchRow, + SearchTermFacet) +from tests.environments import CollectionType +from tests.environments.search_environment import AsyncSearchTestEnvironment +from tests.test_features import EnvironmentFeatures + + +class SearchCollectionTestSuite: + TEST_MANIFEST = [ + 'test_cluster_query_collections', + 'test_scope_query_collections', + 'test_scope_search_fields', + 'test_scope_search_highlight', + ] + + @pytest.mark.asyncio + async def test_cluster_query_collections(self, cb_env): + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10, + scope_name=cb_env.scope.name, + collections=[cb_env.collection.name])) + rows = await cb_env.assert_rows(res, 2, return_rows=True) + + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + @pytest.mark.asyncio + async def test_scope_query_collections(self, cb_env): + q = search.TermQuery('auto') + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10, collections=[cb_env.collection.name])) + rows = await cb_env.assert_rows(res, 2, return_rows=True) + + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, q, SearchOptions(limit=10)) + rows = await cb_env.assert_rows(res, 2, return_rows=True) + + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c in [cb_env.collection.name, cb_env.OTHER_COLLECTION]]) is True + + @pytest.mark.asyncio + async def test_scope_search_fields(self, cb_env): + test_fields = ['make', 'model'] + q = search.TermQuery('auto') + # verify fields works w/in kwargs + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10), + fields=test_fields, + collections=[cb_env.collection.name]) + + fields_with_col = copy(test_fields) + fields_with_col.append('_$c') + rows = await cb_env.assert_rows(res, 1, return_rows=True) + first_entry = rows[0] + assert isinstance(first_entry, SearchRow) + assert isinstance(first_entry.fields, dict) + assert first_entry.fields != {} + assert all(map(lambda f: f in fields_with_col, first_entry.fields.keys())) is True + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + # verify fields works w/in options + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10, + fields=test_fields, + collections=[cb_env.collection.name])) + + rows = await cb_env.assert_rows(res, 1, return_rows=True) + first_entry = rows[0] + assert isinstance(first_entry, SearchRow) + assert isinstance(first_entry.fields, dict) + assert first_entry.fields != {} + assert all(map(lambda f: f in fields_with_col, first_entry.fields.keys())) is True + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + @pytest.mark.asyncio + async def test_scope_search_highlight(self, cb_env): + + q = search.TermQuery('auto') + # check w/in options + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10, highlight_style=HighlightStyle.Html)) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + fragments = rows[0].fragments + assert isinstance(locations, search.SearchRowLocations) + assert isinstance(fragments, dict) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + # check w/in kwargs + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, q, SearchOptions( + limit=10), highlight_style=HighlightStyle.Html, collections=[cb_env.collection.name]) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + fragments = rows[0].fragments + assert isinstance(locations, search.SearchRowLocations) + assert isinstance(fragments, dict) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + +class SearchTestSuite: + TEST_MANIFEST = [ + 'test_bad_search_query', + 'test_cluster_search', + 'test_cluster_search_date_facets', + 'test_cluster_search_disable_scoring', + 'test_cluster_search_facets_fail', + 'test_cluster_search_fields', + 'test_cluster_search_highlight', + 'test_cluster_search_numeric_facets', + 'test_cluster_search_ryow', + 'test_cluster_search_scan_consistency', + 'test_cluster_search_term_facets', + 'test_cluster_search_top_level_facets', + 'test_cluster_search_top_level_facets_kwargs', + 'test_cluster_sort_field', + 'test_cluster_sort_field_multi', + 'test_cluster_sort_geo', + 'test_cluster_sort_id', + 'test_cluster_sort_score', + 'test_cluster_sort_str', + 'test_search_include_locations', + 'test_search_match_operator', + 'test_search_match_operator_fail', + 'test_search_no_include_locations', + 'test_search_raw_query', + 'test_search_timeout', + ] + + @pytest.fixture(scope="class") + def check_disable_scoring_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('search_disable_scoring', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.mark.asyncio + async def test_bad_search_query(self, cb_env): + res = cb_env.cluster.search_query('not-an-index', search.TermQuery('auto')) + with pytest.raises(QueryIndexNotFoundException): + [r async for r in res] + + @pytest.mark.asyncio + async def test_cluster_search(self, cb_env): + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, SearchOptions(limit=10)) + await cb_env.assert_rows(res, 2) + + @pytest.mark.asyncio + async def test_cluster_search_date_facets(self, cb_env): + facet_name = 'last_updated' + facet = search.DateFacet('last_updated', limit=3) + now = datetime.now() + today = datetime(year=now.year, month=now.month, day=now.day) + early_end = (today - timedelta(days=241)) + mid_start = (today - timedelta(days=240)) + mid_end = (today - timedelta(days=121)) + late_start = (today - timedelta(days=120)) + facet.add_range('early', end=f"{early_end.strftime('%Y-%m-%dT%H:%M:%SZ')}") + facet.add_range('mid', start=f"{mid_start.strftime('%Y-%m-%dT%H:%M:%SZ')}", + end=f"{mid_end.strftime('%Y-%m-%dT%H:%M:%SZ')}") + facet.add_range('late', start=f"{late_start.strftime('%Y-%m-%dT%H:%M:%SZ')}") + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + await cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert all(map(lambda ft: isinstance(ft, SearchDateRangeFacet), result_facet.date_ranges)) is True + assert len(result_facet.date_ranges) <= facet.limit + + @pytest.mark.usefixtures('check_disable_scoring_supported') + @pytest.mark.asyncio + async def test_cluster_search_disable_scoring(self, cb_env): + + # verify disable scoring works w/in SearchOptions + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, disable_scoring=True)) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + assert all(map(lambda r: r.score == 0, rows)) is True + + # verify disable scoring works w/in kwargs + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), disable_scoring=True) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + assert all(map(lambda r: r.score == 0, rows)) is True + + # verify setting disable_scoring to False works + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, disable_scoring=False)) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + assert all(map(lambda r: r.score != 0, rows)) is True + + # verify default disable_scoring is False + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10)) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + assert all(map(lambda r: r.score != 0, rows)) is True + + # @TODO: 3.x raises a SearchException... + + def test_cluster_search_facets_fail(self, cb_env): + q = search.TermQuery('auto') + with pytest.raises(ValueError): + cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={'test-facet': None})) + + @pytest.mark.asyncio + async def test_cluster_search_fields(self, cb_env): + test_fields = ['make', 'mode'] + q = search.TermQuery('auto') + # verify fields works w/in kwargs + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), fields=test_fields) + + rows = await cb_env.assert_rows(res, 1, return_rows=True) + first_entry = rows[0] + assert isinstance(first_entry, SearchRow) + assert isinstance(first_entry.fields, dict) + assert first_entry.fields != {} + assert all(map(lambda f: f in test_fields, first_entry.fields.keys())) is True + + # verify fields works w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, fields=test_fields)) + + rows = await cb_env.assert_rows(res, 1, return_rows=True) + first_entry = rows[0] + assert isinstance(first_entry, SearchRow) + assert isinstance(first_entry.fields, dict) + assert first_entry.fields != {} + assert all(map(lambda f: f in test_fields, first_entry.fields.keys())) is True + + @pytest.mark.asyncio + async def test_cluster_search_highlight(self, cb_env): + + q = search.TermQuery('auto') + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, highlight_style=HighlightStyle.Html)) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + fragments = rows[0].fragments + assert isinstance(locations, search.SearchRowLocations) + assert isinstance(fragments, dict) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, SearchOptions( + limit=10), highlight_style=HighlightStyle.Html) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + fragments = rows[0].fragments + assert isinstance(locations, search.SearchRowLocations) + assert isinstance(fragments, dict) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + + @pytest.mark.asyncio + async def test_cluster_search_numeric_facets(self, cb_env): + + facet_name = 'rating' + facet = search.NumericFacet('rating', limit=3) + facet.add_range('low', max=4) + facet.add_range('med', min=4, max=7) + facet.add_range('high', min=7) + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + await cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert all(map(lambda ft: isinstance(ft, SearchNumericRangeFacet), result_facet.numeric_ranges)) is True + assert len(result_facet.numeric_ranges) <= facet.limit + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_cluster_search_ryow(self, cb_env): + key, value = cb_env.get_new_doc() + # need to make sure content is unique + content = str(uuid.uuid4())[:8] + value['description'] = content + q = search.TermQuery(content) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10)) + await cb_env.assert_rows(res, 0) + res = await cb_env.collection.insert(key, value) + ms = MutationState() + ms.add_mutation_token(res.mutation_token()) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, consistent_with=ms)) + await cb_env.assert_rows(res, 1) + + # prior to PYCBC-1477 the SDK _could_ crash w/ this this sort of MS creation + key, value = cb_env.get_new_doc() + # need to make sure content is unique + content = str(uuid.uuid4())[:8] + value['description'] = content + q = search.TermQuery(content) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10)) + await cb_env.assert_rows(res, 0) + res = await cb_env.collection.insert(key, value) + ms = MutationState(res) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, consistent_with=ms)) + await cb_env.assert_rows(res, 1) + + @pytest.mark.asyncio + async def test_cluster_search_term_facets(self, cb_env): + + facet_name = 'model' + facet = search.TermFacet('model', 5) + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + await cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert all(map(lambda ft: isinstance(ft, SearchTermFacet), result_facet.terms)) is True + assert len(result_facet.terms) <= facet.limit + + @pytest.mark.asyncio + async def test_cluster_search_scan_consistency(self, cb_env): + q = search.TermQuery('auto') + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, + scan_consistency=search.SearchScanConsistency.NOT_BOUNDED)) + await cb_env.assert_rows(res, 1) + + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, + scan_consistency=search.SearchScanConsistency.REQUEST_PLUS)) + await cb_env.assert_rows(res, 1) + + with pytest.raises(InvalidArgumentException): + cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, + scan_consistency=search.SearchScanConsistency.AT_PLUS)) + + @pytest.mark.asyncio + async def test_cluster_search_top_level_facets(self, cb_env): + # if the facet limit is omitted, the details of the facets will not be provided + # (i.e. SearchFacetResult.terms is None, + # SearchFacetResult.numeric_ranges is None and SearchFacetResult.date_ranges is None) + facet_name = 'model' + facet = search.TermFacet('model') + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + await cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert result_facet.terms is None + assert result_facet.numeric_ranges is None + assert result_facet.date_ranges is None + + facet_name = 'rating' + facet = search.NumericFacet('rating') + facet.add_range('low', max=4) + facet.add_range('med', min=4, max=7) + facet.add_range('high', min=7) + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + await cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert result_facet.terms is None + assert result_facet.numeric_ranges is None + assert result_facet.date_ranges is None + + @pytest.mark.asyncio + async def test_cluster_search_top_level_facets_kwargs(self, cb_env): + # if the facet limit is omitted, the details of the facets will not be provided + # (i.e. SearchFacetResult.terms is None, + # SearchFacetResult.numeric_ranges is None and SearchFacetResult.date_ranges is None) + facet_name = 'model' + facet = search.TermFacet('model') + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), facets={facet_name: facet}) + + await cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert result_facet.terms is None + assert result_facet.numeric_ranges is None + assert result_facet.date_ranges is None + + facet_name = 'rating' + facet = search.NumericFacet('rating') + facet.add_range('low', max=4) + facet.add_range('med', min=4, max=7) + facet.add_range('high', min=7) + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), facets={facet_name: facet}) + + await cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert result_facet.terms is None + assert result_facet.numeric_ranges is None + assert result_facet.date_ranges is None + + @pytest.mark.asyncio + async def test_cluster_sort_field(self, cb_env): + sort_field = "rating" + q = search.TermQuery('auto') + # field - ascending + sort = search.SortField(field=sort_field, type="number", mode="min", missing="last") + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[sort], fields=[sort_field])) + + rows = await cb_env.assert_rows(res, 1, return_rows=True) + rating = rows[0].fields[sort_field] + for row in rows[1:]: + assert row.fields[sort_field] >= rating + rating = row.fields[sort_field] + + # field - descending + sort.desc = True + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[sort], fields=[sort_field])) + + rows = await cb_env.assert_rows(res, 1, return_rows=True) + rating = rows[0].fields[sort_field] + for row in rows[1:]: + assert rating >= row.fields[sort_field] + rating = row.fields[sort_field] + + @pytest.mark.asyncio + async def test_cluster_sort_field_multi(self, cb_env): + sort_fields = [ + search.SortField(field="rating", type="number", + mode="min", missing="last"), + search.SortField(field="last_updated", type="number", + mode="min", missing="last"), + search.SortScore(), + ] + sort_field_names = ["rating", "last_updated"] + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=sort_fields, fields=sort_field_names)) + await cb_env.assert_rows(res, 1) + + sort_fields = [ + search.SortField(field="rating", type="number", + mode="min", missing="last", desc=True), + search.SortField(field="last_updated", type="number", + mode="min", missing="last"), + search.SortScore(desc=True), + ] + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=sort_fields, fields=sort_field_names)) + await cb_env.assert_rows(res, 1) + + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=["rating", "last_udpated", "-_score"])) + await cb_env.assert_rows(res, 1) + + @pytest.mark.asyncio + async def test_cluster_sort_geo(self, cb_env): + # @TODO: better confirmation on results? + sort_field = "geo" + q = search.TermQuery('auto') + # geo - ascending + sort = search.SortGeoDistance(field=sort_field, location=(37.7749, 122.4194), unit="meters") + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[sort], fields=[sort_field])) + await cb_env.assert_rows(res, 1) + + # geo - descending + sort.desc = True + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[sort], fields=[sort_field])) + await cb_env.assert_rows(res, 1) + + @pytest.mark.asyncio + async def test_cluster_sort_id(self, cb_env): + q = search.TermQuery('auto') + # score - ascending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[search.SortID()])) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + + id = rows[0].id + for row in rows[1:]: + assert row.id >= id + id = row.id + + # score - descending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[search.SortID(desc=True)])) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + + id = rows[0].id + for row in rows[1:]: + assert id >= row.id + id = row.id + + @pytest.mark.asyncio + async def test_cluster_sort_score(self, cb_env): + q = search.TermQuery('auto') + # score - ascending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[search.SortScore()])) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + + score = rows[0].score + for row in rows[1:]: + assert row.score >= score + score = row.score + + # score - descending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[search.SortScore(desc=True)])) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + + score = rows[0].score + for row in rows[1:]: + assert score >= row.score + score = row.score + + @pytest.mark.asyncio + async def test_cluster_sort_str(self, cb_env): + q = search.TermQuery('auto') + # score - ascending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=['_score'])) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + score = rows[0].score + for row in rows[1:]: + assert row.score >= score + score = row.score + + # score - descending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=['-_score'])) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + score = rows[0].score + for row in rows[1:]: + assert score >= row.score + score = row.score + + @pytest.mark.asyncio + async def test_search_include_locations(self, cb_env): + q = search.TermQuery('auto') + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, include_locations=True)) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + assert isinstance(locations, search.SearchRowLocations) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + + # check w/in kwargs + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), include_locations=True) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + assert isinstance(locations, search.SearchRowLocations) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + + @pytest.mark.parametrize('operator, query_terms, expect_rows', + [(search.MatchOperator.AND, "auto deal", True), + (search.MatchOperator.AND, "auto :random:", False), + (search.MatchOperator.OR, "auto deal", True), + (search.MatchOperator.OR, "auto :random:", True)]) + @pytest.mark.asyncio + async def test_search_match_operator(self, cb_env, operator, query_terms, expect_rows): + import random + import string + + random_query_term = "".join(random.choice(string.ascii_letters) + for _ in range(10)) + + if ':random:' in query_terms: + query_terms.replace(':random:', random_query_term) + + q = search.MatchQuery(query_terms, match_operator=operator) + + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, limit=10) + rows = await cb_env.assert_rows(res, 0, return_rows=True) + + if expect_rows: + assert len(rows) > 0 + else: + assert len(rows) == 0 + + def test_search_match_operator_fail(self, cb_env): + with pytest.raises(ValueError): + q = search.MatchQuery('auto deal', match_operator='NOT') + cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, limit=10) + + # @TODO(PYCBC-1296): DIFF between 3.x and 4.x, locations returns None + + @pytest.mark.asyncio + async def test_search_no_include_locations(self, cb_env): + q = search.TermQuery('auto') + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, include_locations=False)) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + assert locations is None + + # check w/in kwargs + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), include_locations=False) + rows = await cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + assert locations is None + + @pytest.mark.asyncio + async def test_search_raw_query(self, cb_env): + query_args = {"match": "auto deal", + "fuzziness": 2, "operator": "and"} + q = search.RawQuery(query_args) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, limit=10) + await cb_env.assert_rows(res, 1) + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_search_timeout(self, cb_env): + from acouchbase.cluster import Cluster + from couchbase.auth import PasswordAuthenticator + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + # Prior to PYCBC-1521, this test would fail as each request would override the cluster level search_timeout. + # If a timeout was not provided in the request, the default 75s timeout would be used. PYCBC-1521 corrects + # this behavior so this test will pass as we are essentially forcing an AmbiguousTimeoutException because + # we are setting the cluster level search_timeout such a small value. + timeout_opts = ClusterTimeoutOptions(search_timeout=timedelta(milliseconds=1)) + cluster = await Cluster.connect(f'{conn_string}', ClusterOptions(auth, timeout_options=timeout_opts)) + # don't need to do this except for older server versions + _ = cluster.bucket(f'{cb_env.bucket.name}') + q = search.TermQuery('auto') + with pytest.raises(AmbiguousTimeoutException): + res = cluster.search_query(cb_env.TEST_INDEX_NAME, q, SearchOptions(limit=10)) + [r async for r in res.rows()] + # if we override the timeout w/in the request the query should succeed. + res = cluster.search_query(cb_env.TEST_INDEX_NAME, q, SearchOptions(limit=10, timeout=timedelta(seconds=10))) + rows = [r async for r in res.rows()] + assert len(rows) > 0 + + +class ClassicSearchCollectionTests(SearchCollectionTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicSearchCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicSearchCollectionTests) if valid_test_method(meth)] + compare = set(SearchCollectionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.NAMED]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + acb_env = AsyncSearchTestEnvironment.from_environment(acb_base_env) + acb_env.enable_search_mgmt() + await acb_env.setup(request.param) + yield acb_env + await acb_env.teardown(request.param) + + +class ClassicSearchTests(SearchTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicSearchTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicSearchTests) if valid_test_method(meth)] + compare = set(SearchTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + acb_env = AsyncSearchTestEnvironment.from_environment(acb_base_env) + acb_env.enable_search_mgmt() + await acb_env.setup(request.param) + yield acb_env + await acb_env.teardown(request.param) diff --git a/acouchbase/tests/searchmgmt_t.py b/acouchbase/tests/searchmgmt_t.py new file mode 100644 index 000000000..1b8ab15e0 --- /dev/null +++ b/acouchbase/tests/searchmgmt_t.py @@ -0,0 +1,486 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import pathlib +from os import path + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import (FeatureUnavailableException, + InvalidArgumentException, + QueryIndexAlreadyExistsException, + SearchIndexNotFoundException) +from couchbase.management.search import SearchIndex + +from ._test_utils import CollectionType, TestEnvironment + + +@pytest.mark.flaky(reruns=5) +class SearchIndexManagementTests: + + IDX_NAME = 'test-fts-index' + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True, + manage_search_indexes=True) + + yield cb_env + + @pytest.fixture(scope="class") + def check_search_index_mgmt_supported(self, cb_env): + cb_env.check_if_feature_supported('search_index_mgmt') + + @pytest.fixture(scope="class", name='test_idx') + def get_test_index(self): + return SearchIndex(name=self.IDX_NAME, source_name='default') + + @pytest_asyncio.fixture() + async def create_test_index(self, cb_env, test_idx): + await cb_env.try_n_times_till_exception(10, 3, + cb_env.sixm.upsert_index, + test_idx, + expected_exceptions=(QueryIndexAlreadyExistsException, )) + + @pytest_asyncio.fixture() + async def drop_test_index(self, cb_env, test_idx): + yield + await cb_env.try_n_times_till_exception(10, 3, + cb_env.sixm.drop_index, + test_idx.name, + expected_exceptions=(SearchIndexNotFoundException, )) + + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_upsert_index(self, cb_env, test_idx): + res = await cb_env.sixm.upsert_index(test_idx) + assert res is None + res = await cb_env.try_n_times(10, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_drop_index(self, cb_env, test_idx): + res = await cb_env.try_n_times(3, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + await cb_env.sixm.drop_index(test_idx.name) + with pytest.raises(SearchIndexNotFoundException): + await cb_env.sixm.drop_index(test_idx.name) + + @pytest.mark.asyncio + async def test_drop_index_fail(self, cb_env, test_idx): + with pytest.raises(SearchIndexNotFoundException): + await cb_env.sixm.drop_index(test_idx.name) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_index(self, cb_env, test_idx): + res = await cb_env.try_n_times(3, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + assert res.name == test_idx.name + + @pytest.mark.asyncio + async def test_get_index_fail_no_index_name(self, cb_env): + with pytest.raises(InvalidArgumentException): + await cb_env.sixm.get_index('') + with pytest.raises(InvalidArgumentException): + await cb_env.sixm.get_index(None) + + @pytest.mark.asyncio + async def test_get_index_fail(self, cb_env): + with pytest.raises(SearchIndexNotFoundException): + await cb_env.sixm.get_index('not-an-index') + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_all_indexes(self, cb_env, test_idx): + # lets add one more + new_idx = SearchIndex(name='new-search-idx', source_name='default') + res = await cb_env.sixm.upsert_index(new_idx) + assert res is None + res = await cb_env.try_n_times(10, 3, cb_env.sixm.get_index, new_idx.name) + assert isinstance(res, SearchIndex) + + indexes = await cb_env.sixm.get_all_indexes() + assert isinstance(indexes, list) + assert len(indexes) >= 2 + assert next((idx for idx in indexes if idx.name == test_idx.name), None) is not None + assert next((idx for idx in indexes if idx.name == new_idx.name), None) is not None + + await cb_env.sixm.drop_index(new_idx.name) + await cb_env.try_n_times_till_exception(10, + 3, + cb_env.sixm.get_index, + new_idx.name, + expected_exceptions=(SearchIndexNotFoundException,)) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_analyze_doc(self, cb_env, test_idx): + if cb_env.server_version_short < 6.5: + pytest.skip((f'FTS analyzeDoc only supported on server versions >= 6.5. ' + f'Using server version: {cb_env.server_version}.')) + # like getting the doc count, this can fail immediately after index + # creation + doc = {"field": "I got text in here"} + analysis = await cb_env.try_n_times( + 5, 2, cb_env.sixm.analyze_document, test_idx.name, doc) + + assert analysis.get('analysis', None) is not None + assert isinstance(analysis.get('analysis'), (list, dict)) + assert analysis.get('status', None) == 'ok' + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_ingestion_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.pause_ingest, + test_idx.name) + assert res is None + + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.resume_ingest, + test_idx.name) + assert res is None + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_query_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.disallow_querying, + test_idx.name) + assert res is None + + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.allow_querying, + test_idx.name) + assert res is None + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_plan_freeze_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.freeze_plan, + test_idx.name) + assert res is None + + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.unfreeze_plan, + test_idx.name) + assert res is None + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_index_stats(self, cb_env, test_idx): + # like getting the doc count, this can fail immediately after index + # creation + stats = await cb_env.try_n_times( + 5, 2, cb_env.sixm.get_index_stats, test_idx.name) + + assert stats is not None + assert isinstance(stats, dict) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_index_doc_count(self, cb_env, test_idx): + # like getting the doc count, this can fail immediately after index + # creation + doc_count = await cb_env.try_n_times( + 5, 2, cb_env.sixm.get_indexed_documents_count, test_idx.name) + + assert doc_count is not None + assert isinstance(doc_count, int) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_all_index_stats(self, cb_env, test_idx): + # like getting the doc count, this can fail immediately after index + # creation + stats = await cb_env.try_n_times( + 5, 2, cb_env.sixm.get_all_index_stats) + + assert stats is not None + assert isinstance(stats, dict) + + +@pytest.mark.flaky(reruns=5) +class ScopeSearchIndexManagementTests: + + IDX_NAME = 'test-fts-index' + TEST_COLLECTION_INDEX_NAME = 'test-search-coll-index' + TEST_COLLECTION_INDEX_PATH = path.join(pathlib.Path(__file__).parent.parent.parent, + 'tests', + 'test_cases', + f'{TEST_COLLECTION_INDEX_NAME}-params-new.json') + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + CollectionType.NAMED, + manage_buckets=True) + + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + cb_env.enable_scope_search_mgmt().enable_search_mgmt() + yield cb_env + cb_env.disable_search_mgmt().disable_scope_search_mgmt() + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest.fixture(scope="class") + def check_search_index_mgmt_supported(self, cb_env): + cb_env.check_if_feature_supported('search_index_mgmt') + cb_env.check_if_feature_supported('scope_search_index_mgmt') + + @pytest.fixture(scope="class", name='test_idx') + def get_test_index(self): + params_json = {} + with open(self.TEST_COLLECTION_INDEX_PATH) as params_file: + input = params_file.read() + params_json = json.loads(input) + mapping_types = params_json.get('mapping', {}).get('types', {}) + if mapping_types and 'test-scope.other-collection' in mapping_types: + del params_json['mapping']['types']['test-scope.other-collection'] + return SearchIndex(name=self.IDX_NAME, source_name='default', params=params_json) + + @pytest_asyncio.fixture() + async def create_test_index(self, cb_env, test_idx): + + await cb_env.try_n_times_till_exception(10, 3, + cb_env.sixm.upsert_index, + test_idx, + expected_exceptions=(QueryIndexAlreadyExistsException, )) + + @pytest_asyncio.fixture() + async def drop_test_index(self, cb_env, test_idx): + yield + await cb_env.try_n_times_till_exception(10, 3, + cb_env.sixm.drop_index, + test_idx.name, + expected_exceptions=(SearchIndexNotFoundException, )) + + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_upsert_index(self, cb_env, test_idx): + res = await cb_env.sixm.upsert_index(test_idx) + assert res is None + res = await cb_env.try_n_times(10, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_drop_index(self, cb_env, test_idx): + res = await cb_env.try_n_times(3, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + await cb_env.sixm.drop_index(test_idx.name) + with pytest.raises(SearchIndexNotFoundException): + await cb_env.sixm.drop_index(test_idx.name) + + @pytest.mark.asyncio + async def test_drop_index_fail(self, cb_env, test_idx): + with pytest.raises(SearchIndexNotFoundException): + await cb_env.sixm.drop_index(test_idx.name) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_index(self, cb_env, test_idx): + res = await cb_env.try_n_times(3, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + assert res.name == test_idx.name + + @pytest.mark.asyncio + async def test_get_index_fail_no_index_name(self, cb_env): + with pytest.raises(InvalidArgumentException): + await cb_env.sixm.get_index('') + with pytest.raises(InvalidArgumentException): + await cb_env.sixm.get_index(None) + + @pytest.mark.asyncio + async def test_get_index_fail(self, cb_env): + with pytest.raises(SearchIndexNotFoundException): + await cb_env.sixm.get_index('not-an-index') + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_all_indexes(self, cb_env, test_idx): + # lets add one more + new_idx = SearchIndex(name='new-search-idx', source_name='default') + res = await cb_env.sixm.upsert_index(new_idx) + assert res is None + res = await cb_env.try_n_times(10, 3, cb_env.sixm.get_index, new_idx.name) + assert isinstance(res, SearchIndex) + + indexes = await cb_env.sixm.get_all_indexes() + assert isinstance(indexes, list) + assert len(indexes) >= 2 + assert next((idx for idx in indexes if idx.name == test_idx.name), None) is not None + assert next((idx for idx in indexes if idx.name == new_idx.name), None) is not None + + await cb_env.sixm.drop_index(new_idx.name) + await cb_env.try_n_times_till_exception(10, + 3, + cb_env.sixm.get_index, + new_idx.name, + expected_exceptions=(SearchIndexNotFoundException,)) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_analyze_doc(self, cb_env, test_idx): + if cb_env.server_version_short < 6.5: + pytest.skip((f'FTS analyzeDoc only supported on server versions >= 6.5. ' + f'Using server version: {cb_env.server_version}.')) + with pytest.raises(FeatureUnavailableException): + await cb_env.sixm.analyze_document(test_idx.name, {"field": "I got text in here"}) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_ingestion_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.pause_ingest, + test_idx.name) + assert res is None + + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.resume_ingest, + test_idx.name) + assert res is None + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_query_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.disallow_querying, + test_idx.name) + assert res is None + + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.allow_querying, + test_idx.name) + assert res is None + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_plan_freeze_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.freeze_plan, + test_idx.name) + assert res is None + + res = await cb_env.try_n_times( + 10, + 3, + cb_env.sixm.unfreeze_plan, + test_idx.name) + assert res is None + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_index_stats(self, cb_env, test_idx): + with pytest.raises(FeatureUnavailableException): + await cb_env.sixm.get_index_stats(test_idx.name) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_index_doc_count(self, cb_env, test_idx): + # like getting the doc count, this can fail immediately after index + # creation + doc_count = await cb_env.try_n_times( + 5, 2, cb_env.sixm.get_indexed_documents_count, test_idx.name) + + assert doc_count is not None + assert isinstance(doc_count, int) + + @pytest.mark.usefixtures("create_test_index") + @pytest.mark.usefixtures("drop_test_index") + @pytest.mark.asyncio + async def test_get_all_index_stats(self, cb_env, test_idx): + # like getting the doc count, this can fail immediately after index + # creation + stats = await cb_env.try_n_times( + 5, 2, cb_env.sixm.get_all_index_stats) + + assert stats is not None + assert isinstance(stats, dict) diff --git a/acouchbase/tests/subdoc_t.py b/acouchbase/tests/subdoc_t.py new file mode 100644 index 000000000..9b968403b --- /dev/null +++ b/acouchbase/tests/subdoc_t.py @@ -0,0 +1,999 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta + +import pytest +import pytest_asyncio + +import couchbase.subdocument as SD +from couchbase.exceptions import (DocumentExistsException, + DocumentNotFoundException, + DocumentUnretrievableException, + InvalidArgumentException, + InvalidValueException, + PathExistsException, + PathMismatchException, + PathNotFoundException) +from couchbase.options import (GetOptions, + LookupInAllReplicasOptions, + LookupInAnyReplicaOptions, + LookupInOptions, + MutateInOptions) +from couchbase.replica_reads import ReadPreference +from couchbase.result import (GetResult, + LookupInReplicaResult, + LookupInResult, + MutateInResult) +from tests.environments import CollectionType +from tests.environments.subdoc_environment import AsyncSubdocTestEnvironment +from tests.environments.test_environment import AsyncTestEnvironment +from tests.mock_server import MockServerType +from tests.test_features import EnvironmentFeatures + + +class SubDocumentTestSuite: + + TEST_MANIFEST = [ + 'test_array_add_unique', + 'test_array_add_unique_create_parents', + 'test_array_add_unique_fail', + 'test_array_append', + 'test_array_append_create_parents', + 'test_array_append_multi_insert', + 'test_array_as_document', + 'test_array_insert', + 'test_array_insert_multi_insert', + 'test_array_prepend', + 'test_array_prepend_create_parents', + 'test_array_prepend_multi_insert', + 'test_count', + 'test_decrement', + 'test_decrement_create_parents', + 'test_increment', + 'test_increment_create_parents', + 'test_insert_create_parents', + 'test_lookup_in_all_replicas_bad_key', + 'test_lookup_in_all_replicas_exists', + 'test_lookup_in_all_replicas_exists_bad_path', + 'test_lookup_in_all_replicas_get', + 'test_lookup_in_all_replicas_get_bad_path', + 'test_lookup_in_all_replicas_get_full', + 'test_lookup_in_all_replicas_multiple_specs', + 'test_lookup_in_all_replicas_with_timeout', + 'test_lookup_in_all_replicas_read_preference', + 'test_lookup_in_any_replica_bad_key', + 'test_lookup_in_any_replica_exists', + 'test_lookup_in_any_replica_exists_bad_path', + 'test_lookup_in_any_replica_get', + 'test_lookup_in_any_replica_get_bad_path', + 'test_lookup_in_any_replica_get_full', + 'test_lookup_in_any_replica_multiple_specs', + 'test_lookup_in_any_replica_with_timeout', + 'test_lookup_in_any_replica_read_preference', + 'test_lookup_in_multiple_specs', + 'test_lookup_in_one_path_not_found', + 'test_lookup_in_simple_exists', + 'test_lookup_in_simple_exists_bad_path', + 'test_lookup_in_simple_get', + 'test_lookup_in_simple_get_bad_path', + 'test_lookup_in_simple_get_spec_as_list', + 'test_lookup_in_simple_long_path', + 'test_lookup_in_simple_with_timeout', + 'test_lookup_in_valid_path_null_content', + 'test_mutate_in_expiry', + 'test_mutate_in_insert_semantics', + 'test_mutate_in_insert_semantics_fail', + 'test_mutate_in_insert_semantics_kwargs', + 'test_mutate_in_preserve_expiry', + 'test_mutate_in_preserve_expiry_fails', + 'test_mutate_in_preserve_expiry_not_used', + 'test_mutate_in_remove', + 'test_mutate_in_remove_blank_path', + 'test_mutate_in_replace_semantics', + 'test_mutate_in_replace_semantics_fail', + 'test_mutate_in_replace_semantics_kwargs', + 'test_mutate_in_replace_full_document', + 'test_mutate_in_simple', + 'test_mutate_in_simple_spec_as_list', + 'test_mutate_in_store_semantics_fail', + 'test_mutate_in_upsert_semantics', + 'test_mutate_in_upsert_semantics_kwargs', + 'test_upsert_create_parents' + ] + + @pytest.fixture(scope="class") + def check_preserve_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('preserve_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_xattr_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('xattr', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def skip_if_go_caves(self, cb_env): + if cb_env.is_mock_server and cb_env.mock_server_type == MockServerType.GoCAVES: + pytest.skip("GoCAVES does not like this operation.") + + @pytest.fixture(scope="class") + def check_replica_read_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('subdoc_replica_read', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_server_groups_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('server_groups', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_array_add_unique(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.array_addunique('array', 6),)) + assert isinstance(result, MutateInResult) + result = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + assert len(val['array']) == 6 + assert 6 in val['array'] + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_array_add_unique_create_parents(self, cb_env): + key, value = cb_env.get_new_doc_by_type('array') + await cb_env.collection.upsert(key, value) + await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + result = await cb_env.collection.mutate_in(key, ( + SD.array_addunique("new.set", "new", create_parents=True), + SD.array_addunique("new.set", "unique"), + SD.array_addunique("new.set", "set"))) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + new_set = result.content_as[dict]["new"]["set"] + assert isinstance(new_set, list) + assert "new" in new_set + assert "unique" in new_set + assert "set" in new_set + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_array_add_unique_fail(self, cb_env): + key = "simple-key" + value = { + "a": "aaa", + "b": [0, 1, 2, 3], + "c": [1.25, 1.5, {"nested": ["str", "array"]}], + } + await cb_env.collection.upsert(key, value) + await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + + with pytest.raises(PathExistsException): + await cb_env.collection.mutate_in(key, (SD.array_addunique("b", 3),)) + + with pytest.raises(InvalidValueException): + await cb_env.collection.mutate_in(key, (SD.array_addunique("b", [4, 5, 6]),)) + + with pytest.raises(InvalidValueException): + await cb_env.collection.mutate_in(key, (SD.array_addunique("b", {"c": "d"}),)) + + with pytest.raises(PathMismatchException): + await cb_env.collection.mutate_in(key, (SD.array_addunique("c", 2),)) + + @pytest.mark.asyncio + async def test_array_append(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.array_append('array', 6),)) + assert isinstance(result, MutateInResult) + result = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + assert len(val['array']) == 6 + assert val['array'][5] == 6 + + @pytest.mark.asyncio + async def test_array_append_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in(key, ( + SD.array_append('new.array', 'Hello,', create_parents=True), + SD.array_append('new.array', 'World!'),)) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + assert result.content_as[dict]['new']['array'] == [ + 'Hello,', 'World!'] + + @pytest.mark.asyncio + async def test_array_append_multi_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.array_append('array', 8, 9, 10),)) + assert isinstance(result, MutateInResult) + result = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + app_res = val['array'][5:] + assert len(app_res) == 3 + assert app_res == [8, 9, 10] + + @pytest.mark.asyncio + async def test_array_as_document(self, cb_env): + key = cb_env.get_existing_doc_by_type('array_only', key_only=True) + result = await cb_env.collection.mutate_in(key, (SD.array_append( + '', 2), SD.array_prepend('', 0), SD.array_insert('[1]', 1),)) + assert isinstance(result, MutateInResult) + result = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[list] + assert isinstance(val, list) + assert len(val) == 3 + assert val[0] == 0 + assert val[1] == 1 + assert val[2] == 2 + + @pytest.mark.asyncio + async def test_array_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.array_insert('array.[2]', 10),)) + assert isinstance(result, MutateInResult) + result = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + assert len(val['array']) == 6 + assert val['array'][2] == 10 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_array_insert_multi_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.array_insert('array.[3]', 6, 7, 8),)) + assert isinstance(result, MutateInResult) + result = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + ins_res = val['array'][3:6] + assert len(ins_res) == 3 + assert ins_res == [6, 7, 8] + + @pytest.mark.asyncio + async def test_array_prepend(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.array_prepend('array', 0),)) + assert isinstance(result, MutateInResult) + result = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + assert len(val['array']) == 6 + assert val['array'][0] == 0 + + @pytest.mark.asyncio + async def test_array_prepend_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in(key, ( + SD.array_prepend('new.array', 'World!', create_parents=True), + SD.array_prepend('new.array', 'Hello,'),)) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + assert result.content_as[dict]['new']['array'] == [ + 'Hello,', 'World!'] + + @pytest.mark.asyncio + async def test_array_prepend_multi_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.array_prepend('array', -2, -1, 0),)) + assert isinstance(result, MutateInResult) + result = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + pre_res = val['array'][:3] + assert len(pre_res) == 3 + assert pre_res == [-2, -1, 0] + + @pytest.mark.asyncio + async def test_count(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.lookup_in(key, (SD.count('array'),)) + assert isinstance(result, LookupInResult) + assert result.content_as[int](0) == 5 + + @pytest.mark.asyncio + async def test_decrement(self, cb_env): + key = cb_env.get_existing_doc_by_type('count', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.decrement('count', 50),)) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + assert result.content_as[dict]['count'] == 50 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_decrement_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.decrement('new.counter', 100, create_parents=True),)) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + assert result.content_as[dict]['new']['counter'] == -100 + + @pytest.mark.asyncio + async def test_increment(self, cb_env): + key = cb_env.get_existing_doc_by_type('count', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.increment('count', 50),)) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + assert result.content_as[dict]['count'] == 150 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_increment_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('count', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.increment('new.counter', 100, create_parents=True),)) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + assert result.content_as[dict]['new']['counter'] == 100 + + @pytest.mark.asyncio + async def test_insert_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.insert('new.path', 'parents created', create_parents=True),)) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + assert result.content_as[dict]['new']['path'] == 'parents created' + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_all_replicas_bad_key(self, cb_env): + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.lookup_in_all_replicas('asdfgh', [SD.exists('batch')]) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_all_replicas_exists(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + results = await cb_env.collection.lookup_in_all_replicas(key, [SD.exists('batch')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.exists(0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_all_replicas_exists_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + results = await cb_env.collection.lookup_in_all_replicas(key, [SD.exists('qzzxy')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert not result.exists(0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_all_replicas_get(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + results = await cb_env.collection.lookup_in_all_replicas(key, [SD.get('batch')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_all_replicas_get_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + results = await cb_env.collection.lookup_in_all_replicas(key, [SD.get('qzzxy')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + with pytest.raises(PathNotFoundException): + result.content_as[str](0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_all_replicas_get_full(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + results = await cb_env.collection.lookup_in_all_replicas(key, [SD.get_full()]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[dict](0) == value + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_all_replicas_multiple_specs(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + results = await cb_env.collection.lookup_in_all_replicas(key, [SD.get('batch'), SD.exists('manufacturer.city')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.exists(1) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_all_replicas_with_timeout(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + opts = LookupInAllReplicasOptions(timeout=timedelta(milliseconds=5000)) + results = await cb_env.collection.lookup_in_all_replicas(key, [SD.get('batch')], opts) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + @pytest.mark.usefixtures('check_server_groups_supported') + async def test_lookup_in_all_replicas_read_preference(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + # No preferred server group was specified in the cluster options so this should raise + # DocumentUnretrievableException + with pytest.raises(DocumentUnretrievableException): + await cb_env.collection.lookup_in_all_replicas( + key, + [SD.get('batch')], + LookupInAllReplicasOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP)) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_any_replica_bad_key(self, cb_env): + with pytest.raises(DocumentUnretrievableException): + await cb_env.collection.lookup_in_any_replica('asdfgh', [SD.exists('batch')]) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_any_replica_exists(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.lookup_in_any_replica(key, [SD.exists('batch')]) + assert isinstance(result, LookupInReplicaResult) + assert result.exists(0) + assert result.is_replica is not None + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_any_replica_exists_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.lookup_in_any_replica(key, [SD.exists('qzzxy')]) + assert isinstance(result, LookupInReplicaResult) + assert not result.exists(0) + assert result.is_replica is not None + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_any_replica_get(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.lookup_in_any_replica(key, [SD.get('batch')]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.is_replica is not None + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_any_replica_get_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.lookup_in_any_replica(key, [SD.get('qzzxy')]) + assert isinstance(result, LookupInReplicaResult) + with pytest.raises(PathNotFoundException): + result.content_as[str](0) + assert result.is_replica is not None + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_any_replica_get_full(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.lookup_in_any_replica(key, [SD.get_full()]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[dict](0) == value + assert result.is_replica is not None + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_any_replica_multiple_specs(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.lookup_in_any_replica(key, [SD.get('batch'), SD.exists('manufacturer.city')]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.exists(1) + assert result.is_replica is not None + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + async def test_lookup_in_any_replica_with_timeout(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + opts = LookupInAnyReplicaOptions(timeout=timedelta(milliseconds=5000)) + result = await cb_env.collection.lookup_in_any_replica(key, [SD.get('batch')], opts) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.is_replica is not None + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_replica_read_supported') + @pytest.mark.usefixtures('check_server_groups_supported') + async def test_lookup_in_any_replica_read_preference(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + # No preferred server group was specified in the cluster options so this should raise + # DocumentUnretrievableException + with pytest.raises(DocumentUnretrievableException): + await cb_env.collection.lookup_in_any_replica( + key, [SD.get('batch')], LookupInAnyReplicaOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP)) + + @pytest.mark.asyncio + @pytest.mark.usefixtures("check_xattr_supported") + async def test_lookup_in_multiple_specs(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.lookup_in(key, (SD.get('$document.exptime', xattr=True), + SD.exists('manufacturer'), + SD.get('manufacturer'), + SD.get('manufacturer.geo.accuracy'),)) + assert isinstance(result, LookupInResult) + assert result.content_as[int](0) == 0 + assert result.exists(1) + assert result.content_as[dict](2) == value['manufacturer'] + assert result.content_as[str](3) == value['manufacturer']['geo']['accuracy'] + + @pytest.mark.asyncio + async def test_lookup_in_one_path_not_found(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.lookup_in( + key, (SD.exists('batch'), SD.exists('qzzxy'),)) + assert isinstance(result, LookupInResult) + assert result.exists(0) + assert not result.exists(1) + # PYCBC-1480, update exists to follow RFC + assert result.content_as[bool](0) is True + assert result.content_as[bool](1) is False + + @pytest.mark.asyncio + async def test_lookup_in_simple_exists(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.lookup_in(key, (SD.exists('batch'),)) + assert isinstance(result, LookupInResult) + assert result.exists(0) + # PYCBC-1480, update exists to follow RFC + assert result.content_as[bool](0) is True + + @pytest.mark.asyncio + async def test_lookup_in_simple_exists_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.lookup_in(key, (SD.exists('qzzxy'),)) + assert isinstance(result, LookupInResult) + assert not result.exists(0) + assert result.content_as[bool](0) is False + + @pytest.mark.asyncio + async def test_lookup_in_simple_get(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.lookup_in(key, (SD.get('batch'),)) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value['batch'] + + @pytest.mark.asyncio + async def test_lookup_in_simple_get_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.lookup_in(key, (SD.get('qzzxy'),)) + assert isinstance(result, LookupInResult) + with pytest.raises(PathNotFoundException): + result.content_as[str](0) + + @pytest.mark.asyncio + async def test_lookup_in_simple_get_spec_as_list(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.lookup_in(key, [SD.get('batch')]) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value['batch'] + + @pytest.mark.asyncio + async def test_lookup_in_simple_long_path(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + result = await cb_env.collection.lookup_in( + key, (SD.get('manufacturer.geo.location.tz'),)) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value['manufacturer']['geo']['location']['tz'] + + @pytest.mark.asyncio + async def test_lookup_in_simple_with_timeout(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.lookup_in(key, + (SD.get('batch'),), + LookupInOptions(timeout=timedelta(milliseconds=5000))) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value['batch'] + + @pytest.mark.asyncio + async def test_lookup_in_valid_path_null_content(self, cb_env): + key, value = cb_env.get_new_doc_by_type('vehicle') + value['empty_field'] = None + await cb_env.collection.upsert(key, value) + res = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert 'empty_field' in res.content_as[dict] + result = await cb_env.collection.lookup_in(key, (SD.get('empty_field'), SD.get('batch'))) + assert isinstance(result, LookupInResult) + assert result.content_as[lambda x: x](0) is None + + @pytest.mark.asyncio + @pytest.mark.usefixtures("check_xattr_supported") + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_expiry(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.mutate_in(key, + (SD.upsert("make", "New Make"), + SD.replace("model", "New Model")), + MutateInOptions(expiry=timedelta(seconds=1000))) + + async def cas_matches(cb, new_cas): + r = await cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + await AsyncTestEnvironment.try_n_times(10, 3, cas_matches, cb_env.collection, result.cas) + + result = await cb_env.collection.get(key, GetOptions(with_expiry=True)) + expires_in = (result.expiry_time - datetime.now()).total_seconds() + assert expires_in > 0 and expires_in < 1021 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_insert_semantics(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.get(key) + + await cb_env.collection.mutate_in(key, + (SD.insert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.INSERT)) + + res = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_insert_semantics_kwargs(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.get(key) + + await cb_env.collection.mutate_in(key, + (SD.insert('new_path', 'im new'),), + insert_doc=True) + + res = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_insert_semantics_fail(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentExistsException): + await cb_env.collection.mutate_in(key, + (SD.insert('new_path', 'im new'),), + insert_doc=True) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_preserve_expiry_supported') + async def test_mutate_in_preserve_expiry(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + await cb_env.collection.mutate_in(key, + (SD.upsert('make', 'New Make'), + SD.replace('model', 'New Model')), + MutateInOptions(expiry=timedelta(seconds=2))) + + expiry_path = '$document.exptime' + res = await AsyncTestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + await cb_env.collection.mutate_in(key, + (SD.upsert('make', 'Updated Make'),), + MutateInOptions(preserve_expiry=True)) + res = await AsyncTestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + await AsyncTestEnvironment.sleep(3) + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.get(key) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_preserve_expiry_supported') + async def test_mutate_in_preserve_expiry_fails(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + with pytest.raises(InvalidArgumentException): + await cb_env.collection.mutate_in( + key, + (SD.insert('c', 'ccc'),), + MutateInOptions(preserve_expiry=True), + ) + + with pytest.raises(InvalidArgumentException): + await cb_env.collection.mutate_in( + key, + (SD.replace('c', 'ccc'),), + MutateInOptions( + expiry=timedelta(seconds=5), + preserve_expiry=True), + ) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_preserve_expiry_supported') + async def test_mutate_in_preserve_expiry_not_used(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + await cb_env.collection.mutate_in( + key, + (SD.upsert('make', 'New Make'), SD.replace('model', 'New Model')), + MutateInOptions(expiry=timedelta(seconds=5)) + ) + + expiry_path = '$document.exptime' + res = await AsyncTestEnvironment.try_n_times( + 10, 3, cb_env.collection.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + await cb_env.collection.mutate_in(key, (SD.upsert('make', 'Updated Make'),)) + res = await AsyncTestEnvironment.try_n_times( + 10, 3, cb_env.collection.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + await AsyncTestEnvironment.sleep(3) + result = await cb_env.collection.get(key) + assert isinstance(result, GetResult) + assert result.content_as[dict]['make'] == 'Updated Make' + + @pytest.mark.asyncio + async def test_mutate_in_remove(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + await cb_env.collection.mutate_in(key, [SD.remove('manufacturer.geo')]) + result = await cb_env.collection.get(key) + assert 'geo' not in result.content_as[dict]['manufacturer'] + + @pytest.mark.asyncio + async def test_mutate_in_remove_blank_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + await cb_env.collection.mutate_in(key, [SD.remove('')]) + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.get(key) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_replace_semantics(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + await cb_env.collection.mutate_in( + key, + (SD.upsert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.REPLACE) + ) + + res = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict]['new_path'] == 'im new' + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_replace_semantics_fail(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.mutate_in( + key, + (SD.upsert('new_path', 'im new'),), + replace_doc=True + ) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_replace_semantics_kwargs(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + await cb_env.collection.mutate_in( + key, + (SD.upsert('new_path', 'im new'),), + replace_doc=True) + + res = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict]['new_path'] == 'im new' + + @pytest.mark.asyncio + async def test_mutate_in_replace_full_document(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + await cb_env.collection.mutate_in( + key, + (SD.replace('', {'make': 'New Make', 'model': 'New Model'}),)) + + res = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict]['make'] == 'New Make' + assert res.content_as[dict]['model'] == 'New Model' + + @pytest.mark.asyncio + async def test_mutate_in_simple(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.mutate_in( + key, + (SD.upsert('make', 'New Make'), SD.replace('model', 'New Model')) + ) + + value['make'] = 'New Make' + value['model'] = 'New Model' + + async def cas_matches(cb, new_cas): + r = await cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + await AsyncTestEnvironment.try_n_times(10, 3, cas_matches, cb_env.collection, result.cas) + + result = await cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.asyncio + async def test_mutate_in_simple_spec_as_list(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = await cb_env.collection.mutate_in( + key, + [SD.upsert('make', 'New Make'), SD.replace('model', 'New Model')] + ) + + value['make'] = 'New Make' + value['model'] = 'New Model' + + async def cas_matches(cb, new_cas): + r = await cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + await AsyncTestEnvironment.try_n_times(10, 3, cas_matches, cb_env.collection, result.cas) + + result = await cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.asyncio + async def test_mutate_in_store_semantics_fail(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(InvalidArgumentException): + await cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + insert_doc=True, upsert_doc=True) + + with pytest.raises(InvalidArgumentException): + await cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + insert_doc=True, replace_doc=True) + + with pytest.raises(InvalidArgumentException): + await cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + upsert_doc=True, replace_doc=True) + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_upsert_semantics(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.get(key) + + await cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.UPSERT)) + + res = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.asyncio + @pytest.mark.usefixtures('skip_if_go_caves') + async def test_mutate_in_upsert_semantics_kwargs(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + await cb_env.collection.get(key) + + await cb_env.collection.mutate_in( + key, + (SD.upsert('new_path', 'im new'),), + upsert_doc=True + ) + + res = await AsyncTestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.asyncio + async def test_upsert_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = await cb_env.collection.mutate_in( + key, (SD.upsert('new.path', 'parents created', create_parents=True),)) + assert isinstance(result, MutateInResult) + result = await cb_env.collection.get(key) + assert result.content_as[dict]['new']['path'] == 'parents created' + + +# For whatever reason GoCAVES is rather flaky w/ subdocument tests +@pytest.mark.flaky(reruns=3, reruns_delay=1) +class ClassicSubDocumentTests(SubDocumentTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicSubDocumentTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicSubDocumentTests) if valid_test_method(meth)] + compare = set(SubDocumentTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + acb_env = AsyncSubdocTestEnvironment.from_environment(acb_base_env) + acb_env.enable_bucket_mgmt() + await acb_env.setup(request.param) + + yield acb_env + + await acb_env.teardown(request.param) diff --git a/acouchbase/tests/transactions_t.py b/acouchbase/tests/transactions_t.py new file mode 100644 index 000000000..c80b355d2 --- /dev/null +++ b/acouchbase/tests/transactions_t.py @@ -0,0 +1,826 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import json +from datetime import timedelta + +import pytest +import pytest_asyncio + +from couchbase.durability import DurabilityLevel, ServerDurability +from couchbase.exceptions import (BucketNotFoundException, + DocumentExistsException, + DocumentNotFoundException, + DocumentUnretrievableException, + FeatureUnavailableException, + ParsingFailedException, + TransactionExpired, + TransactionFailed, + TransactionOperationFailed) +from couchbase.n1ql import QueryProfile, QueryScanConsistency +from couchbase.options import (TransactionConfig, + TransactionGetOptions, + TransactionInsertOptions, + TransactionOptions, + TransactionQueryOptions, + TransactionReplaceOptions) +from couchbase.transactions import TransactionKeyspace, TransactionResult +from couchbase.transcoder import RawBinaryTranscoder +from tests.environments import CollectionType +from tests.environments.test_environment import AsyncTestEnvironment +from tests.test_features import EnvironmentFeatures + + +class TransactionTestSuite: + TEST_MANIFEST = [ + 'test_adhoc', + 'test_bad_query', + 'test_binary', + 'test_binary_kwargs', + 'test_binary_not_supported', + 'test_cleanup_client_attempts', + 'test_cleanup_lost_attempts', + 'test_cleanup_window', + 'test_client_context_id', + 'test_expiration_time', + 'test_get', + 'test_get_lambda_raises_doc_not_found', + 'test_get_inner_exc_doc_not_found', + 'test_get_replica_from_preferred_server_group_unretrievable', + 'test_get_replica_from_preferred_server_group_propagate_unretrievable_exc', + 'test_insert', + 'test_insert_lambda_raises_doc_exists', + 'test_insert_inner_exc_doc_exists', + 'test_max_parallelism', + 'test_metadata_collection', + 'test_metadata_collection_not_found', + 'test_metrics', + 'test_named_params', + 'test_per_txn_config', + 'test_pipeline_batch', + 'test_pipeline_cap', + 'test_positional_params', + 'test_profile_mode', + 'test_query', + 'test_query_lambda_raises_parsing_failure', + 'test_query_inner_exc_parsing_failure', + 'test_query_mode_insert', + 'test_query_mode_remove', + 'test_raw', + 'test_read_only', + 'test_remove', + 'test_remove_fail_bad_cas', + 'test_replace', + 'test_replace_fail_bad_cas', + 'test_rollback', + 'test_rollback_eating_exceptions', + 'test_scan_consistency', + 'test_scope_qualifier', + 'test_timeout', + 'test_transaction_config_durability', + 'test_transaction_result', + ] + + @pytest.fixture(scope='class') + def check_txn_queries_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('txn_queries', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_binary_txns_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('binary_txns', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.fixture(scope='class') + def check_binary_txns_not_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_not_supported('binary_txns', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.fixture(scope='class') + def check_server_groups_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('server_groups', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.mark.parametrize('adhoc', [True, False]) + def test_adhoc(self, adhoc): + cfg = TransactionQueryOptions(adhoc=adhoc) + cfg_adhoc = cfg._base.to_dict().get('adhoc', None) + assert cfg_adhoc is not None + assert cfg_adhoc == adhoc + + @pytest.mark.usefixtures('check_txn_queries_supported') + @pytest.mark.asyncio + async def test_bad_query(self, cb_env): + + async def txn_logic(ctx): + try: + await ctx.query('this wont parse') + pytest.fail('expected bad query to raise exception') + except ParsingFailedException: + pass + except Exception as e: + pytest.fail(f"Expected bad query to raise ParsingFailedException, not {e.__class__.__name__}") + + await cb_env.cluster.transactions.run(txn_logic) + + @pytest.mark.usefixtures('check_binary_txns_supported') + @pytest.mark.asyncio + async def test_binary(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + tc = RawBinaryTranscoder() + value = 'bytes content'.encode('utf-8') + new_content = b'\xFF' + + async def txn_logic(ctx): + await ctx.insert(cb_env.collection, key, value, TransactionInsertOptions(transcoder=tc)) + get_res = await ctx.get(cb_env.collection, key, TransactionGetOptions(transcoder=tc)) + assert get_res.content_as[bytes] == value + replace_res = await ctx.replace(get_res, new_content, TransactionReplaceOptions(transcoder=tc)) + # there's a bug where we don't return the correct content in the replace, so comment this out for now + # assert replace_res.content_as[bytes] == new_content + assert get_res.cas != replace_res.cas + + await cb_env.cluster.transactions.run(txn_logic) + result = await cb_env.collection.get(key, transcoder=tc) + assert result.content_as[bytes] == new_content + + @pytest.mark.usefixtures('check_binary_txns_supported') + @pytest.mark.asyncio + async def test_binary_kwargs(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + tc = RawBinaryTranscoder() + value = 'bytes content'.encode('utf-8') + new_content = b'\xFF' + + async def txn_logic(ctx): + await ctx.insert(cb_env.collection, key, value, transcoder=tc) + get_res = await ctx.get(cb_env.collection, key, transcoder=tc) + assert get_res.content_as[bytes] == value + replace_res = await ctx.replace(get_res, new_content, transcoder=tc) + # there's a bug where we don't return the correct content in the replace, so comment this out for now + # assert replace_res.content_as[bytes] == new_content + assert get_res.cas != replace_res.cas + + await cb_env.cluster.transactions.run(txn_logic) + result = await cb_env.collection.get(key, transcoder=tc) + assert result.content_as[bytes] == new_content + + @pytest.mark.usefixtures('check_binary_txns_not_supported') + @pytest.mark.asyncio + async def test_binary_not_supported(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + tc = RawBinaryTranscoder() + value = 'bytes content'.encode('utf-8') + + async def txn_logic(ctx): + await ctx.insert(cb_env.collection, key, value, TransactionInsertOptions(transcoder=tc)) + + try: + await cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + err_msg = f"Expected to raise FeatureUnavailableException, not {ex.inner_cause.__class__.__name__}" + assert isinstance(ex.inner_cause, FeatureUnavailableException), err_msg + + @pytest.mark.parametrize('cleanup', [False, True]) + def test_cleanup_client_attempts(self, cleanup): + cfg = TransactionConfig(cleanup_client_attempts=cleanup) + cfg_cleanup = cfg._base.to_dict().get('cleanup_client_attempts', None) + assert cfg_cleanup is not None + assert cfg_cleanup is cleanup + + @pytest.mark.parametrize('cleanup', [False, True]) + def test_cleanup_lost_attempts(self, cleanup): + cfg = TransactionConfig(cleanup_lost_attempts=cleanup) + cfg_cleanup = cfg._base.to_dict().get('cleanup_lost_attempts', None) + assert cfg_cleanup is not None + assert cfg_cleanup is cleanup + + @pytest.mark.parametrize('window', [timedelta(seconds=30), timedelta(milliseconds=500)]) + def test_cleanup_window(self, window): + cfg = TransactionConfig(cleanup_window=window) + cfg_window = cfg._base.to_dict().get('cleanup_window', None) + assert cfg_window is not None + assert cfg_window == window.total_seconds() * 1000 # milliseconds + + def test_client_context_id(self): + ctxid = "somestring" + cfg = TransactionQueryOptions(client_context_id=ctxid) + cfg_ctxid = cfg._base.to_dict().get('client_context_id', None) + assert cfg_ctxid is not None + assert cfg_ctxid == ctxid + + @pytest.mark.parametrize('cls', [TransactionConfig, TransactionOptions]) + @pytest.mark.parametrize('exp', [timedelta(seconds=30), timedelta(milliseconds=100)]) + def test_expiration_time(self, cls, exp): + cfg = cls(expiration_time=exp) + # CXXCBC-391, changes make expiration_time an invalid key + cfg_timeout = cfg._base.to_dict().get('expiration_time', None) + assert cfg_timeout is None + cfg_timeout = cfg._base.to_dict().get('timeout', None) + assert cfg_timeout == exp.total_seconds() * 1000*1000*1000 # nanoseconds - and can't use 'is' here + + @pytest.mark.asyncio + async def test_get(self, cb_env): + key, value = cb_env.get_existing_doc() + + async def txn_logic(ctx): + res = await ctx.get(cb_env.collection, key) + assert res.cas > 0 + assert res.id == key + assert res.content_as[dict] == value + + await cb_env.cluster.transactions.run(txn_logic) + + @pytest.mark.asyncio + async def test_get_lambda_raises_doc_not_found(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + num_attempts = 0 + + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + try: + await ctx.get(cb_env.collection, key) + except Exception as ex: + err_msg = f"Expected to raise DocumentNotFoundException, not {ex.__class__.__name__}" + assert isinstance(ex, DocumentNotFoundException), err_msg + + raise Exception('User raised exception.') + + try: + await cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert 'User raised exception.' in str(ex) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.asyncio + async def test_get_inner_exc_doc_not_found(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + num_attempts = 0 + + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + await ctx.get(cb_env.collection, key) + + try: + await cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + assert isinstance(ex.inner_cause, DocumentNotFoundException) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_server_groups_supported') + async def test_get_replica_from_preferred_server_group_unretrievable(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + num_attempts = 0 + + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + with pytest.raises(DocumentUnretrievableException): + await ctx.get_replica_from_preferred_server_group(cb_env.collection, key) + + await cb_env.cluster.transactions.run(txn_logic) + + assert num_attempts == 1 + + @pytest.mark.asyncio + @pytest.mark.usefixtures('check_server_groups_supported') + async def test_get_replica_from_preferred_server_group_propagate_unretrievable_exc(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + num_attempts = 0 + + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + await ctx.get_replica_from_preferred_server_group(cb_env.collection, key) + + try: + await cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + assert isinstance(ex.inner_cause, DocumentUnretrievableException) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.asyncio + async def test_insert(self, cb_env): + key, value = cb_env.get_new_doc() + + async def txn_logic(ctx): + await ctx.insert(cb_env.collection, key, value) + + await cb_env.cluster.transactions.run(txn_logic) + get_result = await cb_env.collection.get(key) + assert get_result.content_as[dict] == value + + @pytest.mark.asyncio + async def test_insert_lambda_raises_doc_exists(self, cb_env): + key, value = cb_env.get_existing_doc() + num_attempts = 0 + + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + try: + await ctx.insert(cb_env.collection, key, value) + except Exception as ex: + err_msg = f"Expected to raise DocumentExistsException, not {ex.__class__.__name__}" + assert isinstance(ex, DocumentExistsException), err_msg + + raise Exception('User raised exception.') + + try: + await cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert 'User raised exception.' in str(ex) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.asyncio + async def test_insert_inner_exc_doc_exists(self, cb_env): + key, value = cb_env.get_existing_doc() + num_attempts = 0 + + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + await ctx.insert(cb_env.collection, key, value) + + try: + await cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + assert isinstance(ex.inner_cause, DocumentExistsException) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + def test_max_parallelism(self): + max = 100 + cfg = TransactionQueryOptions(max_parallelism=max) + cfg_max = cfg._base.to_dict().get('max_parallelism', None) + assert cfg_max is not None + assert cfg_max == max + + @pytest.mark.parametrize('cls', [TransactionOptions, TransactionConfig]) + def test_metadata_collection(self, cls, cb_env): + coll = cb_env.collection + cfg = cls(metadata_collection=TransactionKeyspace(coll=coll)) + cfg_coll = cfg._base.to_dict().get('metadata_collection', None) + assert cfg_coll is not None + assert cfg_coll == f'{coll._scope.bucket_name}.{coll._scope.name}.{coll.name}' + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_metadata_collection_not_found(self, cb_env): + from acouchbase.cluster import AsyncCluster + from couchbase.auth import PasswordAuthenticator + from couchbase.options import ClusterOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + metadata = TransactionKeyspace(bucket='no-bucket', scope='_default', collection='_default') + txn_config = TransactionConfig(metadata_collection=metadata) + cluster = await AsyncCluster.connect(f'{conn_string}', ClusterOptions(auth, transaction_config=txn_config)) + collection = cluster.bucket(cb_env.bucket.name).default_collection() + + async def txn_logic(ctx): + # key should not matter as we should fail when creating the + # transactions object and not actually get to this point + await ctx.get(collection, 'test-key') + + with pytest.raises(BucketNotFoundException): + await cluster.transactions.run(txn_logic) + + @pytest.mark.parametrize('metrics', [True, False]) + def test_metrics(self, metrics): + cfg = TransactionQueryOptions(metrics=metrics) + cfg_metrics = cfg._base.to_dict().get('metrics', None) + assert cfg_metrics is not None + assert cfg_metrics == metrics + + @pytest.mark.parametrize('params', [{'key1': 'thing'}, + {'key1': ['an', 'array']}, + {'key1': 10, 'key2': 'something else'}]) + def test_named_params(self, params): + cfg = TransactionQueryOptions(named_parameters=params) + cfg_params = cfg._base.to_dict().get('named_parameters', None) + assert cfg_params is not None + assert isinstance(cfg_params, dict) + for k, v in params.items(): + assert json.loads(cfg_params[k]) == v + + @pytest.mark.asyncio + async def test_per_txn_config(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + + async def txn_logic(ctx): + await ctx.insert(cb_env.collection, key, {'some': 'thing'}) + await AsyncTestEnvironment.sleep(0.001) + await ctx.get(cb_env.collection, key) + + with pytest.raises(TransactionExpired): + await cb_env.cluster.transactions.run(txn_logic, + TransactionOptions(timeout=timedelta(microseconds=1))) + res = await cb_env.collection.exists(key) + assert res.exists is False + + def test_pipeline_batch(self): + batch = 100 + cfg = TransactionQueryOptions(pipeline_batch=batch) + cfg_batch = cfg._base.to_dict().get('pipeline_batch', None) + assert cfg_batch is not None + assert cfg_batch == batch + + def test_pipeline_cap(self): + cap = 100 + cfg = TransactionQueryOptions(pipeline_cap=cap) + cfg_cap = cfg._base.to_dict().get('pipeline_cap', None) + assert cfg_cap is not None + assert cfg_cap == cap + + @pytest.mark.parametrize('params', [['a', 'b', 'c']]) # , [[1, 2, 3], ['a', 'b', 'c']]]) + def test_positional_params(self, params): + cfg = TransactionQueryOptions(positional_parameters=params) + cfg_params = cfg._base.to_dict().get('positional_parameters', None) + assert cfg_params is not None + assert isinstance(cfg_params, list) + for idx, p in enumerate(cfg_params): + assert params[idx] == json.loads(p) + + @pytest.mark.parametrize('profile', [QueryProfile.OFF, QueryProfile.PHASES, QueryProfile.TIMINGS]) + def test_profile_mode(self, profile): + cfg = TransactionQueryOptions(profile=profile) + cfg_profile = cfg._base.to_dict().get('profile', None) + assert cfg_profile is not None + assert cfg_profile == profile.value + + @pytest.mark.usefixtures('check_txn_queries_supported') + @pytest.mark.asyncio + async def test_query(self, cb_env): + coll = cb_env.collection + key, value = cb_env.get_new_doc() + + async def txn_logic(ctx): + location = f"default:`{coll._scope.bucket_name}`.`{coll._scope.name}`.`{coll.name}`" + await ctx.query( + f'INSERT INTO {location} VALUES("{key}", {json.dumps(value)})', + TransactionQueryOptions(metrics=False)) + + await cb_env.cluster.transactions.run(txn_logic) + res = await cb_env.collection.exists(key) + assert res.exists is True + + @pytest.mark.usefixtures('check_txn_queries_supported') + @pytest.mark.asyncio + async def test_query_lambda_raises_parsing_failure(self, cb_env): + num_attempts = 0 + + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + try: + await ctx.query('This is not N1QL!', TransactionQueryOptions(metrics=False)) + except Exception as ex: + err_msg = f"Expected to raise ParsingFailedException, not {ex.__class__.__name__}" + assert isinstance(ex, ParsingFailedException), err_msg + + raise Exception('User raised exception.') + + try: + await cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert 'User raised exception.' in str(ex) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.usefixtures('check_txn_queries_supported') + @pytest.mark.asyncio + async def test_query_inner_exc_parsing_failure(self, cb_env): + num_attempts = 0 + + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + await ctx.query('This is not N1QL!', TransactionQueryOptions(metrics=False)) + + try: + await cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + assert isinstance(ex.inner_cause, ParsingFailedException) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.usefixtures('check_txn_queries_supported') + @pytest.mark.asyncio + async def test_query_mode_insert(self, cb_env): + coll = cb_env.collection + key, value = cb_env.get_new_doc() + key1, value1 = cb_env.get_new_doc() + await coll.insert(key, value) + + async def txn_logic(ctx): + fdqn = f"`{coll._scope.bucket_name}`.`{coll._scope.name}`.`{coll.name}`" + statement = f'SELECT * FROM {fdqn} WHERE META().id IN $1 ORDER BY META().id ASC' + res = await ctx.query(statement, TransactionQueryOptions(positional_parameters=[[key]])) + assert len(res.rows()) == 1 + assert res.rows()[0].get(f'{coll.name}', {}).get('id') == value.get('id') + await ctx.insert(coll, key1, value1) + + await cb_env.cluster.transactions.run(txn_logic) + get_res = await coll.get(key1) + assert get_res is not None + assert get_res.content_as[dict] == value1 + + @pytest.mark.usefixtures('check_txn_queries_supported') + @pytest.mark.asyncio + async def test_query_mode_remove(self, cb_env): + coll = cb_env.collection + key, value = cb_env.get_new_doc() + key1, value1 = cb_env.get_new_doc() + await coll.insert(key, value) + + async def txn_logic(ctx): + await ctx.insert(coll, key1, value1) + fdqn = f"`{coll._scope.bucket_name}`.`{coll._scope.name}`.`{coll.name}`" + statement = f'SELECT * FROM {fdqn} WHERE META().id IN $1 ORDER BY META().id ASC' + res = await ctx.query(statement, TransactionQueryOptions(positional_parameters=[[key, key1]])) + assert len(res.rows()) == 2 + getRes = await ctx.get(coll, key) + await ctx.remove(getRes) + + await cb_env.cluster.transactions.run(txn_logic) + with pytest.raises(DocumentNotFoundException): + await coll.get(key) + + @pytest.mark.parametrize('raw', [{'key1': 'yo'}, {'key1': 5, 'key2': 'foo'}, {'key': [1, 2, 3]}]) + def test_raw(self, raw): + cfg = TransactionQueryOptions(raw=raw) + cfg_raw = cfg._base.to_dict().get('raw', None) + assert cfg_raw is not None + assert isinstance(cfg_raw, dict) + for k, v in cfg_raw.items(): + assert json.loads(cfg_raw[k]) == raw[k] + + @pytest.mark.parametrize('read_only', [True, False]) + def test_read_only(self, read_only): + cfg = TransactionQueryOptions(read_only=read_only) + cfg_read_only = cfg._base.to_dict().get('read_only', None) + assert cfg_read_only is not None + assert cfg_read_only == read_only + + @pytest.mark.asyncio + async def test_remove(self, cb_env): + key, value = cb_env.get_new_doc() + await cb_env.collection.insert(key, value) + + async def txn_logic(ctx): + get_res = await ctx.get(cb_env.collection, key) + await ctx.remove(get_res) + + await cb_env.cluster.transactions.run(txn_logic) + result = await cb_env.collection.exists(key) + assert result.exists is False + + @pytest.mark.asyncio + async def test_remove_fail_bad_cas(self, cb_env): + key, value = cb_env.get_existing_doc() + num_attempts = 0 + + # txn will retry until timeout + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + rem_res = await ctx.get(cb_env.collection, key) + await ctx.replace(rem_res, {'what': 'new content!'}) + try: + await ctx.remove(rem_res) + except Exception as ex: + assert isinstance(ex, TransactionOperationFailed) + assert 'transaction expired' in ex.message or 'cas_mismatch' in ex.message + + try: + await cb_env.cluster.transactions.run(txn_logic, TransactionOptions(timeout=timedelta(seconds=2))) + except Exception as ex: + assert isinstance(ex, TransactionExpired) + assert 'transaction expired' in ex.message or 'expired in auto' in ex.message + + assert num_attempts > 1 + # txn should fail, so doc should exist + res = await cb_env.collection.get(key) + assert res.content_as[dict] == value + + @pytest.mark.asyncio + async def test_replace(self, cb_env): + key, value = cb_env.get_existing_doc() + new_value = {'some': 'thing else'} + + await cb_env.collection.upsert(key, value) + + async def txn_logic(ctx): + get_res = await ctx.get(cb_env.collection, key) + assert get_res.content_as[dict] == value + replace_res = await ctx.replace(get_res, new_value) + # there's a bug where we don't return the correct content in the replace, so comment this out for now + # assert replace_res.content_as[str] == new_value + assert get_res.cas != replace_res.cas + + await cb_env.cluster.transactions.run(txn_logic) + result = await cb_env.collection.get(key) + assert result.content_as[dict] == new_value + + @pytest.mark.asyncio + async def test_replace_fail_bad_cas(self, cb_env): + key, value = cb_env.get_existing_doc() + num_attempts = 0 + + # txn will retry until timeout + async def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + rem_res = await ctx.get(cb_env.collection, key) + await ctx.replace(rem_res, {'foo': 'bar'}) + try: + await ctx.replace(rem_res, {'foo': 'baz'}) + except Exception as ex: + assert isinstance(ex, TransactionOperationFailed) + assert 'transaction expired' in ex.message or 'cas_mismatch' in ex.message + + try: + await cb_env.cluster.transactions.run(txn_logic, TransactionOptions(timeout=timedelta(seconds=2))) + except Exception as ex: + assert isinstance(ex, TransactionExpired) + assert 'transaction expired' in ex.message or 'expired in auto' in ex.message + + assert num_attempts > 1 + # txn should fail, so doc should have original content + res = await cb_env.collection.get(key) + assert res.content_as[dict] == value + + @pytest.mark.asyncio + async def test_rollback(self, cb_env): + key, value = cb_env.get_new_doc() + + async def txn_logic(ctx): + res = await ctx.insert(cb_env.collection, key, value) + assert res.id == key + assert res.cas > 0 + raise RuntimeError('this should rollback txn') + + with pytest.raises(TransactionFailed): + await cb_env.cluster.transactions.run(txn_logic) + + result = await cb_env.collection.exists(key) + result.exists is False + + @pytest.mark.asyncio + async def test_rollback_eating_exceptions(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = await cb_env.collection.get(key) + cas = result.cas + + async def txn_logic(ctx): + try: + await ctx.insert(cb_env.collection, key, {'this': 'should fail'}) + pytest.fail("insert of existing key should have failed") + except DocumentExistsException: + # just eat the exception + pass + except Exception as e2: + pytest.fail(f"Expected insert to raise TransactionOperationFailed, not {e2.__class__.__name__}") + + try: + await cb_env.cluster.transactions.run(txn_logic) + except Exception as ex: + assert isinstance(ex, TransactionFailed) + # the inner cause should be a DocumentExistsException for this example + # if pytest.fail() occurred this will not be the case, thus failing the test + assert isinstance(ex.inner_cause, DocumentExistsException) + + result = await cb_env.collection.get(key) + assert result.cas == cas + + @pytest.mark.parametrize('cls', [TransactionQueryOptions, TransactionConfig, TransactionOptions]) + @pytest.mark.parametrize('consistency', [QueryScanConsistency.REQUEST_PLUS, + QueryScanConsistency.NOT_BOUNDED, + QueryScanConsistency.AT_PLUS]) + def test_scan_consistency(self, cls, consistency): + cfg = None + try: + cfg = cls(scan_consistency=consistency) + except Exception: + if consistency != QueryScanConsistency.AT_PLUS: + pytest.fail("got unexpected exception creating TransactionConfig", True) + if cfg: + cfg_consistency = cfg._base.to_dict().get('scan_consistency', None) + assert cfg_consistency is not None + assert cfg_consistency == consistency.value + + def test_scope_qualifier(self, cb_env): + pytest.skip('CBD-5091: Pending Transactions changes') + cfg = TransactionQueryOptions(scope=cb_env.collection._scope) + cfg_scope_qualifier = cfg._base.to_dict().get('scope_qualifier', None) + expected = f'default:`{cb_env.collection._scope.bucket_name}`.`{cb_env.collection._scope.name}`' + assert cfg_scope_qualifier is not None + assert cfg_scope_qualifier == expected + bucket, scope = cfg.split_scope_qualifier() + assert bucket == cb_env.collection._scope.bucket_name + assert scope == cb_env.collection._scope.name + + @pytest.mark.parametrize('cls', [TransactionConfig, TransactionOptions]) + @pytest.mark.parametrize('exp', [timedelta(seconds=30), timedelta(milliseconds=100)]) + def test_timeout(self, cls, exp): + cfg = cls(timeout=exp) + cfg_timeout = cfg._base.to_dict().get('timeout', None) + assert cfg_timeout is not None + assert cfg_timeout == exp.total_seconds() * 1000*1000*1000 # nanoseconds - and can't use 'is' here + + @pytest.mark.parametrize('cls', [TransactionConfig, TransactionOptions]) + @pytest.mark.parametrize('level', [DurabilityLevel.NONE, + DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE, + DurabilityLevel.MAJORITY, + DurabilityLevel.PERSIST_TO_MAJORITY]) + def test_transaction_config_durability(self, cls, level): + cfg = cls(durability=ServerDurability(level)) + cfg_level = cfg._base.to_dict().get('durability_level', None) + assert cfg_level is not None + assert DurabilityLevel(cfg_level) is level + + @pytest.mark.asyncio + async def test_transaction_result(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + + async def txn_logic(ctx): + await ctx.insert(cb_env.collection, key, {'some': 'thing'}) + doc = await ctx.get(cb_env.collection, key) + await ctx.replace(doc, {'some': 'thing else'}) + + result = await cb_env.cluster.transactions.run(txn_logic) + assert isinstance(result, TransactionResult) is True + assert result.transaction_id is not None + assert result.unstaging_complete is True + + +class ClassicTransactionTests(TransactionTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicTransactionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicTransactionTests) if valid_test_method(meth)] + compare = set(TransactionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, acb_base_txn_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + EnvironmentFeatures.check_if_feature_supported('txns', + acb_base_txn_env.server_version_short, + acb_base_txn_env.mock_server_type) + + await acb_base_txn_env.setup(request.param, __name__) + yield acb_base_txn_env + await acb_base_txn_env.teardown(request.param, __name__) diff --git a/acouchbase/tests/transcoder_t.py b/acouchbase/tests/transcoder_t.py new file mode 100644 index 000000000..e0554d3ad --- /dev/null +++ b/acouchbase/tests/transcoder_t.py @@ -0,0 +1,968 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta +from typing import Any, Tuple + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import (DocumentLockedException, + DocumentNotFoundException, + ValueFormatException) +from couchbase.options import (GetAndLockOptions, + GetAndTouchOptions, + GetOptions, + InsertOptions, + ReplaceOptions, + UpsertOptions) +from couchbase.transcoder import (LegacyTranscoder, + RawBinaryTranscoder, + RawJSONTranscoder, + RawStringTranscoder, + Transcoder) + +from ._test_utils import (CollectionType, + FakeTestObj, + KVPair, + TestEnvironment) + + +class ZeroFlagsTranscoder(Transcoder): + def encode_value(self, + value, # type: Any + ) -> Tuple[bytes, int]: + return json.dumps(value, ensure_ascii=False).encode('utf-8'), 0 + + def decode_value(self, + value, # type: bytes + flags # type: int + ) -> Any: + # ignoring flags...only for test purposes + return json.loads(value.decode('utf-8')) + + +class DefaultTranscoderTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + manage_buckets=True) + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name="new_kvp") + async def new_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = await cb_env.get_new_key_value() + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5) + @pytest.mark.asyncio + async def test_default_tc_json_upsert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + + await cb.upsert(key, value) + res = await cb.get(key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + @pytest.mark.asyncio + async def test_default_tc_json_insert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb.insert(key, value) + + res = cb.get(key) + res = await cb.get(key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + @pytest.mark.asyncio + async def test_default_tc_json_replace(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + await cb.upsert(key, value) + value['new_content'] = 'new content!' + await cb.replace(key, value) + res = await cb.get(key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + @pytest.mark.asyncio + async def test_default_tc_flags_zero(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + + await cb.upsert(key, value, transcoder=ZeroFlagsTranscoder()) + res = await cb.get(key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + # default TC: no transcoder set in ClusterOptions or KV options + + @pytest.mark.asyncio + async def test_default_tc_string_upsert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = "some string content" + await cb.upsert(key, value) + res = await cb.get(key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == value + + @pytest.mark.asyncio + async def test_default_tc_string_insert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = "some string content" + await cb.insert(key, value) + res = await cb.get(key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == value + + @pytest.mark.asyncio + async def test_default_tc_string_replace(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = "some string content" + await cb.upsert(key, value) + new_content = "new string content" + await cb.replace(key, new_content) + res = await cb.get(key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == new_content + + @pytest.mark.asyncio + async def test_default_tc_binary_upsert(self, cb_env): + cb = cb_env.collection + content = bytes(json.dumps("Here are some bytes"), "utf-8") + with pytest.raises(ValueFormatException): + await cb.upsert('some-test-bytes', content) + + @pytest.mark.asyncio + async def test_default_tc_bytearray_upsert(self, cb_env): + cb = cb_env.collection + content = bytearray(json.dumps("Here are some bytes"), "utf-8") + with pytest.raises(ValueFormatException): + await cb.upsert('some-test-bytes', content) + + @pytest.mark.asyncio + async def test_default_tc_binary_insert(self, cb_env): + cb = cb_env.collection + content = bytes(json.dumps("Here are some bytes"), "utf-8") + with pytest.raises(ValueFormatException): + await cb.insert('somet-test-bytes', content) + + @pytest.mark.asyncio + async def test_default_tc_binary_replace(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = "some string content" + await cb.upsert(key, value) + new_content = bytes(json.dumps("Here are some newer bytes"), "utf-8") + with pytest.raises(ValueFormatException): + await cb.replace(key, new_content) + + +class RawJsonTranscoderTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + transcoder=RawJSONTranscoder()) + + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name="str_kvp") + async def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="bytes_kvp") + async def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="json_kvp") + async def json_value_with_reset(self, cb_env) -> KVPair: + key, value = await cb_env.get_new_key_value() + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_raw_json_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes].decode('utf-8') + + @pytest.mark.asyncio + async def test_raw_json_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + await cb_env.collection.insert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes].decode('utf-8') + + @pytest.mark.asyncio + async def test_raw_json_tc_string_replace(self, cb_env, str_kvp): + key, value = str_kvp + await cb_env.collection.upsert(key, value) + new_content = "new string content" + await cb_env.collection.replace(key, new_content) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes].decode('utf-8') + + @pytest.mark.asyncio + async def test_raw_json_tc_bytes_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_raw_json_tc_bytes_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.insert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_raw_json_tc_bytes_replace(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + new_content = 'new string content'.encode('utf-8') + await cb_env.collection.replace(key, new_content) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_pass_through(self, cb_env, json_kvp): + key, value = json_kvp + json_str = json.dumps(value) + await cb_env.collection.upsert(key, json_str) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] != value + + decoded = json.loads(res.content_as[bytes].decode('utf-8')) + assert decoded == value + + @pytest.mark.asyncio + async def test_raw_json_tc_json_upsert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.upsert(key, value) + + @pytest.mark.asyncio + async def test_raw_json_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.insert(key, value) + + @pytest.mark.asyncio + async def test_raw_json_tc_json_replace(self, cb_env, str_kvp, json_kvp): + key, value = str_kvp + await cb_env.collection.upsert(key, value) + with pytest.raises(ValueFormatException): + await cb_env.collection.replace(key, json_kvp.value) + + +class RawStringTranscoderTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + transcoder=RawStringTranscoder()) + + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name="str_kvp") + async def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="bytes_kvp") + async def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="json_kvp") + async def json_value_with_reset(self, cb_env) -> KVPair: + key, value = await cb_env.get_new_key_value() + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_raw_string_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + @pytest.mark.asyncio + async def test_raw_string_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + await cb_env.collection.insert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + @pytest.mark.asyncio + async def test_raw_string_tc_string_replace(self, cb_env, str_kvp): + key, value = str_kvp + await cb_env.collection.upsert(key, value) + new_content = "new string content" + await cb_env.collection.replace(key, new_content) + res = await cb_env.collection.get(key) + assert isinstance(res.value, str) + assert new_content == res.content_as[str] + + @pytest.mark.asyncio + async def test_raw_string_tc_bytes_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.upsert(key, value) + + @pytest.mark.asyncio + async def test_raw_string_tc_bytes_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.insert(key, value) + + @pytest.mark.asyncio + async def test_raw_string_tc_bytes_replace(self, cb_env, str_kvp, bytes_kvp): + key, value = str_kvp + await cb_env.collection.upsert(key, value) + with pytest.raises(ValueFormatException): + await cb_env.collection.replace(key, bytes_kvp.value) + + @pytest.mark.asyncio + async def test_raw_string_tc_json_upsert(self, cb_env, json_kvp): + key = json_kvp.key + value = json_kvp.value + with pytest.raises(ValueFormatException): + await cb_env.collection.upsert(key, value) + + @pytest.mark.asyncio + async def test_raw_string_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.insert(key, value) + + @pytest.mark.asyncio + async def test_raw_string_tc_json_replace(self, cb_env, str_kvp, json_kvp): + key, value = str_kvp + await cb_env.collection.upsert(key, value) + with pytest.raises(ValueFormatException): + await cb_env.collection.replace(key, json_kvp.value) + + +class RawBinaryTranscoderTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + transcoder=RawBinaryTranscoder()) + + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name="str_kvp") + async def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="bytes_kvp") + async def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="hex_kvp") + async def hex_value_with_reset(self, cb_env) -> KVPair: + key = 'key_hex_bytes' + hex_arr = ['ff0102030405060708090a0b0c0d0e0f', + '101112131415161718191a1b1c1d1e1f', + '202122232425262728292a2b2c2d2e2f', + '303132333435363738393a3b3c3d3e3f'] + value = bytes.fromhex(''.join(hex_arr)) + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="json_kvp") + async def json_value_with_reset(self, cb_env) -> KVPair: + key, value = await cb_env.get_new_key_value() + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_raw_binary_tc_bytes_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_raw_binary_tc_bytes_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.insert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_raw_binary_tc_bytes_replace(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + new_content = 'new string content'.encode('utf-8') + await cb_env.collection.replace(key, new_content) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_raw_binary_tc_hex_upsert(self, cb_env, hex_kvp): + key, value = hex_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_raw_binary_tc_hex_insert(self, cb_env, hex_kvp): + key, value = hex_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_raw_binary_tc_hex_replace(self, cb_env, hex_kvp): + key, value = hex_kvp + await cb_env.collection.upsert(key, value) + new_content = b'\xFF' + await cb_env.collection.replace(key, new_content) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_raw_binary_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.upsert(key, value) + + @pytest.mark.asyncio + async def test_raw_binary_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.insert(key, value) + + @pytest.mark.asyncio + async def test_raw_binary_tc_string_replace(self, cb_env, bytes_kvp, str_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + with pytest.raises(ValueFormatException): + await cb_env.collection.replace(key, str_kvp.value) + + @pytest.mark.asyncio + async def test_raw_binary_tc_json_upsert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.upsert(key, value) + + @pytest.mark.asyncio + async def test_raw_binary_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + await cb_env.collection.insert(key, value) + + @pytest.mark.asyncio + async def test_raw_binary_tc_json_replace(self, cb_env, bytes_kvp, json_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + with pytest.raises(ValueFormatException): + await cb_env.collection.replace(key, json_kvp.value) + + +class LegacyTranscoderTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + request.param, + transcoder=LegacyTranscoder()) + + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name="str_kvp") + async def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="bytes_kvp") + async def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="json_kvp") + async def json_value_with_reset(self, cb_env) -> KVPair: + key, value = await cb_env.get_new_key_value() + yield KVPair(key, value) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="obj_kvp") + async def obj_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_obj' + yield KVPair(key, FakeTestObj()) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_legacy_tc_bytes_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_legacy_tc_bytes_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.insert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_legacy_tc_bytes_replace(self, cb_env, bytes_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + new_content = 'new string content'.encode('utf-8') + await cb_env.collection.replace(key, new_content) + res = await cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + @pytest.mark.asyncio + async def test_legacy_tc_obj_upsert(self, cb_env, obj_kvp): + key, value = obj_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, FakeTestObj) + assert value.PROP == res.value.PROP + assert value.PROP1 == res.value.PROP1 + + @pytest.mark.asyncio + async def test_legacy_tc_obj_insert(self, cb_env, obj_kvp): + key, value = obj_kvp + await cb_env.collection.insert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, FakeTestObj) + assert value.PROP == res.value.PROP + assert value.PROP1 == res.value.PROP1 + + @pytest.mark.asyncio + async def test_legacy_tc_obj_replace(self, cb_env, bytes_kvp, obj_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + await cb_env.collection.replace(key, obj_kvp.value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, FakeTestObj) + assert obj_kvp.value.PROP == res.value.PROP + assert obj_kvp.value.PROP1 == res.value.PROP1 + + @pytest.mark.asyncio + async def test_legacy_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + @pytest.mark.asyncio + async def test_legacy_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + await cb_env.collection.insert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + @pytest.mark.asyncio + async def test_legacy_tc_string_replace(self, cb_env, bytes_kvp, str_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + await cb_env.collection.replace(key, str_kvp.value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, str) + assert str_kvp.value == res.content_as[str] + + @pytest.mark.asyncio + async def test_legacy_tc_json_upsert(self, cb_env, json_kvp): + key, value = json_kvp + await cb_env.collection.upsert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + @pytest.mark.asyncio + async def test_legacy_tc_flags_zero(self, cb_env, json_kvp): + key, value = json_kvp + await cb_env.collection.upsert(key, value, transcoder=ZeroFlagsTranscoder()) + res = await cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + @pytest.mark.asyncio + async def test_legacy_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + await cb_env.collection.insert(key, value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + @pytest.mark.asyncio + async def test_legacy_tc_json_replace(self, cb_env, bytes_kvp, json_kvp): + key, value = bytes_kvp + await cb_env.collection.upsert(key, value) + await cb_env.collection.replace(key, json_kvp.value) + res = await cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert json_kvp.value == res.content_as[dict] + + +class KeyValueOpTranscoderTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + async def couchbase_test_environment(self, couchbase_config, request): + cb_env = await TestEnvironment.get_environment(__name__, couchbase_config, request.param) + + if request.param == CollectionType.NAMED: + await cb_env.try_n_times(5, 3, cb_env.setup_named_collections) + + yield cb_env + if request.param == CollectionType.NAMED: + await cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False) + + @pytest_asyncio.fixture(name="str_kvp") + async def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture(name="bytes_kvp") + async def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + await cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.asyncio + async def test_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + # use RawBinaryTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + await cb_env.collection.upsert(key, value, UpsertOptions( + transcoder=RawBinaryTranscoder())) + with pytest.raises(ValueFormatException): + await cb_env.collection.get(key) + + @pytest.mark.asyncio + async def test_insert(self, cb_env, str_kvp): + key, value = str_kvp + # use RawStringTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + await cb_env.collection.upsert(key, value, InsertOptions(transcoder=RawStringTranscoder())) + with pytest.raises(ValueFormatException): + await cb_env.collection.get(key) + + @pytest.mark.asyncio + async def test_replace(self, cb_env, bytes_kvp): + key, value = bytes_kvp + # use RawBinaryTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + tc = RawBinaryTranscoder() + await cb_env.collection.upsert(key, value, UpsertOptions(transcoder=tc)) + new_content = 'some new bytes content'.encode('utf-8') + await cb_env.collection.replace(key, new_content, ReplaceOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + await cb_env.collection.get(key) + + @pytest.mark.asyncio + async def test_get(self, cb_env, bytes_kvp): + key, value = bytes_kvp + tc = RawBinaryTranscoder() + await cb_env.collection.upsert(key, value, UpsertOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + await cb_env.collection.get(key) + res = await cb_env.collection.get(key, GetOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + + @pytest.mark.asyncio + async def test_get_and_touch(self, cb_env, bytes_kvp): + key, value = bytes_kvp + tc = RawBinaryTranscoder() + await cb_env.collection.upsert(key, value, UpsertOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + await cb_env.collection.get_and_touch(key, timedelta(seconds=30)) + + res = await cb_env.collection.get_and_touch(key, timedelta( + seconds=3), GetAndTouchOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + await cb_env.try_n_times_till_exception( + 10, 3, cb_env.collection.get, key, GetOptions(transcoder=tc), DocumentNotFoundException) + + @pytest.mark.asyncio + async def test_get_and_lock(self, cb_env, bytes_kvp): + key, value = bytes_kvp + tc = RawBinaryTranscoder() + await cb_env.collection.upsert(key, value, UpsertOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + await cb_env.collection.get_and_lock(key, timedelta(seconds=1)) + + await cb_env.try_n_times(10, 1, cb_env.collection.upsert, key, + value, UpsertOptions(transcoder=tc)) + res = await cb_env.collection.get_and_lock(key, timedelta( + seconds=3), GetAndLockOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + # upsert should definitely fail + with pytest.raises(DocumentLockedException): + await cb_env.collection.upsert(key, value, transcoder=tc) + # but succeed eventually + await cb_env.try_n_times(10, 1, cb_env.collection.upsert, key, value, transcoder=tc) diff --git a/acouchbase/tests/usermgmt_t.py b/acouchbase/tests/usermgmt_t.py new file mode 100644 index 000000000..58f9bcfd7 --- /dev/null +++ b/acouchbase/tests/usermgmt_t.py @@ -0,0 +1,720 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import re + +import pytest +import pytest_asyncio + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import (AuthenticationException, + CouchbaseException, + FeatureUnavailableException, + GroupNotFoundException, + InvalidArgumentException, + UserNotFoundException) +from couchbase.management.collections import CollectionSpec +from couchbase.management.options import (DropUserOptions, + GetUserOptions, + UpsertUserOptions) +from couchbase.management.users import (Group, + Role, + User) +from couchbase.options import ClusterOptions + +from ._test_utils import TestEnvironment + + +class UserManagementTests: + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_users=True) + yield cb_env + + @pytest.fixture(scope="class") + def check_collections_supported(self, cb_env): + cb_env.check_if_feature_supported('collections') + + @pytest.fixture(scope="class") + def check_user_groups_supported(self, cb_env): + cb_env.check_if_feature_supported('user_group_mgmt') + + @pytest.mark.asyncio + async def test_internal_user(self, cb_env): + """ + test_internal_user() + Tests create, retrieve, update and removal + of internal (domain_name="local") + Uses *UserOptions() for options + """ + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + initial_user = User(username=username, roles=roles, password=password) + + # create user + await cb_env.um.upsert_user( + User(username=username, roles=roles, password=password), + UpsertUserOptions(domain_name="local")) + + # get user + user_metadata = await cb_env.try_n_times(5, 1, cb_env.um.get_user, username, + GetUserOptions(domain_name="local")) + + # handle 7.0 roles w/ scopes/collections + test_roles = roles + if cb_env.is_feature_supported('collections'): + test_roles = [] + for r in roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + # update user + user = user_metadata.user + user.roles = Role('admin') + user.password = 's3cr3t_pa33w0rd' + + await cb_env.um.upsert_user(user, UpsertUserOptions(domain_name="local")) + + # get user and verify updates + user_metadata = await cb_env.try_n_times(5, 1, cb_env.um.get_user, username, + GetUserOptions(domain_name="local")) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=user.roles) + user_update = user_metadata.user + assert initial_user != user_update + + # remove user + await cb_env.um.drop_user(username, DropUserOptions(domain_name="local")) + await cb_env.try_n_times_till_exception( + 5, + 1, + cb_env.um.get_user, + username, + domain_name="local", + expected_exceptions=UserNotFoundException) + + @pytest.mark.asyncio + async def test_internal_user_kwargs(self, cb_env): + """ + test_internal_user_kwargs() + Tests create, retrieve, update and removal + of internal (domain_name="local") + Uses kwargs for options + """ + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + initial_user = User(username=username, roles=roles, password=password) + + # create user + await cb_env.um.upsert_user(initial_user, domain_name="local") + + # get single user + user_metadata = await cb_env.try_n_times(5, + 1, + cb_env.um.get_user, + username, + domain_name="local") + + # handle 7.0 roles w/ scopes/collections + test_roles = roles + if cb_env.is_feature_supported('collections'): + test_roles = [] + for r in roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + # update user + user = user_metadata.user + user.roles = Role('admin') + user.password = 's3cr3t_pa33w0rd' + + await cb_env.um.upsert_user(user, domain_name="local") + + # get user and verify updates + user_metadata = await cb_env.try_n_times(5, + 1, + cb_env.um.get_user, + username, + domain_name="local") + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=user.roles) + user_update = user_metadata.user + assert initial_user != user_update + + # remove user + await cb_env.um.drop_user(username, domain_name="local") + await cb_env.try_n_times_till_exception( + 5, + 1, + cb_env.um.get_user, + username, + domain_name="local", + expected_exceptions=UserNotFoundException) + + @pytest.mark.asyncio + async def test_user_change_password(self, cb_env): + username = 'change-password-user' + admin_role = Role(name='admin') + original_password = 'original_password' + new_password = 'new_password' + + change_password_user = User(username=username, + display_name="Change Password User", + roles=admin_role, + password=original_password) + # Upsert user + await cb_env.um.upsert_user(change_password_user, UpsertUserOptions(domain_name="local")) + + # Authenticate as change-password-user. + # Done in a while loop to emulate retry + auth = PasswordAuthenticator(username, original_password) + new_cluster = None + while True: + try: + new_cluster = await Cluster.connect(cb_env.cluster._connstr, ClusterOptions(auth)) + except AuthenticationException: + continue + break + + # Change password + await new_cluster.users().change_password(new_password) + + # Assert can authenticate using new password + success_auth = PasswordAuthenticator(username, new_password) + while True: + try: + success_cluster = await Cluster.connect(cb_env.cluster._connstr, ClusterOptions(success_auth)) + except AuthenticationException: + continue + await success_cluster.close() + break + + # Assert cannot authenticate using old password + fail_auth = PasswordAuthenticator(username, original_password) + with pytest.raises(AuthenticationException): + await Cluster.connect(cb_env.cluster._connstr, ClusterOptions(fail_auth)) + + await new_cluster.close() + + @pytest.mark.asyncio + async def test_internal_user_fail(self, cb_env): + """ + test_internal_user() + Tests create, retrieve, update and removal + of internal (domain_name="local") + Uses *UserOptions() for options + """ + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='not-a-bucket'), + Role(name='data_writer', bucket='not-a-bucket') + ] + + with pytest.raises(InvalidArgumentException): + await cb_env.um.upsert_user( + User(username=username, roles=roles, password=password), + UpsertUserOptions(domain_name="local")) + + @pytest.mark.asyncio + async def test_user_display_name(self, cb_env): + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + user = User(username='custom-user', + display_name="Custom User", + roles=roles, + password='s3cr3t') + + # create user + await cb_env.um.upsert_user(user, UpsertUserOptions(domain_name="local")) + + # get user + user_metadata = await cb_env.try_n_times(5, 1, cb_env.um.get_user, + user.username, + GetUserOptions(domain_name="local")) + + assert user_metadata.user.display_name == user.display_name + + await cb_env.um.drop_user(user.username, DropUserOptions(domain_name="local")) + + @pytest.mark.asyncio + async def test_external_user(self, cb_env): + """ + test_external_user() + Tests create, retrieve, update and removal + of external (domain_name="external") + Uses *UserOptions() for options + """ + + username = 'custom-user' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + initial_user = User(username=username, roles=roles) + # create user + await cb_env.um.upsert_user(initial_user, + UpsertUserOptions(domain_name="external")) + + # get user + user_metadata = await cb_env.try_n_times( + 5, 1, cb_env.um.get_user, username, + GetUserOptions(domain_name="external")) + + # handle 7.0 roles w/ scopes/collections + test_roles = roles + if cb_env.is_feature_supported('collections'): + test_roles = [] + for r in roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + # update user + user = user_metadata.user + user.roles = Role('admin') + + await cb_env.um.upsert_user(user, UpsertUserOptions(domain_name="external")) + + # get user and verify updates + user_metadata = await cb_env.try_n_times( + 5, 1, cb_env.um.get_user, username, + GetUserOptions(domain_name="external")) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=user.roles) + user_update = user_metadata.user + assert initial_user != user_update + + # remove user + await cb_env.um.drop_user(username, DropUserOptions(domain_name="external")) + await cb_env.try_n_times_till_exception( + 5, + 1, + cb_env.um.get_user, + username, + domain_name="external", + expected_exceptions=UserNotFoundException) + + @pytest.mark.asyncio + async def test_default_domain(self, cb_env): + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + + await cb_env.um.upsert_user( + User(username=username, password=password, roles=roles)) + + user_metadata = await cb_env.try_n_times(5, 1, cb_env.um.get_user, username) + assert user_metadata is not None + + # handle 7.0 roles w/ scopes/collections + test_roles = roles + if cb_env.is_feature_supported('collections'): + test_roles = [] + for r in roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + users_metadata = await cb_env.um.get_all_users() + assert users_metadata is not None + result = all( + map(lambda um: cb_env.validate_user_and_metadata(um), + users_metadata)) + assert result is True + + await cb_env.um.drop_user(username) + + @pytest.mark.asyncio + async def test_invalid_domain_raises_argument_error(self, cb_env): + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + + # invalid domain generates argument error + with pytest.raises(InvalidArgumentException): + await cb_env.um.get_all_users(domain_name="fake-domain") + + with pytest.raises(InvalidArgumentException): + await cb_env.um.get_user(username, domain_name="fake-domain") + + with pytest.raises(InvalidArgumentException): + await cb_env.um.upsert_user(User(username=username, + password=password, + roles=roles), + domain_name="fake-domain") + + with pytest.raises(InvalidArgumentException): + await cb_env.um.drop_user(username, domain_name="fake-domain") + + @pytest.mark.asyncio + async def test_external_nopassword(self, cb_env): + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + + # password with external generates argument error + with pytest.raises(InvalidArgumentException): + await cb_env.um.upsert_user(User(username=username, + password=password, + roles=roles), + domain_name="external") + + with pytest.raises(InvalidArgumentException): + await cb_env.um.upsert_user(User(username=username, + password=password, + roles=None), + domain_name="external") + + with pytest.raises(InvalidArgumentException): + await cb_env.um.upsert_user(User(username=username, password=password, roles=[]), + domain_name="external") + try: + await cb_env.um.upsert_user( + User(username=username, password=None, roles=roles), + UpsertUserOptions(domain_name="external")) + except InvalidArgumentException: + raise + except CouchbaseException: + pass + finally: + await cb_env.um.drop_user(username, domain_name="external") + + @pytest.mark.usefixtures("check_collections_supported") + @pytest.mark.asyncio + async def test_user_scopes_collections(self, cb_env): + + if cb_env.cm is None: + cm = cb_env.bucket.collections() + + await cm.create_scope('um-test-scope') + for _ in range(3): + scopes = await cm.get_all_scopes() + scope = next((s for s in scopes if s.name == 'um-test-scope'), None) + if scope: + break + await asyncio.sleep(1) + + collection = CollectionSpec('test-collection', scope_name='um-test-scope') + await cm.create_collection(collection) + for _ in range(3): + scopes = await cm.get_all_scopes() + scope = next((s for s in scopes if s.name == 'um-test-scope'), None) + if scope: + coll = next((c for c in scope.collections + if c.name == 'test-collection'), None) + if coll: + break + await asyncio.sleep(1) + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default', scope='um-test-scope'), + Role(name='data_writer', + bucket='default', + scope='um-test-scope', + collection='test-collection') + ] + initial_user = User(username=username, roles=roles, password=password) + + # create user + await cb_env.um.upsert_user(initial_user, domain_name="local") + + # get single user + user_metadata = await cb_env.try_n_times(5, + 1, + cb_env.um.get_user, + username, + domain_name="local") + + test_roles = [] + for r in roles: + if not r.collection: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope=r.scope, + collection='*')) + else: + test_roles.append(r) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + await cb_env.um.drop_user(username) + await cm.drop_collection(collection) + await cm.drop_scope('um-test-scope') + + @pytest.mark.asyncio + async def test_group_feature_not_found(self, cb_env): + if cb_env.is_feature_supported('user_group_mgmt'): + pytest.skip(f'Only test on server versions < 6.5. Using server version: {cb_env.server_version}') + + roles = Role(name='admin') + test_group = Group(name='my-test-group', + roles=roles, + description="test group description") + + with pytest.raises(FeatureUnavailableException): + await cb_env.um.upsert_group(test_group) + with pytest.raises(FeatureUnavailableException): + await cb_env.um.get_all_groups() + with pytest.raises(FeatureUnavailableException): + await cb_env.um.get_group(test_group.name) + with pytest.raises(FeatureUnavailableException): + await cb_env.um.drop_group(test_group.name) + + @pytest.mark.usefixtures("check_user_groups_supported") + @pytest.mark.asyncio + async def test_group(self, cb_env): + roles = Role(name='admin') + test_group = Group(name='my-test-group', + roles=roles, + description="test group description") + # add group + await cb_env.um.upsert_group(test_group) + + # get group + result = await cb_env.try_n_times(5, 1, cb_env.um.get_group, test_group.name) + cb_env.validate_group(result, test_group.roles) + + # remove group + await cb_env.um.drop_group(test_group.name) + await cb_env.try_n_times_till_exception( + 5, + 1, + cb_env.um.get_group, + test_group.name, + expected_exceptions=GroupNotFoundException) + + @pytest.mark.usefixtures("check_user_groups_supported") + @pytest.mark.asyncio + async def test_user_and_groups(self, cb_env): + user_roles = [ + Role(name='query_select', bucket='default'), + Role(name='fts_searcher', bucket='default') + ] + group_roles = [ + Role(name='data_reader', bucket='*'), + Role(name='data_writer', bucket='*') + ] + groups = [ + Group(name='my-test-group', + roles=group_roles, + description="test group description"), + Group(name='my-test-group-1', + roles=Role(name='admin'), + description="test group description") + ] + + # add groups + for group in groups: + await cb_env.um.upsert_group(group) + await cb_env.try_n_times(5, 1, cb_env.um.get_group, group.name) + user_groups = list(map(lambda g: g.name, groups)) + + # add user + test_user = User(username='custom-user', + roles=user_roles, + groups=user_groups, + password='s3cr3t') + await cb_env.um.upsert_user(test_user, domain_name="local") + + # get user + user_metadata = await cb_env.try_n_times(5, + 1, + cb_env.um.get_user, + test_user.username, + domain_name="local") + + # handle 7.0 roles w/ scopes/collections + test_roles = user_roles + if cb_env.is_feature_supported('collections'): + test_roles = [] + for r in user_roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, + user_roles=test_roles, + groups=groups) + + # remove group + remove_group = groups.pop() + await cb_env.um.drop_group(remove_group.name) + await cb_env.try_n_times_till_exception( + 5, + 1, + cb_env.um.get_group, + remove_group.name, + expected_exceptions=GroupNotFoundException) + + # get user to verify roles from removed group are removed + user_metadata = await cb_env.try_n_times(5, + 1, + cb_env.um.get_user, + test_user.username, + domain_name="local") + + # handle 7.0 roles w/ scopes/collections + if cb_env.is_feature_supported('collections'): + test_roles = [] + for r in user_roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, + user_roles=test_roles, + groups=groups) + + # cleanup + await cb_env.um.drop_user(test_user.username, domain_name="local") + for group in groups: + await cb_env.um.drop_group(group.name) + + @pytest.mark.flaky(reruns=5) + @pytest.mark.usefixtures("check_user_groups_supported") + @pytest.mark.asyncio + async def test_get_all_groups(self, cb_env): + roles = [ + Role(name='data_reader', bucket='*'), + Role(name='data_writer', bucket='*') + ] + fresh_group = Group(name='my-test-group', + roles=roles, + description="test group description") + await cb_env.um.upsert_group(fresh_group) + admin_group = Group('admin-test-group', roles=[Role(name='admin')]) + await cb_env.um.upsert_group(admin_group) + all_groups = await cb_env.um.get_all_groups() + # NOTE: we could well have other groups on this server, apart from the one we added, so + # lets be ok with there being more of them. However, the one we added + # _MUST_ be there. + assert len(all_groups) >= 2 + admin_group_dict = admin_group.as_dict() + found = False + for g in all_groups: + if admin_group_dict == g.as_dict(): + found = True + + await cb_env.um.drop_group('my-test-group') + await cb_env.um.drop_group('admin-test-group') + assert found is True + + # @TODO: is this test useful? + # @pytest.mark.usefixtures("check_user_groups_supported") + # @pytest.mark.asyncio + # async def test_timeout(self, cb_env): + # with pytest.raises(AmbiguousTimeoutException): + # await cb_env.um.get_all_groups(timeout=timedelta(seconds=0.1)) + + @pytest.mark.asyncio + async def test_get_roles(self, cb_env): + roles = await cb_env.um.get_roles() + admin_desc = re.compile( + r'.*all cluster features.*web console.*read and write all data.*$') + for rad in reversed(roles): + desc_matches = admin_desc.match(rad.description) + if desc_matches: + assert rad.role.name == 'admin' + assert rad.display_name == 'Full Admin' + return + pytest.fail("No admin role found") + + # see PYCBC-1030 + @pytest.mark.asyncio + async def test_get_roles_all_valid(self, cb_env): + roles = await cb_env.um.get_roles() + for r in roles: + assert r is not None + + @pytest.mark.usefixtures("check_user_groups_supported") + @pytest.mark.asyncio + async def test_missing_group(self, cb_env): + with pytest.raises(GroupNotFoundException): + await cb_env.um.get_group('fred') + + @pytest.mark.asyncio + async def test_missing_user(self, cb_env): + with pytest.raises(UserNotFoundException): + await cb_env.um.get_user('keith') diff --git a/acouchbase/tests/viewmgmt_t.py b/acouchbase/tests/viewmgmt_t.py new file mode 100644 index 000000000..1413e19e3 --- /dev/null +++ b/acouchbase/tests/viewmgmt_t.py @@ -0,0 +1,204 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pathlib +from datetime import timedelta +from os import path + +import pytest +import pytest_asyncio + +from acouchbase.cluster import get_event_loop +from couchbase.exceptions import DesignDocumentNotFoundException +from couchbase.management.options import (GetAllDesignDocumentsOptions, + GetDesignDocumentOptions, + PublishDesignDocumentOptions) +from couchbase.management.views import (DesignDocument, + DesignDocumentNamespace, + View) + +from ._test_utils import TestEnvironment + + +@pytest.mark.flaky(reruns=5, reruns_delay=2) +class ViewIndexManagementTests: + + TEST_VIEW_NAME = 'test-view' + TEST_VIEW_PATH = path.join(pathlib.Path(__file__).parent.parent.parent, + 'tests', + 'test_cases', + f'{TEST_VIEW_NAME}.txt') + + DOCNAME = 'test-ddoc' + + @pytest_asyncio.fixture(scope="class") + def event_loop(self): + loop = get_event_loop() + yield loop + loop.close() + + @pytest_asyncio.fixture(scope="class", name="cb_env") + async def couchbase_test_environment(self, couchbase_config): + cb_env = await TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True, + manage_view_indexes=True) + + yield cb_env + + @pytest.fixture(scope='class') + def test_ddoc(self): + view_data = None + with open(self.TEST_VIEW_PATH) as view_file: + view_data = view_file.read() + + view = View(map=view_data) + ddoc = DesignDocument(name=self.DOCNAME, views={self.TEST_VIEW_NAME: view}) + return ddoc + + @pytest_asyncio.fixture() + async def create_test_view(self, cb_env, test_ddoc): + await cb_env.try_n_times(3, 5, + cb_env.vixm.upsert_design_document, + test_ddoc, + DesignDocumentNamespace.DEVELOPMENT) + + @pytest_asyncio.fixture() + async def drop_test_view(self, cb_env, test_ddoc): + yield + await cb_env.try_n_times_till_exception(10, 1, + cb_env.vixm.drop_design_document, + test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + expected_exceptions=(DesignDocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest_asyncio.fixture() + async def drop_test_view_from_prod(self, cb_env, test_ddoc): + yield + # drop from PROD + try: + await cb_env.vixm.drop_design_document(test_ddoc.name, DesignDocumentNamespace.PRODUCTION) + except DesignDocumentNotFoundException: + pass + except Exception as ex: + raise ex + # now drop from DEV + try: + await cb_env.vixm.drop_design_document(test_ddoc.name, DesignDocumentNamespace.DEVELOPMENT) + except DesignDocumentNotFoundException: + pass + except Exception as ex: + raise ex + + @pytest.mark.usefixtures("drop_test_view") + @pytest.mark.asyncio + async def test_upsert_design_doc(self, cb_env, test_ddoc): + # we started with this already in here, so this isn't really + # necessary...` + await cb_env.vixm.upsert_design_document(test_ddoc, DesignDocumentNamespace.DEVELOPMENT) + + @pytest.mark.usefixtures("create_test_view") + @pytest.mark.asyncio + async def test_drop_design_doc(self, cb_env, test_ddoc): + await cb_env.vixm.drop_design_document(test_ddoc.name, DesignDocumentNamespace.DEVELOPMENT) + + @pytest.mark.asyncio + async def test_drop_design_doc_fail(self, cb_env, test_ddoc): + with pytest.raises(DesignDocumentNotFoundException): + await cb_env.vixm.drop_design_document(test_ddoc.name, DesignDocumentNamespace.PRODUCTION) + + @pytest.mark.usefixtures("create_test_view") + @pytest.mark.usefixtures("drop_test_view") + @pytest.mark.asyncio + async def test_get_design_document_fail(self, cb_env, test_ddoc): + with pytest.raises(DesignDocumentNotFoundException): + await cb_env.vixm.get_design_document(test_ddoc.name, + DesignDocumentNamespace.PRODUCTION, + GetDesignDocumentOptions(timeout=timedelta(seconds=5))) + + @pytest.mark.usefixtures("create_test_view") + @pytest.mark.usefixtures("drop_test_view") + @pytest.mark.asyncio + async def test_get_design_document(self, cb_env, test_ddoc): + ddoc = await cb_env.try_n_times(10, 3, + cb_env.vixm.get_design_document, + test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + GetDesignDocumentOptions(timeout=timedelta(seconds=5))) + assert ddoc is not None + assert ddoc.name == test_ddoc.name + + @pytest.mark.usefixtures("create_test_view") + @pytest.mark.usefixtures("drop_test_view") + @pytest.mark.asyncio + async def test_get_all_design_documents(self, cb_env, test_ddoc): + # should start out in _some_ state. Since we don't know for sure, but we + # do know it does have self.DOCNAME in it in development ONLY, lets assert on that and that + # it succeeds, meaning we didn't get an exception. + # make sure it is there + ddoc = await cb_env.try_n_times(10, 3, + cb_env.vixm.get_design_document, + test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + GetDesignDocumentOptions(timeout=timedelta(seconds=5))) + assert ddoc is not None + # should also be in the get_all response + result = await cb_env.vixm.get_all_design_documents(DesignDocumentNamespace.DEVELOPMENT, + GetAllDesignDocumentsOptions(timeout=timedelta(seconds=10))) + names = [doc.name for doc in result if doc.name == test_ddoc.name] + assert names.count(test_ddoc.name) > 0 + + @pytest.mark.usefixtures("create_test_view") + @pytest.mark.usefixtures("drop_test_view") + @pytest.mark.asyncio + async def test_get_all_design_documents_excludes_namespaces(self, cb_env, test_ddoc): + # we know the test_ddoc.name is _only_ in development, so... + result = await cb_env.vixm.get_all_design_documents(DesignDocumentNamespace.PRODUCTION) + names = [doc.name for doc in result if doc.name == test_ddoc.name] + assert names.count(test_ddoc.name) == 0 + + @pytest.mark.usefixtures("create_test_view") + @pytest.mark.usefixtures("drop_test_view_from_prod") + @pytest.mark.asyncio + async def test_publish_design_doc(self, cb_env, test_ddoc): + # make sure we have the ddoc + ddoc = await cb_env.try_n_times(10, 3, + cb_env.vixm.get_design_document, + test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + GetDesignDocumentOptions(timeout=timedelta(seconds=5))) + assert ddoc is not None + # starts off not in prod + with pytest.raises(DesignDocumentNotFoundException): + await cb_env.vixm.get_design_document(test_ddoc.name, DesignDocumentNamespace.PRODUCTION) + + await cb_env.vixm.publish_design_document(test_ddoc.name, + PublishDesignDocumentOptions(timeout=timedelta(seconds=10))) + # should be in prod now + await cb_env.try_n_times( + 10, + 3, + cb_env.vixm.get_design_document, + test_ddoc.name, + DesignDocumentNamespace.PRODUCTION) + # and still in dev + await cb_env.try_n_times( + 10, + 3, + cb_env.vixm.get_design_document, + test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT) diff --git a/acouchbase/tests/views_t.py b/acouchbase/tests/views_t.py new file mode 100644 index 000000000..1d12cdb24 --- /dev/null +++ b/acouchbase/tests/views_t.py @@ -0,0 +1,323 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta + +import pytest +import pytest_asyncio + +from couchbase.exceptions import (AmbiguousTimeoutException, + DesignDocumentNotFoundException, + InvalidArgumentException) +from couchbase.management.views import DesignDocumentNamespace +from couchbase.options import ViewOptions +from couchbase.views import (ViewMetaData, + ViewOrdering, + ViewRow) +from tests.environments import CollectionType +from tests.environments.views_environment import AsyncViewsTestEnvironment + + +class ViewsTestSuite: + TEST_MANIFEST = [ + 'test_bad_view_query', + 'test_view_query', + 'test_view_query_ascending', + 'test_view_query_descending', + 'test_view_query_endkey_docid', + 'test_view_query_key', + 'test_view_query_keys', + 'test_view_query_raw', + 'test_view_query_raw_fail', + 'test_view_query_startkey_docid', + 'test_view_query_timeout', + ] + + @pytest.mark.asyncio + async def test_bad_view_query(self, cb_env): + view_result = cb_env.bucket.view_query('fake-ddoc', + 'fake-view', + limit=10, + namespace=DesignDocumentNamespace.DEVELOPMENT) + + with pytest.raises(DesignDocumentNotFoundException): + [r async for r in view_result] + + @pytest.mark.asyncio + async def test_view_query(self, cb_env): + expected_count = 10 + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + full_set=True, + namespace=DesignDocumentNamespace.DEVELOPMENT) + + await cb_env.assert_rows(view_result, expected_count) + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + @pytest.mark.asyncio + async def test_view_query_ascending(self, cb_env): + expected_count = 5 + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + limit=expected_count, + namespace=DesignDocumentNamespace.DEVELOPMENT, + order=ViewOrdering.ASCENDING) + + rows = await cb_env.assert_rows(view_result, expected_count, return_rows=True) + row_ids = list(map(lambda r: r.id, rows)) + sorted_row_ids = sorted(row_ids) + assert row_ids == sorted_row_ids + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + @pytest.mark.asyncio + async def test_view_query_descending(self, cb_env): + expected_count = 5 + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + limit=expected_count, + namespace=DesignDocumentNamespace.DEVELOPMENT, + order=ViewOrdering.DESCENDING) + + rows = await cb_env.assert_rows(view_result, expected_count, return_rows=True) + row_ids = list(map(lambda r: r.id, rows)) + sorted_row_ids = sorted(row_ids, reverse=True) + assert row_ids == sorted_row_ids + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + @pytest.mark.asyncio + async def test_view_query_endkey_docid(self, cb_env): + # set the batch id + cb_env.get_batch_id() + keys = cb_env.get_keys() + # take the 2nd-smallest key so that we can purposefully select results of previous key group + key_idx = keys.index(sorted(keys)[1]) + key = keys[key_idx] + key_docids = cb_env.get_docids_by_key(key) + # purposefully select a docid in the middle of key group + endkey_docid = key_docids[4] + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, + endkey=key, + endkey_docid=endkey_docid) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # all docs w/in first key (10 docs) + half of docs w/in next key (5 docs) + expected_count = 15 + rows = await cb_env.assert_rows(view_result, expected_count, True) + # last doc in results should be the endkey and endkey_docid + assert rows[-1].key == key + assert rows[-1].id == endkey_docid + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + @pytest.mark.asyncio + async def test_view_query_key(self, cb_env): + batch_id = cb_env.get_batch_id() + expected_count = 10 + keys = cb_env.get_keys() + key = keys[0] + docids = cb_env.get_docids_by_key(key) + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, + key=key) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + + rows = await cb_env.assert_rows(view_result, expected_count, True) + for row in rows: + assert isinstance(row, ViewRow) + assert isinstance(row.id, str) + assert isinstance(row.key, str) + assert isinstance(row.value, dict) + assert row.key == key + assert row.value['batch'] == batch_id + assert row.value['id'] in docids + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + @pytest.mark.asyncio + async def test_view_query_keys(self, cb_env): + keys = cb_env.get_keys() + expected_count = 20 + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, + keys=keys[:2]) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + + rows = await cb_env.assert_rows(view_result, expected_count, True) + assert all(map(lambda r: r.key in keys, rows)) is True + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + @pytest.mark.asyncio + async def test_view_query_raw(self, cb_env): + # set the batch id + cb_env.get_batch_id() + keys = cb_env.get_keys() + # take the largest key so that we can purposefully narrow the result to 1 record + key_idx = keys.index(max(keys)) + key = keys[key_idx] + key_docids = cb_env.get_docids_by_key(key) + # purposefully use the last doc w/in the key + startkey_docid = key_docids[-1] + # execute a query so we can have pagination + opts = ViewOptions(limit=5, + namespace=DesignDocumentNamespace.DEVELOPMENT) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # need to iterate over the result to execute the query + [r async for r in view_result] + raw = { + 'limit': '5', + 'startkey': json.dumps(key), + 'startkey_docid': startkey_docid, + 'full_set': 'true' + } + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, raw=raw) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # expect only a single record to be returned + expected_count = 1 + rows = await cb_env.assert_rows(view_result, expected_count, True) + assert rows[0].key == key + assert rows[0].id == startkey_docid + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + @pytest.mark.asyncio + async def test_view_query_raw_fail(self, cb_env): + raw = { + 'limit': '5', + # this will fail as it is not encoded JSON + 'startkey': 'view-key', + 'startkey_docid': 'fake-doc-id' + } + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, raw=raw) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + with pytest.raises(InvalidArgumentException): + [r async for r in view_result] + + @pytest.mark.asyncio + async def test_view_query_startkey_docid(self, cb_env): + # set the batch id + cb_env.get_batch_id() + keys = cb_env.get_keys() + # take the largest key so that we can purposefully narrow the result to 1 record + key_idx = keys.index(max(keys)) + key = keys[key_idx] + key_docids = cb_env.get_docids_by_key(key) + # purposefully use the last doc w/in the key + startkey_docid = key_docids[-1] + # execute a query so we can have pagination + opts = ViewOptions(limit=5, + namespace=DesignDocumentNamespace.DEVELOPMENT) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # need to iterate over the result to execute the query + [r async for r in view_result] + opts = ViewOptions(limit=5, + namespace=DesignDocumentNamespace.DEVELOPMENT, + startkey=key, + startkey_docid=startkey_docid) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # expect only a single record to be returned + expected_count = 1 + rows = await cb_env.assert_rows(view_result, expected_count, True) + assert rows[0].key == key + assert rows[0].id == startkey_docid + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.asyncio + async def test_view_query_timeout(self, cb_env): + from acouchbase.cluster import Cluster + from couchbase.auth import PasswordAuthenticator + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + # Prior to PYCBC-1521, this test would fail as each request would override the cluster level views_timeout. + # If a timeout was not provided in the request, the default 75s timeout would be used. PYCBC-1521 corrects + # this behavior so this test will pass as we are essentially forcing an AmbiguousTimeoutException because + # we are setting the cluster level views_timeout such a small value. + timeout_opts = ClusterTimeoutOptions(views_timeout=timedelta(milliseconds=1)) + cluster = await Cluster.connect(f'{conn_string}', ClusterOptions(auth, timeout_options=timeout_opts)) + # don't need to do this except for older server versions + bucket = cluster.bucket(f'{cb_env.bucket.name}') + with pytest.raises(AmbiguousTimeoutException): + res = bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + limit=10, + namespace=DesignDocumentNamespace.DEVELOPMENT) + [r async for r in res.rows()] + # if we override the timeout w/in the request the query should succeed. + res = bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + limit=10, + namespace=DesignDocumentNamespace.DEVELOPMENT, + timeout=timedelta(seconds=10)) + rows = [r async for r in res.rows()] + assert len(rows) > 0 + + +class ClassicViewsTests(ViewsTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicViewsTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicViewsTests) if valid_test_method(meth)] + compare = set(ViewsTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest_asyncio.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + async def couchbase_test_environment(self, acb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + acb_env = AsyncViewsTestEnvironment.from_environment(acb_base_env) + acb_env.enable_views_mgmt() + await acb_env.setup(request.param) + yield acb_env + await acb_env.teardown(request.param) diff --git a/acouchbase/transactions/__init__.py b/acouchbase/transactions/__init__.py new file mode 100644 index 000000000..1a7a642a0 --- /dev/null +++ b/acouchbase/transactions/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .transactions import AttemptContext # noqa: F401 +from .transactions import Transactions # noqa: F401 diff --git a/acouchbase/transactions/transactions.py b/acouchbase/transactions/transactions.py new file mode 100644 index 000000000..edc7d38fb --- /dev/null +++ b/acouchbase/transactions/transactions.py @@ -0,0 +1,363 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from functools import wraps +from typing import (TYPE_CHECKING, + Any, + Awaitable, + Callable, + Coroutine, + Dict, + Optional) + +from couchbase.exceptions import ErrorMapper +from couchbase.exceptions import exception as BaseCouchbaseException +from couchbase.logic.supportability import Supportability +from couchbase.options import (TransactionGetOptions, + TransactionGetReplicaFromPreferredServerGroupOptions, + TransactionInsertOptions, + TransactionQueryOptions, + TransactionReplaceOptions) +from couchbase.transactions import (TransactionGetResult, + TransactionQueryResults, + TransactionResult) +from couchbase.transactions.logic import AttemptContextLogic, TransactionsLogic + +if TYPE_CHECKING: + from asyncio import AbstractEventLoop + + from acouchbase.cluster import AsyncCluster + from acouchbase.collection import AsyncCollection + from couchbase._utils import JSONType, PyCapsuleType + from couchbase.options import TransactionConfig, TransactionOptions + from couchbase.transcoder import Transcoder + +log = logging.getLogger(__name__) + + +class AsyncWrapper: + @staticmethod + def call_async_fn(ftr, self, fn, *args, **kwargs): + try: + fn(self, *args, **kwargs) + except SystemError as e: + ftr.set_exception(e.__cause__) + except Exception as e: + ftr.set_exception(e) + + @classmethod # noqa: C901 + def inject_callbacks(cls, return_cls): # noqa: C901 + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ftr = self._loop.create_future() + tc = kwargs.get('transcoder', self._transcoder) + + def on_ok(res): + log.debug('%s completed, with %s', fn.__name__, res) + # BUG(PYCBC-1476): We might not need this once txn bug is fixed + if isinstance(res, BaseCouchbaseException): + self._loop.call_soon_threadsafe(ftr.set_exception, ErrorMapper.build_exception(res)) + else: + try: + if return_cls is TransactionGetResult: + result = return_cls(res, tc) + else: + result = return_cls(res) if return_cls is not None else None + self._loop.call_soon_threadsafe(ftr.set_result, result) + except Exception as e: + log.error('on_ok raised %s, %s', e, e.__cause__) + self._loop.call_soon_threadsafe(ftr.set_exception, e) + + def on_err(exc): + log.error('%s got on_err called with %s', fn.__name__, exc) + try: + if not exc: + exc = RuntimeError(f'unknown error calling {fn.__name__}') + if isinstance(exc, BaseCouchbaseException): + self._loop.call_soon_threadsafe(ftr.set_exception, ErrorMapper.build_exception(exc)) + else: + self._loop.call_soon_threadsafe(ftr.set_exception, exc) + except Exception as e: + self._loop.call_soon_threadsafe(ftr.set_exception, e) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + AsyncWrapper.call_async_fn(ftr, self, fn, *args, **kwargs) + return ftr + + return wrapped_fn + + return decorator + + +class Transactions(TransactionsLogic): + + def __init__(self, + cluster, # type: AsyncCluster + config # type: TransactionConfig + ): + super().__init__(cluster, config) + + async def run(self, + txn_logic, # type: Callable[[AttemptContextLogic], Coroutine[Any, Any, None]] + transaction_options=None, # type: Optional[TransactionOptions] + **kwargs) -> TransactionResult: + opts = None + if transaction_options: + opts = transaction_options._base + if 'per_txn_config' in kwargs: + Supportability.method_param_deprecated('per_txn_config', 'transaction_options') + opts = kwargs.pop('per_txn_config', None) + + txn_result = await super().run_async(txn_logic, AttemptContext(self._txns, self._loop, self._transcoder, opts)) + return TransactionResult(**txn_result) + + # TODO: make async? + def close(self): + """ + Close the transactions. No transactions can be performed on this instance after this is called. There + is no need to call this, as the transactions close automatically when the transactions object goes out of scope. + + Returns: + None + """ + # stop transactions object -- ideally this is done before closing the cluster. + super().close() + log.info("transactions closed") + + +class AttemptContext(AttemptContextLogic): + + def __init__(self, + ctx, # type: PyCapsuleType + loop, # type: AbstractEventLoop + transcoder, # type: Transcoder + opts # type: Optional[PyCapsuleType] + ): + super().__init__(ctx, transcoder, loop, opts) + + @AsyncWrapper.inject_callbacks(None) + def _new_attempt(self, + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + return super()._new_attempt_async(**kwargs) + + @AsyncWrapper.inject_callbacks(None) + def _rollback(self, + **kwargs # type: Dict[str, Any] + ) -> Awaitable[None]: + return super()._rollback_async(**kwargs) + + @AsyncWrapper.inject_callbacks(TransactionResult) + def _commit(self, + **kwargs # type: Dict[str, Any] + ) -> Awaitable[TransactionResult]: + return super()._commit_async(**kwargs) + + @AsyncWrapper.inject_callbacks(TransactionGetResult) + def _get(self, + coll, # type: AsyncCollection + key, # type: str + **kwargs # type: Dict[str, Any] + ) -> Awaitable[TransactionGetResult]: + return super().get(coll, key, **kwargs) + + def get(self, + coll, # type: AsyncCollection + key, # type: str + options=None, # type: Optional[TransactionGetOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[TransactionGetResult]: + """ + Get a document within this transaction. + + Args: + coll (:class:`couchbase.collection.Collection`): Collection to use to find the document. + key (str): document key. + options (:class:`~couchbase.options.TransactionGetOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TransactionGetOptions` + + Returns: + Awaitable[:class:`couchbase.transactions.TransactionGetResult`]: Document in collection, in a form useful + for passing to other transaction operations. Or `None` if the document was not found. + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + + """ + if 'transcoder' not in kwargs and isinstance(options, TransactionGetOptions): + kwargs['transcoder'] = options.get('transcoder', None) + return self._get(coll, key, **kwargs) + + @AsyncWrapper.inject_callbacks(TransactionGetResult) + def _get_replica_from_preferred_server_group(self, + coll, # type: AsyncCollection + key, # type: str + **kwargs # type: Dict[str, Any] + ) -> TransactionGetResult: + return super().get_replica_from_preferred_server_group(coll, key, **kwargs) + + def get_replica_from_preferred_server_group(self, + coll, # type: AsyncCollection + key, # type: str + options=None, # type: Optional[TransactionGetReplicaFromPreferredServerGroupOptions] # noqa: E501 + **kwargs # type: Dict[str, Any] + ) -> TransactionGetResult: + """ + Get a document within this transaction from any replica in the preferred server group that is specified in + the :class:`couchbase.options.ClusterOptions`. + + Args: + coll (:class:`couchbase.collection.Collection`): Collection to use to find the document. + key (str): document key. + options (:class:`~couchbase.options.TransactionGetReplicaFromPreferredServerGroupOptions`): Optional + parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TransactionGetReplicaFromPreferredServerGroupOptions` + + Returns: + :class:`couchbase.transactions.TransactionGetResult`: Document in collection, in a form useful for passing + to other transaction operations. Or `None` if the document was not found. + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + :class:`couchbase.exceptions.DocumentUnretrievableException`: If the document could not be retrieved from + any replica in the preferred server group. The transaction will not rollback if this exception is + caught. + """ + if 'transcoder' not in kwargs and isinstance(options, TransactionGetReplicaFromPreferredServerGroupOptions): + kwargs['transcoder'] = options.get('transcoder', None) + return self._get_replica_from_preferred_server_group(coll, key, **kwargs) + + @AsyncWrapper.inject_callbacks(TransactionGetResult) + def _insert(self, + coll, # type: AsyncCollection + key, # type: str + value, # type: JSONType + **kwargs # type: Dict[str, Any] + ) -> Awaitable[TransactionGetResult]: + return super().insert(coll, key, value, **kwargs) + + def insert(self, + coll, # type: AsyncCollection + key, # type: str + value, # type: JSONType + options=None, # type: Optional[TransactionInsertOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[TransactionGetResult]: + """ + Insert a new document within a transaction. + + Args: + coll (:class:`couchbase.collection.Collection`): Collection to use to find the document. + key (str): document key. + value (:class:`couchbase._utils.JSONType): Contents of the document. + options (:class:`~couchbase.options.TransactionInsertOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TransactionInsertOptions` + + Returns: + Awaitable[:class:`couchbase.transactions.TransactionGetResult`]: Document in collection, in a form + useful for passing to other transaction operations. + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + + """ + if 'transcoder' not in kwargs and isinstance(options, TransactionInsertOptions): + kwargs['transcoder'] = options.get('transcoder', None) + return self._insert(coll, key, value, **kwargs) + + @AsyncWrapper.inject_callbacks(TransactionGetResult) + def _replace(self, + txn_get_result, # type: TransactionGetResult + value, # type: JSONType + **kwargs # type: Dict[str, Any] + ) -> Awaitable[TransactionGetResult]: + return super().replace(txn_get_result, value, **kwargs) + + def replace(self, + txn_get_result, # type: TransactionGetResult + value, # type: JSONType + options=None, # type: Optional[TransactionReplaceOptions] + **kwargs # type: Dict[str, Any] + ) -> Awaitable[TransactionGetResult]: + """ + Replace the contents of a document within a transaction. + + Args: + txn_get_result (:class:`couchbase.transactions.TransactionGetResult`): Document to replace, gotten from a + previous call to another transaction operation. + value (:class:`couchbase._utils.JSONType): The new contents of the document. + options (:class:`~couchbase.options.TransactionReplaceOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TransactionReplaceOptions` + + Returns: + Awaitable[:class:`couchbase.transactions.TransactionGetResult`]: Document in collection, in a form useful + for passing to other transaction operations. + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + """ + if 'transcoder' not in kwargs and isinstance(options, TransactionReplaceOptions): + kwargs['transcoder'] = options.get('transcoder', None) + return self._replace(txn_get_result, value, **kwargs) + + @AsyncWrapper.inject_callbacks(None) + def remove(self, + txn_get_result, + **kwargs + ): + """ + Remove a document in a transaction. + + Args: + txn_get_result (:class:`couchbase.transactions.TransactionGetResult`): Document to delete. + **kwargs (Dict[str, Any]): currently unused. + Returns: + Awaitable[None] + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + """ + return super().remove(txn_get_result, **kwargs) + + @AsyncWrapper.inject_callbacks(TransactionQueryResults) + def query(self, + query, + options=TransactionQueryOptions(), + **kwargs) -> TransactionQueryResults: + """ + Perform a query within a transaction. + + Args: + query (str): Query to perform. + options (:class:`couchbase.transactions.TransactionQueryOptions`): Query options to use, if any. + **kwargs (Dict[str, Any]): currently unused. + + Returns: + Awaitable[:class:`couchbase.transactions.TransactionQueryResult`] + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + :class:`couchbase.exceptions.CouchbaseException`: If the operation failed, but the transaction will not + necessarily be rolled back, a CouchbaseException other than TransactionOperationFailed will be raised. + If handled, the transaction will not rollback. + """ + return super().query(query, options, **kwargs) diff --git a/acouchbase/views.py b/acouchbase/views.py new file mode 100644 index 000000000..dc8bd69ff --- /dev/null +++ b/acouchbase/views.py @@ -0,0 +1,116 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from typing import Awaitable + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.views import ViewQuery # noqa: F401 +from couchbase.logic.views import ViewRequestLogic, ViewRow + + +class AsyncViewRequest(ViewRequestLogic): + def __init__(self, + connection, + loop, + encoded_query, + **kwargs + ): + num_workers = kwargs.pop('num_workers', 2) + super().__init__(connection, encoded_query, **kwargs) + self._loop = loop + self._tp_executor = ThreadPoolExecutor(num_workers) + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @classmethod + def generate_view_request(cls, connection, loop, encoded_query, **kwargs): + return cls(connection, loop, encoded_query, **kwargs) + + def _get_metadata(self): + try: + views_response = next(self._streaming_result) + self._set_metadata(views_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def execute(self) -> Awaitable[None]: + async def _execute(): + return [r async for r in self] + + return asyncio.create_task(_execute()) + + def __aiter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_query() + + return self + + def _get_next_row(self): + if self.done_streaming is True: + return + + row = next(self._streaming_result) + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopAsyncIteration + + # TODO: until streaming, a dict is returned, no deserializing... + # deserialized_row = self.serializer.deserialize(row) + deserialized_row = row + if issubclass(self.row_factory, ViewRow): + if hasattr(self.row_factory, 'from_json'): + return self.row_factory.from_json(deserialized_row) + return self.row_factory(**deserialized_row) + else: + return deserialized_row + + async def __anext__(self): + try: + return await self._loop.run_in_executor(self._tp_executor, self._get_next_row) + except asyncio.QueueEmpty: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls('Unexpected QueueEmpty exception caught when doing Search query.') + raise excptn + except StopAsyncIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..896d40790 --- /dev/null +++ b/conftest.py @@ -0,0 +1,162 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +pytest_plugins = [ + # "tests.helpers" + 'tests.couchbase_config', + 'tests.environments.test_environment' +] + +_DIAGNOSTIC_TESTS = [ + "acouchbase/tests/bucket_t.py::BucketDiagnosticsTests", + "acouchbase/tests/cluster_t.py::ClusterDiagnosticsTests", + "couchbase/tests/bucket_t.py::ClassicBucketDiagnosticsTests", + "couchbase/tests/cluster_t.py::ClassicClusterDiagnosticsTests", +] + +_KV_TESTS = [ + "acouchbase/tests/collection_t.py::CollectionTests", + "acouchbase/tests/subdoc_t.py::SubDocumentTests", + "acouchbase/tests/mutation_tokens_t.py::MutationTokensEnabledTests", + "acouchbase/tests/binary_collection_t.py::BinaryCollectionTests", + "acouchbase/tests/datastructures_t.py::DatastructuresTests", + "acouchbase/tests/transcoder_t.py::DefaultTranscoderTests", + "couchbase/tests/collection_t.py::ClassicCollectionTests", + "couchbase/tests/collection_multi_t.py::ClassicCollectionMultiTests", + "couchbase/tests/subdoc_t.py::ClassicSubDocumentTests", + "couchbase/tests/mutation_tokens_t.py::ClassicMutationTokensEnabledTests", + "couchbase/tests/binary_collection_t.py::ClassicBinaryCollectionTests", + "couchbase/tests/binary_collection_multi_t.py::ClassicBinaryCollectionMultiTests", + "couchbase/tests/datastructures_t.py::ClassicDatastructuresTests", + "couchbase/tests/datastructures_t.py::ClassicLegacyDatastructuresTests", + "couchbase/tests/transcoder_t.py::ClassicDefaultTranscoderTests", + "txcouchbase/tests/collection_t.py::CollectionTests", + "txcouchbase/tests/subdoc_t.py::SubDocumentTests", + "txcouchbase/tests/mutation_tokens_t.py::MutationTokensEnabledTests", + "txcouchbase/tests/binary_collection_t.py::BinaryCollectionTests", + "txcouchbase/tests/transcoder_t.py::DefaultTranscoderTests", +] + +_STREAMING_TESTS = [ + "acouchbase/tests/query_t.py::ClassicQueryTests", + "acouchbase/tests/query_t.py::ClassicQueryCollectionTests", + "acouchbase/tests/analytics_t.py::ClassicAnalyticsTests", + "acouchbase/tests/analytics_t.py::ClassicAnalyticsCollectionTests", + "acouchbase/tests/search_t.py::ClassicSearchTests", + "acouchbase/tests/search_t.py::ClassicSearchCollectionTests", + "acouchbase/tests/views_t.py::ClassicViewsTests", + "couchbase/tests/analytics_params_t.py::ClassicAnalyticsParamTests", + "couchbase/tests/analytics_t.py::ClassicAnalyticsTests", + "couchbase/tests/analytics_t.py::ClassicAnalyticsCollectionTests", + "couchbase/tests/query_params_t.py::ClassicQueryParamTests", + "couchbase/tests/query_t.py::ClassicQueryTests", + "couchbase/tests/query_t.py::ClassicQueryCollectionTests", + "couchbase/tests/search_params_t.py::ClassicSearchParamTests", + "couchbase/tests/search_params_t.py::ClassicVectorSearchParamTests", + "couchbase/tests/search_t.py::ClassicSearchTests", + "couchbase/tests/search_t.py::ClassicSearchCollectionTests", + "couchbase/tests/views_t.py::ClassicViewsTests", +] + +_MGMT_TESTS = [ + "acouchbase/tests/analyticsmgmt_t.py::AnalyticsManagementTests", + "acouchbase/tests/analyticsmgmt_t.py::AnalyticsManagementLinksTests", + "acouchbase/tests/bucketmgmt_t.py::BucketManagementTests", + "acouchbase/tests/collectionmgmt_t.py::CollectionManagementTests", + "acouchbase/tests/eventingmgmt_t.py::EventingManagementTests", + "acouchbase/tests/eventingmgmt_t.py::ScopeEventingManagementTests", + "acouchbase/tests/querymgmt_t.py::QueryIndexManagementTests", + "acouchbase/tests/querymgmt_t.py::QueryIndexCollectionManagementTests", + "acouchbase/tests/searchmgmt_t.py::SearchIndexManagementTests", + "acouchbase/tests/usermgmt_t.py::UserManagementTests", + "acouchbase/tests/viewmgmt_t.py::ViewIndexManagementTests", + "couchbase/tests/analyticsmgmt_t.py::ClassicAnalyticsManagementTests", + "couchbase/tests/analyticsmgmt_t.py::ClassicAnalyticsManagementLinksTests", + "couchbase/tests/bucketmgmt_t.py::ClassicBucketManagementTests", + "couchbase/tests/collectionmgmt_t.py::ClassicCollectionManagementTests", + "couchbase/tests/eventingmgmt_t.py::ClassicEventingManagementTests", + "couchbase/tests/eventingmgmt_t.py::ClassicScopeEventingManagementTests", + "couchbase/tests/querymgmt_t.py::ClassicQueryIndexManagementTests", + "couchbase/tests/querymgmt_t.py::ClassicQueryIndexCollectionManagementTests", + "couchbase/tests/searchmgmt_t.py::ClassicSearchIndexManagementTests", + "couchbase/tests/usermgmt_t.py::ClassicUserManagementTests", + "couchbase/tests/viewmgmt_t.py::ClassicViewIndexManagementTests" +] + +_SLOW_MGMT_TESTS = [ + "acouchbase/tests/eventingmgmt_t.py::EventingManagementTests", + "acouchbase/tests/eventingmgmt_t.py::ScopeEventingManagementTests", + "couchbase/tests/eventingmgmt_t.py::ClassicEventingManagementTests", + "couchbase/tests/eventingmgmt_t.py::ClassicScopeEventingManagementTests", +] + +_MISC_TESTS = [ + "acouchbase/tests/rate_limit_t.py::RateLimitTests", + "couchbase/tests/connection_t.py::ClassicConnectionTests" + "couchbase/tests/rate_limit_t.py::ClassicRateLimitTests", +] + +_TXNS_TESTS = [ + "acouchbase/tests/transactions_t.py::ClassicTransactionTests", + "couchbase/tests/transactions_t.py:ClassicTransactionTests", +] + + +@pytest.fixture(name="couchbase_config", scope="session") +def get_config(couchbase_test_config): + if couchbase_test_config.mock_server_enabled: + print("Mock server enabled!") + if couchbase_test_config.real_server_enabled: + print("Real server enabled!") + + return couchbase_test_config + + +def pytest_addoption(parser): + parser.addoption( + "--txcouchbase", action="store_true", default=False, help="run txcouchbase tests" + ) + + +def pytest_collection_modifyitems(items): # noqa: C901 + for item in items: + item_details = item.nodeid.split('::') + + item_api = item_details[0].split('/') + if item_api[0] == 'couchbase': + item.add_marker(pytest.mark.pycbc_couchbase) + elif item_api[0] == 'acouchbase': + item.add_marker(pytest.mark.pycbc_acouchbase) + elif item_api[0] == 'txcouchbase': + item.add_marker(pytest.mark.pycbc_txcouchbase) + + test_class_path = '::'.join(item_details[:-1]) + if test_class_path in _DIAGNOSTIC_TESTS: + item.add_marker(pytest.mark.pycbc_diag) + elif test_class_path in _KV_TESTS: + item.add_marker(pytest.mark.pycbc_kv) + elif test_class_path in _STREAMING_TESTS: + item.add_marker(pytest.mark.pycbc_streaming) + elif test_class_path in _MGMT_TESTS: + item.add_marker(pytest.mark.pycbc_mgmt) + elif test_class_path in _MISC_TESTS: + item.add_marker(pytest.mark.pycbc_misc) + elif test_class_path in _TXNS_TESTS: + item.add_marker(pytest.mark.pycbc_txn) + + if test_class_path in _SLOW_MGMT_TESTS: + item.add_marker(pytest.mark.pycbc_slow_mgmt) diff --git a/couchbase-sdk-python-black-duck-manifest.yaml b/couchbase-sdk-python-black-duck-manifest.yaml new file mode 100644 index 000000000..b337207fb --- /dev/null +++ b/couchbase-sdk-python-black-duck-manifest.yaml @@ -0,0 +1,2 @@ +include-projects: + - couchbase-sdk-cxx diff --git a/couchbase/__init__.py b/couchbase/__init__.py index 91d17c954..567e72abb 100644 --- a/couchbase/__init__.py +++ b/couchbase/__init__.py @@ -1,123 +1,187 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. # -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import warnings - -# Pythons > (2.7||3.2) silence deprecation warnings by default. -# Many folks are not happy about this, as it avoids letting them -# know about potential upcoming breaking changes in their code. -# Here we add a warning filter for any deprecation warning thrown -# by Couchbase -warnings.filterwarnings(action='default', - category=DeprecationWarning, - module=r"^couchbase($|\..*)") +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import platform +from functools import partial, partialmethod +from typing import (List, + Optional, + Tuple) -from couchbase.user_constants import * -import couchbase._libcouchbase as _LCB +try: + # Importing the ssl package allows us to utilize some Python voodoo to find OpenSSL. + # This is particularly helpful on M1 macs (PYCBC-1386). + import ssl # noqa: F401 # nopep8 # isort:skip # noqa: E402 + import couchbase.pycbc_core # noqa: F401 # nopep8 # isort:skip # noqa: E402 +except ImportError: + import os # nopep8 # isort:skip # noqa: E402 + import sys # nopep8 # isort:skip # noqa: E402 + # should only need to do this on Windows w/ Python >= 3.8 due to the changes made for how DLLs are resolved + if sys.platform.startswith('win32') and (3, 8) <= sys.version_info: + open_ssl_dir = os.getenv('PYCBC_OPENSSL_DIR') + # if not set by environment, try to use libcrypto and libssl that comes w/ Windows Python install + if not open_ssl_dir: + for p in sys.path: + if os.path.split(p)[-1] == 'DLLs': + open_ssl_dir = p + break + + if open_ssl_dir: + os.add_dll_directory(open_ssl_dir) + else: + print(('PYCBC: Caught import error. ' + 'Most likely due to not finding OpenSSL libraries. ' + 'Set PYCBC_OPENSSL_DIR to location where OpenSSL libraries can be found.')) try: from couchbase._version import __version__ - except ImportError: - __version__ = "0.0.0-could-not-find-git" + __version__ = '0.0.0-could-not-find-version' +PYCBC_VERSION = f'python/{__version__}' +USER_AGENT_EXTRA = '' -def set_json_converters(encode, decode): - """ - Modify the default JSON conversion functions. This affects all - :class:`~couchbase.bucket.Bucket` instances. +try: + USER_AGENT_EXTRA = f'python/{platform.python_version()}' +except Exception: # nosec + pass - These functions will called instead of the default ones (``json.dumps`` - and ``json.loads``) to encode and decode JSON (when :const:`FMT_JSON` is - used). - :param callable encode: Callable to invoke when encoding an object to JSON. - This should have the same prototype as ``json.dumps``, with the - exception that it is only ever passed a single argument. +""" Add support for logging, adding a TRACE level to logging """ +import json # nopep8 # isort:skip # noqa: E402 +import logging # nopep8 # isort:skip # noqa: E402 - :param callable decode: Callable to invoke when decoding an object to JSON. - This should have the same prototype and behavior - as ``json.loads`` with the exception that it is only ever - passed a single argument. +from couchbase.pycbc_core import CXXCBC_METADATA, pycbc_logger, shutdown_logger # nopep8 # isort:skip # noqa: E402 - :return: A tuple of ``(old encoder, old decoder)`` +_PYCBC_LOGGER = pycbc_logger() +_CXXCBC_METADATA_JSON = json.loads(CXXCBC_METADATA) +logging.TRACE = 5 +logging.addLevelName(logging.TRACE, 'TRACE') +logging.Logger.trace = partialmethod(logging.Logger.log, logging.TRACE) +logging.trace = partial(logging.log, logging.TRACE) - No exceptions are raised, and it is the responsibility of the caller to - ensure that the provided functions operate correctly, otherwise exceptions - may be thrown randomly when encoding and decoding values - """ - ret = _LCB._modify_helpers(json_encode=encode, json_decode=decode) - return (ret['json_encode'], ret['json_decode']) +""" -def set_pickle_converters(encode, decode): - """ - Modify the default Pickle conversion functions. This affects all - :class:`~couchbase.bucket.Bucket` instances. +pycbc teardown methods - These functions will be called instead of the default ones - (``pickle.dumps`` and ``pickle.loads``) to encode and decode values to and - from the Pickle format (when :const:`FMT_PICKLE` is used). +""" +import atexit # nopep8 # isort:skip # noqa: E402 - :param callable encode: Callable to invoke when encoding an object to - Pickle. This should have the same prototype as ``pickle.dumps`` with - the exception that it is only ever called with a single argument - :param callable decode: Callable to invoke when decoding a Pickle encoded - object to a Python object. Should have the same prototype as - ``pickle.loads`` with the exception that it is only ever passed a - single argument +def _pycbc_teardown(**kwargs): + """**INTERNAL**""" + global _PYCBC_LOGGER + # if using a console logger we let the natural course of shutdown happen, if using Python logging + # we need a cleaner mechanism to shutdown the C++ logger prior to the Python interpreter starting to finalize + if (_PYCBC_LOGGER + and isinstance(_PYCBC_LOGGER, pycbc_logger) + and not (_PYCBC_LOGGER.is_console_logger() or _PYCBC_LOGGER.is_file_logger())): + shutdown_logger() + _PYCBC_LOGGER = None - :return: A tuple of ``(old encoder, old decoder)`` - No exceptions are raised and it is the responsibility of the caller to - ensure that the provided functions operate correctly. - """ - ret = _LCB._modify_helpers(pickle_encode=encode, pickle_decode=decode) - return (ret['pickle_encode'], ret['pickle_decode']) +atexit.register(_pycbc_teardown) +""" -def enable_logging(): - """ - Enables integration with Python's `logging` module. +Metadata + version methods + +""" +_METADATA_KEYS = ['openssl_default_cert_dir', + 'openssl_default_cert_file', + 'openssl_headers', + 'openssl_runtime', + 'txns_forward_compat_extensions', + 'txns_forward_compat_protocol_version', + 'version'] - This function enables the C library's logging to be propagated to - the Python standard `logging` module. - Calling this function affects any :class:`~.Bucket` objects created - afterwards (but not before). Note that currently this will also - override any ``LCB_LOGLEVEL`` directive inside the environment as - well. +def get_metadata(as_str=False, detailed=False): + metadata = _CXXCBC_METADATA_JSON if detailed is True else { + k: v for k, v in _CXXCBC_METADATA_JSON.items() if k in _METADATA_KEYS} + return json.dumps(metadata) if as_str is True else metadata - The "root" couchbase logger is ``couchbase``. + +def get_transactions_protocol() -> Optional[Tuple[Optional[float], Optional[List[str]]]]: + """Get the transactions protocol version and supported extensions. + + Returns: + Optional[Tuple[Optional[float], Optional[List[str]]]]: The transactions protocol version and + support extensions, if found in the cxx client's metadata. """ - import couchbase._logutil - couchbase._logutil.configure(True) + if not _CXXCBC_METADATA_JSON: + return None + + version = _CXXCBC_METADATA_JSON.get('txns_forward_compat_protocol_version', None) + if version: + version = float(version) + extensions = _CXXCBC_METADATA_JSON.get('txns_forward_compat_extensions', None) + if extensions: + extensions = extensions.split(',') + return version, extensions + + +""" + +Logging methods + +""" + + +def configure_console_logger(): + import os + log_level = os.getenv('PYCBC_LOG_LEVEL', None) + if log_level: + log_file = os.getenv('PYCBC_LOG_FILE', None) + if log_file: + enable_console_logging = 0 if os.getenv('PYCBC_ENABLE_CONSOLE', None) is None else 1 + _PYCBC_LOGGER.create_logger(level=log_level.lower(), + filename=log_file, + enable_console=enable_console_logging) + else: + _PYCBC_LOGGER.create_logger(level=log_level.lower()) + logging.getLogger().debug(get_metadata(as_str=True)) + + +def configure_logging(name, level=logging.INFO, parent_logger=None): + if parent_logger: + name = f'{parent_logger.name}.{name}' + logger = logging.getLogger(name) + if _PYCBC_LOGGER.is_console_logger() or _PYCBC_LOGGER.is_file_logger(): + raise RuntimeError(('Cannot create logger. Another logger has already been ' + 'initialized. Make sure the PYCBC_LOG_LEVEL and PYCBC_LOG_FILE env ' + 'variable are not set if using configure_logging.')) + _PYCBC_LOGGER.configure_logging_sink(logger, level) + logger.debug(get_metadata(as_str=True)) + + +def enable_protocol_logger_to_save_network_traffic_to_file(filename # type: str + ): + """ + **VOLATILE** This API is subject to change at any time. + Exposes the underlying couchbase++ library protocol logger. This method is for logging/debugging + purposes and must be used with caution as network details will be logged to the provided file. -def disable_logging(): - import couchbase._logutil - couchbase._logutil.configure(False) + Args: + filename (str): The name of the file the protocol logger will write to. + Raises: + InvalidArgumentException: If a filename is not provided. + """ + _PYCBC_LOGGER.enable_protocol_logger(filename) -class Couchbase(object): - @classmethod - def connect(self, bucket, **kwargs): - from couchbase.bucket import _depr - from couchbase.connection import Connection - _depr('Couchbase.connect()', 'Bucket()') - return Connection(bucket, **kwargs) +configure_console_logger() diff --git a/couchbase/_bootstrap.py b/couchbase/_bootstrap.py deleted file mode 100644 index 72351dfcb..000000000 --- a/couchbase/_bootstrap.py +++ /dev/null @@ -1,148 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -This module contains the core functionality of '_libcouchbase'. In short, -this contains the convergence between the C module and code written in Python. - -While the _libcouchbase module should never be used directly, in the off chance -that this does happen, ensure this module is loaded as well before anything is -done, otherwise Bad Things May Happen. - -Additionally, this -module contains python functions used exclusively from C. They are here -because it was quicker to write them in Python than it was in C. Do not touch -this file at all. You have been warned -""" -import json -import pickle - -import couchbase.exceptions as E -import couchbase._libcouchbase as C -from couchbase.items import ItemCollection, ItemOptionDict, ItemSequence - - -def _result__repr__(self): - """ - This is used as the `__repr__` function for the :class:`Result` - """ - - details = [] - flags = self.__class__._fldprops - - rcstr = "RC=0x{0:X}".format(self.rc) - if self.rc != 0: - rcstr += "[{0}]".format(self.errstr) - - details.append(rcstr) - - if flags & C.PYCBC_RESFLD_KEY and hasattr(self, 'key'): - details.append("Key={0}".format(repr(self.key))) - - if flags & C.PYCBC_RESFLD_VALUE and hasattr(self, 'value'): - details.append("Value={0}".format(repr(self.value))) - - if flags & C.PYCBC_RESFLD_CAS and hasattr(self, 'cas'): - details.append("CAS=0x{cas:x}".format(cas=self.cas)) - - if flags & C.PYCBC_RESFLD_CAS and hasattr(self, 'flags'): - details.append("Flags=0x{flags:x}".format(flags=self.flags)) - - if flags & C.PYCBC_RESFLD_HTCODE and hasattr(self, "http_status"): - details.append("HTTP={0}".format(self.http_status)) - - if flags & C.PYCBC_RESFLD_URL and hasattr(self, "url"): - details.append("URL={0}".format(self.url)) - - ret = "{0}<{1}>".format(self.__class__.__name__, ', '.join(details)) - return ret - - -def _observeinfo__repr__(self): - constants = ('OBS_PERSISTED', - 'OBS_FOUND', - 'OBS_NOTFOUND', - 'OBS_LOGICALLY_DELETED') - - - flag_str = '' - for c in constants: - if self.flags == getattr(C, c): - flag_str = c - break - - fmstr = ("{cls}<Status=[{status_s} (0x{flags:X})], " - "Master={is_master}, " - "CAS=0x{cas:X}>") - ret = fmstr.format(cls=self.__class__.__name__, - status_s=flag_str, - flags=self.flags, - is_master=bool(self.from_master), - cas=self.cas) - return ret - -def _json_encode_wrapper(*args): - return json.dumps(*args, ensure_ascii=False, separators=(',', ':')) - - -class FMT_AUTO_object_not_a_number(object): - pass - -# TODO: Make this more readable and have PEP8 ignore it. -_FMT_AUTO = FMT_AUTO_object_not_a_number() - - -MAX_URI_LENGTH = 2048 - - -def _view_path_helper(options): - # Assume options are already encoded! - if not options: - return '', '' - - post_body = '' - encoded = options.encoded - - if len(encoded) > MAX_URI_LENGTH: - encoded, post_body = options._long_query_encoded - - return encoded, post_body - - -def run_init(m): - m._init_helpers(result_reprfunc=_result__repr__, - fmt_utf8_flags=C.FMT_UTF8, - fmt_bytes_flags=C.FMT_BYTES, - fmt_json_flags=C.FMT_JSON, - fmt_pickle_flags=C.FMT_PICKLE, - pickle_encode=pickle.dumps, - pickle_decode=pickle.loads, - json_encode=_json_encode_wrapper, - json_decode=json.loads, - lcb_errno_map=E._LCB_ERRNO_MAP, - misc_errno_map=E._EXCTYPE_MAP, - default_exception=E.CouchbaseError, - obsinfo_reprfunc=_observeinfo__repr__, - itmcoll_base_type=ItemCollection, - itmopts_dict_type=ItemOptionDict, - itmopts_seq_type=ItemSequence, - fmt_auto=_FMT_AUTO, - view_path_helper=_view_path_helper) - -run_init(C) - -C.FMT_AUTO = _FMT_AUTO diff --git a/couchbase/_logutil.py b/couchbase/_logutil.py deleted file mode 100644 index 6fae72fd5..000000000 --- a/couchbase/_logutil.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import logging -LEVEL_MAP = ( - logging.DEBUG, # TRACE - logging.DEBUG, # DEBUG - logging.INFO, # INFO - logging.WARN, # WARN - logging.ERROR, # Error - logging.FATAL # Fatal -) - - -def _pylog_log_handler(**kwargs): - pylevel = LEVEL_MAP[kwargs.pop('level')] - logger_name = 'couchbase' + '.' + kwargs['subsys'] - msg = '[{id}] {message} (L:{c_src[1]})'.format(**kwargs) - logging.getLogger(logger_name).log(pylevel, msg) - - -def configure(val): - from couchbase.bucket import Bucket - import couchbase._libcouchbase - if val: - couchbase._libcouchbase.lcb_logging(_pylog_log_handler) - initmsg = 'Initializing Couchbase logging. lcb_version={0}'.format( - Bucket.lcb_version()) - logging.getLogger('couchbase').info(initmsg) - else: - couchbase._libcouchbase.lcb_logging(None) - diff --git a/couchbase/_pyport.py b/couchbase/_pyport.py deleted file mode 100644 index ac1621f17..000000000 --- a/couchbase/_pyport.py +++ /dev/null @@ -1,50 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This module contains various mappings for modules which have had -# their names changed across Python major versions -import sys - -V = sys.version_info[0] - -if V == 3: - import urllib.parse as ulp - from urllib.request import urlopen - from urllib.parse import parse_qs -else: - import urllib as ulp - from urllib2 import urlopen - from urlparse import parse_qs - -long = long if V == 2 else int -xrange = xrange if V == 2 else range -basestring = basestring if V == 2 else str -unicode = unicode if V == 2 else str - -if V == 2: - exec("def PyErr_Restore(cls, obj, bt): raise cls, obj, bt\n") -else: - def PyErr_Restore(cls, obj, bt): - raise obj.with_traceback(bt) - -if V == 2: - def single_dict_key(d): - return d.keys()[0] -else: - def single_dict_key(d): - for k in d.keys(): - return k \ No newline at end of file diff --git a/couchbase/_utils.py b/couchbase/_utils.py new file mode 100644 index 000000000..5ee01cad0 --- /dev/null +++ b/couchbase/_utils.py @@ -0,0 +1,395 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from enum import Enum +from inspect import (Parameter, + Signature, + signature) +from time import time +from typing import (Any, + Callable, + Dict, + List, + Tuple, + Type, + TypeVar, + Union) +from urllib.parse import quote + +from couchbase.exceptions import InvalidArgumentException +from couchbase.pycbc_core import exception + +JSONType = Union[str, int, float, bool, + None, Dict[str, Any], List[Any]] + +PyCapsuleType = TypeVar('PyCapsuleType') + + +def is_null_or_empty( + value # type: str +) -> bool: + return not (value and not value.isspace()) + + +def to_form_str(params # type: Dict[str, Any] + ): + encoded_params = [] + for k, v in params.items(): + encoded_params.append('{0}={1}'.format(quote(k), quote(str(v)))) + + return '&'.join(encoded_params) + + +def identity(x # type: Any + ) -> Any: + return x + + +def timedelta_as_microseconds( + duration, # type: timedelta +) -> int: + if duration and not isinstance(duration, timedelta): + raise InvalidArgumentException( + message="Expected timedelta instead of {}".format(duration) + ) + return int(duration.total_seconds() * 1e6 if duration else 0) + + +def to_microseconds( + timeout # type: Union[timedelta, float, int] +) -> int: + if timeout and not isinstance(timeout, (timedelta, float, int)): + raise InvalidArgumentException(message=("Excepted timeout to be of type " + f"Union[timedelta, float, int] instead of {timeout}")) + if not timeout: + total_us = 0 + elif isinstance(timeout, timedelta): + total_us = int(timeout.total_seconds() * 1e6) + else: + total_us = int(timeout * 1e6) + + return total_us + + +THIRTY_DAYS_IN_SECONDS = 30 * 24 * 60 * 60 + + +def timedelta_as_timestamp( + duration, # type: timedelta +) -> int: + if not isinstance(duration, timedelta): + raise InvalidArgumentException(f'Expected timedelta instead of {duration}') + + # PYCBC-1177 remove deprecated heuristic from PYCBC-948: + seconds = int(duration.total_seconds()) + if seconds < 0: + raise InvalidArgumentException(f'Expected expiry seconds of zero (for no expiry) or greater, got {seconds}.') + + if seconds < THIRTY_DAYS_IN_SECONDS: + return seconds + + return seconds + int(time()) + + +def validate_int(value # type: int + ) -> int: + if not isinstance(value, int): + raise InvalidArgumentException(message='Expected value to be of type int.') + return value + + +def validate_bool(value # type: bool + ) -> int: + if not isinstance(value, bool): + raise InvalidArgumentException(message='Expected value to be of type bool.') + return value + + +def validate_str(value # type: str + ) -> int: + if not isinstance(value, str): + raise InvalidArgumentException(message='Expected value to be of type str.') + return value + + +class Identity: + def __init__(self, + type_, # type: Callable + optional=False # type: bool + ): + self._type = type_ + self._optional = optional + + def __call__(self, x # type: Any + ) -> Any: + if self._optional and x is None: + return x + if not isinstance(x, self._type): + exc = InvalidArgumentException.pycbc_create_exception( + exception(), + "Argument must be of type {} but got {}".format( + self._type, x)) + raise exc + return x + + +class EnumToStr: + def __init__(self, type_, # type: Type[Enum] + conversion_fn=None # type: Callable + ): + self._type = type_ + self._conversion_fn = conversion_fn + + def __call__(self, value # type: Enum + ) -> str: + # TODO: maybe? + if isinstance(value, str) and value in map(lambda x: x.value, self._type): + # TODO: use warning? + # warn("Using deprecated string parameter {}".format(value)) + return value + if not isinstance(value, self._type): + exc = InvalidArgumentException.pycbc_create_exception( + exception(), + "Argument must be of type {} but got {}".format( + self._type, value)) + raise exc + if self._conversion_fn: + return self._conversion_fn(value) + return value.value + + +class StrToEnum: + def __init__(self, type_, # type: Type[Enum] + conversion_fn=None, # type: Callable + optional=False # type: bool + ): + self._type = type_ + self._conversion_fn = conversion_fn + self._optional = optional + + def __call__(self, value # type: str + ) -> Enum: + + if self._optional and value is None: + return value + if self._conversion_fn: + return self._conversion_fn(value) + return self._type(value) + + +NumberType = TypeVar('NumberType', bound=Union[float, int]) + + +class SecondsToTimeDelta: + def __init__(self, type_ # type: Type[timedelta] + ): + self._type = type_ + + def __call__(self, value # type: NumberType + ) -> timedelta: + try: + return self._type(seconds=value) + except (OverflowError, ValueError): + exc = InvalidArgumentException.pycbc_create_exception( + exception(), "Invalid duration arg: {}".format(value)) + raise exc + + +class TimeDeltaToSeconds: + def __init__(self, type_ # type: Union[Type[int], Type[float]] + ): + self._type = type_ + + def __call__(self, td # type: Union[timedelta, float, int] + ) -> Type[NumberType]: + if isinstance(td, (float, int)): + return self._type(td) + return self._type(td.total_seconds()) + +# class TimeDeltaToSeconds: +# def __init__(self, dest_type: Type[NumberType]): +# super( +# Timedelta, +# self).__init__( +# TimedeltaToSeconds(dest_type), +# _seconds_to_timedelta) + + +class ParamTransform: + + def __init__(self, key=None, # type: str + transform=Identity(object) # type: Callable + ) -> None: + self._key = key + self._transform = transform + + @property + def key(self): + return self._key + + @key.setter + def key(self, value): + self._key = value + + @property + def transform(self): + return self._transform + + +class UnidirectionalTransform: + + def __init__(self, key, # type: str + to_dest, # type: ParamTransform + ): + self._key = key + self._to_dest = to_dest + + +class TransformComponents: + + def __init__(self, + to_dest, # type: ParamTransform + from_dest, # type: ParamTransform + default=None # type: Any + ): + self._to_dest = to_dest + self._from_dest = from_dest + self._default = default + + @property + def default(self): + return self._default + + def to_dest_components(self) -> Tuple[str, Callable]: + return self._to_dest.key, self._to_dest.transform + + def from_dest_components(self) -> Tuple[str, Callable]: + return self._from_dest.key, self._from_dest.transform + + +class BidirectionalTransform: + + def __init__(self, key, # type: str + to_dest, # type: ParamTransform + from_dest, # type: ParamTransform + default=None # type: Any + ): + self._key = key + self._to_dest = to_dest + if self._to_dest.key is None: + self._to_dest.key = self._key + self._from_dest = from_dest + if self._from_dest.key is None: + self._from_dest.key = self._key + self._default = default + + def transform_as_dict(self): + return {self._key: TransformComponents(self._to_dest, self._from_dest, self._default)} + + +class BidirectionalMapping: + + def __init__(self, transforms # type: List[BidirectionalTransform] + ): + self._transforms = transforms + self._mapping = {} + for t in self._transforms: + self._mapping.update(t.transform_as_dict()) + + @staticmethod + def convert_to_dest(mapping, # type: Dict[str, Any] + raw_info # type: Dict[str, Any] + ) -> Dict[str, Any]: + converted = {} + for k, v in raw_info.items(): + param_transform = mapping.get(k, TransformComponents( + ParamTransform(k), ParamTransform(k))) + try: + key, transform = param_transform.to_dest_components() + if not key: + key = k + converted[key] = transform(v) + except InvalidArgumentException as e: + raise e + return converted + + @staticmethod + def convert_from_dest(mapping, # type: Dict[str, Any] + raw_info # type: Dict[str, Any] + ) -> Dict[str, Any]: + converted = {} + for k, param_transform in mapping.items(): + key, transform = param_transform.from_dest_components() + if key not in raw_info: + continue + try: + converted[k] = transform(raw_info[key]) + except InvalidArgumentException as e: + raise e + return converted + + def transform_to_dest(self, + data # type: Dict[str, Any] + ) -> Dict[str, Any]: + + # set the defaults + for k in self._mapping.keys(): + if k not in data.keys() and self._mapping[k].default is not None: + data[k] = self._mapping[k].default + return self.convert_to_dest(self._mapping, data) + + def transform_from_dest(self, data): + return self.convert_from_dest(self._mapping, data) + + +class OverloadType(Enum): + DEFAULT = 0 + SECONDARY = 1 + + +class Overload: + def __init__(self, name): + self._name = name + self._sig_map = {} + self._default_fn = None + + def register(self, fn): + sig = signature(fn) + if sig in self._sig_map: + raise TypeError(f'Already registered {self._name} with signature {sig}') + self._sig_map[sig] = fn + + def register_default(self, fn): + if self._default_fn is not None: + raise TypeError(f'Default function already registered for {self._name}') + self._default_fn = fn + + def __call__(self, *args, **kwargs): + for sig, fn in self._sig_map.items(): + params = sig.parameters + try: + bound_args = sig.bind(*args, **kwargs) + except TypeError: + continue + if all(params[arg_name].annotation is Signature.empty or isinstance(arg, params[arg_name].annotation) + for arg_name, arg in bound_args.arguments.items() + if params[arg_name].kind not in [Parameter.VAR_POSITIONAL, Parameter.VAR_KEYWORD]): + return fn(*args, **kwargs) + + # None of the functions in the signature map can be called, call the default one if it exists + if self._default_fn is None: + raise TypeError(f'Unable to find appropriate registered overload for {self._name}') + return self._default_fn(*args, **kwargs) diff --git a/couchbase/admin.py b/couchbase/admin.py deleted file mode 100644 index c5b1b8d63..000000000 --- a/couchbase/admin.py +++ /dev/null @@ -1,318 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -The contents of this module do not have a stable API and are subject to -change -""" -from time import time, sleep - -import couchbase._libcouchbase as LCB -import couchbase.exceptions as E -from couchbase.user_constants import FMT_JSON -from couchbase._pyport import ulp - -METHMAP = { - 'GET': LCB.LCB_HTTP_METHOD_GET, - 'PUT': LCB.LCB_HTTP_METHOD_PUT, - 'POST': LCB.LCB_HTTP_METHOD_POST, - 'DELETE': LCB.LCB_HTTP_METHOD_DELETE -} - - -class NotReadyError(E.CouchbaseError): - """ - Thrown when not all nodes could be ready (internal) - """ - pass - - -class Admin(LCB.Bucket): - """ - An administrative connection to a Couchbase cluster. - - With this object, you can do things which affect the cluster, such as - modifying buckets, allocating nodes, or retrieving information about - the cluster. - - This object should **not** be used to perform Key/Value operations. The - :class:`~.Bucket` is used for that. - """ - def __init__(self, username, password, host='localhost', port=8091, - **kwargs): - - """Connect to a cluster - - :param string username: The administrative username for the cluster, - this is typically ``Administrator`` - :param string password: The administrative password for the cluster, - this is the password you entered when Couchbase was installed - :param string host: The hostname or IP of one of the nodes which is - currently a member of the cluster (or a newly allocated node, if - you wish to operate on that) - :param int port: The management port for the node - - :raise: - :exc:`couchbase.exceptions.AuthError` if incorrect credentials - were supplied - - :exc:`couchbase.exceptions.ConnectError` if there was a problem - establishing a connection to the provided host - - :return: an instance of :class:`Admin` - """ - kwargs.update({ - 'username': username, - 'password': password, - 'connection_string': "http://{0}:{1}".format(host, port), - '_conntype': LCB.LCB_TYPE_CLUSTER - }) - super(Admin, self).__init__(**kwargs) - self._connect() - - def http_request(self, - path, - method='GET', - content=None, - content_type="application/json", - response_format=FMT_JSON): - """ - Perform an administrative HTTP request. This request is sent out to - the administrative API interface (i.e. the "Management/REST API") - of the cluster. - - Note that this is a fairly low level function. You should use one - of the helper methods in this class to perform your task, if - possible. - - :param string path: The path portion (not including the host) of the - rest call to perform. This should also include any encoded arguments. - - :param string method: This is the HTTP method to perform. Currently - supported values are `GET`, `POST`, `PUT`, and `DELETE` - - :param bytes content: Content to be passed along in the request body. - This is only applicable on `PUT` and `POST` methods. - - :param string content_type: Value for the HTTP ``Content-Type`` header. - Currently this is ``application-json``, and should probably not be - set to something else. - - :param int response_format: - Hint about how to format the response. This goes into the - :attr:`~.HttpResult.value` field of the - :class:`~.HttpResult` object. The default is - :const:`~couchbase.FMT_JSON`. - - Note that if the conversion fails, the content will be returned as - ``bytes`` - - :raise: - :exc:`~.ArgumentError` - if the method supplied was incorrect. - :exc:`~.ConnectError` - if there was a problem establishing a connection. - :exc:`~.HTTPError` - if the server responded with a negative reply - - :return: a :class:`~.HttpResult` object. - - .. seealso:: :meth:`bucket_create`, :meth:`bucket_remove` - """ - imeth = None - if not method in METHMAP: - raise E.ArgumentError.pyexc("Unknown HTTP Method", method) - - imeth = METHMAP[method] - return self._http_request(type=LCB.LCB_HTTP_TYPE_MANAGEMENT, - path=path, - method=imeth, - content_type=content_type, - post_data=content, - response_format=response_format) - - def _mk_formstr(self, d): - l = [] - for k, v in d.items(): - l.append('{0}={1}'.format(ulp.quote(k), ulp.quote(str(v)))) - - return '&'.join(l) - - def bucket_create(self, name, bucket_type='couchbase', - bucket_password='', replicas=0, ram_quota=1024, - flush_enabled=False): - """ - Create a new bucket - - :param string name: The name of the bucket to create - :param string bucket_type: The type of bucket to create. This - can either be `couchbase` to create a couchbase style - bucket (which persists data and supports replication) or - `memcached` (which is memory-only and does not support - replication). - :param string bucket_password: The bucket password. This can be - empty to disable authentication. This can be changed later on - using :meth:`bucket_update` - :param int replicas: The number of replicas to use for this - bucket. The maximum number of replicas is currently 3. - This setting can be changed via :meth:`bucket_update` - :param int ram_quota: - The maximum amount of memory (per node) that this bucket - may use, in megabytes. The minimum for this value is 100. - This setting may be changed via :meth:`bucket_update`. - :param bool flush_enabled: - Whether the flush API is enabled. When the flush API is - enabled, any client connected to the bucket is able to - clear its contents. This may be useful in development but - not recommended in production. This setting may be changed - via :meth:`bucket_update` - :return: A :class:`~.HttpResult` - :raise: :exc:`~.HTTPError` if the bucket could not be created. - """ - params = { - 'name': name, - 'type': bucket_type, - 'authType': 'sasl', - 'saslPassword': bucket_password if bucket_password else '', - 'flush_enabled': int(flush_enabled), - 'ramQuotaMB': ram_quota - } - if bucket_type in ('couchbase', 'membase'): - params['replicaNumber'] = replicas - - return self.http_request( - path='/pools/default/buckets', method='POST', - content=self._mk_formstr(params), - content_type='application/x-www-form-urlencoded') - - def bucket_remove(self, name): - """ - Remove an existing bucket from the cluster - - :param string name: The name of the bucket to remove - :return: A :class:`~.HttpResult` - :raise: :exc:`~HTTPError` on error - """ - return self.http_request(path='/pools/default/buckets/' + name, - method='DELETE') - - bucket_delete = bucket_remove - - def bucket_info(self, name): - """ - Retrieve information about the bucket. - - :param string name: The name of the bucket - :return: A :class:`~.HttpResult` object. The result's - :attr:`~.HttpResult.value` attribute contains - A dictionary containing the bucket's information. - The returned object is considered to be opaque, and is - intended primarily for use with :meth:`bucket_update`. - Currently this returns the raw decoded JSON as emitted - by the corresponding server-side API - :raise: :exc:`~.HTTPError` if the request failed - """ - return self.http_request(path='/pools/default/buckets/' + name) - - def wait_ready(self, name, timeout=5.0, sleep_interval=0.2): - """ - Wait for a newly created bucket to be ready. - - :param string name: the name to wait for - :param seconds timeout: the maximum amount of time to wait - :param seconds sleep_interval: the number of time to sleep - between each probe - :raise: :exc:`.CouchbaseError` on internal HTTP error - :raise: :exc:`NotReadyError` if all nodes could not be - ready in time - """ - end = time() + timeout - while True: - try: - info = self.bucket_info(name).value - for node in info['nodes']: - if node['status'] != 'healthy': - raise NotReadyError.pyexc('Not all nodes are healthy') - return # No error and all OK - except E.CouchbaseError: - if time() + sleep_interval > end: - raise - sleep(sleep_interval) - - def bucket_update(self, name, current, bucket_password=None, replicas=None, - ram_quota=None, flush_enabled=None): - """ - Update an existing bucket's settings. - - :param string name: The name of the bucket to update - :param dict current: Current state of the bucket. - This can be retrieve from :meth:`bucket_info` - :param str bucket_password: Change the bucket's password - :param int replicas: The number of replicas for the bucket - :param int ram_quota: The memory available to the bucket - on each node. - :param bool flush_enabled: Whether the flush API should be allowed - from normal clients - :return: A :class:`~.HttpResult` object - :raise: :exc:`~.HTTPError` if the request could not be - completed - - - .. note:: - - The default value for all options in this method is - ``None``. If a value is set to something else, it will - modify the setting. - - - Change the bucket password:: - - adm.bucket_update('a_bucket', adm.bucket_info('a_bucket'), - bucket_password='n3wpassw0rd') - - Enable the flush API:: - - adm.bucket_update('a_bucket', adm.bucket_info('a_bucket'), - flush_enabled=True) - """ - params = {} - current = current.value - - # Merge params - params['authType'] = current['authType'] - if 'saslPassword' in current: - params['saslPassword'] = current['saslPassword'] - - if bucket_password is not None: - params['authType'] = 'sasl' - params['saslPassword'] = bucket_password - - params['replicaNumber'] = ( - replicas if replicas is not None else current['replicaNumber']) - - if ram_quota: - params['ramQuotaMB'] = ram_quota - else: - params['ramQuotaMB'] = current['quota']['ram'] / 1024 / 1024 - - if flush_enabled is not None: - params['flushEnabled'] = int(flush_enabled) - - params['proxyPort'] = current['proxyPort'] - return self.http_request(path='/pools/default/buckets/' + name, - method='POST', - content_type='application/x-www-form-urlencoded', - content=self._mk_formstr(params)) diff --git a/couchbase/analytics.py b/couchbase/analytics.py new file mode 100644 index 000000000..03b6dbf45 --- /dev/null +++ b/couchbase/analytics.py @@ -0,0 +1,117 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.analytics import AnalyticsError # noqa: F401 +from couchbase.logic.analytics import AnalyticsMetaData # noqa: F401 +from couchbase.logic.analytics import AnalyticsMetrics # noqa: F401 +from couchbase.logic.analytics import AnalyticsQuery # noqa: F401 +from couchbase.logic.analytics import AnalyticsScanConsistency # noqa: F401 +from couchbase.logic.analytics import AnalyticsStatus # noqa: F401 +from couchbase.logic.analytics import AnalyticsWarning # noqa: F401 +from couchbase.logic.analytics import AnalyticsRequestLogic +from couchbase.logic.supportability import Supportability + + +class AnalyticsRequest(AnalyticsRequestLogic): + def __init__(self, + connection, + query_params, + row_factory=lambda x: x, + **kwargs + ): + super().__init__(connection, query_params, row_factory=row_factory, **kwargs) + + @classmethod + def generate_analytics_request(cls, connection, query_params, row_factory=lambda x: x, **kwargs): + return cls(connection, query_params, row_factory=row_factory, **kwargs) + + def execute(self): + return [r for r in list(self)] + + def _get_metadata(self): + try: + # @TODO: PYCBC-1524 + analytics_response = next(self._streaming_result) + self._set_metadata(analytics_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def __iter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_query() + + return self + + def _get_next_row(self): + if self.done_streaming is True: + return + + try: + row = next(self._streaming_result) + except StopIteration: + # @TODO: PYCBC-1524 + row = next(self._streaming_result) + + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopIteration + + return self.serializer.deserialize(row) + + def __next__(self): + try: + return self._get_next_row() + except StopIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + +""" +** DEPRECATION NOTICE ** + +The classes below are deprecated for 3.x compatibility. They should not be used. +Instead use: + * All options should be imported from `couchbase.options`. + +""" + +from couchbase.logic.options import AnalyticsOptionsBase # nopep8 # isort:skip # noqa: E402 + + +@Supportability.import_deprecated('couchbase.analytics', 'couchbase.options') +class AnalyticsOptions(AnalyticsOptionsBase): # noqa: F811 + pass diff --git a/couchbase/async/__init__.py b/couchbase/async/__init__.py deleted file mode 100644 index f9fd1170f..000000000 --- a/couchbase/async/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from couchbase.async.bucket import AsyncBucket diff --git a/couchbase/async/async_bucket.py b/couchbase/async/async_bucket.py deleted file mode 120000 index e7b4468f8..000000000 --- a/couchbase/async/async_bucket.py +++ /dev/null @@ -1 +0,0 @@ -bucket.py \ No newline at end of file diff --git a/couchbase/async/bucket.py b/couchbase/async/bucket.py deleted file mode 100644 index b250e42b2..000000000 --- a/couchbase/async/bucket.py +++ /dev/null @@ -1,176 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import couchbase._bootstrap -from couchbase._libcouchbase import ( - AsyncResult, - PYCBC_CONN_F_ASYNC, - PYCBC_CONN_F_ASYNC_DTOR) - -from couchbase.result import AsyncResult -from couchbase.async.view import AsyncViewBase -from couchbase.bucket import Bucket -from couchbase.exceptions import ArgumentError - -class AsyncBucket(Bucket): - """ - This class contains the low-level async implementation of the - :class:`~.Bucket` interface. **This module is not intended to be - used directly by applications**. - - .. warning:: - - Using this module directly may cause odd error messages or - application crashes. Use an existing subclass designated for - your I/O framework (`txcouchbase`, `gcouchbase`, `acouchbase`) - or subclass this module (continue reading) if one does not - already exist. - - Additionally, this module is considered internal API, as - such, the interface is subject to change. - - - An asynchronous bucket must be wired to a so-called `IOPS` - implementation (see :class:`~couchbase.iops.base.IOPS`). The - purpose of the `IOPS` class is to provide the basic I/O wiring - between the module and the underlying event system. - - In non-asynchronous use modes (e.g. the normal asynchronous - `Bucket`), the wiring is done internally within the C library - via an event loop that is "run" for each operation and is - "stopped" whenever all operations complete. - - In order to successfully implement an asynchronous bucket, - rather than running and stopping the event loop for each - operation, it is assumed the event loop is driving the entire - application, and is implicitly run whenever control is returned - to it. - - In Python, two main styles of asynchronous programming exist: - - * Explicit callback-based asynchronous programming (such that - is found in Twisted). This style explicitly makes applications - aware of an event loop (or "reactor") and requests that they - register callbacks for various events. - * Coroutine-based asynchronous programming, that involves - *implicitly* _yielding_ to an event loop. In this style, - the programming style seems to be synchronous, and the actual - event library (for example, `gevent`, or `tulip`) will - implicitly yield to the event loop when the current coroutine - awaits I/O completion. These forms of event loops, are from the - library's perspective, identical to the classic callback-based - event loops (but see below). - - - In both event models, the internal I/O notification system is - callback-based. The main difference is in how the high-level - `Bucket` functions (for example, :meth:`~.Bucket.get` operate: - - In callback-based models, these return objects which allow a - callback to be assigned to them, whereas in coroutine-based - models, these will implicitly yield to other couroutines. - - In both cases, the operations (from this class itself) will - return an object which allows the callback to be set. Subclasses - of this module should ensure that this return value is wrapped - into a suitable object appropriate to whichever event framework - is actually being used. - - Several known subclasses exist: - - * :class:`acouchbase.bucket.Bucket` - this is the Python3/Tulip - based implementation, and uses a hybrid callback/implicit - yield functionality (by returning "future" objects). - * :class:`gcouchbase.bucket.Bucket` - this is the `gevent` - based implementation, and uses an implicit yield model; where - the bucket class will yield to the event loop and return - actual "result" objects - * :class:`txcouchbase.bucket.RawBucket` - this is a thin wrapper - around this class, which returns :class:`~.AsyncResult` objects: - Since Twisted is callback-based, it is possible to return these - raw objects and still remain somewhat idiomatic. - * :class:`txcouchbase.bucket.Bucket` - this wraps the `RawBucket` - class (above) and returns Deferred objects. - - """ - - def __init__(self, iops=None, *args, **kwargs): - """ - Create a new Async Bucket. An async Bucket is an object - which functions like a normal synchronous bucket connection, - except that it returns future objects - (i.e. :class:`~couchbase.result.AsyncResult` - objects) instead of :class:`~couchbase.result.Result`. - These objects are actually :class:`~couchbase.result.MultiResult` - objects which are empty upon retun. As operations complete, this - object becomes populated with the relevant data. - - Note that the AsyncResult object must currently have valid - :attr:`~couchbase.result.AsyncResult.callback` and - :attr:`~couchbase.result.AsyncResult.errback` fields initialized - *after* they are returned from - the API methods. If this is not the case then an exception will be - raised when the callbacks are about to arrive. This behavior is the - primary reason why this interface isn't public, too :) - - :param iops: An :class:`~couchbase.iops.base.IOPS`-interface - conforming object. This object must not be used between two - instances, and is owned by the connection object. - - :param kwargs: Additional arguments to pass to - the :class:`~couchbase.bucket.Bucket` constructor - """ - if not iops: - raise ValueError("Must have IOPS") - - kwargs.setdefault('_flags', 0) - - # Must have an IOPS implementation - kwargs['_iops'] = iops - - # Flags should be async - kwargs['_flags'] |= PYCBC_CONN_F_ASYNC|PYCBC_CONN_F_ASYNC_DTOR - - # Don't lock/unlock GIL as the enter/leave points are not coordinated - # kwargs['unlock_gil'] = False - # This is always set to false in connection.c - - super(AsyncBucket, self).__init__(*args, **kwargs) - - def query(self, *args, **kwargs): - """ - Reimplemented from base class. - - This method does not add additional functionality of the - base class' :meth:`~.Bucket.query` method (all the - functionality is encapsulated in the view class anyway). However it - does require one additional keyword argument - - :param class itercls: A class used for instantiating the view - object. This should be a subclass of - :class:`~couchbase.async.view.AsyncViewBase`. - """ - if not issubclass(kwargs.get('itercls', None), AsyncViewBase): - raise ArgumentError.pyexc("itercls must be defined " - "and must be derived from AsyncViewBase") - - return super(AsyncBucket, self).query(*args, **kwargs) - - def endure(self, key, *args, **kwargs): - res = super(AsyncBucket, self).endure_multi([key], *args, **kwargs) - res._set_single() - return res diff --git a/couchbase/async/events.py b/couchbase/async/events.py deleted file mode 100644 index d8de9ec78..000000000 --- a/couchbase/async/events.py +++ /dev/null @@ -1,82 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file contains various utility classes for scheduling -and destroying events -""" - -class EventQueue(object): - def __init__(self): - self.called = False - self.waiters = [] - - def fire_async(self, event): - """ - Fire this event 'immediately', but in the next event iteration - """ - - def maybe_raise(self, *args, **kwargs): - """ - Given the arguments from '__call__', see if we should raise an error - """ - - def call_single_success(self, event, *args, **kwargs): - """ - Call a single event with success - """ - - def call_single_failure(self, event, *args, **kwargs): - """ - Call a single event with a failure. This will be from within - the 'except' block and thus will have sys.exc_info() available - """ - - def schedule(self, event): - if self.called: - self.fire_async(event) - return - - self.waiters.append(event) - - def __hash__(self): - return hash(self.name) - - def __len__(self): - return len(self.waiters) - - def __iter__(self): - return iter(self.waiters) - - def invoke_waiters(self, *args, **kwargs): - self.called = True - try: - self.maybe_raise(*args, **kwargs) - for event in self.waiters: - try: - self.call_single_success(event, *args, **kwargs) - except: - pass - except: - for event in self.waiters: - try: - self.call_single_failure(event, event, *args, **kwargs) - except Exception as e: - pass - - self.waiters = None - - def __call__(self, *args, **kwargs): - self.invoke_waiters(*args, **kwargs) diff --git a/couchbase/async/n1ql.py b/couchbase/async/n1ql.py deleted file mode 100644 index 1b16ea1fa..000000000 --- a/couchbase/async/n1ql.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -This file contains the base N1QL implementation for async frameworks. -""" - -from couchbase.n1ql import N1QLRequest -from couchbase.async.rowsbase import AsyncRowsBase - - -class AsyncN1QLRequest(AsyncRowsBase, N1QLRequest): - def __init__(self, *args, **kwargs): - N1QLRequest.__init__(self, *args, **kwargs) diff --git a/couchbase/async/rowsbase.py b/couchbase/async/rowsbase.py deleted file mode 100644 index 58af9e77f..000000000 --- a/couchbase/async/rowsbase.py +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -Abstract class for handling async row-based APIs (i.e. N1QL, Views, 2i, etc) -""" - -from couchbase._pyport import PyErr_Restore - - -class AsyncRowsBase(object): - # This class relies on having various properties defined in the base - # class. Unfortunately, abc doesn't allow the proper inheritance - # diagram. - - def __iter__(self): - """ - Unlike our base class, iterating does not make sense here - """ - raise NotImplementedError("Iteration not supported on async view") - - def on_error(self, ex): - """ - Called when there is a failure with the response data - - :param Exception ex: The exception caught. - - This must be implemented in a subclass - """ - raise NotImplementedError("Must be implemented in subclass") - - def on_rows(self, rowiter): - """ - Called when there are more processed views. - - :param iterable rowiter: An iterable which will yield results - as defined by the :class:`RowProcessor` implementation - - This method must be implemented in a subclass - """ - raise NotImplementedError("Must be implemented in subclass") - - def on_done(self): - """ - Called when this request has completed. Once this method is called, - no other methods will be invoked on this object. - - This method must be implemented in a subclass - """ - raise NotImplementedError("Must be implemented in subclass") - - def _callback(self, mres): - """ - This is invoked as the row callback. - If 'rows' is true, then we are a row callback, otherwise - the request has ended and it's time to collect the other data - """ - try: - rows = self._process_payload(self.raw.rows) - if rows: - self.on_rows(rows) - if self.raw.done: - self.on_done() - finally: - if self.raw.done: - self._clear() - - def _errback(self, mres, ex_cls, ex_obj, ex_bt): - try: - PyErr_Restore(ex_cls, ex_obj, ex_bt) - except Exception as e: - self.on_error(e) - self.on_done() - finally: - self._clear() - - def _start(self): - super(AsyncRowsBase, self)._start() - self._mres.callback = self._callback - self._mres.errback = self._errback - - def start(self): - return self._start() \ No newline at end of file diff --git a/couchbase/async/view.py b/couchbase/async/view.py deleted file mode 100644 index 48b81b445..000000000 --- a/couchbase/async/view.py +++ /dev/null @@ -1,38 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -This file contains the view implementation for Async -""" - -from couchbase.views.iterator import View -from couchbase.async.rowsbase import AsyncRowsBase - - -class AsyncViewBase(AsyncRowsBase, View): - def __init__(self, *args, **kwargs): - """ - Initialize a new AsyncViewBase object. This is intended to be - subclassed in order to implement the require methods to be - invoked on error, data, and row events. - - Usage of this class is not as a standalone, but rather as - an ``itercls`` parameter to the - :meth:`~couchbase.connection.Connection.query` method of the - connection object. - """ - View.__init__(self, *args, **kwargs) diff --git a/couchbase/auth.py b/couchbase/auth.py new file mode 100644 index 000000000..e751a0e11 --- /dev/null +++ b/couchbase/auth.py @@ -0,0 +1,172 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from enum import IntEnum +from typing import (Any, + Dict, + Optional) + +from couchbase.exceptions import InvalidArgumentException + + +class Authenticator(dict): + pass + + +class PasswordAuthenticator(Authenticator): + """ + Password authentication mechanism. + + Args: + username (str): Username to use for authentication. + password (str): Password to use for authentication. + cert_path (str, optional): Path of the certificate trust store. Defaults to None. + """ + + def __init__(self, + username, # type: str + password, # type: str + cert_path=None, # type: Optional[str] + **kwargs # type: Dict[str, Any] + ): + """PasswordAuthenticator instance.""" + if not isinstance(username, str): + msg = 'The username must be a str.' + raise InvalidArgumentException(msg) + + if not isinstance(password, str): + msg = 'The password must be a str.' + raise InvalidArgumentException(msg) + + if cert_path is not None and not isinstance(cert_path, str): + msg = 'The cert_path must be a str representing the path to the certificate trust store.' + raise InvalidArgumentException(msg) + + allowed_sasl_mechanisms = kwargs.pop('allowed_sasl_mechanisms', None) + if allowed_sasl_mechanisms is not None: + msg = None + if isinstance(allowed_sasl_mechanisms, str): + allowed_sasl_mechanisms = allowed_sasl_mechanisms.split(',') + if isinstance(allowed_sasl_mechanisms, list): + if not all(map(lambda x: isinstance(x, str), allowed_sasl_mechanisms)): + msg = 'The allowed_sasl_mechanisms must be a list of str SASL mechanisms.' + else: + msg = ('The allowed_sasl_mechanisms must be a list of str SASL mechanisms ' + ' or a comma separated str of SASL mechanisms.') + if msg: + raise InvalidArgumentException(msg) + + self._username = username + self._password = password + self._cert_path = cert_path + self._allowed_sasl_mechanisms = allowed_sasl_mechanisms + + super().__init__(**self.as_dict()) + + def valid_keys(self): + return ['username', 'password', 'cert_path', 'sasl_mech_force', 'allowed_sasl_mechanisms'] + + def as_dict(self): + d = { + 'username': self._username, + 'password': self._password, + 'allowed_sasl_mechanisms': self._allowed_sasl_mechanisms + } + if self._cert_path is not None: + # couchbase++ wants this to be the trust_certificate + d['trust_store_path'] = self._cert_path + + return d + + @staticmethod + def ldap_compatible(username, # type: str + password # type: str + ) -> PasswordAuthenticator: + auth = PasswordAuthenticator(username, password, allowed_sasl_mechanisms=['PLAIN']) + return auth + + +class CertificateAuthenticator(Authenticator): + """ + Certificate authentication mechanism. + + Args: + cert_path (str): Path to the client certificate. Defaults to None. + key_path (str): Path to the client key. Defaults to None. + trust_store_path (str, optional): Path of the certificate trust store. Defaults to None. + """ + + def __init__(self, + cert_path=None, # type: str + key_path=None, # type: str + trust_store_path=None # type: Optional[str] + ): + """CertificateAuthenticator instance.""" + if not isinstance(cert_path, str): + msg = 'The cert_path must be a str representing the path to the client certificate.' + raise InvalidArgumentException(msg) + + if not isinstance(key_path, str): + msg = 'The key_path must be a str representing the path to the client key.' + raise InvalidArgumentException(msg) + + if trust_store_path is not None and not isinstance(trust_store_path, str): + msg = 'The trust_store_path must be a str representing the path to the certificate trust store.' + raise InvalidArgumentException(msg) + + self._trust_store_path = trust_store_path + self._cert_path = cert_path + self._key_path = key_path + + super().__init__(**self.as_dict()) + + def valid_keys(self): + return ['cert_path', 'key_path', 'trust_store_path'] + + def as_dict(self): + d = { + 'cert_path': self._cert_path, + 'key_path': self._key_path + } + if self._trust_store_path is not None: + d['trust_store_path'] = self._trust_store_path + + return d + + +class AuthDomain(IntEnum): + """ + The Authentication domain for a user. + Local: Users managed by Couchbase Server. + External: Users managed by an external resource, eg LDAP. + """ + Local = 0 + External = 1 + + @classmethod + def to_str(cls, value): + if value == cls.External: + return "external" + else: + return "local" + + @classmethod + def from_str(cls, value): + if value == "external": + return cls.External + else: + return cls.Local diff --git a/couchbase/binary_collection.py b/couchbase/binary_collection.py new file mode 100644 index 000000000..108c5c435 --- /dev/null +++ b/couchbase/binary_collection.py @@ -0,0 +1,609 @@ + +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict, + List, + Union) + +from couchbase.result import (CounterResult, + MultiCounterResult, + MultiMutationResult, + MutationResult) + +if TYPE_CHECKING: + from couchbase.options import (AppendMultiOptions, + AppendOptions, + DecrementMultiOptions, + DecrementOptions, + IncrementMultiOptions, + IncrementOptions, + PrependMultiOptions, + PrependOptions) + + +class BinaryCollection: + + def __init__(self, collection): + self._collection = collection + + def increment( + self, + key, # type: str + *opts, # type: IncrementOptions + **kwargs, # type: Any + ) -> CounterResult: + """Increments the ASCII value of the document, specified by the key, by the amount indicated in the delta + option (defaults to 1). + + Args: + key (str): The key of the document to increment. + opts (:class:`~couchbase.options.IncrementOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.IncrementOptions` + + Returns: + :class:`~couchbase.result.CounterResult`: An instance of :class:`~couchbase.result.CounterResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple increment operation:: + + from couchbase.options import IncrementOptions, SignedInt64 + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().increment('counter-doc', IncrementOptions(initial=SignedInt64(100)) + print(f'Counter value: {res.content}') + + Simple increment operation, change default delta:: + + from couchbase.options import IncrementOptions, DeltaValue + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().increment('counter-doc', IncrementOptions(delta=DeltaValue(5)) + print(f'Counter value: {res.content}') + + """ + return self._collection._increment(key, *opts, **kwargs) + + def decrement( + self, + key, # type: str + *opts, # type: DecrementOptions + **kwargs, # type: Any + ) -> CounterResult: + """Decrements the ASCII value of the document, specified by the key, by the amount indicated in the delta + option (defaults to 1). + + Args: + key (str): The key of the document to decrement. + opts (:class:`~couchbase.options.DecrementOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.DecrementOptions` + + Returns: + :class:`~couchbase.result.CounterResult`: An instance of :class:`~couchbase.result.CounterResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple decrement operation:: + + from couchbase.options import DecrementOptions, SignedInt64 + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().decrement('counter-doc', DecrementOptions(initial=SignedInt64(100)) + print(f'Counter value: {res.content}') + + Simple decrement operation, change default delta:: + + from couchbase.options import DecrementOptions, DeltaValue + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().decrement('counter-doc', DecrementOptions(delta=DeltaValue(5)) + print(f'Counter value: {res.content}') + + """ + return self._collection._decrement(key, *opts, **kwargs) + + def append( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: AppendOptions + **kwargs, # type: Any + ) -> MutationResult: + """Appends the specified value to the beginning of document of the specified key. + + Args: + key (str): The key of the document to append to. + value (Union[str, bytes, bytearray]): The value to append to the document. + opts (:class:`~couchbase.options.AppendOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.AppendOptions` + + Returns: + :class:`~couchbase.result.MutationResult`: An instance of :class:`~couchbase.result.MutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple append string operation:: + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().append('string-doc', 'XYZ') + + Simple append binary operation:: + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().append('binary-doc', b'XYZ') + + Simple append operation with options:: + + from datetime import timedelta + + from couchbase.options import AppendOptions + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().append('string-doc', + 'XYZ', + AppendOptions(timeout=timedelta(seconds=2))) + + """ + return self._collection._append(key, value, *opts, **kwargs) + + def prepend( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: PrependOptions + **kwargs, # type: Any + ) -> MutationResult: + """Prepends the specified value to the beginning of document of the specified key. + + Args: + key (str): The key of the document to prepend to. + value (Union[str, bytes, bytearray]): The value to prepend to the document. + opts (:class:`~couchbase.options.PrependOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.PrependOptions` + + Returns: + :class:`~couchbase.result.MutationResult`: An instance of :class:`~couchbase.result.MutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple prepend string operation:: + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().prepend('string-doc', 'ABC') + + Simple prepend binary operation:: + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().prepend('binary-doc', b'ABC') + + Simple prepend operation with options:: + + from datetime import timedelta + + from couchbase.options import PrependOptions + + # ... other code ... + + collection = bucket.default_collection() + res = collection.binary().prepend('string-doc', + 'ABC', + PrependOptions(timeout=timedelta(seconds=2))) + + """ + return self._collection._prepend(key, value, *opts, **kwargs) + + def append_multi( + self, + keys_and_values, # type: Dict[str, Union[str,bytes,bytearray]] + *opts, # type: AppendMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + """For each key-value pair, appends the specified value to the end of the document specified by the key. + + Args: + keys_and_values (Dict[str, Union[str,bytes,bytearray]]): The key-value pairs to use for the multiple + append operations. Each key should correspond to the document to append to and each value should + correspond to the value to append to the document. + opts (:class:`~couchbase.options.AppendMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.AppendMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of + :class:`~couchbase.result.MultiMutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on + the server and the return_exceptions options is False. Otherwise the exception is returned + as a match to the key, but is not raised. + + Examples: + + Simple append-multi string operation:: + + # ... other code ... + + collection = bucket.default_collection() + keys = ['str-doc1', 'str-doc2', 'str-doc3'] + values = ['foo', 'bar', 'baz'] + keys_and_docs = dict(zip(keys, values)) + res = collection.binary().append_multi(keys_and_docs) + + Simple append-multi binary operation:: + + # ... other code ... + + collection = bucket.default_collection() + keys = ['bin-doc1', 'bin-doc2', 'bin-doc3'] + values = [b'foo', b'bar', b'baz'] + keys_and_docs = dict(zip(keys, values)) + res = collection.binary().append_multi(keys_and_docs) + + Simple append-multi operation with options:: + + from datetime import timedelta + + from couchbase.options import AppendMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['bin-doc1', 'bin-doc2', 'bin-doc3'] + values = [b'foo', b'bar', b'baz'] + keys_and_docs = dict(zip(keys, values)) + res = collection.binary().append_multi(keys_and_docs, + AppendMultiOptions(timeout=timedelta(seconds=2))) + + Simple append-multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import AppendMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['bin-doc1', 'bin-doc2', 'bin-doc3'] + values = [b'foo', b'bar', b'baz'] + keys_and_docs = dict(zip(keys, values)) + res = collection.binary().append_multi(keys_and_docs, + AppendMultiOptions(return_exceptions=False)) + + Simple append-multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import AppendMultiOptions, AppendOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['bin-doc1', 'bin-doc2', 'bin-doc3'] + values = [b'foo', b'bar', b'baz'] + keys_and_docs = dict(zip(keys, values)) + per_key_opts = {'bin-doc1': AppendOptions(timeout=timedelta(seconds=10))} + res = collection.binary().append_multi(keys_and_docs, + AppendMultiOptions(return_exceptions=False, + per_key_options=per_key_opts)) + + + """ + return self._collection._append_multi(keys_and_values, *opts, **kwargs) + + def prepend_multi( + self, + keys_and_values, # type: Dict[str, Union[str,bytes,bytearray]] + *opts, # type: PrependMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + """For each key-value pair, prepends the specified value to the beginning of the document specified + by the key. + + Args: + keys_and_values (Dict[str, Union[str,bytes,bytearray]]): The key-value pairs to use for the multiple + prepend operations. Each key should correspond to the document to prepend to and each value should + correspond to the value to prepend to the document. + opts (:class:`~couchbase.options.PrependMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.PrependMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of + :class:`~couchbase.result.MultiMutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a match + to the key, but is not raised. + + Examples: + + Simple prepend-multi string operation:: + + # ... other code ... + + collection = bucket.default_collection() + keys = ['str-doc1', 'str-doc2', 'str-doc3'] + values = ['foo', 'bar', 'baz'] + keys_and_docs = dict(zip(keys, values)) + res = collection.binary().prepend_multi(keys_and_docs) + + Simple prepend-multi binary operation:: + + # ... other code ... + + collection = bucket.default_collection() + keys = ['bin-doc1', 'bin-doc2', 'bin-doc3'] + values = [b'foo', b'bar', b'baz'] + keys_and_docs = dict(zip(keys, values)) + res = collection.binary().prepend_multi(keys_and_docs) + + Simple prepend-multi operation with options:: + + from datetime import timedelta + + from couchbase.options import PrependMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['bin-doc1', 'bin-doc2', 'bin-doc3'] + values = [b'foo', b'bar', b'baz'] + keys_and_docs = dict(zip(keys, values)) + res = collection.binary().prepend_multi(keys_and_docs, + PrependMultiOptions(timeout=timedelta(seconds=2))) + + Simple prepend-multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import PrependMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['bin-doc1', 'bin-doc2', 'bin-doc3'] + values = [b'foo', b'bar', b'baz'] + keys_and_docs = dict(zip(keys, values)) + res = collection.binary().prepend_multi(keys_and_docs, + PrependMultiOptions(return_exceptions=False)) + + Simple prepend-multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import PrependMultiOptions, PrependOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['bin-doc1', 'bin-doc2', 'bin-doc3'] + values = [b'foo', b'bar', b'baz'] + keys_and_docs = dict(zip(keys, values)) + per_key_opts = {'bin-doc1': PrependOptions(timeout=timedelta(seconds=10))} + res = collection.binary().prepend_multi(keys_and_docs, + PrependMultiOptions(return_exceptions=False, + per_key_options=per_key_opts)) + + + """ + return self._collection._prepend_multi(keys_and_values, *opts, **kwargs) + + def increment_multi( + self, + keys, # type: List[str] + *opts, # type: IncrementMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiCounterResult: + """For each key in the provided list, increments the ASCII value of the document, specified by the key, + by the amount indicated in the delta option (defaults to 1). + + Args: + keys (List[str]): The keys to use for the multiple increment operations. Each key should correspond + to the document to increment. + opts (:class:`~couchbase.options.IncrementMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.IncrementMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of + :class:`~couchbase.result.MultiMutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on + the server and the return_exceptions options is False. Otherwise the exception is returned + as a match to the key, but is not raised. + + Examples: + + Simple increment-multi operation, set initial value:: + + from couchbase.options import IncrementMultiOptions, SignedInt64 + + # ... other code ... + + collection = bucket.default_collection() + keys = ['counter_doc1', 'counter_doc1', 'counter_doc1'] + res = collection.binary().increment_multi(keys, IncrementMultiOptions(initial=SignedInt64(100)) + for k, v in res.results.items(): + print(f'Counter doc {k} has value: {v.content}') + + Simple increment-multi operation, change default delta:: + + from couchbase.options import IncrementMultiOptions, DeltaValue + + # ... other code ... + + collection = bucket.default_collection() + keys = ['counter_doc1', 'counter_doc1', 'counter_doc1'] + res = collection.binary().increment_multi(keys, IncrementMultiOptions(delta=DeltaValue(5)) + for k, v in res.results.items(): + print(f'Counter doc {k} has value: {v.content}') + + + Simple increment-multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import IncrementMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['counter_doc1', 'counter_doc1', 'counter_doc1'] + res = collection.binary().increment_multi(keys, + IncrementMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + print(f'Counter doc {k} has value: {v.content}') + + Simple increment-multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import IncrementMultiOptions, IncrementOptions, DeltaValue, SignedInt64 + + # ... other code ... + + collection = bucket.default_collection() + keys = ['counter_doc1', 'counter_doc1', 'counter_doc1'] + per_key_opts = {'counter_doc1': IncrementOptions(delta=DeltaValue(100),initial=SignedInt64(100))} + res = collection.binary().increment_multi(keys, + IncrementMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + print(f'Counter doc {k} has value: {v.content}') + + + """ + return self._collection._increment_multi(keys, *opts, **kwargs) + + def decrement_multi( + self, + keys, # type: List[str] + *opts, # type: DecrementMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiCounterResult: + """For each key in the provided list, decrements the ASCII value of the document, specified by the key, + by the amount indicated in the delta option (defaults to 1). + + Args: + keys (List[str]): The keys to use for the multiple decrement operations. Each key should correspond + to the document to decrement. + opts (:class:`~couchbase.options.DecrementMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.DecrementMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of + :class:`~couchbase.result.MultiMutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + Examples: + + Simple decrement-multi operation, set initial value:: + + from couchbase.options import DecrementMultiOptions, SignedInt64 + + # ... other code ... + + collection = bucket.default_collection() + keys = ['counter_doc1', 'counter_doc1', 'counter_doc1'] + res = collection.binary().decrement_multi(keys, DecrementMultiOptions(initial=SignedInt64(100)) + for k, v in res.results.items(): + print(f'Counter doc {k} has value: {v.content}') + + Simple decrement-multi operation, change default delta:: + + from couchbase.options import DecrementMultiOptions, DeltaValue + + # ... other code ... + + collection = bucket.default_collection() + keys = ['counter_doc1', 'counter_doc1', 'counter_doc1'] + res = collection.binary().decrement_multi(keys, DecrementMultiOptions(delta=DeltaValue(5)) + for k, v in res.results.items(): + print(f'Counter doc {k} has value: {v.content}') + + + Simple decrement-multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import DecrementMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['counter_doc1', 'counter_doc1', 'counter_doc1'] + res = collection.binary().decrement_multi(keys, + DecrementMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + print(f'Counter doc {k} has value: {v.content}') + + Simple decrement-multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import DecrementMultiOptions, DecrementOptions, DeltaValue, SignedInt64 + + # ... other code ... + + collection = bucket.default_collection() + keys = ['counter_doc1', 'counter_doc1', 'counter_doc1'] + per_key_opts = {'counter_doc1': DecrementOptions(delta=DeltaValue(100),initial=SignedInt64(100))} + res = collection.binary().decrement_multi(keys, + DecrementMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + print(f'Counter doc {k} has value: {v.content}') + + + """ + return self._collection._decrement_multi(keys, *opts, **kwargs) diff --git a/couchbase/bucket.py b/couchbase/bucket.py index 3bb5f6257..a67c092ef 100644 --- a/couchbase/bucket.py +++ b/couchbase/bucket.py @@ -1,1473 +1,252 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. # -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from warnings import warn - -import couchbase._bootstrap -import couchbase._libcouchbase as _LCB -from couchbase._libcouchbase import Bucket as _Base - -from couchbase.exceptions import * -from couchbase.user_constants import * -from couchbase.result import * -from couchbase.bucketmanager import BucketManager - -import couchbase.exceptions as exceptions -from couchbase.views.params import make_dvpath, make_options_string -from couchbase.views.iterator import View -from couchbase.n1ql import N1QLQuery, N1QLRequest -from couchbase._pyport import basestring - - -def _depr(fn, usage, stacklevel=3): - """Internal convenience function for deprecation warnings""" - warn('{0} is deprecated. Use {1} instead'.format(fn, usage), - stacklevel=stacklevel, category=DeprecationWarning) - - -class Pipeline(object): - def __init__(self, parent): - """ - .. versionadded:: 1.2.0 - - Creates a new pipeline context. See :meth:`~Bucket.pipeline` - for more details - """ - self._parent = parent - self._results = None - - def __enter__(self): - self._parent._pipeline_begin() - - def __exit__(self, *args): - self._results = self._parent._pipeline_end() - return False - - @property - def results(self): - """ - Contains a list of results for each pipelined operation - executed within the context. The list remains until this - context is reused. - - The elements in the list are either :class:`.Result` - objects (for single operations) or :class:`.MultiResult` - objects (for multi operations) - """ - return self._results - - -class DurabilityContext(object): - def __init__(self, parent, persist_to=-1, replicate_to=-1, timeout=0.0): - self._parent = parent - self._new = { - '_dur_persist_to': persist_to, - '_dur_replicate_to': replicate_to, - '_dur_timeout': int(timeout * 1000000) - } - self._old = {} - - def __enter__(self): - for k, v in self._new.items(): - self._old[k] = getattr(self._parent, k) - setattr(self._parent, k, v) - - def __exit__(self, *args): - for k, v in self._old.items(): - setattr(self._parent, k, v) - return False - - -class Bucket(_Base): - def __init__(self, *args, **kwargs): - """Connect to a bucket. - - :param string connection_string: - The connection string to use for connecting to the bucket. - This is a URI-like string allowing specifying multiple hosts - and a bucket name. - - The format of the connection string is the *scheme* - (``couchbase`` for normal connections, ``couchbases`` for - SSL enabled connections); a list of one or more *hostnames* - delimited by commas; a *bucket* and a set of options. - - like so:: - - couchbase://host1,host2,host3/bucketname?option1=value1&option2=value2 - - If using the SSL scheme (``couchbases``), ensure to specify - the ``certpath`` option to point to the location of the - certificate on the client's filesystem; otherwise connection - may fail with an error code indicating the server's - certificate could not be trusted. - - See :ref:`connopts` for additional connection options. - - :param string password: the password of the bucket - - :param boolean quiet: the flag controlling whether to raise an - exception when the client executes operations on - non-existent keys. If it is `False` it will raise - :exc:`.NotFoundError` exceptions. When - set to `True` the operations will return `None` silently. - - :param boolean unlock_gil: If set (which is the default), the - bucket object will release the python GIL when possible, - allowing other (Python) threads to function in the - background. This should be set to true if you are using - threads in your application (and is the default), as - otherwise all threads will be blocked while couchbase - functions execute. - - You may turn this off for some performance boost and you are - certain your application is not using threads - - :param transcoder: - Set the transcoder object to use. This should conform to the - interface in the documentation (it need not actually be a - subclass). This can be either a class type to instantiate, - or an initialized instance. - :type transcoder: :class:`.Transcoder` - - :param lockmode: The *lockmode* for threaded access. - See :ref:`multiple_threads` for more information. - - :raise: :exc:`.BucketNotFoundError` or :exc:`.AuthError` if - there is no such bucket to connect to, or if invalid - credentials were supplied. - :raise: :exc:`.CouchbaseNetworkError` if the socket wasn't - accessible (doesn't accept connections or doesn't respond - in - :raise: :exc:`.InvalidError` if the connection string - was malformed. - - :return: instance of :class:`~.Bucket` - - - Initialize bucket using default options:: - - from couchbase.bucket import Bucket - cb = Bucket('couchbase:///mybucket') - - Connect to protected bucket:: - - cb = Bucket('couchbase:///protected', password='secret') - - Connect using a list of servers:: - - cb = Bucket('couchbase://host1,host2,host3/mybucket') - - Connect using SSL:: - - cb = Bucket('couchbases://securehost/bucketname?certpath=/var/cb-cert.pem') - - """ - _no_connect_exceptions = kwargs.pop('_no_connect_exceptions', False) - _cntlopts = kwargs.pop('_cntl', {}) - - # The following two blocks adapt some options from 1.x to proper - # connection string (or lcb_cntl_string()) settings. - strcntls = {} - if 'timeout' in kwargs: - _depr('timeout keyword argument', - 'operation_timeout (with float value) in connection string') - strcntls['operation_timeout'] = str(float(kwargs.pop('timeout'))) - - if 'config_cache' in kwargs: - _depr('config_cache keyword argument', - 'config_cache in connection string') - strcntls['config_cache'] = kwargs.pop('config_cache') - - tc = kwargs.get('transcoder') - if isinstance(tc, type): - kwargs['transcoder'] = tc() - - super(Bucket, self).__init__(*args, **kwargs) - # Enable detailed error codes for network errors: - self._cntlstr("detailed_errcodes", "1") - - for ctl, val in strcntls.items(): - self._cntlstr(ctl, val) - - for ctl, val in _cntlopts.items(): - self._cntl(ctl, val) - - try: - self._do_ctor_connect() - except exceptions.CouchbaseError as e: - if not _no_connect_exceptions: - raise - - def _do_ctor_connect(self): - """This should be overidden by subclasses which want to use a - different sort of connection behavior - """ - self._connect() - - def pipeline(self): - """ - Returns a new :class:`Pipeline` context manager. When the - context manager is active, operations performed will return - ``None``, and will be sent on the network when the context - leaves (in its ``__exit__`` method). To get the results of the - pipelined operations, inspect the :attr:`Pipeline.results` - property. - - Operational errors (i.e. negative replies from the server, or - network errors) are delivered when the pipeline exits, but - argument errors are thrown immediately. - - :return: a :class:`Pipeline` object - :raise: :exc:`.PipelineError` if a pipeline is already created - :raise: Other operation-specific errors. - - Scheduling multiple operations, without checking results:: - - with cb.pipeline(): - cb.upsert("key1", "value1") - cb.counter("counter") - cb.upsert_multi({ - "new_key1" : "new_value_1", - "new_key2" : "new_value_2" - }) - - Retrieve the results for several operations:: - - pipeline = cb.pipeline() - with pipeline: - cb.upsert("foo", "bar") - cb.replace("something", "value") - - for result in pipeline.results: - print("Pipeline result: CAS {0}".format(result.cas)) - - .. note:: - - When in pipeline mode, you cannot execute view queries. - Additionally, pipeline mode is not supported on async handles - - .. warning:: - - Pipeline mode should not be used if you are using the same - object concurrently from multiple threads. This only refers - to the internal lock within the object itself. It is safe - to use if you employ your own locking mechanism (for example - a connection pool) - - .. versionadded:: 1.2.0 - - """ - return Pipeline(self) - - # We have these wrappers so that IDEs can do param tooltips and the - # like. we might move this directly into C some day - - def upsert(self, key, value, cas=0, ttl=0, format=None, - persist_to=0, replicate_to=0): - """Unconditionally store the object in Couchbase. - - :param key: - The key to set the value with. By default, the key must be - either a :class:`bytes` or :class:`str` object encodable as - UTF-8. If a custom `transcoder` class is used (see - :meth:`~__init__`), then the key object is passed directly - to the transcoder, which may serialize it how it wishes. - :type key: string or bytes - - :param value: The value to set for the key. The type for `value` - follows the same rules as for `key` - - :param int cas: The _CAS_ value to use. If supplied, the value - will only be stored if it already exists with the supplied - CAS - - :param int ttl: If specified, the key will expire after this - many seconds - - :param int format: If specified, indicates the `format` to use - when encoding the value. If none is specified, it will use - the `default_format` For more info see - :attr:`~.default_format` - - :param int persist_to: - Perform durability checking on this many nodes nodes for - persistence to disk. See :meth:`endure` for more information - - :param int replicate_to: Perform durability checking on this - many replicas for presence in memory. See :meth:`endure` for - more information. - - :raise: :exc:`.ArgumentError` if an argument is supplied that is - not applicable in this context. For example setting the CAS - as a string. - :raise: :exc`.CouchbaseNetworkError` - :raise: :exc:`.KeyExistsError` if the key already exists on the - server with a different CAS value. - :raise: :exc:`.ValueFormatError` if the value cannot be - serialized with chosen encoder, e.g. if you try to store a - dictionary in plain mode. - :return: :class:`~.Result`. - - Simple set:: - - cb.upsert('key', 'value') - - Force JSON document format for value:: - - cb.upsert('foo', {'bar': 'baz'}, format=couchbase.FMT_JSON) - - Perform optimistic locking by specifying last known CAS version:: - - cb.upsert('foo', 'bar', cas=8835713818674332672) - - Several sets at the same time (mutli-set):: - - cb.upsert_multi({'foo': 'bar', 'baz': 'value'}) - - .. seealso:: :meth:`upsert_multi` - """ - return _Base.upsert(self, key, value, cas=cas, ttl=ttl, - format=format, persist_to=persist_to, - replicate_to=replicate_to) - - def insert(self, key, value, ttl=0, format=None, persist_to=0, replicate_to=0): - """Store an object in Couchbase unless it already exists. - - Follows the same conventions as :meth:`upsert` but the value is - stored only if it does not exist already. Conversely, the value - is not stored if the key already exists. - - Notably missing from this method is the `cas` parameter, this is - because `insert` will only succeed if a key does not already - exist on the server (and thus can have no CAS) - - :raise: :exc:`.KeyExistsError` if the key already exists - - .. seealso:: :meth:`upsert`, :meth:`insert_multi` - """ - return _Base.insert(self, key, value, ttl=ttl, format=format, - persist_to=persist_to, replicate_to=replicate_to) - - def replace(self, key, value, cas=0, ttl=0, format=None, - persist_to=0, replicate_to=0): - """Store an object in Couchbase only if it already exists. - - Follows the same conventions as :meth:`upsert`, but the value is - stored only if a previous value already exists. - - :raise: :exc:`.NotFoundError` if the key does not exist - - .. seealso:: :meth:`upsert`, :meth:`replace_multi` - """ - return _Base.replace(self, key, value, ttl=ttl, cas=cas, format=format, - persist_to=persist_to, replicate_to=replicate_to) - - def append(self, key, value, cas=0, format=None, - persist_to=0, replicate_to=0): - """Append a string to an existing value in Couchbase. - - :param string value: The data to append to the existing value. - - Other parameters follow the same conventions as :meth:`upsert`. - - The `format` argument must be one of - :const:`~couchbase.FMT_UTF8` or :const:`~couchbase.FMT_BYTES`. - If not specified, it will be :const:`~.FMT_UTF8` (overriding the - :attr:`default_format` attribute). This is because JSON or - Pickle formats will be nonsensical when random data is appended - to them. If you wish to modify a JSON or Pickle encoded object, - you will need to retrieve it (via :meth:`get`), modify it, and - then store it again (using :meth:`upsert`). - - Additionally, you must ensure the value (and flags) for the - current value is compatible with the data to be appended. For an - example, you may append a :const:`~.FMT_BYTES` value to an - existing :const:`~couchbase.FMT_JSON` value, but an error will - be thrown when retrieving the value using :meth:`get` (you may - still use the :attr:`data_passthrough` to overcome this). - - :raise: :exc:`.NotStoredError` if the key does not exist - - .. seealso:: :meth:`upsert`, :meth:`append_multi` - """ - return _Base.append(self, key, value, cas=cas, format=format, - persist_to=persist_to, replicate_to=replicate_to) - - def prepend(self, key, value, cas=0, format=None, - persist_to=0, replicate_to=0): - """Prepend a string to an existing value in Couchbase. - - .. seealso:: :meth:`append`, :meth:`prepend_multi` - """ - return _Base.prepend(self, key, value, cas=cas, format=format, - persist_to=persist_to, replicate_to=replicate_to) - - def get(self, key, ttl=0, quiet=None, replica=False, no_format=False): - """Obtain an object stored in Couchbase by given key. - - :param string key: The key to fetch. The type of key is the same - as mentioned in :meth:`upsert` - - :param int ttl: If specified, indicates that the key's expiration - time should be *modified* when retrieving the value. - - :param boolean quiet: causes `get` to return None instead of - raising an exception when the key is not found. It defaults - to the value set by :attr:`~quiet` on the instance. In - `quiet` mode, the error may still be obtained by inspecting - the :attr:`~.Result.rc` attribute of the :class:`.Result` - object, or checking :attr:`.Result.success`. - - Note that the default value is `None`, which means to use - the :attr:`quiet`. If it is a boolean (i.e. `True` or - `False`) it will override the `Bucket`-level - :attr:`quiet` attribute. - - :param bool replica: Whether to fetch this key from a replica - rather than querying the master server. This is primarily - useful when operations with the master fail (possibly due to - a configuration change). It should normally be used in an - exception handler like so - - Using the ``replica`` option:: - - try: - res = c.get("key", quiet=True) # suppress not-found errors - catch CouchbaseError: - res = c.get("key", replica=True, quiet=True) - - :param bool no_format: If set to ``True``, then the value will - always be delivered in the :class:`~couchbase.result.Result` - object as being of :data:`~couchbase.FMT_BYTES`. This is a - item-local equivalent of using the :attr:`data_passthrough` - option - - :raise: :exc:`.NotFoundError` if the key does not exist - :raise: :exc:`.CouchbaseNetworkError` - :raise: :exc:`.ValueFormatError` if the value cannot be - deserialized with chosen decoder, e.g. if you try to - retreive an object stored with an unrecognized format - :return: A :class:`~.Result` object - - Simple get:: - - value = cb.get('key').value - - Get multiple values:: - - cb.get_multi(['foo', 'bar']) - # { 'foo' : <Result(...)>, 'bar' : <Result(...)> } - - Inspect the flags:: - - rv = cb.get("key") - value, flags, cas = rv.value, rv.flags, rv.cas - - Update the expiration time:: - - rv = cb.get("key", ttl=10) - # Expires in ten seconds - - .. seealso:: :meth:`get_multi` - """ - - return _Base.get(self, key, ttl=ttl, quiet=quiet, - replica=replica, no_format=no_format) - - def touch(self, key, ttl=0): - """Update a key's expiration time - - :param string key: The key whose expiration time should be - modified - :param int ttl: The new expiration time. If the expiration time - is `0` then the key never expires (and any existing - expiration is removed) - :return: :class:`.OperationResult` - - Update the expiration time of a key :: - - cb.upsert("key", ttl=100) - # expires in 100 seconds - cb.touch("key", ttl=0) - # key should never expire now - - :raise: The same things that :meth:`get` does - - .. seealso:: - :meth:`get` - which can be used to get *and* update the - expiration, - :meth:`touch_multi` - """ - return _Base.touch(self, key, ttl=ttl) - - def lock(self, key, ttl=0): - """Lock and retrieve a key-value entry in Couchbase. - - :param key: A string which is the key to lock. - - :param ttl: a TTL for which the lock should be valid. - While the lock is active, attempts to access the key (via - other :meth:`lock`, :meth:`upsert` or other mutation calls) - will fail with an :exc:`.KeyExistsError`. Note that the - value for this option is limited by the maximum allowable - lock time determined by the server (currently, this is 30 - seconds). If passed a higher value, the server will silently - lower this to its maximum limit. - - - This function otherwise functions similarly to :meth:`get`; - specifically, it will return the value upon success. Note the - :attr:`~.Result.cas` value from the :class:`.Result` object. - This will be needed to :meth:`unlock` the key. - - Note the lock will also be implicitly released if modified by - one of the :meth:`upsert` family of functions when the valid CAS - is supplied - - :raise: :exc:`.TemporaryFailError` if the key is already locked. - :raise: See :meth:`get` for possible exceptions - - Lock a key :: - - rv = cb.lock("locked_key", ttl=5) - # This key is now locked for the next 5 seconds. - # attempts to access this key will fail until the lock - # is released. - - # do important stuff... - - cb.unlock("locked_key", rv.cas) - - Lock a key, implicitly unlocking with :meth:`upsert` with CAS :: - - rv = self.cb.lock("locked_key", ttl=5) - new_value = rv.value.upper() - cb.upsert("locked_key", new_value, rv.cas) - - - Poll and Lock :: - - rv = None - begin_time = time.time() - while time.time() - begin_time < 15: - try: - rv = cb.lock("key", ttl=10) - except TemporaryFailError: - print("Key is currently locked.. waiting") - time.sleep(1) - - if not rv: - raise Exception("Waited too long..") - - # Do stuff.. - - cb.unlock("key", rv.cas) - - - .. seealso:: :meth:`get`, :meth:`lock_multi`, :meth:`unlock` - """ - return _Base.lock(self, key, ttl=ttl) - - def unlock(self, key, cas): - """Unlock a Locked Key in Couchbase. - - This unlocks an item previously locked by :meth:`lock` - - :param key: The key to unlock - :param cas: The cas returned from :meth:`lock`'s - :class:`.Result` object. - - See :meth:`lock` for an example. - - :raise: :exc:`.TemporaryFailError` if the CAS supplied does not - match the CAS on the server (possibly because it was - unlocked by previous call). - - .. seealso:: :meth:`lock` :meth:`unlock_multi` - """ - return _Base.unlock(self, key, cas=cas) +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict) + +from couchbase.collection import Collection +from couchbase.exceptions import ErrorMapper +from couchbase.exceptions import exception as BaseCouchbaseException +from couchbase.logic import BlockingWrapper +from couchbase.logic.bucket import BucketLogic +from couchbase.logic.supportability import Supportability +from couchbase.management.collections import CollectionManager +from couchbase.management.views import ViewIndexManager +from couchbase.result import PingResult, ViewResult +from couchbase.scope import Scope +from couchbase.views import ViewQuery, ViewRequest + +if TYPE_CHECKING: + from couchbase.cluster import Cluster + from couchbase.options import PingOptions, ViewOptions + + +class Bucket(BucketLogic): + """Create a Couchbase Bucket instance. + + Exposes the operations which are available to be performed against a bucket. Namely the ability to + access to Collections as well as performing management operations against the bucket. + + Args: + cluster (:class:`~.Cluster`): A :class:`~.Cluster` instance. + bucket_name (str): Name of the bucket. + + Raises: + :class:`~couchbase.exceptions.BucketNotFoundException`: If provided `bucket_name` cannot be found. - def remove(self, key, cas=0, quiet=None, persist_to=0, replicate_to=0): - """Remove the key-value entry for a given key in Couchbase. - - :param key: A string which is the key to remove. The format and - type of the key follows the same conventions as in - :meth:`upsert` - :type key: string, dict, or tuple/list - - :param int cas: The CAS to use for the removal operation. - If specified, the key will only be removed from the server - if it has the same CAS as specified. This is useful to - remove a key only if its value has not been changed from the - version currently visible to the client. If the CAS on the - server does not match the one specified, an exception is - thrown. - :param boolean quiet: - Follows the same semantics as `quiet` in :meth:`get` - :param int persist_to: If set, wait for the item to be removed - from the storage of at least these many nodes - :param int replicate_to: If set, wait for the item to be removed - from the cache of at least these many nodes - (excluding the master) - :raise: :exc:`.NotFoundError` if the key does not exist. - :raise: :exc:`.KeyExistsError` if a CAS was specified, but - the CAS on the server had changed - :return: A :class:`~.Result` object. - - Simple remove:: - - ok = cb.remove("key").success - - Don't complain if key does not exist:: - - ok = cb.remove("key", quiet=True) - - Only remove if CAS matches our version:: - - rv = cb.get("key") - cb.remove("key", cas=rv.cas) - - Remove multiple keys:: - - oks = cb.remove_multi(["key1", "key2", "key3"]) - - Remove multiple keys with CAS:: - - oks = cb.remove({ - "key1" : cas1, - "key2" : cas2, - "key3" : cas3 - }) - - .. seealso:: :meth:`remove_multi`, :meth:`endure` - for more information on the ``persist_to`` and - ``replicate_to`` options. - """ - return _Base.remove(self, key, cas=cas, quiet=quiet, - persist_to=persist_to, replicate_to=replicate_to) - - def counter(self, key, delta=1, initial=None, ttl=0): - """Increment or decrement the numeric value of an item. - - This method instructs the server to treat the item stored under - the given key as a numeric counter. - - Counter operations require that the stored value - exists as a string representation of a number (e.g. ``123``). If - storing items using the :meth:`upsert` family of methods, and - using the default :const:`couchbase.FMT_JSON` then the value - will conform to this constraint. - - :param string key: A key whose counter value is to be modified - :param int delta: an amount by which the key should be modified. - If the number is negative then this number will be - *subtracted* from the current value. - :param initial: The initial value for the key, if it does not - exist. If the key does not exist, this value is used, and - `delta` is ignored. If this parameter is `None` then no - initial value is used - :type initial: int or `None` - :param int ttl: The lifetime for the key, after which it will - expire - :raise: :exc:`.NotFoundError` if the key does not exist on the - bucket (and `initial` was `None`) - :raise: :exc:`.DeltaBadvalError` if the key exists, but the - existing value is not numeric - :return: A :class:`.Result` object. The current value of the - counter may be obtained by inspecting the return value's - `value` attribute. - - Simple increment:: - - rv = cb.counter("key") - rv.value - # 42 - - Increment by 10:: - - rv = cb.counter("key", delta=10) - - Decrement by 5:: - - rv = cb.counter("key", delta=-5) - - Increment by 20, set initial value to 5 if it does not exist:: - - rv = cb.counter("key", delta=20, initial=5) - - Increment three keys:: - - kv = cb.counter_multi(["foo", "bar", "baz"]) - for key, result in kv.items(): - print "Key %s has value %d now" % (key, result.value) - - .. seealso:: :meth:`counter_multi` - """ - return _Base.counter(self, key, delta=delta, initial=initial, ttl=ttl) - - def incr(self, key, amount=1, **kwargs): - _depr('incr', 'counter') - return self.counter(key, delta=amount, **kwargs) - - def incr_multi(self, keys, amount=1, **kwargs): - _depr('incr_multi', 'counter_multi') - return self.counter_multi(keys, delta=amount, **kwargs) - - def decr(self, key, amount=1, **kwargs): - _depr('decr', 'counter') - return self.counter(key, delta=-amount, **kwargs) - - def decr_multi(self, keys, amount=1, **kwargs): - _depr('decr_multi', 'counter_multi') - return self.counter_multi(keys, delta=-amount, **kwargs) - - def stats(self, keys=None, keystats=False): - """Request server statistics. - - Fetches stats from each node in the cluster. Without a key - specified the server will respond with a default set of - statistical information. It returns the a `dict` with stats keys - and node-value pairs as a value. - - :param keys: One or several stats to query - :type keys: string or list of string - :raise: :exc:`.CouchbaseNetworkError` - :return: `dict` where keys are stat keys and values are - host-value pairs - - Find out how many items are in the bucket:: - - total = 0 - for key, value in cb.stats()['total_items'].items(): - total += value - - Get memory stats (works on couchbase buckets):: - - cb.stats('memory') - # {'mem_used': {...}, ...} - """ - if keys and not isinstance(keys, (tuple, list)): - keys = (keys,) - return self._stats(keys, keystats=keystats) - - def observe(self, key, master_only=False): - """Return storage information for a key. - - It returns a :class:`.ValueResult` object with the ``value`` - field set to a list of :class:`~.ObserveInfo` objects. Each - element in the list responds to the storage status for the key - on the given node. The length of the list (and thus the number - of :class:`~.ObserveInfo` objects) are equal to the number of - online replicas plus the master for the given key. - - :param string key: The key to inspect - :param bool master_only: Whether to only retrieve information - from the master node. - - .. seealso:: :ref:`observe_info` - """ - return _Base.observe(self, key, master_only=master_only) - - def endure(self, key, persist_to=-1, replicate_to=-1, cas=0, - check_removed=False, timeout=5.0, interval=0.010): - """Wait until a key has been distributed to one or more nodes - - By default, when items are stored to Couchbase, the operation is - considered successful if the vBucket master (i.e. the "primary" - node) for the key has successfully stored the item in its - memory. - - In most situations, this is sufficient to assume that the item - has successfully been stored. However the possibility remains - that the "master" server will go offline as soon as it sends - back the successful response and the data is lost. - - The ``endure`` function allows you to provide stricter criteria - for success. The criteria may be expressed in terms of number of - nodes for which the item must exist in that node's RAM and/or on - that node's disk. Ensuring that an item exists in more than one - place is a safer way to guarantee against possible data loss. - - We call these requirements `Durability Constraints`, and thus - the method is called `endure`. - - :param string key: The key to endure. - :param int persist_to: The minimum number of nodes which must - contain this item on their disk before this function - returns. Ensure that you do not specify too many nodes; - otherwise this function will fail. Use the - :attr:`server_nodes` to determine how many nodes exist in - the cluster. - - The maximum number of nodes an item can reside on is - currently fixed to 4 (i.e. the "master" node, and up to - three "replica" nodes). This limitation is current as of - Couchbase Server version 2.1.0. - - If this parameter is set to a negative value, the maximum - number of possible nodes the key can reside on will be used. - - :param int replicate_to: The minimum number of replicas which - must contain this item in their memory for this method to - succeed. As with ``persist_to``, you may specify a negative - value in which case the requirement will be set to the - maximum number possible. - - :param float timeout: A timeout value in seconds before this - function fails with an exception. Typically it should take - no longer than several milliseconds on a functioning cluster - for durability requirements to be satisfied (unless - something has gone wrong). - - :param float interval: The polling interval in seconds to use - for checking the key status on the respective nodes. - Internally, ``endure`` is implemented by polling each server - individually to see if the key exists on that server's disk - and memory. Once the status request is sent to all servers, - the client will check if their replies are satisfactory; if - they are then this function succeeds, otherwise the client - will wait a short amount of time and try again. This - parameter sets this "wait time". - - :param bool check_removed: This flag inverts the check. Instead - of checking that a given key *exists* on the nodes, this - changes the behavior to check that the key is *removed* from - the nodes. - - :param long cas: The CAS value to check against. It is possible - for an item to exist on a node but have a CAS value from a - prior operation. Passing the CAS ensures that only replies - from servers with a CAS matching this parameter are accepted. - - :return: A :class:`~.OperationResult` - :raise: see :meth:`upsert` and :meth:`get` for possible errors - - .. seealso:: :meth:`upsert`, :meth:`endure_multi` - """ - # We really just wrap 'endure_multi' - kv = {key: cas} - rvs = self.endure_multi(keys=kv, persist_to=persist_to, - replicate_to=replicate_to, - check_removed=check_removed, timeout=timeout, - interval=interval) - return rvs[key] - - def durability(self, persist_to=-1, replicate_to=-1, timeout=0.0): - """Returns a context manager which will apply the given - persistence/replication settings to all mutation operations when - active - - :param int persist_to: - :param int replicate_to: - - See :meth:`endure` for the meaning of these two values - - Thus, something like:: - - with cb.durability(persist_to=3): - cb.upsert("foo", "foo_value") - cb.upsert("bar", "bar_value") - cb.upsert("baz", "baz_value") - - is equivalent to:: - - cb.upsert("foo", "foo_value", persist_to=3) - cb.upsert("bar", "bar_value", persist_to=3) - cb.upsert("baz", "baz_value", persist_to=3) - - - .. versionadded:: 1.2.0 - - .. seealso:: :meth:`endure` - """ - return DurabilityContext(self, persist_to, replicate_to, timeout) - - def upsert_multi(self, keys, ttl=0, format=None, persist_to=0, replicate_to=0): - """ - Write multiple items to the cluster. Multi version of :meth:`upsert` - - :param dict keys: A dictionary of keys to set. The keys are the - keys as they should be on the server, and the values are the - values for the keys to be stored. - - `keys` may also be a :class:`~.ItemCollection`. If using a - dictionary variant for item collections, an additional - `ignore_cas` parameter may be supplied with a boolean value. - If not specified, the operation will fail if the CAS value - on the server does not match the one specified in the - `Item`'s `cas` field. - :param int ttl: If specified, sets the expiration value - for all keys - :param int format: If specified, this is the conversion format - which will be used for _all_ the keys. - :param int persist_to: Durability constraint for persistence. - Note that it is more efficient to use :meth:`endure_multi` - on the returned :class:`~couchbase.result.MultiResult` than - using these parameters for a high volume of keys. Using - these parameters however does save on latency as the - constraint checking for each item is performed as soon as it - is successfully stored. - :param int replicate_to: Durability constraints for replication. - See notes on the `persist_to` parameter for usage. - :return: A :class:`~.MultiResult` object, which is a - `dict`-like object - - The multi methods are more than just a convenience, they also - save on network performance by batch-scheduling operations, - reducing latencies. This is especially noticeable on smaller - value sizes. + """ - .. seealso:: :meth:`upsert` - """ - return _Base.upsert_multi(self, keys, ttl=ttl, format=format, - persist_to=persist_to, - replicate_to=replicate_to) + def __init__(self, + cluster, # type: Cluster + bucket_name # type: str + ): + super().__init__(cluster, bucket_name) + self._open_bucket() - def insert_multi(self, keys, ttl=0, format=None, persist_to=0, replicate_to=0): - """Add multiple keys. Multi variant of :meth:`insert` + @BlockingWrapper.block(True) + def _open_bucket(self, **kwargs): + ret = super()._open_or_close_bucket(open_bucket=True, **kwargs) + if isinstance(ret, BaseCouchbaseException): + raise ErrorMapper.build_exception(ret) + self._set_connected(ret) - .. seealso:: :meth:`insert`, :meth:`upsert_multi`, :meth:`upsert` - """ - return _Base.insert_multi(self, keys, ttl=ttl, format=format, - persist_to=persist_to, - replicate_to=replicate_to) + @BlockingWrapper.block(True) + def _close_bucket(self, **kwargs): + super()._open_or_close_bucket(open_bucket=False, **kwargs) + self._destroy_connection() - def replace_multi(self, keys, ttl=0, format=None, - persist_to=0, replicate_to=0): - """Replace multiple keys. Multi variant of :meth:`replace` - - .. seealso:: :meth:`replace`, :meth:`upsert_multi`, :meth:`upsert` - """ - return _Base.replace_multi(self, keys, ttl=ttl, format=format, - persist_to=persist_to, - replicate_to=replicate_to) - - def append_multi(self, keys, format=None, persist_to=0, replicate_to=0): - """Append to multiple keys. Multi variant of :meth:`append`. + def close(self): + """Shuts down this bucket instance. Cleaning up all resources associated with it. .. warning:: + Use of this method is almost *always* unnecessary. Bucket resources should be cleaned + up once the bucket instance falls out of scope. However, in some applications tuning resources + is necessary and in those types of applications, this method might be beneficial. - If using the `Item` interface, use the :meth:`append_items` - and :meth:`prepend_items` instead, as those will - automatically update the :attr:`.Item.value` - property upon successful completion. - - .. seealso:: :meth:`append`, :meth:`upsert_multi`, :meth:`upsert` """ - return _Base.append_multi(self, keys, format=format, - persist_to=persist_to, - replicate_to=replicate_to) + # only close if we are connected + if self.connected: + self._close_bucket() - def prepend_multi(self, keys, format=None, persist_to=0, replicate_to=0): - """Prepend to multiple keys. Multi variant of :meth:`prepend` - - .. seealso:: :meth:`prepend`, :meth:`upsert_multi`, :meth:`upsert` - """ - return _Base.prepend_multi(self, keys, format=format, - persist_to=persist_to, - replicate_to=replicate_to) + def default_scope(self) -> Scope: + """Creates a :class:`~.scope.Scope` instance of the default scope. - def get_multi(self, keys, ttl=0, quiet=None, replica=False, no_format=False): - """Get multiple keys. Multi variant of :meth:`get` + Returns: + :class:`~.scope.Scope`: A :class:`~.scope.Scope` instance of the default scope. - :param keys: keys the keys to fetch - :type keys: :ref:`iterable<argtypes>` - :param int ttl: Set the expiration for all keys when retrieving - :param boolean replica: - Whether the results should be obtained from a replica - instead of the master. See :meth:`get` for more information - about this parameter. - :return: A :class:`~.MultiResult` object. This is a dict-like - object and contains the keys (passed as) `keys` as the - dictionary keys, and :class:`~.Result` objects as values """ - return _Base.get_multi(self, keys, ttl=ttl, quiet=quiet, - replica=replica, no_format=no_format) + return self.scope(Scope.default_name()) - def touch_multi(self, keys, ttl=0): - """Touch multiple keys. Multi variant of :meth:`touch` + def scope(self, name # type: str + ) -> Scope: + """Creates a :class:`~couchbase.scope.Scope` instance of the specified scope. - :param keys: the keys to touch - :type keys: :ref:`iterable<argtypes>`. - ``keys`` can also be a dictionary with values being - integers, in which case the value for each key will be used - as the TTL instead of the global one (i.e. the one passed to - this function) - :param int ttl: The new expiration time - :return: A :class:`~.MultiResult` object + Args: + name (str): Name of the scope to reference. - Update three keys to expire in 10 seconds :: + Returns: + :class:`~couchbase.scope.Scope`: A :class:`~couchbase.scope.Scope` instance of the specified scope. - cb.touch_multi(("key1", "key2", "key3"), ttl=10) - - Update three keys with different expiration times :: - - cb.touch_multi({"foo" : 1, "bar" : 5, "baz" : 10}) - - .. seealso:: :meth:`touch` - """ - return _Base.touch_multi(self, keys, ttl=ttl) - - def lock_multi(self, keys, ttl=0): - """Lock multiple keys. Multi variant of :meth:`lock` - - :param keys: the keys to lock - :type keys: :ref:`iterable<argtypes>` - :param int ttl: The lock timeout for all keys - - :return: a :class:`~.MultiResult` object - - .. seealso:: :meth:`lock` - """ - return _Base.lock_multi(self, keys, ttl=ttl) - - def unlock_multi(self, keys): - """Unlock multiple keys. Multi variant of :meth:`unlock` - - :param dict keys: the keys to unlock - :return: a :class:`~couchbase.result.MultiResult` object - - The value of the ``keys`` argument should be either the CAS, or - a previously returned :class:`Result` object from a :meth:`lock` - call. Effectively, this means you may pass a - :class:`~.MultiResult` as the ``keys`` argument. - - Thus, you can do something like :: - - keys = (....) - rvs = cb.lock_multi(keys, ttl=5) - # do something with rvs - cb.unlock_multi(rvs) - - .. seealso:: :meth:`unlock` - """ - return _Base.unlock_multi(self, keys) - - def observe_multi(self, keys, master_only=False): - """Multi-variant of :meth:`observe`""" - return _Base.observe_multi(self, keys, master_only=master_only) - - def endure_multi(self, keys, persist_to=-1, replicate_to=-1, - timeout=5.0, interval=0.010, check_removed=False): - """Check durability requirements for multiple keys - - :param keys: The keys to check - - The type of keys may be one of the following: - * Sequence of keys - * A :class:`~couchbase.result.MultiResult` object - * A ``dict`` with CAS values as the dictionary value - * A sequence of :class:`~couchbase.result.Result` objects - - :return: A :class:`~.MultiResult` object - of :class:`~.OperationResult` items. - - .. seealso:: :meth:`endure` """ - return _Base.endure_multi(self, keys, persist_to=persist_to, - replicate_to=replicate_to, - timeout=timeout, interval=interval, - check_removed=check_removed) + return Scope(self, name) - def rget(self, key, replica_index=None, quiet=None): - """Get an item from a replica node + def collection(self, collection_name # type: str + ) -> Collection: + """Creates a :class:`~couchbase.collection.Collection` instance of the specified collection. - :param string key: The key to fetch - :param int replica_index: The replica index to fetch. - If this is ``None`` then this method will return once any - replica responds. Use :attr:`configured_replica_count` to - figure out the upper bound for this parameter. + Args: + collection_name (str): Name of the collection to reference. - The value for this parameter must be a number between 0 and - the value of :attr:`configured_replica_count`-1. - :param boolean quiet: Whether to suppress errors when the key is - not found + Returns: + :class:`~couchbase.collection.Collection`: A :class:`~couchbase.collection.Collection` instance of the specified collection. - This method (if `replica_index` is not supplied) functions like - the :meth:`get` method that has been passed the `replica` - parameter:: + """ # noqa: E501 + scope = self.default_scope() + return scope.collection(collection_name) - c.get(key, replica=True) + def default_collection(self) -> Collection: + """Creates a :class:`~couchbase.collection.Collection` instance of the default collection. - .. seealso:: :meth:`get` :meth:`rget_multi` - """ - if replica_index is not None: - return _Base._rgetix(self, key, replica=replica_index, quiet=quiet) - else: - return _Base._rget(self, key, quiet=quiet) - - def rget_multi(self, keys, replica_index=None, quiet=None): - if replica_index is not None: - return _Base._rgetix_multi(self, keys, - replica=replica_index, quiet=quiet) - else: - return _Base._rget_multi(self, keys, quiet=quiet) - - def _view(self, ddoc, view, - use_devmode=False, - params=None, - unrecognized_ok=False, - passthrough=False): - """Internal method to Execute a view (MapReduce) query + Returns: + :class:`~couchbase.collection.Collection`: A :class:`~couchbase.collection.Collection` instance of the default collection. + """ # noqa: E501 + scope = self.default_scope() + return scope.collection(Collection.default_name()) - :param string ddoc: Name of the design document - :param string view: Name of the view function to execute - :param params: Extra options to pass to the view engine - :type params: string or dict - :return: a :class:`~couchbase.result.HttpResult` object. - """ - - if params: - if not isinstance(params, str): - params = make_options_string( - params, - unrecognized_ok=unrecognized_ok, - passthrough=passthrough) - else: - params = "" + @BlockingWrapper.block(PingResult) + def ping(self, + *opts, # type: PingOptions + **kwargs # type: Dict[str, Any] + ) -> PingResult: + """Performs a ping operation against the bucket. - ddoc = self._mk_devmode(ddoc, use_devmode) - url = make_dvpath(ddoc, view) + params + The ping operation pings the services which are specified + (or all services if none are specified). Returns a report which describes the outcome of + the ping operations which were performed. - ret = self._http_request(type=_LCB.LCB_HTTP_TYPE_VIEW, - path=url, - method=_LCB.LCB_HTTP_METHOD_GET, - response_format=FMT_JSON) - return ret + Args: + opts (:class:`~couchbase.options.PingOptions`): Optional parameters for this operation. - @staticmethod - def _mk_devmode(n, use_devmode): - if n.startswith('dev_') or not use_devmode: - return n - return 'dev_' + n + Returns: + :class:`~couchbase.result.PingResult`: A report which describes the outcome of the ping operations + which were performed. - def bucket_manager(self): - """ - Returns a :class:`~.BucketManager` object which may be used to - perform management operations on the current bucket. These - operations may create/modify design documents and flush the - bucket """ - return BucketManager(self) - - def query(self, design, view, use_devmode=False, itercls=View, **kwargs): - """ - Query a pre-defined MapReduce view, passing parameters. - - This method executes a view on the cluster. It accepts various - parameters for the view and returns an iterable object - (specifically, a :class:`~.View`). - - :param string design: The design document - :param string view: The view function contained within the design - document - :param boolean use_devmode: Whether the view name should be - transformed into a development-mode view. See documentation - on :meth:`~.BucketManager.design_create` for more - explanation. - :param kwargs: Extra arguments passed to the :class:`~.View` - object constructor. - :param itercls: Subclass of 'view' to use. This parameter is - mainly used by async modules - :param kwargs: Additional parameters passed to the - :class:`~.View` constructor. See that class' - documentation for accepted parameters. - - .. seealso:: - - :class:`~.View` - contains more extensive documentation and examples - - :class:`~.Query` - contains documentation on the available query options + return super().ping(*opts, **kwargs) - :class:`~.SpatialQuery` - contains documentation on the available query options - for Geospatial views. + def view_query(self, + design_doc, # type: str + view_name, # type: str + *view_options, # type: ViewOptions + **kwargs # type: Dict[str, Any] + ) -> ViewResult: + """Executes a View query against the bucket. .. note:: - To query a spatial view, you must explicitly use the - :class:`.SpatialQuery`. Passing key-value view parameters - in ``kwargs`` is not supported for spatial views. - - """ - design = self._mk_devmode(design, use_devmode) - return itercls(self, design, view, **kwargs) - - def n1ql_query(self, query, itercls=N1QLRequest, **kwargs): - """ - Execute a N1QL query. - - :param query: The query to execute. This may either be a - :class:`.N1QLQuery` object, or a string (which will be - implicitly converted to one). - :param kwargs: Arguments for :class:`.N1QLRequest`. - :return: An iterator which yields rows. The returned - """ - if not isinstance(query, N1QLQuery): - query = N1QLQuery(query) - - return itercls(query, self, **kwargs) - - def __repr__(self): - return ('<{modname}.{cls} bucket={bucket}, nodes={nodes} at 0x{oid:x}>' - ).format(modname=__name__, cls=self.__class__.__name__, - nodes=self.server_nodes, bucket=self.bucket, - oid=id(self)) + The query is executed lazily in that it is executed once iteration over the + :class:`~.result.ViewResult` begins. - # "items" interface - def append_items(self, items, **kwargs): - """ - Method to append data to multiple :class:`~.Item` objects. - - This method differs from the normal :meth:`append_multi` in that - each `Item`'s `value` field is updated with the appended data - upon successful completion of the operation. - - - :param items: The item dictionary. The value for each key should - contain a ``fragment`` field containing the object to append - to the value on the server. - :type items: :class:`~couchbase.items.ItemOptionDict`. - - The rest of the options are passed verbatim to - :meth:`append_multi` - - .. seealso:: :meth:`append_multi`, :meth:`append` - """ - rv = self.append_multi(items, **kwargs) - # Assume this is an 'ItemOptionDict' - for k, v in items.dict.items(): - if k.success: - k.value += v['fragment'] - - return rv - - def prepend_items(self, items, **kwargs): - """Method to prepend data to multiple :class:`~.Item` objects. - .. seealso:: :meth:`append_items` - """ - rv = self.prepend_multi(items, **kwargs) - for k, v in items.dict.items(): - if k.success: - k.value = v['fragment'] + k.value - - return rv - - @property - def closed(self): - """Returns True if the object has been closed with :meth:`_close`""" - return self._privflags & _LCB.PYCBC_CONN_F_CLOSED - - def _get_timeout_common(self, op): - return self._cntl(op, value_type='timeout') - - def _set_timeout_common(self, op, value): - value = float(value) - if value <= 0: - raise ValueError('Timeout must be greater than 0') - - self._cntl(op, value_type='timeout', value=value) - - @property - def timeout(self): - return self._get_timeout_common(_LCB.LCB_CNTL_OP_TIMEOUT) - - @timeout.setter - def timeout(self, value): - self._set_timeout_common(_LCB.LCB_CNTL_OP_TIMEOUT, value) - - @property - def views_timeout(self): - return self._get_timeout_common(_LCB.LCB_CNTL_VIEW_TIMEOUT) - - @views_timeout.setter - def views_timeout(self, value): - self._set_timeout_common(_LCB.LCB_CNTL_VIEW_TIMEOUT, value) - - _OLDOPS = { 'set': 'upsert', 'add': 'insert', 'delete': 'remove'} - for o, n in _OLDOPS.items(): - for variant in ('', '_multi'): - oldname = o + variant - newname = n + variant - - try: - dst = locals()[n + variant] - except KeyError: - dst = getattr(_Base, n + variant) - - def mkmeth(oldname, newname, _dst): - def _tmpmeth(self, *args, **kwargs): - _depr(oldname, newname) - return _dst(self, *args, **kwargs) - return _tmpmeth - - locals().update({oldname: mkmeth(oldname, newname, dst)}) - - """ - Lists the names of all the memcached operations. This is useful - for classes which want to wrap all the methods - """ - _MEMCACHED_OPERATIONS = ('upsert', 'get', 'insert', 'append', 'prepend', - 'replace', 'remove', 'counter', 'touch', - 'lock', 'unlock', 'endure', - 'observe', 'rget', 'stats', - 'set', 'add', 'delete') - - _MEMCACHED_NOMULTI = ('stats') - - @classmethod - def _gen_memd_wrappers(cls, factory): - """Generates wrappers for all the memcached operations. - :param factory: A function to be called to return the wrapped - method. It will be called with two arguments; the first is - the unbound method being wrapped, and the second is the name - of such a method. - - The factory shall return a new unbound method - - :return: A dictionary of names mapping the API calls to the - wrapped functions - """ - d = {} - for n in cls._MEMCACHED_OPERATIONS: - for variant in (n, n + "_multi"): - try: - d[variant] = factory(getattr(cls, variant), variant) - except AttributeError: - if n in cls._MEMCACHED_NOMULTI: - continue - raise - return d + .. seealso:: + * :class:`~.management.ViewIndexManager`: for how to manage query indexes - def _cntl(self, *args, **kwargs): - """Low-level interface to the underlying C library's settings. via - ``lcb_cntl()``. + Args: + design_doc (str): The name of the design document containing the view to execute. + view_name (str): The name of the view to execute. + view_options (:class:`~.options.ViewOptions`): Optional parameters for the view query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~.options.ViewOptions` - This method accepts an opcode and an - optional value. Constants are intentionally not defined for - the various opcodes to allow saner error handling when an - unknown opcode is not used. + Returns: + :class:`~.result.ViewResult`: An instance of a :class:`~.result.ViewResult` which + provides access to iterate over the query results and access metadata about the query. - .. warning:: + Examples: + Simple view query:: - If you pass the wrong parameters to this API call, your - application may crash. For this reason, this is not a public - API call. Nevertheless it may be used sparingly as a - workaround for settings which may have not yet been exposed - directly via a supported API + from couchbase.management.views import DesignDocumentNamespace - :param int op: Type of cntl to access. These are defined in - libcouchbase's ``cntl.h`` header file + # ... other code ... - :param value: An optional value to supply for the operation. - If a value is not passed then the operation will return the - current value of the cntl without doing anything else. - otherwise, it will interpret the cntl in a manner that makes - sense. If the value is a float, it will be treated as a - timeout value and will be multiplied by 1000000 to yield the - microsecond equivalent for the library. If the value is a - boolean, it is treated as a C ``int`` + view_result = bucket.view_query('ddoc-name', + 'view-name', + limit=10, + namespace=DesignDocumentNamespace.DEVELOPMENT) - :param value_type: String indicating the type of C-level value - to be passed to ``lcb_cntl()``. The possible values are: + for row in view_result.rows(): + print(f'Found row: {row}') - * ``"string"`` - NUL-terminated `const char`. - Pass a Python string - * ``"int"`` - C ``int`` type. Pass a Python int - * ``"uint32_t"`` - C ``lcb_uint32_t`` type. - Pass a Python int - * ``"unsigned"`` - C ``unsigned int`` type. - Pass a Python int - * ``"float"`` - C ``float`` type. Pass a Python float - * ``"timeout"`` - The number of seconds as a float. This is - converted into microseconds within the extension library. - - :return: If no `value` argument is provided, retrieves the - current setting (per the ``value_type`` specification). - Otherwise this function returns ``None``. """ - return _Base._cntl(self, *args, **kwargs) + # If the view_query was provided a timeout we will use that value for the streaming timeout + # when the streaming object is created in the bindings. If the view_query does not specify a + # timeout, the streaming_timeout defaults to cluster's view_timeout (set here). If the cluster + # also does not specify a view_timeout we set the streaming_timeout to + # couchbase::core::timeout_defaults::view_timeout when the streaming object is created in the bindings. + streaming_timeout = self.streaming_timeouts.get('view_timeout', None) + query = ViewQuery.create_view_query_object(self.name, design_doc, view_name, *view_options, **kwargs) + return ViewResult(ViewRequest.generate_view_request(self.connection, + query.as_encodable(), + default_serializer=self.default_serializer, + streaming_timeout=streaming_timeout)) - def _cntlstr(self, key, value): + def collections(self) -> CollectionManager: """ - Low-level interface to the underlying C library's settings. - via ``lcb_cntl_string()``. - - This method accepts a key and a value. It can modify the same - sort of settings as the :meth:`~._cntl` method, but may be a - bit more convenient to follow in code. - - .. warning:: - - See :meth:`~._cntl` for warnings. + Get a :class:`~couchbase.management.collections.CollectionManager` which can be used to manage the scopes and collections + of this bucket. - :param string key: The setting key - :param string value: The setting value + Returns: + :class:`~couchbase.management.collections.CollectionManager`: A :class:`~couchbase.management.collections.CollectionManager` instance. + """ # noqa: E501 + return CollectionManager(self.connection, self.name) - See the API documentation for libcouchbase for a list of - acceptable setting keys. + def view_indexes(self) -> ViewIndexManager: """ - return _Base._cntlstr(self, key, value) + Get a :class:`~couchbase.management.views.ViewIndexManager` which can be used to manage the view design documents + and views of this bucket. - @staticmethod - def lcb_version(): - return _LCB.lcb_version() + Returns: + :class:`~couchbase.management.views.ViewIndexManager`: A :class:`~couchbase.management.views.ViewIndexManager` instance. + """ # noqa: E501 + return ViewIndexManager(self.connection, self.name) - def design_get(self, *args, **kwargs): - _depr('design_get', 'bucket_manager().design_get') - return self.bucket_manager().design_get(*args, **kwargs) - def design_create(self, *args, **kwargs): - _depr('design_create', 'bucket_manager().design_create') - return self.bucket_manager().design_create(*args, **kwargs) +""" +** DEPRECATION NOTICE ** - def design_publish(self, *args, **kwargs): - _depr('design_publish', 'bucket_manager().design_publish') - return self.bucket_manager().design_publish(*args, **kwargs) +The classes below are deprecated for 3.x compatibility. They should not be used. +Instead use: + * All options should be imported from `couchbase.options`. + * All view Enums should be imported from `couchbase.views`. - def design_delete(self, *args, **kwargs): - _depr('design_delete', 'bucket_manager().design_delete') - return self.bucket_manager().design_delete(*args, **kwargs) +""" - def flush(self): - """ - Clears the bucket's contents. +from couchbase.logic.options import PingOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import ViewOptionsBase # nopep8 # isort:skip # noqa: E402 - .. note:: - This functionality requires that the flush option be - enabled for the bucket by the cluster administrator. You - can enable flush on the bucket using the administrative - console (See http://docs.couchbase.com/admin/admin/UI/ui-data-buckets.html) +@Supportability.import_deprecated('couchbase.bucket', 'couchbase.options') # noqa: F811 +class PingOptions(PingOptionsBase): # noqa: F811 + pass - .. note:: - This is a destructive operation, as it will clear all the - data from the bucket. +@Supportability.import_deprecated('couchbase.bucket', 'couchbase.options') # noqa: F811 +class ViewOptions(ViewOptionsBase): # noqa: F811 + pass - .. note:: - A successful execution of this method means that the bucket - will have started the flush process. This does not - necessarily mean that the bucket is actually empty. - """ - path = '/pools/default/buckets/{0}/controller/doFlush' - path = path.format(self.bucket) - return self._http_request(type=_LCB.LCB_HTTP_TYPE_MANAGEMENT, - path=path, method=_LCB.LCB_HTTP_METHOD_POST) +from couchbase.views import ViewScanConsistency # nopep8 # isort:skip # noqa: E402, F401 +from couchbase.views import ViewOrdering # nopep8 # isort:skip # noqa: E402, F401 +from couchbase.views import ViewErrorMode # nopep8 # isort:skip # noqa: E402, F401 diff --git a/couchbase/bucketmanager.py b/couchbase/bucketmanager.py deleted file mode 100644 index 03005d17b..000000000 --- a/couchbase/bucketmanager.py +++ /dev/null @@ -1,282 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import time - -import couchbase._libcouchbase as _LCB -import couchbase.exceptions as exceptions -from couchbase.exceptions import CouchbaseError, ArgumentError -from couchbase.views.params import Query, SpatialQuery, STALE_OK -from couchbase._pyport import single_dict_key - -class BucketManager(object): - """ - The `BucketManager` class allows access to common maintenance APIs related - to a :class:`~couchbase.bucket.Bucket` object. It is normally returned via - the :meth:`~couchbase.bucket.Bucket.bucket_manager` method - """ - def __init__(self, cb): - self._cb = cb - - def _http_request(self, **kwargs): - return self._cb._http_request(**kwargs) - - def _mk_devmode(self, *args): - return self._cb._mk_devmode(*args) - - def _view(self, *args, **kwargs): - return self._cb._view(*args, **kwargs) - - def _doc_rev(self, res): - """ - Returns the rev id from the header - """ - jstr = res.headers['X-Couchbase-Meta'] - jobj = json.loads(jstr) - return jobj['rev'] - - def _poll_vq_single(self, dname, use_devmode, ddresp): - """ - Initiate a view query for a view located in a design document - :param ddresp: The design document to poll (as JSON) - :return: True if successful, False if no views. - """ - vname = None - query = None - v_mr = ddresp.get('views', {}) - v_spatial = ddresp.get('spatial', {}) - if v_mr: - vname = single_dict_key(v_mr) - query = Query() - elif v_spatial: - vname = single_dict_key(v_spatial) - query = SpatialQuery() - - if not vname: - return False - - query.stale = STALE_OK - query.limit = 1 - - for r in self._cb.query(dname, vname, use_devmode=use_devmode, - query=query): - pass - return True - - def _design_poll(self, name, mode, oldres, timeout=5, use_devmode=False): - """ - Poll for an 'async' action to be complete. - :param string name: The name of the design document - :param string mode: One of ``add`` or ``del`` to indicate whether - we should check for addition or deletion of the document - :param oldres: The old result from the document's previous state, if - any - :param float timeout: How long to poll for. If this is 0 then this - function returns immediately - :type oldres: :class:`~couchbase.result.HttpResult` - """ - if not timeout: - return True - - if timeout < 0: - raise ArgumentError.pyexc("Interval must not be negative") - - t_end = time.time() + timeout - old_rev = None - - if oldres: - old_rev = self._doc_rev(oldres) - - while time.time() < t_end: - try: - cur_resp = self.design_get(name, use_devmode=use_devmode) - if old_rev and self._doc_rev(cur_resp) == old_rev: - continue - - try: - if not self._poll_vq_single( - name, use_devmode, cur_resp.value): - continue - return True - - except CouchbaseError: - continue - - except CouchbaseError: - if mode == 'del': - # Deleted, whopee! - return True - - raise exceptions.TimeoutError.pyexc( - "Wait time for design action completion exceeded") - - def design_create(self, name, ddoc, use_devmode=True, syncwait=0): - """ - Store a design document - - :param string name: The name of the design - :param ddoc: The actual contents of the design document - - :type ddoc: string or dict - If ``ddoc`` is a string, it is passed, as-is, to the server. - Otherwise it is serialized as JSON, and its ``_id`` field is set to - ``_design/{name}``. - - :param bool use_devmode: - Whether a *development* mode view should be used. Development-mode - views are less resource demanding with the caveat that by default - they only operate on a subset of the data. Normally a view will - initially be created in 'development mode', and then published - using :meth:`design_publish` - - :param float syncwait: - How long to poll for the action to complete. Server side design - operations are scheduled and thus this function may return before - the operation is actually completed. Specifying the timeout here - ensures the client polls during this interval to ensure the - operation has completed. - - :raise: :exc:`couchbase.exceptions.TimeoutError` if ``syncwait`` was - specified and the operation could not be verified within the - interval specified. - - :return: An :class:`~couchbase.result.HttpResult` object. - - .. seealso:: :meth:`design_get`, :meth:`design_delete`, - :meth:`design_publish` - - """ - name = self._cb._mk_devmode(name, use_devmode) - - fqname = "_design/{0}".format(name) - if not isinstance(ddoc, dict): - ddoc = json.loads(ddoc) - - ddoc = ddoc.copy() - ddoc['_id'] = fqname - ddoc = json.dumps(ddoc) - - existing = None - if syncwait: - try: - existing = self.design_get(name, use_devmode=False) - except CouchbaseError: - pass - - ret = self._cb._http_request( - type=_LCB.LCB_HTTP_TYPE_VIEW, path=fqname, - method=_LCB.LCB_HTTP_METHOD_PUT, post_data=ddoc, - content_type="application/json") - - self._design_poll(name, 'add', existing, syncwait, - use_devmode=use_devmode) - return ret - - def design_get(self, name, use_devmode=True): - """ - Retrieve a design document - - :param string name: The name of the design document - :param bool use_devmode: Whether this design document is still in - "development" mode - - :return: A :class:`~couchbase.result.HttpResult` containing - a dict representing the format of the design document - - :raise: :exc:`couchbase.exceptions.HTTPError` if the design does not - exist. - - .. seealso:: :meth:`design_create` - - """ - name = self._mk_devmode(name, use_devmode) - - existing = self._http_request(type=_LCB.LCB_HTTP_TYPE_VIEW, - path="_design/" + name, - method=_LCB.LCB_HTTP_METHOD_GET, - content_type="application/json") - return existing - - def design_publish(self, name, syncwait=0): - """ - Convert a development mode view into a production mode views. - Production mode views, as opposed to development views, operate on the - entire cluster data (rather than a restricted subset thereof). - - :param string name: The name of the view to convert. - - Once the view has been converted, ensure that all functions (such as - :meth:`design_get`) have the ``use_devmode`` parameter disabled, - otherwise an error will be raised when those functions are used. - - Note that the ``use_devmode`` option is missing. This is intentional - as the design document must currently be a development view. - - :return: An :class:`~couchbase.result.HttpResult` object. - - :raise: :exc:`couchbase.exceptions.HTTPError` if the design does not - exist - - .. seealso:: :meth:`design_create`, :meth:`design_delete`, - :meth:`design_get` - """ - existing = self.design_get(name, use_devmode=True) - rv = self.design_create(name, existing.value, use_devmode=False, - syncwait=syncwait) - self.design_delete(name, use_devmode=True, - syncwait=syncwait) - self._design_poll(name, 'add', None, - timeout=syncwait, use_devmode=False) - return rv - - def design_delete(self, name, use_devmode=True, syncwait=0): - """ - Delete a design document - - :param string name: The name of the design document to delete - :param bool use_devmode: Whether the design to delete is a development - mode design doc. - - :param float syncwait: Timeout for operation verification. See - :meth:`design_create` for more information on this parameter. - - :return: An :class:`HttpResult` object. - - :raise: :exc:`couchbase.exceptions.HTTPError` if the design does not - exist - :raise: :exc:`couchbase.exceptions.TimeoutError` if ``syncwait`` was - specified and the operation could not be verified within the - specified interval. - - .. seealso:: :meth:`design_create`, :meth:`design_get` - - """ - name = self._mk_devmode(name, use_devmode) - existing = None - if syncwait: - try: - existing = self.design_get(name, use_devmode=False) - except CouchbaseError: - pass - - ret = self._http_request(type=_LCB.LCB_HTTP_TYPE_VIEW, - path="_design/" + name, - method=_LCB.LCB_HTTP_METHOD_DELETE) - - self._design_poll(name, 'del', existing, syncwait) - return ret diff --git a/couchbase/cluster.py b/couchbase/cluster.py new file mode 100644 index 000000000..82cde23b9 --- /dev/null +++ b/couchbase/cluster.py @@ -0,0 +1,873 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import time +from datetime import timedelta +from typing import (TYPE_CHECKING, + Any, + Dict) + +from couchbase.analytics import AnalyticsQuery, AnalyticsRequest +from couchbase.bucket import Bucket +from couchbase.diagnostics import ClusterState, ServiceType +from couchbase.exceptions import ErrorMapper, UnAmbiguousTimeoutException +from couchbase.exceptions import exception as BaseCouchbaseException +from couchbase.logic import BlockingWrapper +from couchbase.logic.cluster import ClusterLogic +from couchbase.logic.supportability import Supportability +from couchbase.management.analytics import AnalyticsIndexManager +from couchbase.management.buckets import BucketManager +from couchbase.management.eventing import EventingFunctionManager +from couchbase.management.queries import QueryIndexManager +from couchbase.management.search import SearchIndexManager +from couchbase.management.users import UserManager +from couchbase.n1ql import N1QLQuery, N1QLRequest +from couchbase.options import PingOptions, forward_args +from couchbase.result import (AnalyticsResult, + ClusterInfoResult, + DiagnosticsResult, + PingResult, + QueryResult, + SearchResult) +from couchbase.search import (FullTextSearchRequest, + SearchQueryBuilder, + SearchRequest) +from couchbase.transactions import Transactions + +if TYPE_CHECKING: + from couchbase.options import (AnalyticsOptions, + ClusterOptions, + DiagnosticsOptions, + QueryOptions, + SearchOptions, + WaitUntilReadyOptions) + from couchbase.search import SearchQuery + + +class Cluster(ClusterLogic): + """Create a Couchbase Cluster instance. + + The cluster instance exposes the operations which are available to be performed against a cluster. + + .. note:: + Although creating an instance of :class:`.Cluster` is allowed, it is recommended to + use the Cluster's static :meth:`.Cluster.connect` method. See :meth:`.Cluster.connect` for connect + for examples. + + Args: + connstr (str): + The connection string to use for connecting to the cluster. + This is a URI-like string allowing specifying multiple hosts. + + The format of the connection string is the *scheme* + (``couchbase`` for normal connections, ``couchbases`` for + SSL enabled connections); a list of one or more *hostnames* + delimited by commas + options (:class:`~couchbase.options.ClusterOptions`): Global options to set for the cluster. + Some operations allow the global options to be overriden by passing in options to the + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + overrride provided :class:`~couchbase.options.ClusterOptions` + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If no :class:`~couchbase.auth.Authenticator` + is provided. Also raised if an invalid `ClusterOption` is provided. + :class:`~couchbase.exceptions.AuthenticationException`: If provided :class:`~couchbase.auth.Authenticator` + has incorrect credentials. + + """ + + def __init__(self, + connstr, # type: str + *options, # type: ClusterOptions + **kwargs, # type: Dict[str, Any] + ) -> Cluster: + + super().__init__(connstr, *options, **kwargs) + self._connect() + + @BlockingWrapper.block(True) + def _connect(self, **kwargs): + ret = super()._connect_cluster(**kwargs) + if isinstance(ret, BaseCouchbaseException): + raise ErrorMapper.build_exception(ret) + self._set_connection(ret) + + @BlockingWrapper.block(True) + def _close_cluster(self): + super()._close_cluster() + super()._destroy_connection() + + @property + def transactions(self) -> Transactions: + """ + :class:`~couchbase.transactions.Transactions`: A Transactions instance which can be used to + perform transactions on this cluster. + """ + if not self._transactions: + self._transactions = Transactions(self, self._transaction_config) + return self._transactions + + def close(self): + """Shuts down this cluster instance. Cleaning up all resources associated with it. + + .. warning:: + Use of this method is almost *always* unnecessary. Cluster resources should be cleaned + up once the cluster instance falls out of scope. However, in some applications tuning resources + is necessary and in those types of applications, this method might be beneficial. + + """ + if self.connected: + self._close_cluster() + + def bucket(self, bucket_name) -> Bucket: + """Creates a Bucket instance to a specific bucket. + + .. seealso:: + :class:`.bucket.Bucket` + + Args: + bucket_name (str): Name of the bucket to reference + + Returns: + :class:`~couchbase.bucket.Bucket`: A bucket instance + + Raises: + RuntimeError: If called prior to the cluster being connected. + :class:`~couchbase.exceptions.BucketNotFoundException`: If provided `bucket_name` cannot + be found. + + """ + if not self.connected: + raise RuntimeError("Cluster not yet connected.") + + return Bucket(self, bucket_name) + + def cluster_info(self) -> ClusterInfoResult: + """Retrieve the Couchbase cluster information + + .. note:: + If using Couchbase Server version < 6.6, a bucket *must* be opened prior to calling + `cluster.cluster_info()`. If a bucket is not opened a + :class:`~couchbase.exceptions.ServiceUnavailableException` will be raised. + + + Returns: + :class:`~couchbase.result.ClusterInfoResult`: Information about the connected cluster. + + Raises: + RuntimeError: If called prior to the cluster being connected. + :class:`~couchbase.exceptions.ServiceUnavailableException`: If called prior to connecting + to a bucket if using server version < 6.6. + + """ + if not self.connected: + raise RuntimeError( + "Cluster is not connected, cannot get cluster info.") + cluster_info = None + cluster_info = self._get_cluster_info() + self._cluster_info = cluster_info + return cluster_info + + @BlockingWrapper.block(ClusterInfoResult) + def _get_cluster_info(self): + return super()._get_cluster_info() + + @BlockingWrapper.block(PingResult) + def ping(self, + *opts, # type: PingOptions + **kwargs # type: Any + ) -> PingResult: + """Performs a ping operation against the cluster. + + The ping operation pings the services which are specified + (or all services if none are specified). Returns a report which describes the outcome of + the ping operations which were performed. + + Args: + opts (:class:`~couchbase.options.PingOptions`): Optional parameters for this operation. + + Returns: + :class:`~couchbase.result.PingResult`: A report which describes the outcome of the ping operations + which were performed. + + """ + return super().ping(*opts, **kwargs) + + @BlockingWrapper.block(DiagnosticsResult) + def diagnostics(self, + *opts, # type: DiagnosticsOptions + **kwargs # type: Dict[str, Any] + ) -> DiagnosticsResult: + """Performs a diagnostic operation against the cluster. + + The diagnostic operations returns a report about the current active connections with the cluster. + Includes information about remote and local addresses, last activity, and other diagnostics information. + + Args: + opts (:class:`~couchbase.options.DiagnosticsOptions`): Optional parameters for this operation. + + Returns: + :class:`~couchbase.result.DiagnosticsResult`: A report which describes current active connections + with the cluster. + + """ + + return super().diagnostics(*opts, **kwargs) + + def wait_until_ready(self, + timeout, # type: timedelta + *opts, # type: WaitUntilReadyOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Wait until the cluster is ready for use. + + Check the current connections to see if the desired state has been reached. If not, + perform a ping against the specified services. The ping operation will be performed + repeatedly with a slight delay in between until the specified timeout has been reached + or the cluster is ready for use, whichever comes first. + + .. seealso:: + * :class:`~couchbase.diagnostics.ServiceType` + * :class:`~couchbase.diagnostics.ClusterState` + + Args: + timeout (timedelta): Amount of time to wait for cluster to be ready before a + :class:`~couchbase.exceptions.UnAmbiguousTimeoutException` is raised. + opts (:class:`~couchbase.options.WaitUntilReadyOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.WaitUntilReadyOptions` + + Raises: + :class:`~couchbase.exceptions.UnAmbiguousTimeoutException`: If the specified timeout is reached prior to + the cluster being ready for use. + + Example: + + Wait until the cluster is ready to use KV and query services:: + + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.diagnostics import ServiceType + from couchbase.options import WaitUntilReadyOptions + + auth = PasswordAuthenticator('username', 'password') + cluster = Cluster.connect('couchbase://localhost', ClusterOptions(auth)) + + cluster.wait_until_ready(timedelta(seconds=3), + WaitUntilReadyOptions(service_types=[ServiceType.KeyValue, ServiceType.Query])) + + """ + + final_args = forward_args(kwargs, *opts) + service_types = final_args.get("service_types", None) + if not service_types: + service_types = [ServiceType(st.value) for st in ServiceType] + + desired_state = final_args.get("desired_state", ClusterState.Online) + service_types_set = set(map(lambda st: st.value if isinstance(st, ServiceType) else st, service_types)) + + # @TODO: handle units + timeout_millis = timeout.total_seconds() * 1000 + + interval_millis = float(50) + start = time.perf_counter() + time_left = timeout_millis + while True: + + diag_res = self.diagnostics() + endpoint_svc_types = set(map(lambda st: st.value, diag_res.endpoints.keys())) + if not endpoint_svc_types.issuperset(service_types_set): + self.ping(PingOptions(service_types=service_types)) + diag_res = self.diagnostics() + + if diag_res.state == desired_state: + break + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((time.perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise UnAmbiguousTimeoutException(message="Desired state not found.") + + time.sleep(interval_millis / 1000) + + def query(self, + statement, # type: str + *options, # type: QueryOptions + **kwargs # type: Dict[str, Any] + ) -> QueryResult: + """Executes a N1QL query against the cluster. + + .. note:: + The query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.QueryResult` begins. + + .. seealso:: + * :class:`~couchbase.management.queries.QueryIndexManager`: for how to manage query indexes + * :meth:`couchbase.Scope.query`: For how to execute scope-level queries. + + Args: + statement (str): The N1QL statement to execute. + options (:class:`~couchbase.options.QueryOptions`): Optional parameters for the query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.QueryOptions` + + Returns: + :class:`~couchbase.result.QueryResult`: An instance of a :class:`~couchbase.result.QueryResult` which + provides access to iterate over the query results and access metadata and metrics about the query. + + Examples: + Simple query:: + + q_res = cluster.query('SELECT * FROM `travel-sample` WHERE country LIKE 'United%' LIMIT 2;') + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple query with positional parameters:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = cluster.query(q_str, QueryOptions(positional_parameters=['United%', 5])) + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple query with named parameters:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = cluster.query(q_str, QueryOptions(named_parameters={'country': 'United%', 'lim':2})) + for row in q_res.rows(): + print(f'Found row: {row}') + + Retrieve metadata and/or metrics from query:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = cluster.query(q_str, QueryOptions(metrics=True)) + for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'Query metadata: {q_res.metadata()}') + print(f'Query metrics: {q_res.metadata().metrics()}') + + """ + + # If the n1ql_query was provided a timeout we will use that value for the streaming timeout + # when the streaming object is created in the bindings. If the n1ql_query does not specify a + # timeout, the streaming_timeout defaults to cluster's query_timeout (set here). If the cluster + # also does not specify a query_timeout we set the streaming_timeout to + # couchbase::core::timeout_defaults::query_timeout when the streaming object is created in the bindings. + streaming_timeout = self.streaming_timeouts.get('query_timeout', None) + query = N1QLQuery.create_query_object(statement, *options, **kwargs) + return QueryResult(N1QLRequest.generate_n1ql_request(self.connection, + query.params, + default_serializer=self.default_serializer, + streaming_timeout=streaming_timeout)) + + def analytics_query(self, # type: Cluster + statement, # type: str + *options, # type: AnalyticsOptions + **kwargs # type: Dict[str, Any] + ) -> AnalyticsResult: + """Executes an analaytics query against the cluster. + + .. note:: + The analytics query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.AnalyticsResult` begins. + + .. seealso:: + + * :class:`~couchbase.management.analytics.AnalyticsIndexManager`: for how to manage analytics dataverses, datasets, indexes and links. + * :meth:`.Scope.analytics_query`: for how to execute scope-level analytics queries + + Args: + statement (str): The analytics SQL++ statement to execute. + options (:class:`~couchbase.options.AnalyticsOptions`): Optional parameters for the analytics query + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.AnalyticsOptions` + + Returns: + :class:`~couchbase.result.AnalyticsResult`: An instance of a + :class:`~couchbase.result.AnalyticsResult` which provides access to iterate over the analytics + query results and access metadata and metrics about the analytics query. + + Examples: + .. note:: + Be sure to setup the necessary dataverse(s), dataset(s) for your analytics queries. + See :analytics_intro:`Analytics Introduction <>` in Couchbase Server docs. + + Simple analytics query:: + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = cluster.analytics_query(q_str) + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple analytics query with positional parameters:: + + from couchbase.options import AnalyticsOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = cluster.analytics_query(q_str, AnalyticsOptions(positional_parameters=['United%', 5])) + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple analytics query with named parameters:: + + from couchbase.options import AnalyticsOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = cluster.analytics_query(q_str, + AnalyticsOptions(named_parameters={'country': 'United%', 'lim':2})) + for row in q_res.rows(): + print(f'Found row: {row}') + + Retrieve metadata and/or metrics from analytics query:: + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = cluster.analytics_query(q_str) + for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'Analytics query metadata: {q_res.metadata()}') + print(f'Analytics query metrics: {q_res.metadata().metrics()}') + + """ # noqa: E501 + # If the analytics_query was provided a timeout we will use that value for the streaming timeout + # when the streaming object is created in the bindings. If the analytics_query does not specify a + # timeout, the streaming_timeout defaults to cluster's analytics_timeout (set here). If the cluster + # also does not specify an analytics_timeout we set the streaming_timeout to + # couchbase::core::timeout_defaults::analytics_timeout when the streaming object is created in the bindings. + streaming_timeout = self.streaming_timeouts.get('analytics_timeout', None) + query = AnalyticsQuery.create_query_object(statement, *options, **kwargs) + return AnalyticsResult(AnalyticsRequest.generate_analytics_request(self.connection, + query.params, + default_serializer=self.default_serializer, + streaming_timeout=streaming_timeout)) + + def search_query(self, + index, # type: str + query, # type: SearchQuery + *options, # type: SearchOptions + **kwargs # type: Dict[str, Any] + ) -> SearchResult: + """Executes an search query against the cluster. + + .. note:: + The search query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.SearchResult` begins. + + .. seealso:: + * :class:`~couchbase.management.search.SearchIndexManager`: for how to manage search indexes. + + Args: + index (str): Name of the search query to use. + query (:class:`~couchbase.search.SearchQuery`): Type of search query to perform. + options (:class:`~couchbase.options.SearchOptions`): Optional parameters for the search query + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.SearchOptions` + + Returns: + :class:`~couchbase.result.SearchResult`: An instance of a + :class:`~couchbase.result.SearchResult` which provides access to iterate over the search + query results and access metadata and metrics about the search query. + + Examples: + + .. note:: + Be sure to create a search index prior to executing search queries. Also, if an application + desires to utilize search row locations, highlighting, etc. make sure the search index is + setup appropriately. See :search_create_idx:`Creating Indexes <>` in Couchbase Server docs. + + Simple search query:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + query = search.TermQuery('home') + q_res = cluster.search_query('travel-sample-index', + query, + SearchOptions(limit=10)) + + for row in q_res.rows(): + print(f'Found row: {row}') + + + Simple search query with facets:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + facet_name = 'activity' + facet = search.TermFacet('activity') + query = search.TermQuery('home') + q_res = cluster.search_query('travel-sample-index', + query, + SearchOptions(limit=10, facets={facet_name: facet})) + + for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'facets: {q_res.facets()}') + + + Simple search query with fields and locations:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + search_fields = ['name', 'activity'] + query = search.TermQuery('home') + q_res = cluster.search_query('travel-sample-index', + query, + SearchOptions(limit=10, + include_locations=True, + fields=search_fields)) + + for row in q_res.rows(): + print(f'Found row: {row}') + print(f'Fields: {row.fields}') + print(f'Locations: {row.locations}') + + """ + # If the search_query was provided a timeout we will use that value for the streaming timeout + # when the streaming object is created in the bindings. If the search_query does not specify a + # timeout, the streaming_timeout defaults to cluster's search_timeout (set here). If the cluster + # also does not specify a search_timeout we set the streaming_timeout to + # couchbase::core::timeout_defaults::search_timeout when the streaming object is created in the bindings. + streaming_timeout = self.streaming_timeouts.get('search_timeout', None) + query = SearchQueryBuilder.create_search_query_object(index, query, *options, **kwargs) + return SearchResult(FullTextSearchRequest.generate_search_request(self.connection, + query.as_encodable(), + default_serializer=self.default_serializer, + streaming_timeout=streaming_timeout)) + + def search(self, + index, # type: str + request, # type: SearchRequest + *options, # type: SearchOptions + **kwargs, # type: Dict[str, Any] + ) -> SearchResult: + """Executes an search against the cluster. + + .. note:: + The search is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.SearchResult` begins. + + .. seealso:: + * :class:`~couchbase.management.search.SearchIndexManager`: for how to manage search indexes. + * :meth:`~couchbase.scope.Scope.search`: for how to execute scope-level search + + Args: + index (str): Name of the search index to use. + request (:class:`~couchbase.search.SearchRequest`): Type of search request to perform. + options (:class:`~couchbase.options.SearchOptions`): Optional parameters for the search query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.SearchOptions` + + Returns: + :class:`~couchbase.result.SearchResult`: An instance of a + :class:`~couchbase.result.SearchResult` which provides access to iterate over the search + query results and access metadata and metrics about the search query. + + Examples: + + .. note:: + Be sure to create a search index prior to executing a search. Also, if an application + desires to utilize search row locations, highlighting, etc. make sure the search index is + setup appropriately. See :search_create_idx:`Creating Indexes <>` in Couchbase Server docs. + + Simple search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + request = search.SearchRequest.create(search.TermQuery('home')) + q_res = cluster.search('travel-sample-index', + request, + SearchOptions(limit=10)) + + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple vector search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + from couchbase.vector_search import VectorQuery, VectorSearch + + # ... other code ... + + # NOTE: the vector is expected to be type List[float], set the vector to the appropriate value, this is an example. + vector = [-0.014653487130999565, -0.008658270351588726, 0.017129190266132355, -0.015563474968075752] + request = search.SearchRequest.create(VectorSearch.from_vector_query(VectorQuery('vector_field', vector))) + q_res = cluster.search('travel-sample-vector-index', + request, + SearchOptions(limit=10)) + + for row in q_res.rows(): + print(f'Found row: {row}') + + Combine search and vector search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + from couchbase.vector_search import VectorQuery, VectorSearch + + # ... other code ... + + # NOTE: the vector is expected to be type List[float], set the vector to the appropriate value, this is an example. + vector_search = VectorSearch.from_vector_query(VectorQuery('vector_field', [-0.014653487130999565, + -0.008658270351588726, + 0.017129190266132355, + -0.015563474968075752])) + request = search.SearchRequest.create(search.MatchAllQuery()).with_vector_search(vector_search) + q_res = cluster.search('travel-sample-vector-index', + request, + SearchOptions(limit=10)) + + for row in q_res.rows(): + print(f'Found row: {row}') + """ # noqa: E501 + + # If the search_query was provided a timeout we will use that value for the streaming timeout + # when the streaming object is created in the bindings. If the search_query does not specify a + # timeout, the streaming_timeout defaults to cluster's search_timeout (set here). If the cluster + # also does not specify a search_timeout we set the streaming_timeout to + # couchbase::core::timeout_defaults::search_timeout when the streaming object is created in the bindings. + streaming_timeout = self.streaming_timeouts.get('search_timeout', None) + query = SearchQueryBuilder.create_search_query_from_request(index, request, *options, **kwargs) + return SearchResult(FullTextSearchRequest.generate_search_request(self.connection, + query.as_encodable(), + default_serializer=self.default_serializer, + streaming_timeout=streaming_timeout)) + + def buckets(self) -> BucketManager: + """ + Get a :class:`~couchbase.management.buckets.BucketManager` which can be used to manage the buckets + of this cluster. + + Returns: + :class:`~couchbase.management.buckets.BucketManager`: A :class:`~couchbase.management.buckets.BucketManager` instance. + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return BucketManager(self.connection) + + def users(self) -> UserManager: + """ + Get a :class:`~couchbase.management.users.UserManager` which can be used to manage the users + of this cluster. + + Returns: + :class:`~couchbase.management.users.UserManager`: A :class:`~couchbase.management.users.UserManager` instance. + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return UserManager(self.connection) + + def query_indexes(self) -> QueryIndexManager: + """ + Get a :class:`~couchbase.management.queries.QueryIndexManager` which can be used to manage the query + indexes of this cluster. + + Returns: + :class:`~couchbase.management.queries.QueryIndexManager`: A :class:`~couchbase.management.queries.QueryIndexManager` instance. + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return QueryIndexManager(self.connection) + + def analytics_indexes(self) -> AnalyticsIndexManager: + """ + Get a :class:`~couchbase.management.analytics.AnalyticsIndexManager` which can be used to manage the analytics + dataverses, dataset, indexes and links of this cluster. + + Returns: + :class:`~couchbase.management.analytics.AnalyticsIndexManager`: An :class:`~couchbase.management.analytics.AnalyticsIndexManager` instance. + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return AnalyticsIndexManager(self.connection) + + def search_indexes(self) -> SearchIndexManager: + """ + Get a :class:`~couchbase.management.search.SearchIndexManager` which can be used to manage the search + indexes of this cluster. + + Returns: + :class:`~couchbase.management.search.SearchIndexManager`: A :class:`~couchbase.management.search.SearchIndexManager` instance. + + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return SearchIndexManager(self.connection) + + def eventing_functions(self) -> EventingFunctionManager: + """ + Get a :class:`~couchbase.management.eventing.EventingFunctionManager` which can be used to manage the + eventing functions of this cluster. + + .. note:: + Eventing function management is an **uncommitted** API that is unlikely to change, + but may still change as final consensus on its behavior has not yet been reached. + + Returns: + :class:`~couchbase.management.eventing.EventingFunctionManager`: An :class:`~couchbase.management.eventing.EventingFunctionManager` instance. + + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return EventingFunctionManager(self.connection) + + @staticmethod + def connect(connstr, # type: str + *options, # type: ClusterOptions + **kwargs, # type: Dict[str, Any] + ) -> Cluster: + """Create a Couchbase Cluster and connect + + Args: + connstr (str): + The connection string to use for connecting to the cluster. + This is a URI-like string allowing specifying multiple hosts. + + The format of the connection string is the *scheme* + (``couchbase`` for normal connections, ``couchbases`` for + SSL enabled connections); a list of one or more *hostnames* + delimited by commas + options (:class:`~couchbase.options.ClusterOptions`): Global options to set for the cluster. + Some operations allow the global options to be overriden by passing in options to the + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + overrride provided :class:`~couchbase.options.ClusterOptions` + + Returns: + :class:`.Cluster`: If successful, a connect Couchbase Cluster instance. + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If no :class:`~couchbase.auth.Authenticator` + is provided. Also raised if an invalid `ClusterOption` is provided. + :class:`~couchbase.exceptions.AuthenticationException`: If provided :class:`~couchbase.auth.Authenticator` + has incorrect credentials. + + + Examples: + Initialize cluster using default options:: + + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.options import ClusterOptions + + auth = PasswordAuthenticator('username', 'password') + cluster = Cluster.connect('couchbase://localhost', ClusterOptions(auth)) + + Connect using SSL:: + + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.options import ClusterOptions + + auth = PasswordAuthenticator('username', 'password', cert_path='/path/to/cert') + cluster = Cluster.connect('couchbases://localhost', ClusterOptions(auth)) + + Initialize cluster using with global timeout options:: + + from datetime import timedelta + + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + + auth = PasswordAuthenticator('username', 'password') + timeout_opts = ClusterTimeoutOptions(kv_timeout=timedelta(seconds=10), + query_timeout=timedelta(seconds=120)) + cluster = Cluster.connect('couchbase://localhost', ClusterOptions(auth, timeout_options=timeout_opts)) + + """ + cluster = Cluster(connstr, *options, **kwargs) + return cluster + + +""" +** DEPRECATION NOTICE ** + +The classes below are deprecated for 3.x compatibility. They should not be used. +Instead use: + * All options should be imported from `couchbase.options`. + * All options Enums should be imported from `couchbase.options`. + * All N1QL Enums should be imported from `couchbase.n1ql`. + +""" + +from couchbase.logic.options import ClusterOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import ClusterTimeoutOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import ClusterTracingOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import DiagnosticsOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import QueryOptionsBase # nopep8 # isort:skip # noqa: E402 + + +@Supportability.import_deprecated('couchbase.cluster', 'couchbase.options') # noqa: F811 +class ClusterOptions(ClusterOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.cluster', 'couchbase.options') # noqa: F811 +class ClusterTimeoutOptions(ClusterTimeoutOptionsBase): + pass + + +@Supportability.import_deprecated('couchbase.cluster', 'couchbase.options') # noqa: F811 +class ClusterTracingOptions(ClusterTracingOptionsBase): + pass + + +@Supportability.import_deprecated('couchbase.cluster', 'couchbase.options') # noqa: F811 +class DiagnosticsOptions(DiagnosticsOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.cluster', 'couchbase.options') # noqa: F811 +class QueryOptions(QueryOptionsBase): # noqa: F811 + pass + + +from couchbase.n1ql import QueryScanConsistency # nopep8 # isort:skip # noqa: E402, F401 +from couchbase.n1ql import QueryProfile # nopep8 # isort:skip # noqa: E402, F401 +from couchbase.options import Compression # nopep8 # isort:skip # noqa: E402, F401 diff --git a/couchbase/collection.py b/couchbase/collection.py new file mode 100644 index 000000000..765faf46c --- /dev/null +++ b/couchbase/collection.py @@ -0,0 +1,3148 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from copy import copy +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Tuple, + Union) + +from couchbase.binary_collection import BinaryCollection +from couchbase.datastructures import (CouchbaseList, + CouchbaseMap, + CouchbaseQueue, + CouchbaseSet) +from couchbase.exceptions import (DocumentExistsException, + ErrorMapper, + InvalidArgumentException, + PathExistsException, + QueueEmpty) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.kv_range_scan import RangeScanRequest +from couchbase.logic import (BlockingWrapper, + decode_replicas, + decode_value) +from couchbase.logic.collection import CollectionLogic +from couchbase.logic.supportability import Supportability +from couchbase.management.queries import CollectionQueryIndexManager +from couchbase.options import (AppendMultiOptions, + DecrementMultiOptions, + ExistsMultiOptions, + GetAllReplicasMultiOptions, + GetAndLockMultiOptions, + GetAnyReplicaMultiOptions, + GetMultiOptions, + IncrementMultiOptions, + InsertMultiOptions, + LockMultiOptions, + PrependMultiOptions, + RemoveMultiOptions, + ReplaceMultiOptions, + ScanOptions, + TouchMultiOptions, + UnlockMultiOptions, + UpsertMultiOptions, + forward_args, + get_valid_multi_args) +from couchbase.pycbc_core import (binary_multi_operation, + kv_multi_operation, + operations) +from couchbase.result import (CounterResult, + ExistsResult, + GetReplicaResult, + GetResult, + LookupInReplicaResult, + LookupInResult, + MultiCounterResult, + MultiExistsResult, + MultiGetReplicaResult, + MultiGetResult, + MultiMutationResult, + MutateInResult, + MutationResult, + OperationResult, + ScanResultIterable) +from couchbase.subdocument import (array_addunique, + array_append, + array_prepend, + count) +from couchbase.subdocument import get as subdoc_get +from couchbase.subdocument import remove as subdoc_remove +from couchbase.subdocument import replace +from couchbase.subdocument import upsert as subdoc_upsert +from couchbase.transcoder import Transcoder + +if TYPE_CHECKING: + from datetime import timedelta + + from couchbase._utils import JSONType + from couchbase.kv_range_scan import ScanType + from couchbase.options import (AppendOptions, + DecrementOptions, + ExistsOptions, + GetAndLockOptions, + GetAndTouchOptions, + GetAnyReplicaOptions, + GetOptions, + IncrementOptions, + InsertOptions, + LookupInAllReplicasOptions, + LookupInAnyReplicaOptions, + LookupInOptions, + MutateInOptions, + MutationMultiOptions, + NoValueMultiOptions, + PrependOptions, + RemoveOptions, + ReplaceOptions, + TouchOptions, + UnlockOptions, + UpsertOptions) + from couchbase.result import MultiResultType + from couchbase.subdocument import Spec + + +class Collection(CollectionLogic): + + def __init__(self, scope, name): + super().__init__(scope, name) + + def get(self, + key, # type: str + *opts, # type: GetOptions + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + """Retrieves the value of a document from the collection. + + Args: + key (str): The key for the document to retrieve. + opts (:class:`~couchbase.options.GetOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetOptions` + + Returns: + :class:`~couchbase.result.GetResult`: An instance of :class:`~couchbase.result.GetResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple get operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + res = collection.get('airline_10') + print(f'Document value: {res.content_as[dict]}') + + + Simple get operation with options:: + + from datetime import timedelta + from couchbase.options import GetOptions + + # ... other code ... + + res = collection.get('airline_10', GetOptions(timeout=timedelta(seconds=2))) + print(f'Document value: {res.content_as[dict]}') + + """ + + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_internal(key, **final_args) + + @BlockingWrapper.block_and_decode(GetResult) + def _get_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + """ **Internal Operation** + + Internal use only. Use :meth:`Collection.get` instead. + """ + return super().get(key, **kwargs) + + def get_any_replica(self, + key, # type: str + *opts, # type: GetAnyReplicaOptions + **kwargs, # type: Dict[str, Any] + ) -> GetReplicaResult: + """Retrieves the value of a document from the collection leveraging both active and all available replicas returning + the first available. + + Args: + key (str): The key for the document to retrieve. + opts (:class:`~couchbase.options.GetAnyReplicaOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAnyReplicaOptions` + + Returns: + :class:`~couchbase.result.GetReplicaResult`: An instance of :class:`~couchbase.result.GetReplicaResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentUnretrievableException`: If the key provided does not exist + on the server. + + Examples: + + Simple get_any_replica operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + res = collection.get_any_replica('airline_10') + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + + + Simple get_any_replica operation with options:: + + from datetime import timedelta + from couchbase.options import GetAnyReplicaOptions + + # ... other code ... + + res = collection.get_any_replica('airline_10', GetAnyReplicaOptions(timeout=timedelta(seconds=5))) + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + + """ # noqa: E501 + + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_any_replica_internal(key, **final_args) + + @BlockingWrapper.block_and_decode(GetReplicaResult) + def _get_any_replica_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> GetReplicaResult: + """ **Internal Operation** + + Internal use only. Use :meth:`Collection.get_any_replica` instead. + """ + return super().get_any_replica(key, **kwargs) + + def get_all_replicas(self, + key, # type: str + *opts, # type: GetAllReplicasOptions + **kwargs, # type: Dict[str, Any] + ) -> Iterable[GetReplicaResult]: + """Retrieves the value of a document from the collection returning both active and all available replicas. + + Args: + key (str): The key for the document to retrieve. + opts (:class:`~couchbase.options.GetAllReplicasOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAllReplicasOptions` + + Returns: + Iterable[:class:`~couchbase.result.GetReplicaResult`]: A stream of + :class:`~couchbase.result.GetReplicaResult` representing both active and replicas of the document retrieved. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple get_all_replicas operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + result = collection.get_all_replicas('airline_10') + for res in results: + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + + + Simple get_all_replicas operation with options:: + + from datetime import timedelta + from couchbase.options import GetAllReplicasOptions + + # ... other code ... + + result = collection.get_all_replicas('airline_10', GetAllReplicasOptions(timeout=timedelta(seconds=10))) + for res in result: + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + + Stream get_all_replicas results:: + + from datetime import timedelta + from couchbase.options import GetAllReplicasOptions + + # ... other code ... + + result = collection.get_all_replicas('airline_10', GetAllReplicasOptions(timeout=timedelta(seconds=10))) + while True: + try: + res = next(result) + print(f'Document is replica: {res.is_replica}') + print(f'Document value: {res.content_as[dict]}') + except StopIteration: + print('Done streaming replicas.') + break + + """ + + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_all_replicas_internal(key, **final_args) + + @BlockingWrapper.block_and_decode(GetReplicaResult) + def _get_all_replicas_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Iterable[GetReplicaResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`Collection.get_all_replicas` instead. + """ + return super().get_all_replicas(key, **kwargs) + + @BlockingWrapper.block(ExistsResult) + def exists( + self, + key, # type: str + *opts, # type: ExistsOptions + **kwargs, # type: Dict[str, Any] + ) -> ExistsResult: + """Checks whether a specific document exists or not. + + Args: + key (str): The key for the document to check existence. + opts (:class:`~couchbase.options.ExistsOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.ExistsOptions` + + Returns: + :class:`~couchbase.result.ExistsResult`: An instance of :class:`~couchbase.result.ExistsResult`. + + Examples: + + Simple exists operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_10' + res = collection.exists(key) + print(f'Document w/ key - {key} {"exists" if res.exists else "does not exist"}') + + + Simple exists operation with options:: + + from datetime import timedelta + from couchbase.options import ExistsOptions + + # ... other code ... + + key = 'airline_10' + res = collection.exists(key, ExistsOptions(timeout=timedelta(seconds=2))) + print(f'Document w/ key - {key} {"exists" if res.exists else "does not exist"}') + + """ + return super().exists(key, *opts, **kwargs) + + @BlockingWrapper.block(MutationResult) + def insert( + self, # type: "Collection" + key, # type: str + value, # type: JSONType + *opts, # type: InsertOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + """Inserts a new document to the collection, failing if the document already exists. + + Args: + key (str): Document key to insert. + value (JSONType): The value of the document to insert. + opts (:class:`~couchbase.options.InsertOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.InsertOptions` + + Returns: + :class:`~couchbase.result.MutationResult`: An instance of :class:`~couchbase.result.MutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentExistsException`: If the document already exists on the + server. + + Examples: + + Simple insert operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_8091' + airline = { + "type": "airline", + "id": 8091, + "callsign": "CBS", + "iata": None, + "icao": None, + "name": "Couchbase Airways", + } + res = collection.insert(key, doc) + + + Simple insert operation with options:: + + from couchbase.durability import DurabilityLevel, ServerDurability + from couchbase.options import InsertOptions + + # ... other code ... + + key = 'airline_8091' + airline = { + "type": "airline", + "id": 8091, + "callsign": "CBS", + "iata": None, + "icao": None, + "name": "Couchbase Airways", + } + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = collection.insert(key, doc, InsertOptions(durability=durability)) + + """ + return super().insert(key, value, *opts, **kwargs) + + @BlockingWrapper.block(MutationResult) + def upsert( + self, + key, # type: str + value, # type: JSONType + *opts, # type: UpsertOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + """Upserts a document to the collection. This operation succeeds whether or not the document already exists. + + Args: + key (str): Document key to upsert. + value (JSONType): The value of the document to upsert. + opts (:class:`~couchbase.options.UpsertOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.UpsertOptions` + + Returns: + :class:`~couchbase.result.MutationResult`: An instance of :class:`~couchbase.result.MutationResult`. + + Examples: + + Simple upsert operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_8091' + airline = { + "type": "airline", + "id": 8091, + "callsign": "CBS", + "iata": None, + "icao": None, + "name": "Couchbase Airways", + } + res = collection.upsert(key, doc) + + + Simple upsert operation with options:: + + from couchbase.durability import DurabilityLevel, ServerDurability + from couchbase.options import UpsertOptions + + # ... other code ... + + key = 'airline_8091' + airline = { + "type": "airline", + "id": 8091, + "callsign": "CBS", + "iata": None, + "icao": None, + "name": "Couchbase Airways", + } + durability = ServerDurability(level=DurabilityLevel.MAJORITY) + res = collection.upsert(key, doc, InsertOptions(durability=durability)) + + """ + return super().upsert(key, value, *opts, **kwargs) + + @BlockingWrapper.block(MutationResult) + def replace(self, + key, # type: str + value, # type: JSONType + *opts, # type: ReplaceOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + """Replaces the value of an existing document. Failing if the document does not exist. + + Args: + key (str): Document key to replace. + value (JSONType): The value of the document to replace. + opts (:class:`~couchbase.options.ReplaceOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.ReplaceOptions` + + Returns: + :class:`~couchbase.result.MutationResult`: An instance of :class:`~couchbase.result.MutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the document does not exist on the + server. + + Examples: + + Simple replace operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_8091' + res = collection.get(key) + content = res.content_as[dict] + airline["name"] = "Couchbase Airways!!" + res = collection.replace(key, doc) + + + Simple replace operation with options:: + + from couchbase.durability import DurabilityLevel, ServerDurability + from couchbase.options import ReplaceOptions + + # ... other code ... + + key = 'airline_8091' + res = collection.get(key) + content = res.content_as[dict] + airline["name"] = "Couchbase Airways!!" + durability = ServerDurability(level=DurabilityLevel.MAJORITY) + res = collection.replace(key, doc, InsertOptions(durability=durability)) + + """ + return super().replace(key, value, *opts, **kwargs) + + @BlockingWrapper.block(MutationResult) + def remove(self, + key, # type: str + *opts, # type: RemoveOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + """Removes an existing document. Failing if the document does not exist. + + Args: + key (str): Key for the document to remove. + opts (:class:`~couchbase.options.RemoveOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.RemoveOptions` + + Returns: + :class:`~couchbase.result.MutationResult`: An instance of :class:`~couchbase.result.MutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the document does not exist on the + server. + + Examples: + + Simple remove operation:: + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + res = collection.remove('airline_10') + + + Simple remove operation with options:: + + from couchbase.durability import DurabilityLevel, ServerDurability + from couchbase.options import RemoveOptions + + # ... other code ... + + durability = ServerDurability(level=DurabilityLevel.MAJORITY) + res = collection.remove('airline_10', RemoveOptions(durability=durability)) + + """ + return super().remove(key, *opts, **kwargs) + + @BlockingWrapper.block(MutationResult) + def touch(self, + key, # type: str + expiry, # type: timedelta + *opts, # type: TouchOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + """Updates the expiry on an existing document. + + Args: + key (str): Key for the document to touch. + expiry (timedelta): The new expiry for the document. + opts (:class:`~couchbase.options.TouchOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TouchOptions` + + Returns: + :class:`~couchbase.result.MutationResult`: An instance of :class:`~couchbase.result.MutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the document does not exist on the + server. + + Examples: + + Simple touch operation:: + + from datetime import timedelta + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + res = collection.touch('airline_10', timedelta(seconds=300)) + + + Simple touch operation with options:: + + from datetime import timedelta + + from couchbase.options import TouchOptions + + # ... other code ... + + res = collection.touch('airline_10', + timedelta(seconds=300), + TouchOptions(timeout=timedelta(seconds=2))) + + """ + return super().touch(key, expiry, *opts, **kwargs) + + def get_and_touch(self, + key, # type: str + expiry, # type: timedelta + *opts, # type: GetAndTouchOptions + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + """Retrieves the value of the document and simultanously updates the expiry time for the same document. + + Args: + key (str): The key for the document retrieve and set expiry time. + expiry (timedelta): The new expiry to apply to the document. + opts (:class:`~couchbase.options.GetAndTouchOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAndTouchOptions` + + Returns: + :class:`~couchbase.result.GetResult`: An instance of :class:`~couchbase.result.GetResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple get and touch operation:: + + from datetime import timedelta + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_10' + res = collection.get_and_touch(key, timedelta(seconds=20)) + print(f'Document w/ updated expiry: {res.content_as[dict]}') + + + Simple get and touch operation with options:: + + from datetime import timedelta + from couchbase.options import GetAndTouchOptions + + # ... other code ... + + key = 'airline_10' + res = collection.get_and_touch(key, + timedelta(seconds=20), + GetAndTouchOptions(timeout=timedelta(seconds=2))) + print(f'Document w/ updated expiry: {res.content_as[dict]}') + + """ + # add to kwargs for conversion to int + kwargs["expiry"] = expiry + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_and_touch_internal(key, **final_args) + + @BlockingWrapper.block_and_decode(GetResult) + def _get_and_touch_internal(self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + """ **Internal Operation** + + Internal use only. Use :meth:`Collection.get_and_touch` instead. + + """ + return super().get_and_touch(key, **kwargs) + + def get_and_lock( + self, + key, # type: str + lock_time, # type: timedelta + *opts, # type: GetAndLockOptions + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + """Locks a document and retrieves the value of that document at the time it is locked. + + Args: + key (str): The key for the document to lock and retrieve. + lock_time (timedelta): The amount of time to lock the document. + opts (:class:`~couchbase.options.GetAndLockOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAndLockOptions` + + Returns: + :class:`~couchbase.result.GetResult`: An instance of :class:`~couchbase.result.GetResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple get and lock operation:: + + from datetime import timedelta + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_10' + res = collection.get_and_lock(key, timedelta(seconds=20)) + print(f'Locked document: {res.content_as[dict]}') + + + Simple get and lock operation with options:: + + from datetime import timedelta + from couchbase.options import GetAndLockOptions + + # ... other code ... + + key = 'airline_10' + res = collection.get_and_lock(key, + timedelta(seconds=20), + GetAndLockOptions(timeout=timedelta(seconds=2))) + print(f'Locked document: {res.content_as[dict]}') + + """ + # add to kwargs for conversion to int + kwargs["lock_time"] = lock_time + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_and_lock_internal(key, **final_args) + + @BlockingWrapper.block_and_decode(GetResult) + def _get_and_lock_internal(self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + """ **Internal Operation** + + Internal use only. Use :meth:`Collection.get_and_lock` instead. + + """ + return super().get_and_lock(key, **kwargs) + + @BlockingWrapper.block(None) + def unlock(self, + key, # type: str + cas, # type: int + *opts, # type: UnlockOptions + **kwargs, # type: Dict[str, Any] + ) -> None: + """Unlocks a previously locked document. + + Args: + key (str): The key for the document to unlock. + cas (int): The CAS of the document, used to validate lock ownership. + opts (:class:`couchbaseoptions.UnlockOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.UnlockOptions` + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + :class:`~couchbase.exceptions.DocumentLockedException`: If the provided cas is invalid. + + Examples: + + Simple unlock operation:: + + from datetime import timedelta + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + key = 'airline_10' + res = collection.get_and_lock(key, timedelta(seconds=5)) + collection.unlock(key, res.cas) + # this should be okay once document is unlocked + collection.upsert(key, res.content_as[dict]) + + """ + return super().unlock(key, cas, *opts, **kwargs) + + def lookup_in( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInOptions + **kwargs, # type: Dict[str, Any] + ) -> LookupInResult: + """Performs a lookup-in operation against a document, fetching individual fields or information + about specific fields inside the document value. + + Args: + key (str): The key for the document look in. + spec (Iterable[:class:`~couchbase.subdocument.Spec`]): A list of specs describing the data to fetch + from the document. + opts (:class:`~couchbase.options.LookupInOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.LookupInOptions` + + Returns: + :class:`~couchbase.result.LookupInResult`: An instance of :class:`~couchbase.result.LookupInResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple look-up in operation:: + + import couchbase.subdocument as SD + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('hotel') + + key = 'hotel_10025' + res = collection.lookup_in(key, (SD.get("geo"),)) + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + + Simple look-up in operation with options:: + + from datetime import timedelta + + import couchbase.subdocument as SD + from couchbase.options import LookupInOptions + + # ... other code ... + + key = 'hotel_10025' + res = collection.lookup_in(key, + (SD.get("geo"),), + LookupInOptions(timeout=timedelta(seconds=2))) + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + """ + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_internal(key, spec, **final_args) + + @BlockingWrapper.block_and_decode(LookupInResult) + def _lookup_in_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Dict[str, Any] + ) -> LookupInResult: + """ **Internal Operation** + + Internal use only. Use :meth:`Collection.lookup_in` instead. + + """ + return super().lookup_in(key, spec, **kwargs) + + def lookup_in_any_replica( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInAnyReplicaOptions + **kwargs, # type: Dict[str, Any] + ) -> LookupInReplicaResult: + """Performs a lookup-in operation against a document, fetching individual fields or information + about specific fields inside the document value. It leverages both active and all available replicas + returning the first available + + Args: + key (str): The key for the document look in. + spec (Iterable[:class:`~couchbase.subdocument.Spec`]): A list of specs describing the data to fetch + from the document. + opts (:class:`~couchbase.options.LookupInAnyReplicaOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.LookupInAnyReplicaOptions` + + Returns: + :class:`~couchbase.result.LookupInReplicaResult`: An instance of :class:`~couchbase.result.LookupInReplicaResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentUnretrievableException`: If the key provided does not exist + on the server. + + Examples: + + Simple lookup_in_any_replica operation:: + + import couchbase.subdocument as SD + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('hotel') + + key = 'hotel_10025' + res = collection.lookup_in_any_replica(key, (SD.get("geo"),)) + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + + Simple lookup_in_any_replica operation with options:: + + from datetime import timedelta + + import couchbase.subdocument as SD + from couchbase.options import LookupInAnyReplicaOptions + + # ... other code ... + + key = 'hotel_10025' + res = collection.lookup_in_any_replica(key, + (SD.get("geo"),), + LookupInAnyReplicaOptions(timeout=timedelta(seconds=2))) + print(f'Document is replica: {res.is_replica}') + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + """ # noqa: E501 + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_any_replica_internal(key, spec, **final_args) + + @BlockingWrapper.block_and_decode(LookupInReplicaResult) + def _lookup_in_any_replica_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Dict[str, Any] + ) -> LookupInReplicaResult: + """ **Internal Operation** + + Internal use only. Use :meth:`Collection.lookup_in` instead. + + """ + return super().lookup_in_any_replica(key, spec, **kwargs) + + def lookup_in_all_replicas( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInAllReplicasOptions + **kwargs, # type: Any + ) -> Iterable[LookupInReplicaResult]: + """Performs a lookup-in operation against a document, fetching individual fields or information + about specific fields inside the document value, returning results from both active and all available replicas + + Args: + key (str): The key for the document look in. + spec (Iterable[:class:`~couchbase.subdocument.Spec`]): A list of specs describing the data to fetch + from the document. + opts (:class:`~couchbase.options.LookupInAllReplicasOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.LookupInAllReplicasOptions` + + Returns: + Iterable[:class:`~couchbase.result.LookupInReplicaResult`]: A stream of + :class:`~couchbase.result.LookupInReplicaResult` representing both active and replicas of the sub-document + retrieved. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple lookup_in_all_replicas operation:: + + import couchbase.subdocument as SD + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('hotel') + + key = 'hotel_10025' + results = collection.lookup_in_all_replicas(key, (SD.get("geo"),)) + for res in results: + print(f'Document is replica: {res.is_replica}') + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + + Simple lookup_in_all_replicas operation with options:: + + import couchbase.subdocument as SD + from datetime import timedelta + from couchbase.options import LookupInAllReplicasOptions + + # ... other code ... + + key = 'hotel_10025' + results = collection.lookup_in_all_replicas(key, + (SD.get("geo"),), + LookupInAllReplicasOptions(timeout=timedelta(seconds=2))) + + for res in results: + print(f'Document is replica: {res.is_replica}') + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + + Stream lookup_in_all_replicas results:: + + from datetime import timedelta + from couchbase.options import GetAllReplicasOptions + + # ... other code ... + + key = 'hotel_10025' + results = collection.lookup_in_all_replicas(key, + (SD.get("geo"),), + LookupInAllReplicasOptions(timeout=timedelta(seconds=2))) + while True: + try: + res = next(results) + print(f'Document is replica: {res.is_replica}') + print(f'Hotel {key} coordinates: {res.content_as[dict](0)}') + except StopIteration: + print('Done streaming replicas.') + break + + """ + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_all_replicas_internal(key, spec, **final_args) + + @BlockingWrapper.block_and_decode(LookupInReplicaResult) + def _lookup_in_all_replicas_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Dict[str, Any] + ) -> Iterable[LookupInReplicaResult]: + """ **Internal Operation** + + Internal use only. Use :meth:`Collection.lookup_in` instead. + + """ + return super().lookup_in_all_replicas(key, spec, **kwargs) + + @BlockingWrapper.block(MutateInResult) + def mutate_in( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: MutateInOptions + **kwargs, # type: Dict[str, Any] + ) -> MutateInResult: + """Performs a mutate-in operation against a document. Allowing atomic modification of specific fields + within a document. Also enables access to document extended-attributes (i.e. xattrs). + + Args: + key (str): The key for the document look in. + spec (Iterable[:class:`~couchbase.subdocument.Spec`]): A list of specs describing the operations to + perform on the document. + opts (:class:`~couchbase.options.MutateInOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.MutateInOptions` + + Returns: + :class:`~couchbase.result.MutateInResult`: An instance of :class:`~couchbase.result.MutateInResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + Examples: + + Simple mutate-in operation:: + + import couchbase.subdocument as SD + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('hotel') + + key = 'hotel_10025' + res = collection.mutate_in(key, (SD.replace("city", "New City"),)) + + + Simple mutate-in operation with options:: + + from datetime import timedelta + + import couchbase.subdocument as SD + from couchbase.options import MutateInOptions + + # ... other code ... + + key = 'hotel_10025' + res = collection.mutate_in(key, + (SD.replace("city", "New City"),), + MutateInOptions(timeout=timedelta(seconds=2))) + + """ + return super().mutate_in(key, spec, *opts, **kwargs) + + def scan(self, scan_type, # type: ScanType + *opts, # type: ScanOptions + **kwargs, # type: Dict[str, Any] + ) -> ScanResultIterable: + """Execute a key-value range scan operation from the collection. + + .. note:: + Use this API for low concurrency batch queries where latency is not a critical as the system may have to scan a lot of documents to find the matching documents. + For low latency range queries, it is recommended that you use SQL++ with the necessary indexes. + + Args: + scan_type (:class:`~couchbase.kv_range_scan.ScanType`): Either a :class:`~couchbase.kv_range_scan.RangeScan`, + :class:`~couchbase.kv_range_scan.PrefixScan` or + :class:`~couchbase.kv_range_scan.SamplingScan` instance. + opts (:class:`~couchbase.options.ScanOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.ScanOptions` + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If scan_type is not either a RangeScan or SamplingScan instance. + :class:`~couchbase.exceptions.InvalidArgumentException`: If sort option is provided and is incorrect type. + :class:`~couchbase.exceptions.InvalidArgumentException`: If consistent_with option is provided and is not a + + Returns: + :class:`~couchbase.result.ScanResultIterable`: An instance of :class:`~couchbase.result.ScanResultIterable`. + + Examples: + + Simple range scan operation:: + + from couchbase.kv_range_scan import RangeScan + from couchbase.options import ScanOptions + + # ... other code ... + + bucket = cluster.bucket('travel-sample') + collection = bucket.scope('inventory').collection('airline') + + scan_type = RangeScan(ScanTerm('airline-00'), ScanTerm('airline-99')) + scan_iter = collection.scan(scan_type, ScanOptions(ids_only=True)) + + for res in scan_iter: + print(res) + + + """ # noqa: E501 + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + final_args['transcoder'] = self.default_transcoder + scan_args = super().build_scan_args(scan_type, **final_args) + range_scan_request = RangeScanRequest(**scan_args) + return ScanResultIterable(range_scan_request) + + def binary(self) -> BinaryCollection: + """Creates a BinaryCollection instance, allowing access to various binary operations + possible against a collection. + + .. seealso:: + :class:`~couchbase.binary_collection.BinaryCollection` + + Returns: + :class:`~couchbase.binary_collection.BinaryCollection`: A BinaryCollection instance. + + """ + return BinaryCollection(self) + + @BlockingWrapper.block(MutationResult) + def _append( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: AppendOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + """ **Internal Operation** + + Internal use only. Use :meth:`.BinaryCollection.append` instead. + + """ + return super().append(key, value, *opts, **kwargs) + + @BlockingWrapper.block(MutationResult) + def _prepend( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: PrependOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + """ **Internal Operation** + + Internal use only. Use :meth:`.BinaryCollection.prepend` instead. + + """ + return super().prepend(key, value, *opts, **kwargs) + + @BlockingWrapper.block(CounterResult) + def _increment( + self, + key, # type: str + *opts, # type: IncrementOptions + **kwargs, # type: Dict[str, Any] + ) -> CounterResult: + """ **Internal Operation** + + Internal use only. Use :meth:`.BinaryCollection.increment` instead. + + """ + return super().increment(key, *opts, **kwargs) + + @BlockingWrapper.block(CounterResult) + def _decrement( + self, + key, # type: str + *opts, # type: DecrementOptions + **kwargs, # type: Dict[str, Any] + ) -> CounterResult: + """ **Internal Operation** + + Internal use only. Use :meth:`.BinaryCollection.decrement` instead. + + """ + return super().decrement(key, *opts, **kwargs) + + def couchbase_list(self, key # type: str + ) -> CouchbaseList: + """Returns a CouchbaseList permitting simple list storage in a document. + + .. seealso:: + :class:`~couchbase.datastructures.CouchbaseList` + + Returns: + :class:`~couchbase.datastructures.CouchbaseList`: A CouchbaseList instance. + + """ + return CouchbaseList(key, self) + + @BlockingWrapper._dsop(create_type='list') + def list_append(self, key, # type: str + value, # type: JSONType + create=False, # type: Optional[bool] + **kwargs, # type: Dict[str, Any] + ) -> OperationResult: + """Add an item to the end of a list. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseList.append` + instead. + + Args: + key (str): The key for the list document. + value (JSONType): The value to append to the list. + create (bool, optional): Whether the list should be created if it does not exist. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`couchbase~.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + """ + op = array_append('', value) + sd_res = self.mutate_in(key, (op,), **kwargs) + return OperationResult(sd_res.cas, sd_res.mutation_token()) + + @BlockingWrapper._dsop(create_type='list') + def list_prepend(self, key, # type: str + value, # type: JSONType + create=False, # type: Optional[bool] + **kwargs, # type: Dict[str, Any] + ) -> OperationResult: + """ Add an item to the beginning of a list. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseList.prepend` + instead. + + Args: + key (str): The key for the list document. + value (JSONType): The value to prepend to the list. + create (bool, optional): Whether the list should be created if it does not exist. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + """ + op = array_prepend('', value) + sd_res = self.mutate_in(key, (op,), **kwargs) + return OperationResult(sd_res.cas, sd_res.mutation_token()) + + @BlockingWrapper._dsop() + def list_set(self, key, # type: str + index, # type: int + value, # type: JSONType + **kwargs # type: Dict[str, Any] + ) -> OperationResult: + """Sets an item within a list at a given position. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseList.set_at` + instead. + + Args: + key (str): The key for the list document. + index (int): The position to replace. + value (JSONType): The value to prepend to the list. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + IndexError: If the index is out of bounds. + + """ + + op = replace(f'[{index}]', value) + sd_res = self.mutate_in(key, (op,), **kwargs) + return OperationResult(sd_res.cas, sd_res.mutation_token()) + + @BlockingWrapper._dsop() + def list_get(self, key, # type: str + index, # type: int + **kwargs # type: Dict[str, Any] + ) -> Any: + """Get a specific element within a list. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseList.get_at` + instead. + + Args: + key (str): The key for the list document. + index (int): The position to retrieve. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Any: The value of the element at the specified index. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + IndexError: If the index is out of bounds. + + """ + op = subdoc_get(f'[{index}]') + sd_res = self.lookup_in(key, (op,), **kwargs) + return sd_res.value[0].get("value", None) + + @BlockingWrapper._dsop() + def list_remove(self, key, # type: str + index, # type: int + **kwargs # type: Dict[str, Any] + ) -> OperationResult: + """Remove the element at a specific index from a list. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseList.remove_at` + instead. + + Args: + key (str): The key for the list document. + index (int): The position to remove. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + IndexError: If the index is out of bounds. + + """ + + op = subdoc_remove(f'[{index}]') + sd_res = self.mutate_in(key, (op,), **kwargs) + return OperationResult(sd_res.cas, sd_res.mutation_token()) + + @BlockingWrapper._dsop() + def list_size(self, key, # type: str + **kwargs # type: Dict[str, Any] + ) -> int: + """Returns the number of items in the list. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseList.size` + instead. + + Args: + key (str): The key for the list document. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + int: The number of items in the list. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + """ + + op = count('') + sd_res = self.lookup_in(key, (op,), **kwargs) + return sd_res.value[0].get("value", None) + + def couchbase_map(self, key # type: str + ) -> CouchbaseMap: + """Returns a CouchbaseMap permitting simple map storage in a document. + + .. seealso:: + :class:`~couchbase.datastructures.CouchbaseMap` + + Returns: + :class:`~couchbase.datastructures.CouchbaseMap`: A CouchbaseMap instance. + + """ + return CouchbaseMap(key, self) + + @BlockingWrapper._dsop(create_type='dict') + def map_add(self, + key, # type: str + mapkey, # type: str + value, # type: Any + create=False, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ) -> OperationResult: + """Set a value for a key in a map. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseMap.add` + instead. + + Args: + key (str): The key for the map document. + mapkey (str): The key in the map to set. + value (Any): The value to use. + create (bool, optional): Whether the map should be created if it does not exist. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + """ + op = subdoc_upsert(mapkey, value) + sd_res = self.mutate_in(key, (op,), **kwargs) + return OperationResult(sd_res.cas, sd_res.mutation_token()) + + @BlockingWrapper._dsop() + def map_get(self, + key, # type: str + mapkey, # type: str + **kwargs # type: Dict[str, Any] + ) -> Any: + """Retrieve a value from a map. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseMap.get` + instead. + + Args: + key (str): The key for the map document. + mapkey (str): The key in the map to set. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Any: The value of the specified key. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + """ + op = subdoc_get(mapkey) + sd_res = self.lookup_in(key, (op,), **kwargs) + return sd_res.value[0].get("value", None) + + @BlockingWrapper._dsop() + def map_remove(self, + key, # type: str + mapkey, # type: str + **kwargs # type: Dict[str, Any] + ) -> OperationResult: + """Remove an item from a map. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseMap.remove` + instead. + + Args: + key (str): The key for the map document. + mapkey (str): The key in the map to set. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + """ + op = subdoc_remove(mapkey) + sd_res = self.mutate_in(key, (op,), **kwargs) + return OperationResult(sd_res.cas, sd_res.mutation_token()) + + @BlockingWrapper._dsop() + def map_size(self, + key, # type: str + **kwargs # type: Dict[str, Any] + ) -> int: + """Get the number of items in the map. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseMap.remove` + instead. + + Args: + key (str): The key for the map document. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + int: The number of items in the map. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + + """ + op = count('') + sd_res = self.lookup_in(key, (op,), **kwargs) + return sd_res.value[0].get("value", None) + + def couchbase_set(self, key # type: str + ) -> CouchbaseSet: + """Returns a CouchbaseSet permitting simple map storage in a document. + + .. seealso:: + :class:`~couchbase.datastructures.CouchbaseSet` + + Returns: + :class:`~couchbase.datastructures.CouchbaseSet`: A CouchbaseSet instance. + + """ + return CouchbaseSet(key, self) + + @BlockingWrapper._dsop(create_type='list') + def set_add(self, + key, # type: str + value, # type: Any + create=False, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ) -> Optional[OperationResult]: + """Add an item to a set if the item does not yet exist. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseSet.add` + instead. + + Args: + key (str): The key for the set document. + value (Any): The value to add to the set. + create (bool, optional): Whether the set should be created if it does not exist. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + """ + op = array_addunique('', value) + try: + sd_res = self.mutate_in(key, (op,), **kwargs) + return OperationResult(sd_res.cas, sd_res.mutation_token()) + except PathExistsException: + pass + + @BlockingWrapper._dsop() + def set_remove(self, + key, # type: str + value, # type: Any + **kwargs # type: Dict[str, Any] + ) -> Optional[OperationResult]: + """Remove an item from a set. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseSet.remove` + instead. + + Args: + key (str): The key for the set document. + value (Any): The value to remove from the set. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + """ + while True: + rv = self.get(key, **kwargs) + try: + ix = rv.value.index(value) + kwargs['cas'] = rv.cas + return self.list_remove(key, ix, **kwargs) + except DocumentExistsException: + pass + except ValueError: + return + + def set_size(self, + key, # type: str + **kwargs # type: Dict[str, Any] + ) -> int: + """Get the length of a set. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseSet.size` + instead. + + Args: + key (str): The key for the set document. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + int: The length of a set. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + """ + return self.list_size(key, **kwargs) + + def set_contains(self, + key, # type: str + value, # type: Any + **kwargs # type: Dict[str, Any] + ) -> bool: + """Determine if an item exists in a set + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseSet.contains` + instead. + + Args: + key (str): The key for the set document. + value (Any): The value to check for. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + bool: True if the set contains the specified value. False othwerwise. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + """ + rv = self.get(key, **kwargs) + return value in rv.value + + def couchbase_queue(self, key # type: str + ) -> CouchbaseQueue: + """Returns a CouchbaseQueue permitting simple map storage in a document. + + .. seealso:: + :class:`~couchbase.datastructures.CouchbaseQueue` + + Returns: + :class:`~couchbase.datastructures.CouchbaseQueue`: A CouchbaseQueue instance. + + """ + return CouchbaseQueue(key, self) + + @BlockingWrapper._dsop(create_type='list') + def queue_push(self, + key, # type: str + value, # type: Any + create=False, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ) -> OperationResult: + """Add an item to the end of a queue. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseQueue.push` + instead. + + Args: + key (str): The key for the queue document. + value (Any): The value to add. + create (bool, optional): Whether the queue should be created if it does not exist. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + """ + return self.list_prepend(key, value, **kwargs) + + @BlockingWrapper._dsop() + def queue_pop(self, + key, # type: str + **kwargs # type: Dict[str, Any] + ) -> OperationResult: + """Remove and return the first item queue. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseQueue.pop` + instead. + + Args: + key (str): The key for the queue document. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`~couchbase.result.OperationResult`: An instance of :class:`~couchbase.result.OperationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + """ + while True: + try: + itm = self.list_get(key, -1, **kwargs) + except IndexError: + raise QueueEmpty + + kwargs.update({k: v for k, v in getattr( + itm, '__dict__', {}).items() if k in {'cas'}}) + try: + self.list_remove(key, -1, **kwargs) + return itm + except DocumentExistsException: + pass + except IndexError: + raise QueueEmpty + + @BlockingWrapper._dsop() + def queue_size(self, + key # type: str + ) -> int: + """Get the length of a queue. + + .. warning:: + This method is deprecated and will be removed in a future version. Use :meth:`.CouchbaseQueue.size` + instead. + + Args: + key (str): The key for the queue document. + + Returns: + int: The length of the queue. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist + on the server. + """ + return self.list_size(key) + + def _get_multi_mutation_transcoded_op_args( + self, + keys_and_docs, # type: Dict[str, JSONType] + *opts, # type: MutationMultiOptions + **kwargs, # type: Any + ) -> Tuple[Dict[str, Any], bool]: + + if not isinstance(keys_and_docs, dict): + raise InvalidArgumentException(message='Expected keys_and_docs to be a dict.') + + opts_type = kwargs.pop('opts_type', None) + if not opts_type: + raise InvalidArgumentException(message='Expected options type is missing.') + + final_args = get_valid_multi_args(opts_type, kwargs, *opts) + per_key_args = final_args.pop('per_key_options', None) + op_transcoder = final_args.pop('transcoder', self.default_transcoder) + op_args = {} + for key, value in keys_and_docs.items(): + op_args[key] = copy(final_args) + # per key args override global args + if per_key_args and key in per_key_args: + key_transcoder = per_key_args.pop('transcoder', op_transcoder) + op_args[key].update(per_key_args[key]) + transcoded_value = key_transcoder.encode_value(value) + else: + transcoded_value = op_transcoder.encode_value(value) + op_args[key]['value'] = transcoded_value + + if isinstance(opts_type, ReplaceMultiOptions): + for k, v in op_args.items(): + expiry = v.get('expiry', None) + preserve_expiry = v.get('preserve_expiry', False) + if expiry and preserve_expiry is True: + raise InvalidArgumentException( + message=("The expiry and preserve_expiry options cannot " + f"both be set for replace operations. Multi-op key: {k}.") + ) + + return_exceptions = final_args.pop('return_exceptions', True) + return op_args, return_exceptions + + def _get_multi_op_args( + self, + keys, # type: List[str] + *opts, # type: NoValueMultiOptions + **kwargs, # type: Any + ) -> Tuple[Dict[str, Any], bool, Dict[str, Transcoder]]: + if not isinstance(keys, list): + raise InvalidArgumentException(message='Expected keys to be a list.') + + opts_type = kwargs.pop('opts_type', None) + if not opts_type: + raise InvalidArgumentException(message='Expected options type is missing.') + + final_args = get_valid_multi_args(opts_type, kwargs, *opts) + op_transcoder = final_args.pop('transcoder', self.default_transcoder) + per_key_args = final_args.pop('per_key_options', None) + op_args = {} + key_transcoders = {} + for key in keys: + op_args[key] = copy(final_args) + # per key args override global args + if per_key_args and key in per_key_args: + key_transcoder = per_key_args.pop('transcoder', op_transcoder) + key_transcoders[key] = key_transcoder + op_args[key].update(per_key_args[key]) + else: + key_transcoders[key] = op_transcoder + + return_exceptions = final_args.pop('return_exceptions', True) + return op_args, return_exceptions, key_transcoders + + def get_multi(self, + keys, # type: List[str] + *opts, # type: GetMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiGetResult: + """For each key in the provided list, retrieve the document associated with the key. + + Args: + keys (List[str]): The keys to use for the multiple get operations. + opts (:class:`~couchbase.options.GetMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetMultiOptions` + + Returns: + :class:`~couchbase.result.MultiGetResult`: An instance of + :class:`~couchbase.result.MultiGetResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + Examples: + + Simple get-multi operation:: + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_multi(keys) + for k, v in res.results.items(): + print(f'Doc {k} has value: {v.content_as[dict]}') + + Simple get-multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import GetMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_multi(keys, + GetMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + print(f'Doc {k} has value: {v.content_as[dict]}') + + Simple get-multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import GetMultiOptions, GetOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + per_key_opts = {'doc1': GetOptions(timeout=timedelta(seconds=10))} + res = collection.get_multi(keys, + GetMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + print(f'Doc {k} has value: {v.content_as[dict]}') + + + """ + op_args, return_exceptions, transcoders = self._get_multi_op_args(keys, + *opts, + opts_type=GetMultiOptions, + **kwargs) + op_type = operations.GET.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + for k, v in res.raw_result.items(): + if k == 'all_okay': + continue + if isinstance(v, CouchbaseBaseException): + continue + value = v.raw_result.get('value', None) + flags = v.raw_result.get('flags', None) + tc = transcoders[k] + v.raw_result['value'] = decode_value(tc, value, flags) + + return MultiGetResult(res, return_exceptions) + + def get_any_replica_multi(self, + keys, # type: List[str] + *opts, # type: GetAnyReplicaMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiGetReplicaResult: + """For each key in the provided list, retrieve the document associated with the key from the collection + leveraging both active and all available replicas returning the first available. + + Args: + keys (List[str]): The keys to use for the multiple get operations. + opts (:class:`~couchbase.options.GetAnyReplicaMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAnyReplicaMultiOptions` + + Returns: + :class:`~couchbase.result.MultiGetReplicaResult`: An instance of + :class:`~couchbase.result.MultiGetReplicaResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentUnretrievableException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + Examples: + + Simple get_any_replica_multi operation:: + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_any_replica_multi(keys) + for k, v in res.results.items(): + if v.is_replica: + print(f'Replica doc {k} has value: {v.content_as[dict]}') + else: + print(f'Active doc {k} has value: {v.content_as[dict]}') + + Simple get_any_replica_multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import GetAnyReplicaMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_any_replica_multi(keys, + GetAnyReplicaMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + if v.is_replica: + print(f'Replica doc {k} has value: {v.content_as[dict]}') + else: + print(f'Active doc {k} has value: {v.content_as[dict]}') + + Simple get_any_replica_multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import GetAnyReplicaMultiOptions, GetAnyReplicaOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + per_key_opts = {'doc1': GetAnyReplicaOptions(timeout=timedelta(seconds=10))} + res = collection.get_any_replica_multi(keys, + GetAnyReplicaMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + if v.is_replica: + print(f'Replica doc {k} has value: {v.content_as[dict]}') + else: + print(f'Active doc {k} has value: {v.content_as[dict]}') + + """ + op_args, return_exceptions, transcoders = self._get_multi_op_args(keys, + *opts, + opts_type=GetAnyReplicaMultiOptions, + **kwargs) + op_type = operations.GET_ANY_REPLICA.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + for k, v in res.raw_result.items(): + if k == 'all_okay': + continue + if isinstance(v, CouchbaseBaseException): + continue + value = v.raw_result.get('value', None) + flags = v.raw_result.get('flags', None) + tc = transcoders[k] + v.raw_result['value'] = decode_value(tc, value, flags) + + return MultiGetReplicaResult(res, return_exceptions) + + def get_all_replicas_multi(self, + keys, # type: List[str] + *opts, # type: GetAllReplicasMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiGetReplicaResult: + """For each key in the provided list, retrieve the document from the collection returning both + active and all available replicas. + + Args: + keys (List[str]): The keys to use for the multiple get operations. + opts (:class:`~couchbase.options.GetAllReplicasMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAllReplicasMultiOptions` + + Returns: + :class:`~couchbase.result.MultiGetReplicaResult`: An instance of + :class:`~couchbase.result.MultiGetReplicaResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + Examples: + + Simple get_all_replicas_multi operation:: + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_all_replicas_multi(keys) + for k, docs in res.results.items(): + for doc in docs: + if doc.is_replica: + print(f'Replica doc {k} has value: {doc.content_as[dict]}') + else: + print(f'Active doc {k} has value: {doc.content_as[dict]}') + + Simple get_all_replicas_multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import GetAllReplicasMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_all_replicas_multi(keys, + GetAllReplicasMultiOptions(return_exceptions=False)) + for k, docs in res.results.items(): + for doc in docs: + if doc.is_replica: + print(f'Replica doc {k} has value: {doc.content_as[dict]}') + else: + print(f'Active doc {k} has value: {doc.content_as[dict]}') + + Simple get_all_replicas_multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import GetAllReplicasMultiOptions, GetAllReplicasOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + per_key_opts = {'doc1': GetAllReplicasOptions(timeout=timedelta(seconds=10))} + res = collection.get_all_replicas_multi(keys, + GetAllReplicasMultiOptions(per_key_options=per_key_opts)) + for k, docs in res.results.items(): + for doc in docs: + if doc.is_replica: + print(f'Replica doc {k} has value: {doc.content_as[dict]}') + else: + print(f'Active doc {k} has value: {doc.content_as[dict]}') + + """ + op_args, return_exceptions, transcoders = self._get_multi_op_args(keys, + *opts, + opts_type=GetAllReplicasMultiOptions, + **kwargs) + op_type = operations.GET_ALL_REPLICAS.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + + # all the successful results will be streamed_results, so lets + # pop those off the main result dict and re-add the key back + # transformed into a List[GetReplicaResult] + result_keys = [] + for k, v in res.raw_result.items(): + if k == 'all_okay' or isinstance(v, CouchbaseBaseException): + continue + result_keys.append(k) + + for k in result_keys: + value = res.raw_result.pop(k) + tc = transcoders[k] + res.raw_result[k] = list(r for r in decode_replicas(tc, value, GetReplicaResult)) + + return MultiGetReplicaResult(res, return_exceptions) + + def lock_multi(self, + keys, # type: List[str] + lock_time, # type: timedelta + *opts, # type: LockMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiGetResult: + """ + .. warning:: + This method is deprecated and will be deprecated in a future release. + Use :meth:`~couchbase.collection.Collection.get_and_lock_multi` instead. + """ + Supportability.method_deprecated('lock_multi', 'get_and_lock_multi') + return self.get_and_lock_multi(keys, lock_time, *opts, **kwargs) + + def get_and_lock_multi(self, + keys, # type; List[str] + lock_time, # type: timedelta + *opts, # type: GetAndLockMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiGetResult: + """For each key in the provided list, lock the document associated with the key. + + Args: + keys (List[str]): The keys to use for the multiple lock operations. + lock_time (timedelta): The amount of time to lock the documents. + opts (:class:`~couchbase.options.GetAndLockMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.GetAndLockMultiOptions` + + Returns: + :class:`~couchbase.result.MultiGetResult`: An instance of + :class:`~couchbase.result.MultiGetResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + Examples: + + Simple get_and_lock_multi operation:: + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_and_lock_multi(keys, timedelta(seconds=20)) + for k, v in res.results.items(): + print(f'Locked document: key={k}, content={v.content_as[str]}') + + Simple get_and_lock_multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import GetAndLockMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_and_lock_multi(keys, + timedelta(seconds=20), + GetAndLockMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + print(f'Locked document: key={k}, content={v.content_as[str]}') + + Simple get_and_lock_multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import GetAndLockMultiOptions, GetAndLockOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + per_key_opts = {'doc1': GetAndLockOptions(timeout=timedelta(seconds=10))} + res = collection.get_and_lock_multi(keys, + timedelta(seconds=20), + GetAndLockMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + print(f'Locked document: key={k}, content={v.content_as[str]}') + + """ + kwargs["lock_time"] = lock_time + op_args, return_exceptions, transcoders = self._get_multi_op_args(keys, + *opts, + opts_type=LockMultiOptions, + **kwargs) + op_type = operations.GET_AND_LOCK.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + for k, v in res.raw_result.items(): + if k == 'all_okay': + continue + if isinstance(v, CouchbaseBaseException): + continue + value = v.raw_result.get('value', None) + flags = v.raw_result.get('flags', None) + tc = transcoders[k] + v.raw_result['value'] = decode_value(tc, value, flags) + + return MultiGetResult(res, return_exceptions) + + def exists_multi(self, + keys, # type: List[str] + *opts, # type: ExistsMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiExistsResult: + """For each key in the provided list, check if the document associated with the key exists. + + Args: + keys (List[str]): The keys to use for the multiple exists operations. + opts (:class:`~couchbase.options.ExistsMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.ExistsMultiOptions` + + Returns: + :class:`~couchbase.result.MultiExistsResult`: An instance of :class:`~couchbase.result.MultiExistsResult`. + + Examples: + + Simple exists_multi operation:: + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.exists_multi(keys) + for k, v in res.results.items(): + print(f'Doc with key={k} {"exists" if v.exists else "does not exist"}') + + Simple exists_multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import ExistsMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.exists_multi(keys, + ExistsMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + print(f'Doc with key={k} {"exists" if v.exists else "does not exist"}') + + Simple exists_multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import ExistsMultiOptions, ExistsOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + per_key_opts = {'doc1': ExistsOptions(timeout=timedelta(seconds=10))} + res = collection.exists_multi(keys, + ExistsMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + print(f'Doc with key={k} {"exists" if v.exists else "does not exist"}') + """ # noqa: E501 + op_args, return_exceptions, _ = self._get_multi_op_args(keys, + *opts, + opts_type=ExistsMultiOptions, + **kwargs) + op_type = operations.EXISTS.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiExistsResult(res, return_exceptions) + + def insert_multi(self, + keys_and_docs, # type: Dict[str, JSONType] + *opts, # type: InsertMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + """For each key, value pair in the provided dict, inserts a new document to the collection, + failing if the document already exists. + + Args: + keys_and_docs (Dict[str, JSONType]): The keys and values/docs to use for the multiple insert operations. + opts (:class:`~couchbase.options.InsertMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.InsertMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of :class:`~couchbase.result.MultiMutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentExistsException`: If the key provided already exists on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + Examples: + + Simple insert_multi operation:: + + collection = bucket.default_collection() + keys_and_docs = { + 'doc1': {'foo': 'bar', 'id': 'doc1'}, + 'doc2': {'bar': 'baz', 'id': 'doc2'}, + 'doc3': {'baz': 'qux', 'id': 'doc3'}, + 'doc4': {'qux': 'quux', 'id': 'doc4'} + } + res = collection.insert_multi(keys_and_docs) + for k, v in res.results.items(): + print(f'Doc inserted: key={k}, cas={v.cas}') + + Simple insert_multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import InsertMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys_and_docs = { + 'doc1': {'foo': 'bar', 'id': 'doc1'}, + 'doc2': {'bar': 'baz', 'id': 'doc2'}, + 'doc3': {'baz': 'qux', 'id': 'doc3'}, + 'doc4': {'qux': 'quux', 'id': 'doc4'} + } + res = collection.insert_multi(keys_and_docs, + InsertMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + print(f'Doc inserted: key={k}, cas={v.cas}') + + Simple insert_multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import InsertMultiOptions, InsertOptions + + # ... other code ... + + collection = bucket.default_collection() + keys_and_docs = { + 'doc1': {'foo': 'bar', 'id': 'doc1'}, + 'doc2': {'bar': 'baz', 'id': 'doc2'}, + 'doc3': {'baz': 'qux', 'id': 'doc3'}, + 'doc4': {'qux': 'quux', 'id': 'doc4'} + } + per_key_opts = {'doc1': InsertOptions(timeout=timedelta(seconds=10))} + res = collection.insert_multi(keys_and_docs, + InsertMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + print(f'Doc inserted: key={k}, cas={v.cas}') + + """ # noqa: E501 + op_args, return_exceptions = self._get_multi_mutation_transcoded_op_args(keys_and_docs, + *opts, + opts_type=InsertMultiOptions, + **kwargs) + op_type = operations.INSERT.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiMutationResult(res, return_exceptions) + + def upsert_multi(self, + keys_and_docs, # type: Dict[str, JSONType] + *opts, # type: UpsertMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + """For each key, value pair in the provided dict, upserts a document to the collection. This operation + succeeds whether or not the document already exists. + + Args: + keys_and_docs (Dict[str, JSONType]): The keys and values/docs to use for the multiple upsert operations. + opts (:class:`~couchbase.options.UpsertMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.UpsertMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of :class:`~couchbase.result.MultiMutationResult`. + + Examples: + + Simple upsert_multi operation:: + + collection = bucket.default_collection() + keys_and_docs = { + 'doc1': {'foo': 'bar', 'id': 'doc1'}, + 'doc2': {'bar': 'baz', 'id': 'doc2'}, + 'doc3': {'baz': 'qux', 'id': 'doc3'}, + 'doc4': {'qux': 'quux', 'id': 'doc4'} + } + res = collection.upsert_multi(keys_and_docs) + for k, v in res.results.items(): + print(f'Doc upserted: key={k}, cas={v.cas}') + + Simple upsert_multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import UpsertMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys_and_docs = { + 'doc1': {'foo': 'bar', 'id': 'doc1'}, + 'doc2': {'bar': 'baz', 'id': 'doc2'}, + 'doc3': {'baz': 'qux', 'id': 'doc3'}, + 'doc4': {'qux': 'quux', 'id': 'doc4'} + } + res = collection.upsert_multi(keys_and_docs, + UpsertMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + print(f'Doc upserted: key={k}, cas={v.cas}') + + Simple upsert_multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import UpsertMultiOptions, UpsertOptions + + # ... other code ... + + collection = bucket.default_collection() + keys_and_docs = { + 'doc1': {'foo': 'bar', 'id': 'doc1'}, + 'doc2': {'bar': 'baz', 'id': 'doc2'}, + 'doc3': {'baz': 'qux', 'id': 'doc3'}, + 'doc4': {'qux': 'quux', 'id': 'doc4'} + } + per_key_opts = {'doc1': UpsertOptions(timeout=timedelta(seconds=10))} + res = collection.upsert_multi(keys_and_docs, + UpsertMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + print(f'Doc upserted: key={k}, cas={v.cas}') + + """ # noqa: E501 + op_args, return_exceptions = self._get_multi_mutation_transcoded_op_args(keys_and_docs, + *opts, + opts_type=UpsertMultiOptions, + **kwargs) + op_type = operations.UPSERT.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiMutationResult(res, return_exceptions) + + def replace_multi(self, + keys_and_docs, # type: Dict[str, JSONType] + *opts, # type: ReplaceMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + """For each key, value pair in the provided dict, replaces the value of a document in the collection. + This operation fails if the document does not exist. + + Args: + keys_and_docs (Dict[str, JSONType]): The keys and values/docs to use for the multiple replace operations. + opts (:class:`~couchbase.options.ReplaceMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.ReplaceMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of :class:`~couchbase.result.MultiMutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + Examples: + + Simple replace_multi operation:: + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_multi(keys) + keys_and_docs = {} + for k, v in res.results.items(): + # assuming document is JSON + content = v.content_as[dict] + content['foo'] = 'bar' + keys_and_docs[k] = content + res = collection.replace_multi(keys_and_docs) + for k, v in res.results.items(): + print(f'Doc replaced: key={k}, cas={v.cas}') + + Simple replace_multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import ReplaceMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_multi(keys) + keys_and_docs = {} + for k, v in res.results.items(): + # assuming document is JSON + content = v.content_as[dict] + content['foo'] = 'bar' + keys_and_docs[k] = content + res = collection.replace_multi(keys_and_docs, + ReplaceMultiOptions(return_exceptions=False)) + for k, v in res.results.items(): + print(f'Doc replaced: key={k}, cas={v.cas}') + + Simple replace_multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import ReplaceMultiOptions, ReplaceOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.get_multi(keys) + keys_and_docs = {} + for k, v in res.results.items(): + # assuming document is JSON + content = v.content_as[dict] + content['foo'] = 'bar' + keys_and_docs[k] = content + per_key_opts = {'doc1': ReplaceOptions(timeout=timedelta(seconds=10))} + res = collection.replace_multi(keys_and_docs, + ReplaceMultiOptions(per_key_options=per_key_opts)) + for k, v in res.results.items(): + print(f'Doc replaced: key={k}, cas={v.cas}') + + """ # noqa: E501 + op_args, return_exceptions = self._get_multi_mutation_transcoded_op_args(keys_and_docs, + *opts, + opts_type=ReplaceMultiOptions, + **kwargs) + op_type = operations.REPLACE.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiMutationResult(res, return_exceptions) + + def remove_multi(self, + keys, # type: List[str] + *opts, # type: RemoveMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + """For each key in the provided list, remove the existing document. This operation fails + if the document does not exist. + + Args: + keys (List[str]): The keys to use for the multiple remove operations. + opts (:class:`~couchbase.options.RemoveMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.RemoveMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of :class:`~couchbase.result.MultiMutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + Examples: + + Simple remove_multi operation:: + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.remove_multi(keys) + + Simple remove_multi operation, raise an Exception if an Exception occurs:: + + from couchbase.options import RemoveMultiOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + res = collection.remove_multi(keys, + RemoveMultiOptions(return_exceptions=False)) + + Simple remove_multi operation, individual key options:: + + from datetime import timedelta + + from couchbase.options import RemoveMultiOptions, RemoveOptions + + # ... other code ... + + collection = bucket.default_collection() + keys = ['doc1', 'doc2', 'doc3'] + per_key_opts = {'doc1': RemoveOptions(timeout=timedelta(seconds=10))} + res = collection.remove_multi(keys, + RemoveMultiOptions(per_key_options=per_key_opts)) + + """ # noqa: E501 + op_args, return_exceptions, _ = self._get_multi_op_args(keys, + *opts, + opts_type=RemoveMultiOptions, + **kwargs) + op_type = operations.REMOVE.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiMutationResult(res, return_exceptions) + + def touch_multi(self, + keys, # type: List[str] + expiry, # type: timedelta + *opts, # type: TouchMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + """For each key in the provided list, update the expiry on an existing document. This operation fails + if the document does not exist. + + Args: + keys (List[str]): The keys to use for the multiple touch operations. + expiry (timedelta): The new expiry for the document. + opts (:class:`~couchbase.options.TouchMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TouchMultiOptions` + + Returns: + :class:`~couchbase.result.MultiMutationResult`: An instance of + :class:`~couchbase.result.MultiMutationResult`. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + """ + kwargs['expiry'] = expiry + op_args, return_exceptions, _ = self._get_multi_op_args(keys, + *opts, + opts_type=TouchMultiOptions, + **kwargs) + op_type = operations.TOUCH.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiMutationResult(res, return_exceptions) + + def unlock_multi(self, # noqa: C901 + keys, # type: Union[MultiResultType, Dict[str, int]] + *opts, # type: UnlockMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> Dict[str, Union[None, CouchbaseBaseException]]: + """For each result in the provided :class:`~couchbase.result.MultiResultType` in the provided list, + unlocks a previously locked document. This operation fails if the document does not exist. + + Args: + keys (Union[MultiResultType, Dict[str, int]]): The result from a previous multi operation. + opts (:class:`~couchbase.options.UnlockMultiOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.UnlockMultiOptions` + + Returns: + Dict[str, Union[None, CouchbaseBaseException]]: Either None if operation successful or an Exception + if the operation was unsuccessful + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the key provided does not exist on the + server and the return_exceptions options is False. Otherwise the exception is returned as a + match to the key, but is not raised. + + :class:`~couchbase.exceptions.DocumentLockedException`: If the provided cas is invalid and the + return_exceptions options is False. Otherwise the exception is returned as a match to the key, + but is not raised. + + """ + op_keys_cas = {} + if isinstance(keys, dict): + if not all(map(lambda k: isinstance(k, str), keys.keys())): + raise InvalidArgumentException('If providing keys of type dict, all values must be type int.') + if not all(map(lambda v: isinstance(v, int), keys.values())): + raise InvalidArgumentException('If providing keys of type dict, all values must be type int.') + op_keys_cas = copy(keys) + elif isinstance(keys, (MultiGetResult, MultiMutationResult)): + for k, v in keys.results.items(): + op_keys_cas[k] = v.cas + else: + raise InvalidArgumentException( + 'keys type must be Union[MultiGetResult, MultiMutationResult, Dict[str, int].') + + op_args, return_exceptions, _ = self._get_multi_op_args(list(op_keys_cas.keys()), + *opts, + opts_type=UnlockMultiOptions, + **kwargs) + + for k, v in op_args.items(): + v['cas'] = op_keys_cas[k] + + op_type = operations.UNLOCK.value + res = kv_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + output = {} + for k, v in res.raw_result.items(): + if k == 'all_okay': + continue + if isinstance(v, CouchbaseBaseException): + if not return_exceptions: + raise ErrorMapper.build_exception(v) + else: + output[k] = ErrorMapper.build_exception(v) + else: + output[k] = None + + return output + + def _get_multi_counter_op_args( + self, + keys, # type: List[str] + *opts, # type: Union[IncrementMultiOptions, DecrementMultiOptions] + **kwargs, # type: Any + ) -> Tuple[Dict[str, Any], bool]: + if not isinstance(keys, list): + raise InvalidArgumentException(message='Expected keys to be a list.') + + opts_type = kwargs.pop('opts_type', None) + if not opts_type: + raise InvalidArgumentException(message='Expected options type is missing.') + + final_args = get_valid_multi_args(opts_type, kwargs, *opts) + + global_delta, global_initial = self._get_and_validate_delta_initial(final_args) + final_args['delta'] = int(global_delta) + final_args['initial'] = int(global_initial) + + per_key_args = final_args.pop('per_key_options', None) + op_args = {} + for key in keys: + op_args[key] = copy(final_args) + # per key args override global args + if per_key_args and key in per_key_args: + # need to validate delta/initial if provided per key + delta = per_key_args[key].get('delta', None) + initial = per_key_args[key].get('initial', None) + self._validate_delta_initial(delta=delta, initial=initial) + if delta: + per_key_args[key]['delta'] = int(delta) + if initial: + per_key_args[key]['initial'] = int(initial) + op_args[key].update(per_key_args[key]) + + return_exceptions = final_args.pop('return_exceptions', True) + return op_args, return_exceptions + + def _get_multi_binary_mutation_op_args( + self, + keys_and_docs, # type: Dict[str, Union[str, bytes, bytearray]] + *opts, # type: Union[AppendMultiOptions, PrependMultiOptions] + **kwargs, # type: Any + ) -> Tuple[Dict[str, Any], bool]: + + if not isinstance(keys_and_docs, dict): + raise InvalidArgumentException(message='Expected keys_and_docs to be a dict.') + + opts_type = kwargs.pop('opts_type', None) + if not opts_type: + raise InvalidArgumentException(message='Expected options type is missing.') + + parsed_keys_and_docs = {} + for k, v in keys_and_docs.items(): + if isinstance(v, str): + value = v.encode("utf-8") + elif isinstance(v, bytearray): + value = bytes(v) + else: + value = v + + if not isinstance(value, bytes): + raise ValueError( + "The value provided must of type str, bytes or bytearray.") + + parsed_keys_and_docs[k] = value + + final_args = get_valid_multi_args(opts_type, kwargs, *opts) + per_key_args = final_args.pop('per_key_options', None) + op_args = {} + for key, value in parsed_keys_and_docs.items(): + op_args[key] = copy(final_args) + # per key args override global args + if per_key_args and key in per_key_args: + op_args[key].update(per_key_args[key]) + op_args[key]['value'] = value + + return_exceptions = final_args.pop('return_exceptions', True) + return op_args, return_exceptions + + def _append_multi( + self, + keys_and_values, # type: Dict[str, Union[str,bytes,bytearray]] + *opts, # type: AppendMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + op_args, return_exceptions = self._get_multi_binary_mutation_op_args(keys_and_values, + *opts, + opts_type=AppendMultiOptions, + **kwargs) + op_type = operations.APPEND.value + res = binary_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiMutationResult(res, return_exceptions) + + def _prepend_multi( + self, + keys_and_values, # type: Dict[str, Union[str,bytes,bytearray]] + *opts, # type: PrependMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiMutationResult: + op_args, return_exceptions = self._get_multi_binary_mutation_op_args(keys_and_values, + *opts, + opts_type=PrependMultiOptions, + **kwargs) + op_type = operations.PREPEND.value + res = binary_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiMutationResult(res, return_exceptions) + + def _increment_multi( + self, + keys, # type: List[str] + *opts, # type: IncrementMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiCounterResult: + op_args, return_exceptions = self._get_multi_counter_op_args(keys, + *opts, + opts_type=IncrementMultiOptions, + **kwargs) + op_type = operations.INCREMENT.value + res = binary_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiCounterResult(res, return_exceptions) + + def _decrement_multi( + self, + keys, # type: List[str] + *opts, # type: DecrementMultiOptions + **kwargs, # type: Dict[str, Any] + ) -> MultiCounterResult: + op_args, return_exceptions = self._get_multi_counter_op_args(keys, + *opts, + opts_type=DecrementMultiOptions, + **kwargs) + op_type = operations.DECREMENT.value + res = binary_multi_operation( + **self._get_connection_args(), + op_type=op_type, + op_args=op_args + ) + return MultiCounterResult(res, return_exceptions) + + def query_indexes(self) -> CollectionQueryIndexManager: + """ + Get a :class:`~couchbase.management.queries.CollectionQueryIndexManager` which can be used to manage the query + indexes of this cluster. + + Returns: + :class:`~couchbase.management.queries.CollectionQueryIndexManager`: A :class:`~couchbase.management.queries.CollectionQueryIndexManager` instance. + """ # noqa: E501 + return CollectionQueryIndexManager(self.connection, self._scope.bucket_name, self._scope.name, self.name) + + @staticmethod + def default_name(): + return "_default" + + +""" +** DEPRECATION NOTICE ** + +The classes below are deprecated for 3.x compatibility. They should not be used. +Instead use: + * All options should be imported from `couchbase.options`. + * All constrained int classes should be imported from `couchbase.options`. + * Scope object should be imported from `couchbase.scope`. + * Do not use the `CBCollection` class, use Collection instead. + +""" + +from couchbase.logic.options import AppendOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import DecrementOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import DeltaValueBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import DurabilityOptionBlockBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import ExistsOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import GetAllReplicasOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import GetAndLockOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import GetAndTouchOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import GetAnyReplicaOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import GetOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import IncrementOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import InsertOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import LookupInOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import OptionsTimeoutBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import PrependOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import RemoveOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import ReplaceOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import TouchOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import UnlockOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.options import UpsertOptionsBase # nopep8 # isort:skip # noqa: E402 +from couchbase.logic.scope import ScopeLogic # nopep8 # isort:skip # noqa: E402 + +from couchbase.options import ConstrainedInt # nopep8 # isort:skip # noqa: E402, F401 +from couchbase.options import SignedInt64 # nopep8 # isort:skip # noqa: E402, F401 + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.scope') +class Scope(ScopeLogic): + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class AppendOptions(AppendOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class DecrementOptions(DecrementOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class DeltaValue(DeltaValueBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class DurabilityOptionBlock(DurabilityOptionBlockBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class ExistsOptions(ExistsOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class GetAllReplicasOptions(GetAllReplicasOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class GetAndTouchOptions(GetAndTouchOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class GetAndLockOptions(GetAndLockOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class GetAnyReplicaOptions(GetAnyReplicaOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class GetOptions(GetOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class IncrementOptions(IncrementOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class InsertOptions(InsertOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class LookupInOptions(LookupInOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class OptionsTimeout(OptionsTimeoutBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class PrependOptions(PrependOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class RemoveOptions(RemoveOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class ReplaceOptions(ReplaceOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class TouchOptions(TouchOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class UnlockOptions(UnlockOptionsBase): # noqa: F811 + pass + + +@Supportability.import_deprecated('couchbase.collection', 'couchbase.options') # noqa: F811 +class UpsertOptions(UpsertOptionsBase): # noqa: F811 + pass + + +@Supportability.class_deprecated('couchbase.collection.Collection') +class CBCollection(Collection): + pass diff --git a/couchbase/connection.py b/couchbase/connection.py deleted file mode 100644 index 15aa081fb..000000000 --- a/couchbase/connection.py +++ /dev/null @@ -1,10 +0,0 @@ -from couchbase.bucket import Bucket, _depr -from couchbase.user_constants import * -from couchbase.connstr import convert_1x_args - -_depr('couchbase.connection', 'couchbase.bucket') - -class Connection(Bucket): - def __init__(self, bucket='default', **kwargs): - kwargs = convert_1x_args(bucket, **kwargs) - super(Connection, self).__init__(**kwargs) diff --git a/couchbase/connstr.py b/couchbase/connstr.py deleted file mode 100644 index 99cd6f6f8..000000000 --- a/couchbase/connstr.py +++ /dev/null @@ -1,181 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from __future__ import print_function -import copy - -try: - from urlparse import urlparse, parse_qs - from urllib import urlencode -except ImportError: - from urllib.parse import urlparse, parse_qs, urlencode - - -class ConnectionString(object): - """ - This class parses the connection string and may be passed - to the :class:`~.Bucket` constructor instead of a raw - string. - - .. note:: - - Raw strings passed to the :class:`~.Bucket` constructor - are *not* first converted into a `ConnectionString` - object. This is an internal implementation detail, but - may be helpful to know in case of bugs encountered in - this class. - - The :meth:`encode` method can be used to check the encoded - form of the connection string. - """ - - def __init__(self, bucket='default', - hosts=None, options=None, scheme='couchbase'): - """ - Create a new ConnectionString object. - - This is an alternative form to manually inputting - a connection string. - - :param string bucket: The bucket to connect to - :param list hosts: A list of hosts to which the initial - connection should be attempted - :param dict options: A dictionary of options. These options - are passed verbatim to the C library. - :param string scheme: The scheme to be used when connecting. - This scheme is used to interpret the initial default - port to use for each node. - """ - - #: The bucket to connect to. See :meth:`~.__init__` - self.bucket = bucket - - #: Options for the connection. See :meth:`~.__init__` - self.options = copy.copy(options) if options else {} - - #: List of hosts. See :meth:`~.__init__` - self.hosts = copy.copy(hosts) if hosts else [] - - #: The scheme to be used. See :meth:`~.__init__` - self.scheme = scheme - - @classmethod - def parse(cls, ss): - """ - Parses an existing connection string - - This method will return a :class:`~.ConnectionString` object - which will allow further inspection on the input parameters. - - :param string ss: The existing connection string - :return: A new :class:`~.ConnectionString` object - """ - - up = urlparse(ss) - path = up.path - - if '?' in path: - path, query = up.path.split('?') - else: - query = "" - - if path.startswith('/'): - path = path[1:] - - bucket = path - options = parse_qs(query) - scheme = up.scheme - hosts = up.netloc.split(',') - return cls(bucket=bucket, options=options, hosts=hosts, scheme=scheme) - - @property - def implicit_port(self): - if self.scheme == 'http': - return 8091 - elif self.scheme == 'couchbase': - return 11210 - elif self.scheme == 'couchbases': - return 11207 - else: - return -1 - - def encode(self): - """ - Encodes the current state of the object into a string. - - :return: The encoded string - """ - opt_dict = {} - for k, v in self.options.items(): - opt_dict[k] = v[0] - - ss = '{scheme}://{hosts}/{bucket}?{options}'.format( - scheme=self.scheme, hosts=','.join(self.hosts), bucket=self.bucket, - options=urlencode(opt_dict)) - return ss - - def __str__(self): - return self.encode() - - -def _fmthost(host, port): - if port is not None: - return '{0}:{1}'.format(host, port) - else: - return host - - -def _build_connstr(host, port, bucket): - """ - Converts a 1.x host:port specification to a connection string - """ - hostlist = [] - if isinstance(host, (tuple, list)): - for curhost in host: - if isinstance(curhost, (list, tuple)): - hostlist.append(_fmthost(*curhost)) - else: - hostlist.append(curhost) - else: - hostlist.append(_fmthost(host, port)) - - return 'http://{0}/{1}'.format(','.join(hostlist), bucket) - - -def convert_1x_args(bucket, **kwargs): - """ - Converts arguments for 1.x constructors to their 2.x forms - """ - host = kwargs.pop('host', 'localhost') - port = kwargs.pop('port', None) - if not 'connstr' in kwargs and 'connection_string' not in kwargs: - kwargs['connection_string'] = _build_connstr(host, port, bucket) - return kwargs - - -if __name__ == "__main__": - sample = "couchbase://host1:111,host2:222,host3:333/default?op_timeout=4.2" - cs = ConnectionString.parse(sample) - print("Hosts", cs.hosts) - print("Implicit Port", cs.implicit_port) - print("Bucket", cs.bucket) - print("Options", cs.options) - - cs.bucket = "Hi" - print("Encoded again", cs) - - kwargs = convert_1x_args('beer-sample', host=[('192.168.37.101',8091)]) - print(kwargs) diff --git a/couchbase/constants.py b/couchbase/constants.py new file mode 100644 index 000000000..3eec01e84 --- /dev/null +++ b/couchbase/constants.py @@ -0,0 +1,22 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# flake8: noqa +from couchbase.pycbc_core import (FMT_BYTES, + FMT_COMMON_MASK, + FMT_JSON, + FMT_LEGACY_MASK, + FMT_PICKLE, + FMT_UTF8) diff --git a/couchbase/datastructures.py b/couchbase/datastructures.py new file mode 100644 index 000000000..f4279831e --- /dev/null +++ b/couchbase/datastructures.py @@ -0,0 +1,633 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time +from datetime import timedelta +from typing import (TYPE_CHECKING, + Any, + Dict, + Generator, + List, + Optional) + +from couchbase.exceptions import (CasMismatchException, + DocumentNotFoundException, + InvalidArgumentException, + PathExistsException, + PathNotFoundException, + QueueEmpty, + UnAmbiguousTimeoutException) +from couchbase.logic.wrappers import BlockingWrapper +from couchbase.options import MutateInOptions +from couchbase.subdocument import (array_addunique, + array_append, + array_prepend, + count) +from couchbase.subdocument import exists as subdoc_exists +from couchbase.subdocument import get as subdoc_get +from couchbase.subdocument import (remove, + replace, + upsert) + +if TYPE_CHECKING: + from couchbase._utils import JSONType + from couchbase.collection import Collection + + +class CouchbaseList: + """ + CouchbaseList provides a simplified interface for storing lists within a Couchbase document. + + Args: + key (str): Document key to use for the list. + collection (:class:`~.collection.Collection`): The :class:`~.collection.Collection` where the + list belongs + + """ + + def __init__(self, key, # type: str + collection # type: Collection + ) -> None: + self._key = key + self._collection = collection + self._full_list = None + + @BlockingWrapper.datastructure_op(create_type=list) + def _get(self) -> List: + """ + Get the entire list. + """ + + return self._collection.get(self._key) + + @BlockingWrapper.datastructure_op(create_type=list) + def append(self, value # type: JSONType + ) -> None: + """Add an item to the end of the list. + + Args: + value (JSONType): The value to add. + + """ + op = array_append('', value) + self._collection.mutate_in(self._key, (op,)) + + @BlockingWrapper.datastructure_op(create_type=list) + def prepend(self, value # type: JSONType + ) -> None: + """Add an item to the beginning of the list. + + Args: + value (JSONType): The value to add. + + """ + op = array_prepend('', value) + self._collection.mutate_in(self._key, (op,)) + + def set_at(self, index, # type: int + value # type: JSONType + ) -> None: + """Sets an item within a list at a specified index. + + Args: + index (int): The index to retrieve. + value (JSONType): The value to set. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the index is out of range. + + """ + try: + op = replace(f'[{index}]', value) + self._collection.mutate_in(self._key, (op,)) + except PathNotFoundException: + raise InvalidArgumentException(message=f'Index: {index} is out of range.') from None + + @BlockingWrapper.datastructure_op(create_type=list) + def get_at(self, index # type: int + ) -> Any: + """Retrieves the item at a specific index in the list. + + Args: + index (int): The index to retrieve. + + Returns: + Any: The value of the element at the specified index. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the index is out of range. + + """ + try: + op = subdoc_get(f'[{index}]') + sdres = self._collection.lookup_in(self._key, (op,)) + return sdres.value[0].get("value", None) + except PathNotFoundException: + raise InvalidArgumentException(message=f'Index: {index} is out of range.') from None + + def remove_at(self, index # type: int + ) -> None: + """Removes an item at a specific index from the list. + + Args: + index (int): The index to remove. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the index is out of range. + + """ + try: + op = remove(f'[{index}]') + self._collection.mutate_in(self._key, (op,)) + except PathNotFoundException: + raise InvalidArgumentException(message=f'Index: {index} is out of range.') from None + + @BlockingWrapper.datastructure_op(create_type=list) + def size(self) -> int: + """Returns the number of items in the list. + + Returns: + int: The number of items in the list. + + """ + op = count('') + sdres = self._collection.lookup_in(self._key, (op,)) + return sdres.value[0].get("value", None) + + @BlockingWrapper.datastructure_op(create_type=list) + def index_of(self, value # type: JSONType + ) -> int: + """Returns the index of a specific value from the list. + + Args: + value (JSONType): The value to search for. + + Returns: + int: The index of the value in the list. Returns -1 if value is not found. + + """ + list_ = self._get() + for idx, val in enumerate(list_.content_as[list]): + if val == value: + return idx + + return -1 + + def get_all(self) -> List[Any]: + """Returns the entire list of items in this list. + + Returns: + int: The entire list. + + """ + + list_ = self._get() + return list_.content_as[list] + + def clear(self) -> None: + """Clears the list. + + Raises: + :class:`~couchbase.exceptions.DocumentNotFoundException`: If the list does not already exist. + """ + try: + self._collection.remove(self._key) + except DocumentNotFoundException: + pass + + def __iter__(self): + list_ = self._get() + self._full_list = (v for v in list_.content_as[list]) + return self + + def __next__(self): + return next(self._full_list) + + +class CouchbaseMap: + """ + CouchbaseMap provides a simplified interface for storing a map within a Couchbase document. + + Args: + key (str): Document key to use for the map. + collection (:class:`~.collection.Collection`): The :class:`~.collection.Collection` where the + map belongs + + """ + + def __init__(self, key, # type: str + collection # type: Collection + ) -> None: + self._key = key + self._collection = collection + self._full_map = None + + @BlockingWrapper.datastructure_op(create_type=dict) + def _get(self) -> Dict[str, Any]: + """ + Get the entire map. + """ + return self._collection.get(self._key) + + @BlockingWrapper.datastructure_op(create_type=dict) + def add(self, + mapkey, # type: str + value # type: Any + ) -> None: + """Sets a specific key to the specified value in the map. + + Args: + mapkey (str): The key to set. + value (JSONType): The value to set. + + """ + op = upsert(mapkey, value) + self._collection.mutate_in(self._key, (op,)) + + @BlockingWrapper.datastructure_op(create_type=dict) + def get(self, + mapkey, # type: str + ) -> Any: + """Fetches a specific key from the map. + + Args: + mapkey (str): The key to fetch. + + Returns: + Any: The value of the specified key. + + """ + op = subdoc_get(mapkey) + sd_res = self._collection.lookup_in(self._key, (op,)) + return sd_res.value[0].get("value", None) + + def remove(self, + mapkey # type: str + ) -> None: + """Removes a specific key from the map. + + Args: + mapkey (str): The key in the map to remove. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the key is not in the map. + + """ + try: + op = remove(mapkey) + self._collection.mutate_in(self._key, (op,)) + except PathNotFoundException: + raise InvalidArgumentException(message=f'Key: {mapkey} is not in the map.') from None + + @BlockingWrapper.datastructure_op(create_type=dict) + def size(self) -> int: + """Returns the number of items in the map. + + Returns: + int: The number of items in the map. + + """ + op = count('') + sd_res = self._collection.lookup_in(self._key, (op,)) + return sd_res.value[0].get("value", None) + + @BlockingWrapper.datastructure_op(create_type=dict) + def exists(self, + key # type: str + ) -> bool: + """Checks whether a specific key exists in the map. + + Args: + key (str): The key to set. + + Returns: + bool: True if the key exists in the map, False otherwise. + + """ + op = subdoc_exists(key) + sd_res = self._collection.lookup_in(self._key, (op,)) + return sd_res.exists(0) + + def keys(self) -> List[str]: + """Returns a list of all the keys which exist in the map. + + Returns: + List[str]: A list of all the keys that exist in the map. + """ + + map_ = self._get() + return list(map_.content_as[dict].keys()) + + def values(self) -> List[Any]: + """Returns a list of all the values which exist in the map. + + Returns: + List[Any]: A list of all the values that exist in the map. + """ + + map_ = self._get() + return list(map_.content_as[dict].values()) + + def get_all(self) -> Dict[str, Any]: + """Retrieves the entire map. + + Returns: + Dict[str, Any]: The entire CouchbaseMap. + """ + map_ = self._get() + return map_.content_as[dict] + + def clear(self) -> None: + """Clears the map. + """ + try: + self._collection.remove(self._key) + except DocumentNotFoundException: + pass + + def items(self) -> Generator: + """Provides mechanism to loop over the entire map. + + Returns: + Generator: A generator expression for the map + """ + + map_ = self._get() + return ((k, v) for k, v in map_.content_as[dict].items()) + + +class CouchbaseSet: + """ + CouchbaseSet provides a simplified interface for storing a set within a Couchbase document. + + Args: + key (str): Document key to use for the set. + collection (:class:`~.collection.Collection`): The :class:`~.collection.Collection` where the + set belongs. + + """ + + def __init__(self, + key, # type: str + collection # type: Collection + ) -> None: + self._key = key + self._collection = collection + + @BlockingWrapper.datastructure_op(create_type=list) + def _get(self) -> List: + """ + Get the entire set. + """ + return self._collection.get(self._key) + + @BlockingWrapper.datastructure_op(create_type=list) + def add(self, + value # type: Any + ) -> bool: + """Adds a new item to the set. Returning whether the item already existed in the set or not. + + Args: + value (Any): + + Returns: + bool: True if the value was added, False otherwise (meaning the value already + exists in the set). + + """ + try: + op = array_addunique('', value) + self._collection.mutate_in(self._key, (op,)) + return True + except PathExistsException: + return False + + def remove(self, # noqa: C901 + value, # type: Any + timeout=None # type: Optional[timedelta] + ) -> None: + """Removes a specific value from the set. + + Args: + value (Any): The value to remove + timeout (timedelta, optional): Amount of time allowed when attempting + to remove the value. Defaults to 10 seconds. + + """ + + if timeout is None: + timeout = timedelta(seconds=10) + + timeout_millis = timeout.total_seconds() * 1000 + + interval_millis = float(50) + start = time.perf_counter() + time_left = timeout_millis + while True: + sd_res = self._get() + list_ = sd_res.content_as[list] + val_idx = -1 + for idx, v in enumerate(list_): + if v == value: + val_idx = idx + break + + if val_idx >= 0: + try: + op = remove(f'[{val_idx}]') + self._collection.mutate_in(self._key, (op,), MutateInOptions(cas=sd_res.cas)) + break + except CasMismatchException: + pass + else: + break + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((time.perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise UnAmbiguousTimeoutException(message=f"Unable to remove {value} from the CouchbaseSet.") + + time.sleep(interval_millis / 1000) + + @BlockingWrapper.datastructure_op(create_type=list) + def contains(self, + value # type: Any + ) -> bool: + """Returns whether a specific value already exists in the set. + + Args: + value (Any): The value to check for existence. + + Returns: + bool: True if the specified value exists in the set. False otherwise. + + """ + list_ = self._get().content_as[list] + return value in list_ + + @BlockingWrapper.datastructure_op(create_type=list) + def size(self) -> int: + """Returns the number of items in the set. + + Returns: + int: The number of items in the set. + + """ + op = count('') + sd_res = self._collection.lookup_in(self._key, (op,)) + return sd_res.value[0].get("value", None) + + def clear(self) -> None: + """Clears the set. + """ + try: + self._collection.remove(self._key) + except DocumentNotFoundException: + pass + + @BlockingWrapper.datastructure_op(create_type=list) + def values(self) -> List[Any]: + """Returns a list of all the values which exist in the set. + + Returns: + List[Any]: The values that exist in the set. + """ + + list_ = self._get() + return list_.content_as[list] + + +class CouchbaseQueue: + """ + CouchbaseQueue provides a simplified interface for storing a queue within a Couchbase document. + + Args: + key (str): Document key to use for the queue. + collection (:class:`~.collection.Collection`): The :class:`~.collection.Collection` where the + queue belongs. + + """ + + def __init__(self, + key, # type: str + collection # type: Collection + ) -> None: + self._key = key + self._collection = collection + self._full_queue = None + + @BlockingWrapper.datastructure_op(create_type=list) + def _get(self) -> List: + """ + Get the entire queuee. + """ + return self._collection.get(self._key) + + @BlockingWrapper.datastructure_op(create_type=list) + def push(self, + value # type: JSONType + ) -> None: + """Adds a new item to the back of the queue. + + Args: + value (JSONType): The value to push onto the queue. + + """ + op = array_prepend('', value) + self._collection.mutate_in(self._key, (op,)) + + def pop(self, + timeout=None # type: Optional[timedelta] + ) -> Any: + """Removes an item from the front of the queue. + + Args: + timeout (timedelta, optional): Amount of time allowed when attempting + to remove the value. Defaults to 10 seconds. + + + Returns: + Any: The value that was removed from the front of the queue. + """ + + if timeout is None: + timeout = timedelta(seconds=10) + + timeout_millis = timeout.total_seconds() * 1000 + + interval_millis = float(50) + start = time.perf_counter() + time_left = timeout_millis + while True: + try: + op = subdoc_get('[-1]') + sd_res = self._collection.lookup_in(self._key, (op,)) + val = sd_res.value[0].get("value", None) + + try: + op = remove('[-1]') + self._collection.mutate_in(self._key, (op,), MutateInOptions(cas=sd_res.cas)) + return val + except CasMismatchException: + pass + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((time.perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise UnAmbiguousTimeoutException(message="Unable to pop from the CouchbaseQueue.") + + time.sleep(interval_millis / 1000) + except PathNotFoundException: + raise QueueEmpty('No items to remove from the queue') + + @BlockingWrapper.datastructure_op(create_type=list) + def size(self) -> int: + """Returns the number of items in the queue. + + Returns: + int: The number of items in the queue. + + """ + op = count('') + sd_res = self._collection.lookup_in(self._key, (op,)) + return sd_res.value[0].get("value", None) + + def clear(self) -> None: + """Clears the queue. + """ + try: + self._collection.remove(self._key) + except DocumentNotFoundException: + pass + + def __iter__(self): + list_ = self._get() + self._full_queue = (v for v in list_.content_as[list]) + return self + + def __next__(self): + return next(self._full_queue) diff --git a/couchbase/diagnostics.py b/couchbase/diagnostics.py new file mode 100644 index 000000000..7a9dac873 --- /dev/null +++ b/couchbase/diagnostics.py @@ -0,0 +1,161 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from enum import Enum +from typing import (Any, + Dict, + Optional) + + +class EndpointState(Enum): + Disconnected = "disconnected" + Connecting = "connecting" + Connected = "connected" + Disconnecting = "disconnecting" + + +class ClusterState(Enum): + Online = "online" + Degraded = "degraded" + Offline = "offline" + + +class ServiceType(Enum): + View = "views" + KeyValue = "kv" + Query = "query" + Search = "search" + Analytics = "analytics" + Management = "mgmt" + + +class PingState(Enum): + OK = 'ok' + TIMEOUT = 'timeout' + ERROR = 'error' + + +class EndpointDiagnosticsReport: + def __init__(self, + service_type, # type: ServiceType + source # type: Dict[str, Any] + ): + self._src = source + self._service_type = service_type + + @property + def type(self) -> ServiceType: + """**DEPRECATED** user service_type + + Endpoint point service type + + Returns: + ServiceType: Endpoint Service Type + """ + return self._service_type + + @property + def service_type(self) -> ServiceType: + return self._service_type + + @property + def id(self) -> str: + return self._src.get('id', None) + + @property + def local(self) -> str: + return self._src.get('local', None) + + @property + def remote(self) -> str: + return self._src.get('remote', None) + + @property + def namespace(self) -> str: + # was 'scope', now 'namespace' + return self._src.get('namespace', None) + + @property + def last_activity(self) -> timedelta: + """**DEPRECATED** user last_activity_us + + Endpoint point last activity in us + + Returns: + timedelta: last activity in us + """ + return timedelta(microseconds=self._src.get('last_activity_us', None)) + + @property + def last_activity_us(self) -> timedelta: + return timedelta(microseconds=self._src.get('last_activity_us', None)) + + @property + def state(self) -> EndpointState: + return EndpointState(self._src.get('state', None)) + + def as_dict(self) -> dict: + return self._src + + +class EndpointPingReport: + + def __init__(self, + service_type, # type: ServiceType + source # type: Dict[str, Any] + ): + self._src_ping = source + self._service_type = service_type + + @property + def service_type(self) -> ServiceType: + return self._service_type + + @property + def id(self) -> str: + return self._src_ping.get('id', None) + + @property + def local(self) -> str: + return self._src_ping.get('local', None) + + @property + def remote(self) -> str: + return self._src_ping.get('remote', None) + + @property + def namespace(self) -> Optional[str]: + # was 'scope', now 'namespace' + return self._src_ping.get( + 'namespace', self._src_ping.get('scope', None)) + + @property + def error(self) -> Optional[str]: + return self._src_ping.get('error', None) + + @property + def latency(self) -> timedelta: + return timedelta(microseconds=self._src_ping.get('latency_us', None)) + + @property + def state(self) -> PingState: + return PingState(self._src_ping.get('state', None)) + + def as_dict(self) -> Dict[str, Any]: + return self._src_ping + + def __repr__(self): + return "EndpointPingReport:{}".format(self._src_ping) diff --git a/couchbase/durability.py b/couchbase/durability.py new file mode 100644 index 000000000..0889ccd71 --- /dev/null +++ b/couchbase/durability.py @@ -0,0 +1,181 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from enum import IntEnum +from typing import (Dict, + Optional, + TypeVar, + Union) + + +class ReplicateTo(IntEnum): + """ + Specify the number of nodes to wait for replication. + """ + NONE = 0 # Do not apply replication requirements. + ONE = 1 # Wait for replication to atleast one replica nodes + TWO = 2 # Wait for replication to atleast two replica nodes + THREE = 3 # Wait for replication to all three replica nodes. + + +class PersistTo(IntEnum): + """ + Specify the number of nodes to wait for persistance. + + Use `~couchbase.durability.PersistToExtended` for extended functionality. The + `~couchbase.durability.PersistToExtended` is the preferred enum for to use for the 4.x SDK. + """ + NONE = 0 # Do not apply persistance requirements. + ONE = 2 # cxx: Wait for persistence to at least one node. + TWO = 3 # cxx: Wait for persistence to at least two nodes. + THREE = 4 # cxx: Wait for persistence to at least three nodes. + + +class PersistToExtended(IntEnum): + """ + Extends the `~couchbase.durability.PersistTo` enum to match the cxx client options. This is the + preferred Enum, wihle the `~couchbase.durability.PersistTo` enum is around for 3.x compatibility. + """ + NONE = 0 # Do not apply persistance requirements. + ACTIVE = 1 # Wait for persistence to active node. + ONE = 2 # Wait for persistence to at least one node. + TWO = 3 # Wait for persistence to at least two nodes. + THREE = 4 # Wait for persistence to at least three nodes. + # This is maximum possible persistence requirement, that includes active and all three replica nodes. + FOUR = 5 # Wait for persistence to at least all nodes. + + +class Durability(IntEnum): + """Synchronous Durability Level + + **DEPRECATED** Use `DurabilityLevel` + """ + NONE = 0 + MAJORITY = 1 + MAJORITY_AND_PERSIST_TO_ACTIVE = 2 + PERSIST_TO_MAJORITY = 3 + + +class DurabilityLevel(IntEnum): + NONE = 0 + MAJORITY = 1 + MAJORITY_AND_PERSIST_TO_ACTIVE = 2 + PERSIST_TO_MAJORITY = 3 + + # def to_server_str(self): + # if self.name == 'MAJORITY_AND_PERSIST_TO_ACTIVE': + # return 'majorityAndPersistActive' + # elif self.name == 'NONE': + # return 'none' + # elif self.name == 'MAJORITY': + # return 'majority' + # elif self.name == 'PERSIST_TO_MAJORITY': + # return 'persistToMajority' + # else: + # return 'none' + + @classmethod + def to_server_str(cls, value): + if value == cls.MAJORITY_AND_PERSIST_TO_ACTIVE: + return 'majorityAndPersistActive' + elif value == cls.NONE: + return 'none' + elif value == cls.MAJORITY: + return 'majority' + elif value == cls.PERSIST_TO_MAJORITY: + return 'persistToMajority' + else: + return 'none' + + @classmethod + def from_server_str(cls, value): + if value == 'majorityAndPersistActive': + return cls.MAJORITY_AND_PERSIST_TO_ACTIVE + elif value == 'none': + return cls.NONE + elif value == 'majority': + return cls.MAJORITY + elif value == 'persistToMajority': + return cls.PERSIST_TO_MAJORITY + else: + return cls.NONE + + +class ClientDurability: + + def __init__(self, + replicate_to=ReplicateTo.NONE, # type: ReplicateTo + persist_to=PersistTo.NONE # type: Union[PersistTo, PersistToExtended] + ): + # type: (...) -> None + """ + Client Durability + + :param persist_to: If set, wait for the item to be removed + from the storage of at least these many nodes + + :param replicate_to: If set, wait for the item to be removed + from the cache of at least these many nodes + (excluding the master) + """ + self._replicate_to = replicate_to + self._persist_to = persist_to + + @property + def replicate_to(self) -> ReplicateTo: + return self._replicate_to + + @property + def persist_to(self) -> Union[PersistTo, PersistToExtended]: + return self._persist_to + + +class ServerDurability: + + def __init__(self, # type: ServerDurability + level, # type: DurabilityLevel + ): + # type: (...) -> None + """ + Server-based Durability (Synchronous Replication) + + :param Durability level: durability level + """ + self._level = level + + @property + def level(self) -> DurabilityLevel: + return self._level + + +DurabilityType = TypeVar('DurabilityType', bound=Union[ClientDurability, ServerDurability]) + + +class DurabilityParser: + @staticmethod + def parse_durability(durability # type: DurabilityType + ) -> Optional[Union[int, Dict[str, int]]]: + if isinstance(durability, ClientDurability): + return { + 'replicate_to': durability.replicate_to.value, + 'persist_to': durability.persist_to.value + } + + if isinstance(durability, ServerDurability): + return durability.level.value + + return None diff --git a/couchbase/encryption/__init__.py b/couchbase/encryption/__init__.py new file mode 100644 index 000000000..ee7528b41 --- /dev/null +++ b/couchbase/encryption/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .crypto_manager import CryptoManager # noqa: F401 +from .decrypter import Decrypter # noqa: F401 +from .encrypter import Encrypter # noqa: F401 +from .encryption_result import EncryptionResult # noqa: F401 +from .key import Key # noqa: F401 +from .keyring import Keyring # noqa: F401 diff --git a/couchbase/encryption/crypto_manager.py b/couchbase/encryption/crypto_manager.py new file mode 100644 index 000000000..7daa1fd49 --- /dev/null +++ b/couchbase/encryption/crypto_manager.py @@ -0,0 +1,105 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Optional, Union + + +class CryptoManager(ABC): + """Interface a CryptoManager must implement + + """ + + _DEFAULT_ENCRYPTER_ALIAS = "__DEFAULT__" + + @abstractmethod + def encrypt(self, + plaintext, # type: Union[str, bytes, bytearray] + encrypter_alias=None, # type: Optional[str] + ) -> dict: + """Encrypts the given plaintext using the given encrypter alias. + + Args: + plaintext (Union[str, bytes, bytearray]): Input to be encrypted + encrypter_alias (str, optional): Alias of encrypter to use, if None, default alias is used. + + Returns: + Dict: A :class:`~couchbase.encryption.EncryptionResult` as a dict + + Raises: + :class:`~couchbase.exceptions.EncryptionFailureException` + """ + pass + + @abstractmethod + def decrypt(self, + encrypted, # type: dict + ) -> bytes: + """Decrypts the given encrypted result based on the 'alg' key in the encrypted result. + + Args: + encrypted (Dict): A dict containing encryption information, must have an 'alg' key. + + Returns: + bytes: A decrypted result based on the given encrypted input. + + Raises: + :class:`~couchbase.exceptions.DecryptionFailureException` + + """ + pass + + @abstractmethod + def mangle(self, + field_name, # type: str + ) -> str: + """Mangles provided JSON field name. + + Args: + field_name (str): JSON field name to be mangled. + + Returns: + str: The mangled field name. + """ + pass + + @abstractmethod + def demangle(self, + field_name, # type: str + ) -> str: + """Demangles provided JSON field name. + + Args: + field_name (str): JSON field name to be demangled. + + Returns: + str: The demangled field name. + """ + pass + + @abstractmethod + def is_mangled(self, + field_name, # type: str + ) -> bool: + """Checks if provided JSON field name has been mangled. + + Args: + field_name (str): JSON field name to check if mangled. + + Returns: + bool: True if the field is mangled, False otherwise. + + """ + pass diff --git a/couchbase/encryption/decrypter.py b/couchbase/encryption/decrypter.py new file mode 100644 index 000000000..d15b2a811 --- /dev/null +++ b/couchbase/encryption/decrypter.py @@ -0,0 +1,65 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + from couchbase.encryption import EncryptionResult, Keyring + + +class Decrypter(ABC): + """Interface a Decrypter must implement + + """ + + def __init__(self, + keyring, # type: Keyring + alg=None, # type: Optional[str] + ): + self._keyring = keyring + self._alg = alg + + @property + def keyring(self): + return self._keyring + + def algorithm(self) -> str: + """Provides name of algorithm associated with Decrypter + + Returns: + str: The name of the description algorithm + """ + return self._alg + + @abstractmethod + def decrypt(self, + encrypted, # type: EncryptionResult + ) -> bytes: + """Decrypts the given :class:`~couchbase.encryption.EncryptionResult` ciphertext. + + The Decrypter's algorithm should match the `alg` property of the given + :class:`~couchbase.encryption.EncryptionResult` + + Args: + encrypted (:class:`~couchbase.encryption.EncryptionResult`): The encrypted value to decrypt. + + Returns: + bytes: The decrypted ciphertext. + + Raises: + :class:`~couchbase.exceptions.InvalidCryptoKeyException` + :class:`~couchbase.exceptions.InvalidCipherTextException` + """ diff --git a/couchbase/encryption/encrypter.py b/couchbase/encryption/encrypter.py new file mode 100644 index 000000000..a42c41ef5 --- /dev/null +++ b/couchbase/encryption/encrypter.py @@ -0,0 +1,55 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import Union + +from couchbase.encryption import EncryptionResult, Keyring + + +class Encrypter(ABC): + def __init__(self, + keyring, # type: Keyring + key, # type: str + ): + self._keyring = keyring + self._key = key + + @property + def keyring(self) -> Keyring: + return self._keyring + + @property + def key(self) -> str: + return self._key + + @abstractmethod + def encrypt(self, + plaintext, # type: Union[str, bytes, bytearray] + ) -> EncryptionResult: + """Encrypts the given plaintext + + Args: + plaintext (Union[str, bytes, bytearray]): The plaintext to be encrypted + + Returns: + :class:``~couchbase.encryption.EncryptionResult`: a :class:`~couchbase.encryption.EncryptionResult` + containing all the necessary information required for decryption. + + Raises: + :class:`~couchbase.exceptions.InvalidCryptoKeyException`: If the :class:`.Encrypter` has an invalid + key for encryption. + """ + pass diff --git a/couchbase/encryption/encryption_result.py b/couchbase/encryption/encryption_result.py new file mode 100644 index 000000000..ce1175aee --- /dev/null +++ b/couchbase/encryption/encryption_result.py @@ -0,0 +1,119 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import base64 +from typing import (Any, + Optional, + Union) + +from couchbase.exceptions import CryptoKeyNotFoundException, InvalidArgumentException + + +class EncryptionResult: + def __init__(self, + alg, # type: str + kid=None, # type: Optional[str] + ciphertext=None, # type: Optional[str] + **kwargs, # type: Optional[Any] + ): + self._map = {"alg": alg} + + if kid: + self._map["kid"] = kid + + if ciphertext and self._valid_base64(ciphertext): + self._map["ciphertext"] = ciphertext + + if kwargs: + self._map.update(**kwargs) + + @classmethod + def new_encryption_result_from_dict(cls, + values, # type: dict + ) -> EncryptionResult: + + alg = values.pop("alg", None) + if not alg: + raise InvalidArgumentException( + "EncryptionResult must include alg property." + ) + + return EncryptionResult(alg, **values) + + def put(self, + key, # type: str + val # type: Any + ): + self._map[key] = val + + def put_and_base64_encode(self, + key, # type: str + val, # type: bytes + ): + if not isinstance(val, bytes): + raise ValueError("Provided value must be of type bytes.") + self._map[key] = base64.b64encode(val) + + def get(self, + key, # type: str + ) -> Any: + val = self._map.get(key, None) + if not val: + raise CryptoKeyNotFoundException( + message="No mapping to EncryptionResult value found for key: '{}'.".format( + key) + ) + + return val + + def algorithm(self) -> str: + return self._map["alg"] + + def get_with_base64_decode(self, + key, # type: str + ) -> bytes: + val = self._map.get(key, None) + if not val: + raise CryptoKeyNotFoundException( + message="No mapping to EncryptionResult value found for key: '{}'.".format( + key) + ) + + return base64.b64decode(val) + + def asdict(self) -> dict: + return self._map + + def _valid_base64(self, + val, # type: Union[str, bytes, bytearray] + ) -> bool: + try: + if isinstance(val, str): + bytes_val = bytes(val, "ascii") + elif isinstance(val, bytes): + bytes_val = val + elif isinstance(val, bytearray): + bytes_val = val + else: + raise ValueError( + "Provided value must be of type str, bytes or bytearray" + ) + + return base64.b64encode(base64.b64decode(bytes_val)) == bytes_val + + except Exception as ex: # noqa: F841 + return False diff --git a/couchbase/encryption/key.py b/couchbase/encryption/key.py new file mode 100644 index 000000000..8b73c5fc0 --- /dev/null +++ b/couchbase/encryption/key.py @@ -0,0 +1,33 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Union + + +class Key: + def __init__(self, + id, # type: str + bytes_, # type: Union[bytes, bytearray] + ): + self._id = id + self._bytes = bytes_ if isinstance(bytes_, bytes) else bytes(bytes_) + + @property + def id(self) -> str: + return self._id + + @property + def bytes(self) -> bytes: + return self._bytes diff --git a/couchbase/encryption/keyring.py b/couchbase/encryption/keyring.py new file mode 100644 index 000000000..74aca9c3f --- /dev/null +++ b/couchbase/encryption/keyring.py @@ -0,0 +1,39 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from couchbase.encryption import Key + + +class Keyring(ABC): + @abstractmethod + def get_key(self, + key_id, # type: str + ) -> Key: + """Returns requested key + + Args: + keyid (str): Key ID to retrieve + + Returns: + :class:`~couchbase.encryption.Key`: The corresponding :class:`~couchbase.encryption.Key` + of the provided key_id. + + Raises: + :raises :class:`~couchbase.exceptions.CryptoKeyNotFoundException` + """ diff --git a/couchbase/exceptions.py b/couchbase/exceptions.py index 04813841f..4f0bf5ec7 100644 --- a/couchbase/exceptions.py +++ b/couchbase/exceptions.py @@ -1,545 +1,2076 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. # -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re +import sys +from collections import defaultdict +from enum import Enum +from string import Template +from typing import (Any, + Dict, + Optional, + Set, + Union) + +from couchbase.pycbc_core import exception + + +class CouchbaseErrorCategory(Enum): + common = "couchbase.common" + key_value = "couchbase.key_value" + query = "couchbase.query" + search = "couchbase.search" + view = "couchbase.view" + analytics = "couchbase.analytics" + management = "couchbase.management" + network = "couchbase.network" + other = "other" + + +class ErrorContext: + def __init__(self, **kwargs): + self._base = kwargs -import couchbase._libcouchbase as C + @property + def last_dispatched_to(self) -> Optional[str]: + return self._base.get("last_dispatched_to", None) -class CouchbaseError(Exception): - """Base exception for Couchbase errors + @property + def last_dispatched_from(self) -> Optional[str]: + return self._base.get("last_dispatched_from", None) - This is the base class for all exceptions thrown by Couchbase + @property + def retry_attempts(self) -> int: + return self._base.get("retry_attempts", None) - **Exception Attributes** + @property + def retry_reasons(self) -> Set[str]: + return self._base.get("retry_reasons", None) - .. py:attribute:: rc + @staticmethod + def from_dict(**kwargs): + # type: (...) -> ErrorContext + klass = kwargs.get("context_type", "ErrorContext") + cl = getattr(sys.modules[__name__], klass) + return cl(**kwargs) - The return code which caused the error + def _get_base(self): + return self._base - A :class:`~couchbase.result.MultiResult` object, if this - exception was thrown as part of a multi-operation. This contains - all the operations (including ones which may not have failed) + def __repr__(self): + return f'ErrorContext({self._base})' - .. py:attribute:: inner_cause - If this exception was triggered by another exception, it is - present here. +class HTTPErrorContext(ErrorContext): + _HTTP_EC_KEYS = ["client_context_id", "method", "path", "http_status", + "http_body"] - .. py:attribute:: key + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._http_err_ctx = {k: v for k, + v in kwargs.items() if k in self._HTTP_EC_KEYS} - If applicable, this is the key which failed. + @property + def method(self) -> Optional[str]: + return self._http_err_ctx.get("method", None) - .. py:attribute:: csrc_info + @property + def response_code(self) -> Optional[int]: + return self._http_err_ctx.get("http_status", None) - A tuple of (`file`, `line`) pointing to a location in the C - source code where the exception was thrown (if applicable) + @property + def path(self) -> Optional[str]: + return self._http_err_ctx.get("path", None) - .. py:attribute:: categories + @property + def response_body(self) -> Optional[str]: + return self._http_err_ctx.get("http_body", None) - An integer representing a set of bits representing various error - categories for the specific error as returned by libcouchbase. + @property + def client_context_id(self) -> Optional[str]: + return self._http_err_ctx.get("client_context_id", None) - .. py:attribute:: is_data + def __repr__(self): + return f'HTTPErrorContext({self._http_err_ctx})' - True if this error is a negative reply from the server - (see :exc:`CouchbaseDataError`) - .. py:attribute:: is_transient +class KeyValueErrorContext(ErrorContext): + _KV_EC_KEYS = ['key', 'bucket_name', 'scope_name', 'collection_name', + 'opaque', 'status_code', 'error_map_info', 'extended_error_info', + 'retry_attempts', 'retry_reasons'] - True if this error was likely caused by a transient condition - (see :exc:`CouchbaseTransientError`) + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._kv_err_ctx = {k: v for k, + v in kwargs.items() if k in self._KV_EC_KEYS} - .. py:attribute:: is_fatal + @property + def key(self) -> Optional[str]: + return self._kv_err_ctx.get("key", None) - True if this error indicates a likely fatal condition for the client. - See :exc:`CouchbaseFatalError` + @property + def bucket_name(self) -> Optional[str]: + return self._kv_err_ctx.get("bucket_name", None) - .. py:attribute:: is_network + @property + def scope_name(self) -> Optional[str]: + return self._kv_err_ctx.get("scope_name", None) - True if errors were received during TCP transport. - See :exc:`CouchbaseNetworkError` + @property + def collection_name(self) -> Optional[str]: + return self._kv_err_ctx.get("collection_name", None) + def __repr__(self): + return "KeyValueErrorContext:{}".format(self._kv_err_ctx) - """ - @classmethod - def rc_to_exctype(cls, rc): - """ - Map an error code to an exception +class ManagementErrorContext(ErrorContext): + _MGMT_EC_KEYS = ["client_context_id", "content", "path", "http_status"] - :param int rc: The error code received for an operation + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._mgmt_err_ctx = {k: v for k, + v in kwargs.items() if k in self._MGMT_EC_KEYS} - :return: a subclass of :class:`CouchbaseError` - """ - try: - return _LCB_ERRNO_MAP[rc] - except KeyError: - newcls = _mk_lcberr(rc) - _LCB_ERRNO_MAP[rc] = newcls - return newcls + @property + def response_code(self) -> Optional[int]: + return self._mgmt_err_ctx.get("http_status", None) - @classmethod - def _can_derive(cls, rc): - """ - Determines if the given error code is logically derived from this class - :param int rc: the error code to check - :return: a boolean indicating if the code is derived from this exception - """ - return issubclass(cls.rc_to_exctype(rc), cls) - - def __init__(self, params=None): - if isinstance(params, str): - params = {'message': params} - elif isinstance(params, CouchbaseError): - self.__dict__.update(params.__dict__) - return - - self.rc = params.get('rc', 0) - self.all_results = params.get('all_results', {}) - self.result = params.get('result', None) - self.inner_cause = params.get('inner_cause', None) - self.csrc_info = params.get('csrc_info', ()) - self.key = params.get('key', None) - self.objextra = params.get('objextra', None) - self.message = params.get('message', None) + @property + def path(self) -> Optional[str]: + return self._mgmt_err_ctx.get("path", None) - @classmethod - def pyexc(cls, message=None, obj=None, inner=None): - return cls({'message': message, - 'objextra': obj, - 'inner_cause': inner}) + @property + def content(self) -> Optional[str]: + return self._mgmt_err_ctx.get("content", None) @property - def categories(self): - """ - Gets the exception categories (as a set of bits) - """ - return C._get_errtype(self.rc) + def client_context_id(self) -> Optional[str]: + return self._mgmt_err_ctx.get("client_context_id", None) + + def __repr__(self): + return f'ManagementErrorContext({self._mgmt_err_ctx})' + + +class TransactionsErrorContext(ErrorContext): + _TXN_EC_KEYS = ["failure_type"] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.txn_err_ctx = {k: v for k, + v in kwargs.items() if k in self._TXN_EC_KEYS} @property - def is_transient(self): - return self.categories & C.LCB_ERRTYPE_TRANSIENT + def failure_type(self) -> Optional[str]: + return self.txn_err_ctx.get("failure_type", None) @property - def is_fatal(self): - return self.categories & C.LCB_ERRTYPE_FATAL + def last_dispatched_to(self) -> Optional[str]: + return None @property - def is_network(self): - return self.categories & C.LCB_ERRTYPE_NETWORK + def last_dispatched_from(self) -> Optional[str]: + return None @property - def is_data(self): - return self.categories & C.LCB_ERRTYPE_DATAOP + def retry_attempts(self) -> int: + return None - def split_results(self): - """ - Convenience method to separate failed and successful results. + @property + def retry_reasons(self) -> Set[str]: + return None + + def __str__(self): + return f'TransactionsErrorContext{{{self.txn_err_ctx}}}' - .. versionadded:: 2.0.0 - This function will split the results of the failed operation - (see :attr:`.all_results`) into "good" and "bad" dictionaries. +class AnalyticsErrorContext(HTTPErrorContext): + _ANALYTICS_EC_KEYS = ["first_error_code", "first_error_message", "statement", "parameters"] - The intent is for the application to handle any successful - results in a success code path, and handle any failed results - in a "retry" code path. For example + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._analytics_err_ctx = {k: v for k, + v in kwargs.items() if k in self._ANALYTICS_EC_KEYS} - .. code-block:: python + @property + def first_error_code(self) -> Optional[int]: + return self._analytics_err_ctx.get("first_error_code", None) - try: - cb.add_multi(docs) - except CouchbaseTransientError as e: - # Temporary failure or server OOM - _, fail = e.split_results() + @property + def first_error_message(self) -> Optional[str]: + return self._analytics_err_ctx.get("first_error_message", None) + + @property + def statement(self) -> Optional[str]: + return self._analytics_err_ctx.get("statement", None) + + @property + def parameters(self) -> Optional[str]: + return self._analytics_err_ctx.get("parameters", None) + + def __repr__(self): + return f'AnalyticsErrorContext({self._get_base()})' + + +class QueryErrorContext(HTTPErrorContext): + _QUERY_EC_KEYS = ["first_error_code", "first_error_message", "statement", "parameters"] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._query_err_ctx = {k: v for k, + v in kwargs.items() if k in self._QUERY_EC_KEYS} + + @property + def first_error_code(self) -> Optional[int]: + return self._query_err_ctx.get("first_error_code", None) + + @property + def first_error_message(self) -> Optional[str]: + return self._query_err_ctx.get("first_error_message", None) + + @property + def statement(self) -> Optional[str]: + return self._query_err_ctx.get("statement", None) + + @property + def parameters(self) -> Optional[str]: + return self._query_err_ctx.get("parameters", None) + + def __repr__(self): + return f'QueryErrorContext({self._get_base()})' + + +class SearchErrorContext(HTTPErrorContext): + _SEARCH_EC_KEYS = ["index_name", "query", "parameters"] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._search_err_ctx = {k: v for k, + v in kwargs.items() if k in self._SEARCH_EC_KEYS} + + @property + def index_name(self) -> Optional[str]: + return self._search_err_ctx.get("index_name", None) + + @property + def query(self) -> Optional[str]: + return self._search_err_ctx.get("query", None) - # Sleep for a bit to reduce the load on the server - time.sleep(0.5) + @property + def parameters(self) -> Optional[str]: + return self._search_err_ctx.get("parameters", None) + + def __repr__(self): + return f'SearchErrorContext({self._get_base()})' + + +class ViewErrorContext(HTTPErrorContext): + _VIEW_EC_KEYS = ["design_document_name", "view_name", "query_string"] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._view_err_ctx = {k: v for k, + v in kwargs.items() if k in self._VIEW_EC_KEYS} + + @property + def design_document_name(self) -> Optional[str]: + return self._view_err_ctx.get("design_document_name", None) + + @property + def view_name(self) -> Optional[str]: + return self._view_err_ctx.get("view_name", None) + + @property + def query_string(self) -> Optional[str]: + return self._view_err_ctx.get("query_string", None) + + def __repr__(self): + return f'ViewErrorContext({self._get_base()})' + + +class SubdocumentErrorContext(KeyValueErrorContext): + _SUBDOC_EC_KEYS = ["first_error_path", "first_error_index", "deleted"] + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._subdoc_err_ctx = {k: v for k, + v in kwargs.items() if k in self._SUBDOC_EC_KEYS} + + @property + def first_error_path(self) -> Optional[str]: + return self._subdoc_err_ctx.get("first_error_path", None) + + @property + def first_error_index(self) -> Optional[int]: + return self._subdoc_err_ctx.get("first_error_index", None) + + @property + def deleted(self) -> bool: + return self._subdoc_err_ctx.get("deleted", False) + + def __repr__(self): + return f'SubdocumentErrorContext({self._get_base()})' + + +ErrorContextType = Union[AnalyticsErrorContext, + ErrorContext, + HTTPErrorContext, + KeyValueErrorContext, + ManagementErrorContext, + QueryErrorContext, + SearchErrorContext, + SubdocumentErrorContext, + TransactionsErrorContext, + ViewErrorContext] + + +class CouchbaseException(Exception): + def __init__(self, + base=None, # type: Optional[exception] + message=None, # type: Optional[str] + context=None, # type: Optional[ErrorContextType] + error_code=None, # type: Optional[int] + exc_info=None # type: Optional[Dict[str, Any]] + ): + self._base = base + self._context = context + self._message = message + self._error_code = error_code + self._exc_info = exc_info + super().__init__(message) - # Try to add only the failed results again - cb.add_multi(fail) + @property + def error_code(self) -> Optional[int]: + if self._error_code: + return self._error_code + if self._base: + return self._base.err() - Of course, in the example above, the second retry may fail as - well, and a more robust implementation is left as an exercise - to the reader. + @property + def context(self) -> ErrorContextType: + """ + ** DEPRECATED ** use error_context - :return: A tuple of ( `ok`, `bad` ) dictionaries. + Returns: + Union[ErrorContext, HTTPErrorContext, KeyValueErrorContext]: Exception's error context object """ + return self.error_context - ret_ok, ret_fail = {}, {} - for v in self.all_results.values(): - if v.success: - ret_ok[v.key] = v + @property + def error_context(self) -> ErrorContextType: + if not self._context: + if self._base: + base_ec = self._base.error_context() or dict() else: - ret_fail[v.key] = v + base_ec = dict() + self._context = ErrorContext.from_dict(**base_ec) + return self._context - return ret_ok, ret_fail + @property + def message(self) -> Optional[str]: + """ + **VOLATILE** This API is subject to change at any time. - def __str__(self): + Returns: + Optional[str]: Exception's error message, if it exists. + """ + if self._message: + return self._message + if self._base: + return self._base.strerror().replace('_', ' ') + + @property + def inner_cause(self) -> Optional[Exception]: + """ + **VOLATILE** This API is subject to change at any time. + + Returns: + Optional[Exception]: Exception's inner cause, if it exists. + """ + return self._exc_info.get('inner_cause', None) + + @classmethod + def pycbc_create_exception(cls, base=None, message=None): + return cls(base=base, message=message) + + def __repr__(self): + from couchbase._utils import is_null_or_empty details = [] + if self._base: + details.append( + "ec={}, category={}".format( + self._base.err(), + self._base.err_category())) + if not is_null_or_empty(self._message): + details.append("message={}".format(self._message)) + else: + details.append("message={}".format(self._base.strerror())) + else: + if not is_null_or_empty(self._message): + details.append(f'message={self._message}') + if self._context: + details.append(f'context={self._context}') + if self._exc_info and 'cinfo' in self._exc_info: + details.append('C Source={0}:{1}'.format(*self._exc_info['cinfo'])) + if self._exc_info and 'inner_cause' in self._exc_info: + details.append('Inner cause={0}'.format(self._exc_info['inner_cause'])) + return "<{}>".format(", ".join(details)) + + def __str__(self): + return self.__repr__() + + +# common errors +class InternalServerFailureException(CouchbaseException): + """ Raised when the server service provides error w/in below ranges. + Query: Error range 5xxx + Analytics: Error range 25xxx + KV: error code ERR_INTERNAL (0x84) + Search: HTTP 500 + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class TimeoutException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class MissingConnectionException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class AmbiguousTimeoutException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class TemporaryFailException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) - if self.key: - details.append("Key={0}".format(repr(self.key))) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class UnAmbiguousTimeoutException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class RequestCanceledException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class InvalidArgumentException(CouchbaseException): + """ Raised when a provided argmument has an invalid value + and/or invalid type. + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class AuthenticationException(CouchbaseException): + """Indicates that an error occurred authenticating the user to the cluster.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class CasMismatchException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +CASMismatchException = CasMismatchException - if self.rc: - details.append("RC=0x{0:X}[{1}]".format( - self.rc, C._strerror(self.rc))) - if self.message: - details.append(self.message) - if self.all_results: - details.append("Results={0}".format(len(self.all_results))) - if self.inner_cause: - details.append("inner_cause={0}".format(self.inner_cause)) +class BucketNotFoundException(CouchbaseException): + """Indicates that the bucket being referenced does not exist.""" - if self.csrc_info: - details.append("C Source=({0},{1})".format(*self.csrc_info)) + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) - if self.objextra: - details.append("OBJ={0}".format(repr(self.objextra))) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" - s = "<{0}>".format(", ".join(details)) - return s + def __str__(self): + return self.__repr__() + + +class ValueFormatException(CouchbaseException): + """Failed to decode or encode value""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class HTTPException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class ServiceUnavailableException(CouchbaseException): + """ Raised if tt can be determined from the config unambiguously that a + given service is not available. + I.e. no query node in the config, or a memcached bucket is accessed + and views or n1ql queries should be performed + """ + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) -class InternalSDKError(CouchbaseError): + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class FeatureUnavailableException(CouchbaseException): + """Raised when feature that is not available with the current server version is used.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class InternalSDKException(CouchbaseException): """ This means the SDK has done something wrong. Get support. (this doesn't mean *you* didn't do anything wrong, it does mean you should not be seeing this message) """ -class CouchbaseInternalError(InternalSDKError): - pass + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) -class CouchbaseNetworkError(CouchbaseError): - """ - Base class for network-related errors. These indicate issues in the low - level connectivity - """ + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" -class CouchbaseInputError(CouchbaseError): - """ - Base class for errors possibly caused by malformed input - """ + def __str__(self): + return self.__repr__() -class CouchbaseTransientError(CouchbaseError): - """ - Base class for errors which are likely to go away with time - """ +# kv errors -class CouchbaseFatalError(CouchbaseError): - """ - Base class for errors which are likely fatal and require reinitialization - of the instance - """ -class CouchbaseDataError(CouchbaseError): - """ - Base class for negative replies received from the server. These errors - indicate that the server could not satisfy the request because of certain - data constraints (such as an item not being present, or a CAS mismatch) - """ +class DeltaInvalidException(CouchbaseException): + """Indicates the delta value specified for an operation is too large.""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) -class ArgumentError(CouchbaseError): - """Invalid argument + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" - A given argument is invalid or must be set - """ + def __str__(self): + return self.__repr__() -class ValueFormatError(CouchbaseError): - """Failed to decode or encode value""" +class DocumentExistsException(CouchbaseException): + """Indicates that the referenced document exists already, but the operation was not expecting it to exist.""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) -# The following exceptions are derived from libcouchbase -class AuthError(CouchbaseError): - """Authentication failed + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" - You provided an invalid username/password combination. - """ + def __str__(self): + return self.__repr__() -class DeltaBadvalError(CouchbaseError): - """The given value is not a number +class DocumentLockedException(CouchbaseException): + """Indicates that the referenced document could not be used as it is currently locked, + likely by another actor in the system.""" - The server detected that operation cannot be executed with - requested arguments. For example, when incrementing not a number. - """ + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" -class TooBigError(CouchbaseError): - """Object too big + def __str__(self): + return self.__repr__() - The server reported that this object is too big - """ +class DocumentNotLockedException(CouchbaseException): + """Indicates that the referenced document is not locked, generally raised when an + unlock operation is being performed.""" -class BusyError(CouchbaseError): - """The cluster is too busy + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) - The server is too busy to handle your request right now. - please back off and try again at a later time. - """ + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + def __str__(self): + return self.__repr__() -class InternalError(CouchbaseError): - """Internal Error - Internal error inside the library. You would have - to destroy the instance and create a new one to recover. - """ +class DocumentNotFoundException(CouchbaseException): + """Indicates that the referenced document does not exist.""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) -class InvalidError(CouchbaseError): - """Invalid arguments specified""" + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + def __str__(self): + return self.__repr__() -class NoMemoryError(CouchbaseError): - """The server ran out of memory""" +class DocumentNotJsonException(CouchbaseException): + """Indicates a sub-document operation was attempted on a non-JSON document.""" -class RangeError(CouchbaseError): - """An invalid range specified""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" -class LibcouchbaseError(CouchbaseError): - """A generic error""" + def __str__(self): + return self.__repr__() -class TemporaryFailError(CouchbaseError): - """Temporary failure (on server) +class DocumentUnretrievableException(CouchbaseException): + """Indicates that the referenced document does not exist and therefore no replicas are found.""" - The server tried to perform the requested operation, but failed - due to a temporary constraint. Retrying the operation may work. + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) - This error may also be delivered if the key being accessed was - locked. + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" - .. seealso:: + def __str__(self): + return self.__repr__() - :meth:`couchbase.bucket.Bucket.lock` - :meth:`couchbase.bucket.Bucket.unlock` - """ +class DurabilityImpossibleException(CouchbaseException): + """Given durability requirements are impossible to achieve""" -class KeyExistsError(CouchbaseError): - """The key already exists (with another CAS value) + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) - This exception may be thrown during an ``add()`` operation - (if the key already exists), or when a CAS is supplied - and the server-side CAS differs. - """ + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + def __str__(self): + return self.__repr__() -class NotFoundError(CouchbaseError): - """The key does not exist""" +class DurabilityInvalidLevelException(CouchbaseException): + """Given durability level is invalid""" -class DlopenFailedError(CouchbaseError): - """Failed to open shared object""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" -class DlsymFailedError(CouchbaseError): - """Failed to locate the requested symbol in the shared object""" + def __str__(self): + return self.__repr__() -class NetworkError(CouchbaseNetworkError): - """Network error +class DurabilitySyncWriteAmbiguousException(CouchbaseException): + """There is a synchronous mutation pending for given key + The SyncWrite request has not completed in the specified time and has ambiguous + result - it may Succeed or Fail; but the final value is not yet known""" - A network related problem occured (name lookup, - read/write/connect etc) - """ + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" -class NotMyVbucketError(CouchbaseError): - """The vbucket is not located on this server + def __str__(self): + return self.__repr__() - The server who received the request is not responsible for the - object anymore. (This happens during changes in the cluster - topology) - """ +class DurabilitySyncWriteInProgressException(CouchbaseException): + """Returned if an attempt is made to mutate a key which already has a + SyncWrite pending. Client would typically retry (possibly with backoff). + Similar to ELOCKED""" -class NotStoredError(CouchbaseError): - """The object was not stored on the server""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" -class NotSupportedError(CouchbaseError): - """Not supported + def __str__(self): + return self.__repr__() - The server doesn't support the requested command. This error - differs from :exc:`couchbase.exceptions.UnknownCommandError` by - that the server knows about the command, but for some reason - decided to not support it. - """ +class InvalidValueException(CouchbaseException): + """Indicates the provided value was invalid for the operation.""" -class UnknownCommandError(CouchbaseError): - """The server doesn't know what that command is""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" -class UnknownHostError(CouchbaseNetworkError): - """The server failed to resolve the requested hostname""" + def __str__(self): + return self.__repr__() -class ProtocolError(CouchbaseNetworkError): - """Protocol error +class NumberTooBigException(CouchbaseException): + def __init__(self, message=None, **kwargs): + """Indicates existing number is outside the valid range for arithmetic operations.""" + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) - There is something wrong with the datastream received from - the server - """ + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() -class TimeoutError(CouchbaseError): - """The operation timed out""" +class PathExistsException(CouchbaseException): + """Indicates that the reference path already existed, but the operation expected that it did not.""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) -class ConnectError(CouchbaseNetworkError): - """Failed to connect to the requested server""" + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + def __str__(self): + return self.__repr__() -class BucketNotFoundError(CouchbaseError): - """The requested bucket does not exist""" +class PathInvalidException(CouchbaseException): + """Indicates that the reference path was not syntactically correct.""" -class ClientNoMemoryError(CouchbaseError): - """The client ran out of memory""" + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" -class ClientTemporaryFailError(CouchbaseError): - """Temporary failure (on client) + def __str__(self): + return self.__repr__() - The client encountered a temporary error (retry might resolve - the problem) - """ +class PathMismatchException(CouchbaseException): + """Indicates that the referenced path made incorrect assumptions about the structure of a document, + for instance attempting to access a field as an object when in fact it is an array.""" -class BadHandleError(CouchbaseError): - """Invalid handle type + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) - The requested operation isn't allowed for given type. - """ + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class PathNotFoundException(CouchbaseException): + """Indicates that the reference path was not found.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class PathTooBigException(CouchbaseException): + """Indicates that the reference path is too long, or contains too many independent components.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class RangeScanCompletedException(CouchbaseException): + """Indicates the range scan has completed.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() -class HTTPError(CouchbaseError): - """HTTP error""" - - -class ObjectThreadError(CouchbaseError): - """Thrown when access from multiple threads is detected""" - - -class ViewEngineError(CouchbaseError): - """Thrown for inline errors during view queries""" - -class ObjectDestroyedError(CouchbaseError): - """Object has been destroyed. Pending events are invalidated""" - - -class PipelineError(CouchbaseError): - """Illegal operation within pipeline state""" - -_LCB_ERRCAT_MAP = { - C.LCB_ERRTYPE_NETWORK: CouchbaseNetworkError, - C.LCB_ERRTYPE_INPUT: CouchbaseInputError, - C.LCB_ERRTYPE_TRANSIENT: CouchbaseTransientError, - C.LCB_ERRTYPE_FATAL: CouchbaseFatalError, - C.LCB_ERRTYPE_DATAOP: CouchbaseDataError, - C.LCB_ERRTYPE_INTERNAL: CouchbaseInternalError -} - -_LCB_ERRNO_MAP = { - C.LCB_AUTH_ERROR: AuthError, - C.LCB_DELTA_BADVAL: DeltaBadvalError, - C.LCB_E2BIG: TooBigError, - C.LCB_EBUSY: BusyError, - C.LCB_ENOMEM: NoMemoryError, - C.LCB_ETMPFAIL: TemporaryFailError, - C.LCB_KEY_EEXISTS: KeyExistsError, - C.LCB_KEY_ENOENT: NotFoundError, - C.LCB_DLOPEN_FAILED: DlopenFailedError, - C.LCB_DLSYM_FAILED: DlsymFailedError, - C.LCB_NETWORK_ERROR: NetworkError, - C.LCB_NOT_MY_VBUCKET: NotMyVbucketError, - C.LCB_NOT_STORED: NotStoredError, - C.LCB_NOT_SUPPORTED: NotSupportedError, - C.LCB_UNKNOWN_HOST: UnknownHostError, - C.LCB_PROTOCOL_ERROR: ProtocolError, - C.LCB_ETIMEDOUT: TimeoutError, - C.LCB_CONNECT_ERROR: ConnectError, - C.LCB_BUCKET_ENOENT: BucketNotFoundError, - C.LCB_EBADHANDLE: BadHandleError, - C.LCB_INVALID_HOST_FORMAT: InvalidError, - C.LCB_INVALID_CHAR: InvalidError, - C.LCB_EINVAL: InvalidError, - C.LCB_DURABILITY_ETOOMANY: ArgumentError, - C.LCB_DUPLICATE_COMMANDS: ArgumentError, - C.LCB_CLIENT_ETMPFAIL: ClientTemporaryFailError, - C.LCB_HTTP_ERROR: HTTPError -} - -def _mk_lcberr(rc, name=None, default=CouchbaseError, docstr="", extrabase=[]): +class PathTooDeepException(CouchbaseException): + """Indicates that the reference path contains too many levels to parse.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class ValueTooDeepException(CouchbaseException): """ - Create a new error class derived from the appropriate exceptions. - :param int rc: libcouchbase error code to map - :param str name: The name of the new exception - :param class default: Default exception to return if no categories are found - :return: a new exception derived from the appropriate categories, or the - value supplied for `default` + Indicates that the value provided, if inserted into the document, + would cause the document to become too deep for the server to accept. """ - categories = C._get_errtype(rc) - if not categories: - return default - - bases = extrabase[::] - - for cat, base in _LCB_ERRCAT_MAP.items(): - if cat & categories: - bases.append(base) - - if name is None: - name = "LCB_0x{0:0X} (generated, catch: {1})".format( - rc, ", ".join(x.__name__ for x in bases)) - - d = { '__doc__' : docstr } - - if not bases: - bases = [CouchbaseError] - - return type(name, tuple(bases), d) - -# Reinitialize the exception classes again. -for rc, oldcls in _LCB_ERRNO_MAP.items(): - # Determine the new reparented error category for this - newname = "_{0}_0x{1:0X} (generated, catch {0})".format(oldcls.__name__, rc) - newcls = _mk_lcberr(rc, name=newname, default=None, docstr=oldcls.__doc__, - extrabase=[oldcls]) - if not newcls: - # No categories for this type, fall back to existing one - continue - - _LCB_ERRNO_MAP[rc] = newcls - -_EXCTYPE_MAP = { - C.PYCBC_EXC_ARGUMENTS: ArgumentError, - C.PYCBC_EXC_ENCODING: ValueFormatError, - C.PYCBC_EXC_INTERNAL: InternalSDKError, - C.PYCBC_EXC_HTTP: HTTPError, - C.PYCBC_EXC_THREADING: ObjectThreadError, - C.PYCBC_EXC_DESTROYED: ObjectDestroyedError, - C.PYCBC_EXC_PIPELINE: PipelineError -} + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class MutationTokenOutdatedException(CouchbaseException): + """Indicates that the vbucket uuid requirements do not align with the server.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# @TODO: How to Deprecate? + + +SubdocCantInsertValueException = InvalidValueException +SubdocPathMismatchException = PathMismatchException + +# Query Exceptions + + +class ParsingFailedException(CouchbaseException): + """ + Raised when the query service is unable to parse a N1QL query + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class AlreadyQueriedException(CouchbaseException): + """ + Raised when query (N1QL, Search, Analytics or Views) results + have already been iterated over. + """ + + def __init__(self, message='Previously iterated over results.', **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class KeyspaceNotFoundException(CouchbaseException): + """Keyspace not found (collection or bucket does not exist)""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Search Exceptions + + +class NoChildrenException(CouchbaseException): + """ + Compound query is missing children" + """ + + def __init__(self, message='No child queries.', **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Bucket Mgmt + + +class BucketAlreadyExistsException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class BucketDoesNotExistException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class BucketNotFlushableException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Scope/Collection mgmt + + +class CollectionAlreadyExistsException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class CollectionNotFoundException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class UnsupportedOperation(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class ScopeAlreadyExistsException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class ScopeNotFoundException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# User mgmt + + +class GroupNotFoundException(CouchbaseException): + """ The RBAC Group was not found""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class UserNotFoundException(CouchbaseException): + """ The RBAC User was not found""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Query index mgmt + + +class QueryIndexAlreadyExistsException(CouchbaseException): + """ The query index already exists""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class QueryIndexNotFoundException(CouchbaseException): + """ The query index was not found""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class WatchQueryIndexTimeoutException(CouchbaseException): + """Unable to find all requested indexes online within specified timeout""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Search Index mgmttest_bad_scope_query + + +class SearchIndexNotFoundException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Analytics mgmt + + +class DataverseAlreadyExistsException(CouchbaseException): + """Raised when attempting to create dataverse when it already exists""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class DataverseNotFoundException(CouchbaseException): + """Raised when attempting to drop a dataverse which does not exist""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class DatasetNotFoundException(CouchbaseException): + """Raised when attempting to drop a dataset which does not exist.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class DatasetAlreadyExistsException(CouchbaseException): + """Raised when attempting to create a dataset which already exists""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class AnalyticsLinkExistsException(CouchbaseException): + """Raised when attempting to create an analytics link which already exists""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class AnalyticsLinkNotFoundException(CouchbaseException): + """Raised when attempting to replace or drop an analytics link that does not exists""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +# Views mgmt + + +class DesignDocumentNotFoundException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Eventing function mgmt + + +class EventingFunctionNotFoundException(CouchbaseException): + """Raised when an eventing function is not found""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class EventingFunctionCompilationFailureException(CouchbaseException): + """Raised when compilation of an eventing function failed""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class EventingFunctionIdenticalKeyspaceException(CouchbaseException): + """Raised when the source and metadata keyspaces are the same""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class EventingFunctionNotBootstrappedException(CouchbaseException): + """Raised when an eventing function is deployed but not “fully” bootstrapped""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class EventingFunctionNotDeployedException(CouchbaseException): + """Raised when an eventing function is not deployed, but the action expects it to be deployed""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class EventingFunctionNotUnDeployedException(CouchbaseException): + """Raised when an eventing function is deployed, but the action expects it to be undeployed""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class EventingFunctionAlreadyDeployedException(CouchbaseException): + """Raised when an eventing function has already been deployed""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class EventingFunctionCollectionNotFoundException(CouchbaseException): + """Raised when collection in specified keyspace is not found""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# python client only errors + + +class FeatureNotFoundException(CouchbaseException): + """Thrown when feature is not supported by server version.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class InvalidIndexException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class MissingTokenException(CouchbaseException): + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class QueueEmpty(CouchbaseException): + """ + Thrown if a datastructure queue is empty + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class UnsuccessfulOperationException(CouchbaseException): + """Thrown when a specific pycbc_core operation is unsuccessful.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Ratelimiting + + +class RateLimitedException(CouchbaseException): + """ + Rate limited exceptions are an uncommitted API that is unlikely to change, + but may still change as final consensus on its behavior has not yet been reached. + + The server decided that the caller must be rate limited due to + exceeding a rate threshold.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class QuotaLimitedException(CouchbaseException): + """ + Quota limited exceptions are an uncommitted API that is unlikely to change, + but may still change as final consensus on its behavior has not yet been reached. + + The server decided that the caller must be limited due to exceeding + a quota threshold.""" + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Transaction errors + + +class TransactionException(CouchbaseException): + """ + Base class for any transaction-related exception + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + # handle inheritance structure + if type(self).__name__ != 'TransactionException': + return super().__repr__() + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class TransactionOperationFailed(TransactionException): + """ + Indicates a transaction operation failed. + + The transaction will be rolled back no matter what, but this error can give some context as to why it failed. + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class TransactionFailed(TransactionException): + """ + The transaction failed and was rolled back. + + No actors can see any changes made by this transaction. + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class TransactionExpired(TransactionException): + """ + The transaction could not be fully completed in the configured timeout. + + It is in an undefined state, but it unambiguously did not reach the commit point. No actors will be able to see the + contents of this transaction.The transaction exceeded the expiry set in the TransactionConfig, and was rolled back. + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + + +class TransactionCommitAmbiguous(TransactionException): + """ + The transaction expired at the point of trying to commit it. It is ambiguous whether the transaction has committed + or not. Actors may be able to see the content of this transaction. + + This error is result of inevitable and unavoidable edge cases when working with unreliable networks. For example, + consider an ordinary mutation being made over the network to any database. The mutation could succeed on the + database-side, and then just before the result is returned to the client, the network connection drops. The client + cannot receive the success result and will timeout - it is ambiguous to it whether the mutation succeeded or not. + + The transactions layer will work to resolve the ambiguity up until the transaction expires, but if unable to resolve + it in that time, it is forced to raise this error. The transaction may or may not have been successful, and + error-handling of this is highly application-dependent. + + The asynchronous cleanup process will try to complete the transaction: roll it back if it didn't commit, roll it + forwards if it did. + """ + + def __init__(self, message=None, **kwargs): + if message and isinstance(message, str) and 'message' not in kwargs: + kwargs['message'] = message + super().__init__(**kwargs) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return self.__repr__() + +# Field Level Encryption Exceptions + +# @TODO: Need to look at FLE library to make updates here + + +class CryptoException(CouchbaseException): + def __init__(self, params=None, + message="Generic Cryptography exception", **kwargs): + params = params or {} + param_dict = params.get("objextra") or defaultdict(lambda: "unknown") + params["message"] = Template(message).safe_substitute(**param_dict) + super(CryptoException, self).__init__(params=params, **kwargs) + + +class EncryptionFailureException(CryptoException): + def __init__(self, params=None, + message="Generic encryption failure.", **kwargs): + super(EncryptionFailureException, self).__init__( + params=params, message=message, **kwargs + ) + + +class DecryptionFailureException(CryptoException): + def __init__(self, params=None, + message="Generic decryption failure.", **kwargs): + super(DecryptionFailureException, self).__init__( + params=params, message=message, **kwargs + ) + + +class CryptoKeyNotFoundException(CryptoException): + def __init__(self, message): + self._message = message + super(CryptoKeyNotFoundException, self).__init__(message=message) + + def __str__(self): + return "{}: {}".format(self.__class__.__name__, self._message) + + +class InvalidCryptoKeyException(CryptoException): + def __init__(self, message): + self._message = message + super(InvalidCryptoKeyException, self).__init__(message=message) + + def __str__(self): + return "{}: {}".format(self.__class__.__name__, self._message) + + +class EncrypterNotFoundException(CryptoException): + def __init__(self, message): + self._message = message + super(EncrypterNotFoundException, self).__init__(message=message) + + def __str__(self): + return "{}: {}".format(self.__class__.__name__, self._message) + + +class DecrypterNotFoundException(CryptoException): + def __init__(self, message): + self._message = message + super(DecrypterNotFoundException, self).__init__(message=message) + + def __str__(self): + return "{}: {}".format(self.__class__.__name__, self._message) + + +class EncrypterAlreadyExistsException(CryptoException): + def __init__(self, message): + self._message = message + super(EncrypterAlreadyExistsException, self).__init__(message=message) + + def __str__(self): + return "{}: {}".format(self.__class__.__name__, self._message) + + +class DecrypterAlreadyExistsException(CryptoException): + def __init__(self, message): + self._message = message + super(DecrypterAlreadyExistsException, self).__init__(message=message) + + def __str__(self): + return "{}: {}".format(self.__class__.__name__, self._message) + + +class InvalidCipherTextException(CryptoException): + def __init__(self, message): + self._message = message + super(InvalidCipherTextException, self).__init__(message=message) + + def __str__(self): + return "{}: {}".format(self.__class__.__name__, self._message) + + +# CXX Error Map + + +# @TODO: do we need this? I think w/ the initial 4.x rewrite it was +# possible to pass in an excpetion w/ a result object, but I +# do not think we do this anymore. Need to confirm. +CLIENT_ERROR_MAP = dict( + { + 2: RequestCanceledException, + 3: InvalidArgumentException, + 9: CasMismatchException, + 13: AmbiguousTimeoutException, + 14: UnAmbiguousTimeoutException, + 101: DocumentNotFoundException, + 103: DocumentLockedException, + 105: DocumentExistsException, + 113: PathNotFoundException, + 114: PathMismatchException, + 119: InvalidValueException, + 123: PathExistsException, + } +) + + +class ExceptionMap(Enum): + RequestCanceledException = 2 + InvalidArgumentException = 3 + ServiceUnavailableException = 4 + InternalServerFailureException = 5 + AuthenticationException = 6 + TemporaryFailException = 7 + ParsingFailedException = 8 + CasMismatchException = 9 + BucketNotFoundException = 10 + CollectionNotFoundException = 11 + UnsupportedOperation = 12 + AmbiguousTimeoutException = 13 + UnAmbiguousTimeoutException = 14 + FeatureUnavailableException = 15 + ScopeNotFoundException = 16 + QueryIndexNotFoundException = 17 + QueryIndexAlreadyExistsException = 18 + RateLimitedException = 21 + QuotaLimitedException = 22 + DocumentNotFoundException = 101 + DocumentUnretrievableException = 102 + DocumentLockedException = 103 + DocumentExistsException = 105 + DurabilityInvalidLevelException = 107 + DurabilityImpossibleException = 108 + DurabilitySyncWriteAmbiguousException = 109 + DurabilitySyncWriteInProgressException = 110 + PathNotFoundException = 113 + PathMismatchException = 114 + PathInvalidException = 115 + PathTooBigException = 116 + PathTooDeepException = 117 + ValueTooDeepException = 118 + InvalidValueException = 119 + DocumentNotJsonException = 120 + NumberTooBigException = 121 + DeltaInvalidException = 122 + PathExistsException = 123 + DocumentNotLockedException = 131 + MutationTokenOutdatedException = 133 + RangeScanCompletedException = 134 + DatasetNotFoundException = 303 + DataverseNotFoundException = 304 + DatasetAlreadyExistsException = 305 + DataverseAlreadyExistsException = 306 + DesignDocumentNotFoundException = 502 + InternalSDKException = 5000 + HTTPException = 5001 + UnsuccessfulOperationException = 5002 + + +PYCBC_ERROR_MAP = {e.value: getattr(sys.modules[__name__], e.name) for e in ExceptionMap} + +KV_ERROR_CONTEXT_MAPPING = {'key_value_locked': DocumentLockedException, + 'key_value_temporary_failure': TemporaryFailException} + +QUERY_ERROR_MAPPING = {r'.*Keyspace not found.*': KeyspaceNotFoundException, + r'.*Scope not found.*': ScopeNotFoundException, + r'.*No index available.*': QueryIndexNotFoundException, + r'.*[iI]ndex.*not found.*': QueryIndexNotFoundException, + r'.*[iI]ndex.*already exists.*': QueryIndexAlreadyExistsException} + + +class ErrorMapper: + @staticmethod + def _process_mapping(compiled_map, # type: Dict[str, CouchbaseException] + err_content # type: str + ) -> Optional[CouchbaseException]: + matches = None + for pattern, exc_class in compiled_map.items(): + try: + matches = pattern.match(err_content) + except Exception: # nosec + pass + if matches: + return exc_class + + return None + + @staticmethod # noqa: C901 + def _parse_http_response_body(compiled_map, # type: Dict[str, CouchbaseException] # noqa: C901 + response_body # type: str + ) -> Optional[CouchbaseException]: + + err_text = None + try: + http_body = json.loads(response_body) + except json.decoder.JSONDecodeError: + return None + + if isinstance(http_body, str): + exc_class = ErrorMapper._process_mapping(compiled_map, http_body) + if exc_class is not None: + return exc_class + elif isinstance(http_body, dict) and http_body.get("errors", None) is not None: + errors = http_body.get("errors") + if isinstance(errors, list): + for err in errors: + err_text = f"{err.get('code', None)} {err.get('msg', None)}" + if err_text: + exc_class = ErrorMapper._process_mapping(compiled_map, err_text) + if exc_class is not None: + return exc_class + err_text = None + else: + err_text = errors.get("name", None) + # eventing function mgmt cases + elif isinstance(http_body, dict) and http_body.get('name', None) is not None: + exc = ErrorMapper._process_mapping(compiled_map, http_body.get('name', None)) + if exc is not None: + return exc + + if err_text is not None: + exc_class = ErrorMapper._process_mapping(compiled_map, err_text) + return exc_class + + return None + + @staticmethod + def _parse_http_context(err_ctx, # type: HTTPErrorContext + mapping=None, # type: Dict[str, CouchbaseException] + err_info=None # type: Dict[str, Any] + ) -> Optional[CouchbaseException]: + from couchbase._utils import is_null_or_empty + + compiled_map = {} + if mapping: + compiled_map = {{str: re.compile}.get( + type(k), lambda x: x)(k): v for k, v in mapping.items()} + + exc_msg = err_info.get('error_message', None) if err_info else None + if not is_null_or_empty(exc_msg): + exc_class = ErrorMapper._process_mapping(compiled_map, exc_msg) + if exc_class is not None: + return exc_class + + if not is_null_or_empty(err_ctx.response_body): + err_text = err_ctx.response_body + exc_class = ErrorMapper._process_mapping(compiled_map, err_text) + if exc_class is not None: + return exc_class + + exc_class = ErrorMapper._parse_http_response_body(compiled_map, err_text) + if exc_class is not None: + return exc_class + + return None + + @staticmethod + def _parse_kv_context(err_ctx, # type: KeyValueErrorContext + mapping, # type: Dict[str, CouchbaseException] + err_content=None # type: str + ) -> Optional[CouchbaseException]: + from couchbase._utils import is_null_or_empty + + compiled_map = {{str: re.compile}.get( + type(k), lambda x: x)(k): v for k, v in mapping.items()} + + if not is_null_or_empty(err_content): + exc_class = ErrorMapper._process_mapping(compiled_map, err_content) + if exc_class is not None: + return exc_class + + if err_ctx.retry_reasons is not None: + for rr in err_ctx.retry_reasons: + exc_class = ErrorMapper._process_mapping(compiled_map, rr) + if exc_class is not None: + return exc_class + + return None + + @classmethod + def build_exception(cls, + base_exc, # type: exception + mapping=None, # type: Dict[str, CouchbaseException] + ) -> CouchbaseException: + exc_class = None + err_ctx = None + ctx = base_exc.error_context() + if ctx is None: + exc_class = PYCBC_ERROR_MAP.get(base_exc.err(), CouchbaseException) + err_info = base_exc.error_info() + else: + err_ctx = ErrorContext.from_dict(**ctx) + err_info = base_exc.error_info() + + if isinstance(err_ctx, HTTPErrorContext): + exc_class = ErrorMapper._parse_http_context(err_ctx, mapping, err_info=err_info) + + if isinstance(err_ctx, KeyValueErrorContext): + if mapping is None: + mapping = KV_ERROR_CONTEXT_MAPPING + exc_class = ErrorMapper._parse_kv_context(err_ctx, mapping) + + if isinstance(err_ctx, QueryErrorContext): + if mapping is None: + mapping = QUERY_ERROR_MAPPING + exc_class = ErrorMapper._parse_http_context(err_ctx, mapping) + + if exc_class is None: + exc_class = PYCBC_ERROR_MAP.get(base_exc.err(), CouchbaseException) + + exc = exc_class(base=base_exc, exc_info=err_info, context=err_ctx) + return exc diff --git a/couchbase/experimental.py b/couchbase/experimental.py deleted file mode 100644 index 9332da6fe..000000000 --- a/couchbase/experimental.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -_USE_EXPERIMENTAL_APIS = False -def enable(): - """ - Enable usage of experimental APIs bundled with Couchbase. - """ - global _USE_EXPERIMENTAL_APIS - _USE_EXPERIMENTAL_APIS = True - -def enabled_or_raise(): - if _USE_EXPERIMENTAL_APIS: - return - - raise ImportError( - "Your application has requested use of an unstable couchbase " - "client API. Use " - "couchbase.experimental.enable() to enable experimental APIs. " - "Experimental APIs are subject to interface, behavior, and " - "stability changes. Use at your own risk") diff --git a/couchbase/iops/__init__.py b/couchbase/iops/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/couchbase/iops/base.py b/couchbase/iops/base.py deleted file mode 100644 index 3e4831b32..000000000 --- a/couchbase/iops/base.py +++ /dev/null @@ -1,140 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -This file is here for example purposes only. It demonstrates the basic -IOPS API. - -This is not yet considered stable interface, although this is currently -the only means by which an external event loop can be integrated with -Couchbase through Python -""" - -from couchbase._libcouchbase import ( - PYCBC_EVACTION_WATCH, - PYCBC_EVACTION_UNWATCH, - PYCBC_EVACTION_CLEANUP, - LCB_READ_EVENT, - LCB_WRITE_EVENT, - LCB_RW_EVENT, - IOEvent, - TimerEvent, - Event -) - -class IOPS(object): - def __init__(self): - """ - The IOPS class is intended as an efficient and multiplexing - manager of one or more :class:`Event` objects. - - As this represents an interface with methods only, - there is no required behavior in the constructor of this object - """ - - def update_event(self, event, action, flags): - """ - This method shall perform an action modifying an event. - - :param event: An :class:`IOEvent` object which shall have its - watcher settings modified. The ``IOEvent`` object is an object - which provides a ``fileno()`` method. - - :param int action: one of: - - * ``PYCBC_EVACTION_WATCH``: Watch this file for events - * ``PYCBC_EVACTION_UNWATCH``: Remove this file from all watches - * ``PYCBC_EVACTION_CLEANUP``: Destroy any references to this object - - :param int flags: Event details, this indicates which events this - file should be watched for. This is only applicable if ``action`` - was ``PYCBC_EVACTION_WATCH``. It can a bitmask of the following: - - * ``LCB_READ_EVENT``: Watch this file until it becomes readable - * ``LCB_WRITE_EVENT``: Watch this file until it becomes writeable - - If the action is to watch the event for readability or writeability, - the ``IOPS`` implementation shall schedule the underlying event system - to call one of the ``ready_r``, ``ready_w`` or ``ready_rw`` methods - (for readbility, writeability or both readability and writability - respectively) at such a time when the underlying reactor/event loop - implementation has signalled it being so. - - Event watchers are non-repeatable. This means that once the event - has been delivered, the ``IOEvent`` object shall be removed from a - watching state. The extension shall call this method again for each - time an event is requested. - - This method must be implemented - """ - - def update_timer(self, timer, action, usecs): - """ - This method shall schedule or unschedule a timer. - - :param timer: A :class:`TimerEvent` object. - :param action: See :meth:`update_event` for meaning - :param usecs: A relative offset in microseconds when this timer - shall be fired. - - This method follows the same semantics as :meth:`update_event`, - except that there is no file. - - When the underlying event system shall invoke the timer, the - ``TimerEvent`` ``ready`` method shall be called with ``0`` as its - argument. - - Like ``IOEvents``, ``TimerEvents`` are non-repeatable. - - This method must be implemented - """ - - def io_event_factory(self): - """ - Returns a new instance of :class:`IOEvent`. - - This method is optional, and is useful in case an implementation - wishes to utilize its own subclass of ``IOEvent``. - - As with most Python subclasses, the user should ensure that the - base implementation's ``__init__`` is called. - """ - - def timer_event_factory(self): - """ - Returns a new instance of :class:`TimerEvent`. Like the - :meth:`io_event_factory`, this is optional - """ - - def start_watching(self): - """ - Called by the extension when all scheduled IO events have been - submitted. Depending on the I/O model, this method can either - drive the event loop until :meth:`stop_watching` is called, or - do nothing. - - This method must be implemented - """ - - def stop_watching(self): - """ - Called by the extension when it no longer needs to wait for events. - Its function is to undo anything which was done in the - :meth:`start_watching` method - - This method must be implemented - """ diff --git a/couchbase/iops/select.py b/couchbase/iops/select.py deleted file mode 100644 index 4011b0244..000000000 --- a/couchbase/iops/select.py +++ /dev/null @@ -1,184 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from __future__ import absolute_import - -import select -import time - -import couchbase._libcouchbase as LCB -from couchbase._libcouchbase import ( - Event, TimerEvent, IOEvent, - LCB_READ_EVENT, LCB_WRITE_EVENT, LCB_RW_EVENT, - PYCBC_EVSTATE_ACTIVE, - PYCBC_EVACTION_WATCH, - PYCBC_EVACTION_UNWATCH -) - -class SelectTimer(TimerEvent): - def __init__(self): - super(SelectTimer, self).__init__() - self.pydata = 0 - - @property - def exptime(self): - return self.pydata - - @exptime.setter - def exptime(self, val): - self.pydata = val - - def activate(self, usecs): - self.exptime = time.time() + usecs / 1000000 - - def deactivate(self): - pass - - @property - def active(self): - return self.state == PYCBC_EVSTATE_ACTIVE - - # Rich comparison operators implemented - __cmp__ not used in Py3 - def __lt__(self, other): return self.exptime < other.exptime - def __le__(self, other): return self.exptime <= other.exptime - def __gt__(self, other): return self.exptime > other.exptime - def __ge__(self, other): return self.exptime >= other.exptime - def __ne__(self, other): return self.exptime != other.exptime - def __eq__(self, other): return self.exptime == other.exptime - - -class SelectIOPS(object): - def __init__(self): - self._do_watch = False - self._ioevents = set() - self._timers = [] - - # Active readers and writers - self._evwr = set() - self._evrd = set() - - - - def _unregister_timer(self, timer): - timer.deactivate() - if timer in self._timers: - self._timers.remove(timer) - - def _unregister_event(self, event): - try: - self._evrd.remove(event) - except KeyError: - pass - try: - self._evwr.remove(event) - except KeyError: - pass - try: - self._ioevents.remove(event) - except KeyError: - pass - - def update_timer(self, timer, action, usecs): - if action == PYCBC_EVACTION_UNWATCH: - self._unregister_timer(timer) - return - - if timer.active: - self._unregister_timer(timer) - timer.activate(usecs) - self._timers.append(timer) - - def update_event(self, event, action, flags, fd=None): - if action == PYCBC_EVACTION_UNWATCH: - self._unregister_event(event) - return - - elif action == PYCBC_EVACTION_WATCH: - if flags & LCB_READ_EVENT: - self._evrd.add(event) - else: - try: - self._evrd.remove(event) - except KeyError: - pass - - if flags & LCB_WRITE_EVENT: - self._evwr.add(event) - else: - try: - self._evwr.remove(event) - except KeyError: - pass - - def _poll(self): - rin = self._evrd - win = self._evwr - ein = list(rin) + list(win) - - self._timers.sort() - mintime = self._timers[0].exptime - time.time() - if mintime < 0: - mintime = 0 - - if not (rin or win or ein): - time.sleep(mintime) - rout = tuple() - wout = tuple() - eout = tuple() - else: - rout, wout, eout = select.select(rin, win, ein, mintime) - - now = time.time() - - ready_events = {} - for ev in rout: - ready_events[ev] = LCB_READ_EVENT - - for ev in wout: - if ev in ready_events: - ready_events[ev] |= LCB_WRITE_EVENT - else: - ready_events[ev] = LCB_WRITE_EVENT - - for ev in eout: - ready_events[ev] = LCB_RW_EVENT - - for ev, flags in ready_events.items(): - if ev.state == PYCBC_EVSTATE_ACTIVE: - ev.ready(flags) - - for timer in self._timers[:]: - if not timer.active: - continue - - if timer.exptime > now: - continue - - timer.ready(0) - - def start_watching(self): - if self._do_watch: - return - - self._do_watch = True - while self._do_watch: - self._poll() - - def stop_watching(self): - self._do_watch = False - - def timer_event_factory(self): - return SelectTimer() diff --git a/couchbase/items.py b/couchbase/items.py deleted file mode 100644 index 7df065bdc..000000000 --- a/couchbase/items.py +++ /dev/null @@ -1,202 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This module contains various collections to be used with items. These provide -# various means by which multiple operations can have their options customized - -# Each of these collections yields an iterator consisting of a 3-tuple: -# (item, {options}) -# The CAS, Format, and Value are all expected to be inside the Item itself; - - -from couchbase._libcouchbase import Item as _Item - -class Item(_Item): - def __init__(self, key=None, value=None): - """ - Construct a new `Item` object. - - :param string key: The key to initialize this item with - :param object value: The value to initialize this item with - - The `Item` class is a sublcass of a - :class:`~couchbase.result.ValueResult`. - Its members are all writeable and accessible from this object. - - .. warning:: - - As the item build-in properties (such as ``key``, ``value``, - ``cas``, etc.) - are implemented directly in C and are not exposed in the item's - ``__dict__`` field, you cannot override these fields in a subclass - to be a ``property`` or some other custom data descriptor. - - To confuse matters even more, if you do implement these properties - as descriptors, they will be visible from your own code, but *not* - from the implementation code. You have been warned. - - In short, don't override these properties. - - Here's an example of what you should *not* do:: - - class MyItem(Item): - # ... - @property - def key(self): - return self._key - - @key.setter - def key(self, newkey): - self._key = key - - To use this class with the :class:`couchbase.bucket.Bucket` - API methods, you must take care to: - - 1. Use only the ``*_multi`` methods - 2. Pass one of the :class:`ItemCollection` objects to these methods. - This will let the API know to enable special handling for - the :class:`Item` API. - - """ - - super(Item, self).__init__() - self.key = key - self.value = value - - def as_itcoll(self, **kwargs): - """ - Convenience method to return an instance of a :class:`ItemCollection` - containing only this item. This would then be used like so:: - - cb.upsert_multi(itm.as_itcoll()) - - Or use it with options:: - - cb.upsert_multi(itm.as_itcoll(ignore_cas=True)) - - :param kwargs: Extra operation-specific options. - - :return: An :class:`ItemCollection` instance - """ - if not kwargs: - return ItemSequence([self]) - else: - return ItemOptionDict({self: kwargs}) - -class ItemCollection(object): - """ - The base class for a collection of Items. - """ - def __len__(self): - raise NotImplementedError() - - def __iter__(self): - """ - This iterator is mainly intended for the internal API use; it - yields a tuple of (item, options). - In the case of a :class:`ItemSequence` which does not store - options, the second element is always `None` - """ - - def dict_items(self): - """ - Iterator which returns a tuple of ``(item, options)`` - for each item in this collection. - """ - return iter(self) - -class ItemOptionDict(ItemCollection): - def __init__(self, d=None): - """ - A simple mapping of :class:`Item` objects to optional dictionaries - of values. - - The keys and values for the options dictionary depends on the command - being used. See the appropriate command for more options - - :param dict d: A dictionary of item -> option, or None. - """ - if d is None: - d = {} - self._d = d - - @property - def dict(self): - """ - Return the actual dict object - """ - return self._d - - def add(self, itm, **options): - """ - Convenience method to add an item together with a series of options. - - :param itm: The item to add - :param options: keyword arguments which will be placed in the item's - option entry. - - If the item already exists, it (and its options) will be overidden. Use - :attr:`dict` instead to update options - - """ - if not options: - options = None - self._d[itm] = options - - def create_and_add(self, key, value=None, cas=0, **options): - """ - Creates and adds an item. - :param key: The key to use for the item - :param value: The value to use for the item - :param options: Additional operation-specific options - """ - itm = Item(key, value) - itm.cas = cas - return self.add(itm, **options) - - def __iter__(self): - for p in self._d.items(): - yield p - - def __len__(self): - return len(self._d) - -class ItemSequence(ItemCollection): - def __init__(self, obj): - """ - Create a new :class:`ItemSequence` object - - :param seq: A sequence containing the items - :type seq: An iterable or a single item - """ - self._seq = [ obj ] if isinstance(obj, Item) else obj - # Verify this is indeed a sequence - len(self._seq) - - @property - def sequence(self): - """ - The actual sequence object passed in - """ - return self._seq - - def __len__(self): - return len(self._seq) - - def __iter__(self): - for e in self._seq: - yield (e, None) diff --git a/couchbase/kv_range_scan.py b/couchbase/kv_range_scan.py new file mode 100644 index 000000000..326a92428 --- /dev/null +++ b/couchbase/kv_range_scan.py @@ -0,0 +1,62 @@ +# Copyright 2021, Couchbase, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any, Dict + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ExceptionMap, + RangeScanCompletedException) +from couchbase.logic.kv_range_scan import PrefixScan # noqa: F401 +from couchbase.logic.kv_range_scan import RangeScan # noqa: F401 +from couchbase.logic.kv_range_scan import SamplingScan # noqa: F401 +from couchbase.logic.kv_range_scan import ScanTerm # noqa: F401 +from couchbase.logic.kv_range_scan import ScanType # noqa: F401 +from couchbase.logic.kv_range_scan import RangeScanRequestLogic + + +class RangeScanRequest(RangeScanRequestLogic): + def __init__(self, + **kwargs, # type: Dict[str, Any] + ): + super().__init__(**kwargs) + + def __iter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_scan() + + return self + + def __next__(self): + try: + return self._get_next_row() + # We can stop iterator when we receive RangeScanCompletedException + except RangeScanCompletedException: + self._done_streaming = True + raise StopIteration + except StopIteration: + self._done_streaming = True + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/couchbase/logic/__init__.py b/couchbase/logic/__init__.py new file mode 100644 index 000000000..a7057f2f2 --- /dev/null +++ b/couchbase/logic/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .wrappers import BlockingWrapper # noqa: F401 +from .wrappers import decode_replicas # noqa: F401 +from .wrappers import decode_value # noqa: F401 diff --git a/couchbase/logic/analytics.py b/couchbase/logic/analytics.py new file mode 100644 index 000000000..18b4156f3 --- /dev/null +++ b/couchbase/logic/analytics.py @@ -0,0 +1,473 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta +from enum import Enum +from typing import (Any, + Dict, + List, + Optional, + Union) + +from couchbase._utils import to_microseconds +from couchbase.exceptions import ErrorMapper, InvalidArgumentException +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.options import AnalyticsOptionsBase +from couchbase.options import AnalyticsOptions, UnsignedInt64 +from couchbase.pycbc_core import analytics_query +from couchbase.serializer import DefaultJsonSerializer, Serializer +from couchbase.tracing import CouchbaseSpan + + +class AnalyticsScanConsistency(Enum): + """ + For use with :attr:`~.AnalyticsQuery.consistency`, will allow cached + values to be returned. This will improve performance but may not + reflect the latest data in the server. + """ + NOT_BOUNDED = "not_bounded" + REQUEST_PLUS = "request_plus" + + +class AnalyticsStatus(Enum): + """ + Represents the status of an analytics query. + """ + RUNNING = "running" + SUCCESS = "success" + ERRORS = "errors" + COMPLETED = "completed" + STOPPED = "stopped" + TIMEOUT = "timeout" + CLOSED = "closed" + FATAL = "fatal" + ABORTED = "aborted" + UNKNOWN = "unknown" + + +class AnalyticsProblem(object): + def __init__(self, raw): + self._raw = raw + + def code(self) -> int: + return self._raw.get("code", None) + + def message(self) -> str: + return self._raw.get("message", None) + + +class AnalyticsWarning(AnalyticsProblem): + def __init__(self, analytics_warning): + super().__init__(analytics_warning) + + def __repr__(self): + return "AnalyticsWarning:{}".format(super()._raw) + + +class AnalyticsError(AnalyticsProblem): + def __init__(self, analytics_error): + super().__init__(analytics_error) + + def __repr__(self): + return "AnalyticsError:{}".format(super()._raw) + + +class AnalyticsMetrics(object): + + def __init__(self, raw # type: Dict[str, Any] + ) -> None: + self._raw = raw + + def elapsed_time(self) -> timedelta: + us = self._raw.get("elapsed_time") / 1000 + return timedelta(microseconds=us) + + def execution_time(self) -> timedelta: + us = self._raw.get("execution_time") / 1000 + return timedelta(microseconds=us) + + def result_count(self) -> UnsignedInt64: + return UnsignedInt64(self._raw.get("result_count", 0)) + + def result_size(self) -> UnsignedInt64: + return UnsignedInt64(self._raw.get("result_size", 0)) + + def error_count(self) -> UnsignedInt64: + return UnsignedInt64(self._raw.get("error_count", 0)) + + def processed_objects(self) -> UnsignedInt64: + return UnsignedInt64(self._raw.get("processed_objects", 0)) + + def warning_count(self) -> UnsignedInt64: + return UnsignedInt64(self._raw.get("warning_count", 0)) + + def __repr__(self): + return "AnalyticsMetrics:{}".format(self._raw) + + +class AnalyticsMetaData: + def __init__(self, raw # type: Dict[str, Any] + ) -> None: + if raw is not None: + self._raw = raw.get('metadata', None) + sig = self._raw.get('signature', None) + if sig is not None: + self._raw['signature'] = json.loads(sig) + else: + self._raw = None + + def request_id(self) -> str: + return self._raw.get("request_id", None) + + def client_context_id(self) -> str: + return self._raw.get("client_context_id", None) + + def status(self) -> AnalyticsStatus: + return AnalyticsStatus[self._raw.get("status", "unknown").upper()] + + def signature(self) -> Optional[Dict[str, Any]]: + return self._raw.get("signature", None) + + def warnings(self) -> List[AnalyticsWarning]: + return list( + map(AnalyticsWarning, self._raw.get("warnings", [])) + ) + + def errors(self) -> List[AnalyticsError]: + return list( + map(AnalyticsError, self._raw.get("errors", [])) + ) + + def metrics(self) -> Optional[AnalyticsMetrics]: + if "metrics" in self._raw: + return AnalyticsMetrics(self._raw.get("metrics", {})) + return None + + def __repr__(self): + return "AnalyticsMetaData:{}".format(self._raw) + + +class AnalyticsQuery: + + _VALID_OPTS = { + 'timeout': {'timeout': lambda x: x}, + 'read_only': {'readonly': lambda x: x}, + 'scan_consistency': {'consistency': lambda x: x.value}, + 'client_context_id': {'client_context_id': lambda x: x}, + 'priority': {'priority': lambda x: x}, + 'query_context': {'query_context': lambda x: x}, + 'serializer': {'serializer': lambda x: x}, + 'raw': {'raw': lambda x: x}, + 'positional_parameters': {}, + 'named_parameters': {}, + 'span': {'span': lambda x: x} + } + + def __init__(self, query, *args, **kwargs): + + self._adhoc = True + self._params = {"statement": query} + self._raw = None + if args: + self._add_pos_args(*args) + if kwargs: + self._set_named_args(**kwargs) + + def _set_named_args(self, **kv): + """ + Set a named parameter in the query. The named field must + exist in the query itself. + + :param kv: Key-Value pairs representing values within the + query. These values should be stripped of their leading + `$` identifier. + + """ + # named_params = {} + # for k in kv: + # named_params["${0}".format(k)] = json.dumps(kv[k]) + # couchbase++ wants all args JSONified + named_params = {f'${k}': json.dumps(v) for k, v in kv.items()} + + self._params["named_parameters"] = named_params + return self + + def _add_pos_args(self, *args): + """ + Set values for *positional* placeholders (``$1,$2,...``) + + :param args: Values to be used + """ + arg_array = self._params.setdefault("positional_parameters", []) + # couchbase++ wants all args JSONified + json_args = [json.dumps(arg) for arg in args] + arg_array.extend(json_args) + + def set_option(self, name, value): + """ + Set a raw option in the query. This option is encoded + as part of the query parameters without any client-side + verification. Use this for settings not directly exposed + by the Python client. + + :param name: The name of the option + :param value: The value of the option + """ + self._params[name] = value + + @property + def params(self): + return self._params + + @property + def timeout(self) -> Optional[float]: + value = self._params.get('timeout', None) + if not value: + return None + value = value[:-1] + return float(value) + + @timeout.setter + def timeout(self, value # type: Union[timedelta,float,int] + ) -> None: + if not value: + self._params.pop('timeout', 0) + else: + total_us = to_microseconds(value) + self.set_option('timeout', total_us) + + @property + def metrics(self): + return self._params.get("metrics", True) + + @metrics.setter + def metrics(self, value): + self.set_option("metrics", value) + + @property + def priority(self): + return self._params.get("priority", False) + + @priority.setter + def priority(self, value): + self.set_option("priority", value) + + @property + def statement(self): + return self._params["statement"] + + @property + def consistency(self): + return self._params.get( + "scan_consistency", AnalyticsScanConsistency.NOT_BOUNDED.value + ) + + @consistency.setter + def consistency(self, value): + self._params["scan_consistency"] = value + + @property + def client_context_id(self) -> Optional[str]: + return self._params.get('client_context_id', None) + + @client_context_id.setter + def client_context_id(self, value # type: str + ) -> None: + self.set_option('client_context_id', value) + + @property + def readonly(self): + value = self._params.get("readonly", False) + return value + + @readonly.setter + def readonly(self, value): + self._params["readonly"] = value + + @property + def query_context(self) -> Optional[str]: + return self._params.get('scope_qualifier', None) + + @query_context.setter + def query_context(self, value # type: str + ) -> None: + self.set_option('scope_qualifier', value) + + @property + def serializer(self): + return self._params.get("serializer", None) + + @serializer.setter + def serializer(self, value): + if not issubclass(value.__class__, Serializer): + raise InvalidArgumentException('Serializer should implement Serializer interface.') + self._params["serializer"] = value + + @property + def raw(self) -> Optional[Dict[str, Any]]: + return self._params.get('raw', None) + + @raw.setter + def raw(self, value # type: Dict[str, Any] + ) -> None: + if not isinstance(value, dict): + raise TypeError("Raw option must be of type Dict[str, Any].") + for k in value.keys(): + if not isinstance(k, str): + raise TypeError("key for raw value must be str") + raw_params = {f'{k}': json.dumps(v) for k, v in value.items()} + self.set_option('raw', raw_params) + + @property + def span(self) -> Optional[CouchbaseSpan]: + return self._params.get('span', None) + + @span.setter + def span(self, value # type: CouchbaseSpan + ): + if not issubclass(value.__class__, CouchbaseSpan): + raise InvalidArgumentException('Span should implement CouchbaseSpan interface.') + self.set_option('span', value) + + @classmethod + def create_query_object(cls, statement, *options, **kwargs): + # lets make a copy of the options, and update with kwargs... + opt = AnalyticsOptions() + # TODO: is it possible that we could have [AnalyticsOptions, AnalyticsOptions, ...]?? + # If so, why??? + opts = list(options) + for o in opts: + if isinstance(o, (AnalyticsOptions, AnalyticsOptionsBase)): + opt = o + opts.remove(o) + args = opt.copy() + args.update(kwargs) + + # now lets get positional parameters. Actual positional + # params OVERRIDE positional_parameters + positional_parameters = args.pop("positional_parameters", []) + if opts and len(opts) > 0: + positional_parameters = opts + + # now the named parameters. NOTE: all the kwargs that are + # not VALID_OPTS must be named parameters, and the kwargs + # OVERRIDE the list of named_parameters + new_keys = list(filter(lambda x: x not in cls._VALID_OPTS, args.keys())) + named_parameters = args.pop("named_parameters", {}) + for k in new_keys: + named_parameters[k] = args[k] + + query = cls(statement, *positional_parameters, **named_parameters) + # now lets try to setup the options. + # but for now we will use the existing N1QLQuery. Could be we can + # add to it, etc... + + # default to True on analytics metrics + query.metrics = args.get("metrics", True) + + for k, v in ((k, args[k]) for k in (args.keys() & cls._VALID_OPTS)): + for target, transform in cls._VALID_OPTS[k].items(): + setattr(query, target, transform(v)) + return query + + +class AnalyticsRequestLogic: + def __init__(self, + connection, + query_params, + row_factory=lambda x: x, + **kwargs + ): + + self._connection = connection + self._query_params = query_params + self.row_factory = row_factory + self._streaming_result = None + self._default_serializer = kwargs.pop('default_serializer', DefaultJsonSerializer()) + self._serializer = None + self._started_streaming = False + self._streaming_timeout = kwargs.pop('streaming_timeout', None) + self._done_streaming = False + self._metadata = None + + @property + def params(self) -> Dict[str, Any]: + return self._query_params + + @property + def serializer(self) -> Serializer: + if self._serializer: + return self._serializer + + serializer = self.params.get('serializer', None) + if not serializer: + serializer = self._default_serializer + + self._serializer = serializer + return self._serializer + + @property + def started_streaming(self) -> bool: + return self._started_streaming + + @property + def done_streaming(self) -> bool: + return self._done_streaming + + def metadata(self): + # @TODO: raise if query isn't complete? + return self._metadata + + def _set_metadata(self, analytics_response): + if isinstance(analytics_response, CouchbaseBaseException): + raise ErrorMapper.build_exception(analytics_response) + + self._metadata = AnalyticsMetaData(analytics_response.raw_result.get('value', None)) + + def _submit_query(self, **kwargs): + if self.done_streaming: + return + + self._started_streaming = True + analytics_kwargs = { + 'conn': self._connection, + } + analytics_kwargs.update(self.params) + + streaming_timeout = self.params.get('timeout', self._streaming_timeout) + if streaming_timeout: + analytics_kwargs['streaming_timeout'] = streaming_timeout + + # this is for txcouchbase... + callback = kwargs.pop('callback', None) + if callback: + analytics_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + analytics_kwargs['errback'] = errback + + self._streaming_result = analytics_query(**analytics_kwargs) + + def __iter__(self): + raise NotImplementedError( + 'Cannot use synchronous iterator, are you using `async for`?' + ) + + def __aiter__(self): + raise NotImplementedError( + 'Cannot use asynchronous iterator.' + ) diff --git a/couchbase/logic/bucket.py b/couchbase/logic/bucket.py new file mode 100644 index 000000000..b8c7ba469 --- /dev/null +++ b/couchbase/logic/bucket.py @@ -0,0 +1,136 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict, + Optional) + +from couchbase.diagnostics import ServiceType +from couchbase.exceptions import InvalidArgumentException +from couchbase.options import forward_args +from couchbase.pycbc_core import (diagnostics_operation, + open_or_close_bucket, + operations) +from couchbase.result import PingResult +from couchbase.serializer import Serializer +from couchbase.transcoder import Transcoder + +if TYPE_CHECKING: + from couchbase.options import PingOptions + + +class BucketLogic: + def __init__(self, cluster, bucket_name): + self._cluster = cluster + self._connection = cluster.connection + self._bucket_name = bucket_name + self._connected = False + + @property + def connection(self): + """ + **INTERNAL** + """ + return self._connection + + @property + def streaming_timeouts(self): + """ + **INTERNAL** + """ + return self._cluster.streaming_timeouts + + @property + def default_transcoder(self) -> Optional[Transcoder]: + return self._cluster.default_transcoder + + @property + def default_serializer(self) -> Optional[Serializer]: + return self._cluster.default_serializer + + @property + def connected(self) -> bool: + """ + bool: Indicator on if the bucket has been connected or not. + """ + return self._connected + + @property + def name(self): + """ + str: The name of this :class:`~.Bucket` instance. + """ + return self._bucket_name + + def _open_or_close_bucket(self, open_bucket=True, **kwargs): + if not self._connection: + raise RuntimeError("No cluster connection") + + bucket_kwargs = { + "open_bucket": 1 if open_bucket is True else 0 + } + + callback = kwargs.pop('callback', None) + if callback: + bucket_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + bucket_kwargs['errback'] = errback + + return open_or_close_bucket(self._connection, self._bucket_name, **bucket_kwargs) + + def _set_connected(self, value): + self._connected = value + + def _destroy_connection(self): + del self._cluster + del self._connection + + def ping(self, + *opts, # type: PingOptions + **kwargs # type: Dict[str,Any] + ) -> Optional[PingResult]: + + ping_kwargs = { + 'conn': self._connection, + 'op_type': operations.PING.value + } + + callback = kwargs.pop('callback', None) + if callback: + ping_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + ping_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *opts) + service_types = final_args.get("service_types", None) + if not service_types: + service_types = list( + map(lambda st: st.value, [ServiceType(st.value) for st in ServiceType])) + + if not isinstance(service_types, list): + raise InvalidArgumentException("Service types must be a list/set.") + + service_types = list(map(lambda st: st.value if isinstance(st, ServiceType) else st, service_types)) + final_args["service_types"] = service_types + # TODO: tracing + # final_args.pop("span", None) + + ping_kwargs.update(final_args) + return diagnostics_operation(**ping_kwargs) diff --git a/couchbase/logic/cluster.py b/couchbase/logic/cluster.py new file mode 100644 index 000000000..6ee3a8535 --- /dev/null +++ b/couchbase/logic/cluster.py @@ -0,0 +1,536 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import logging +import warnings +from typing import (TYPE_CHECKING, + Any, + Dict, + List, + Optional, + Tuple, + Union) +from urllib.parse import parse_qs, urlparse + +from couchbase import USER_AGENT_EXTRA +from couchbase.auth import CertificateAuthenticator, PasswordAuthenticator +from couchbase.diagnostics import ServiceType +from couchbase.exceptions import InvalidArgumentException +from couchbase.options import (ClusterOptions, + ClusterTimeoutOptions, + ClusterTracingOptions, + TLSVerifyMode, + TransactionConfig, + forward_args, + get_valid_args) +from couchbase.pycbc_core import (close_connection, + cluster_mgmt_operations, + create_connection, + diagnostics_operation, + get_connection_info, + management_operation, + mgmt_operations, + operations) +from couchbase.result import (ClusterInfoResult, + DiagnosticsResult, + PingResult) +from couchbase.serializer import DefaultJsonSerializer, Serializer +from couchbase.transcoder import JSONTranscoder, Transcoder + +if TYPE_CHECKING: + from couchbase.options import DiagnosticsOptions, PingOptions + +log = logging.getLogger(__name__) + + +class ClusterLogic: + + _LEGACY_CONNSTR_QUERY_ARGS = { + 'ssl': {'tls_verify': TLSVerifyMode.to_str}, + 'certpath': {'cert_path': lambda x: x}, + 'cert_path': {'cert_path': lambda x: x}, + 'truststorepath': {'trust_store_path': lambda x: x}, + 'trust_store_path': {'trust_store_path': lambda x: x}, + 'sasl_mech_force': {'sasl_mech_force': lambda x: x.split(',') if isinstance(x, str) else x} + } + + def __init__(self, # noqa: C901 + connstr, # type: str + *options, # type: ClusterOptions + **kwargs # type: Dict[str, Any] + ) -> ClusterLogic: + + # parse query string prior to parsing ClusterOptions + connection_str, query_opts, legacy_opts = self._parse_connection_string(connstr) + self._connstr = connection_str + + kwargs.update(query_opts) + cluster_opts = get_valid_args(ClusterOptions, kwargs, *options) + # add legacy options after parsing ClusterOptions to keep logic separate + cluster_opts.update(self._parse_legacy_query_options(**legacy_opts)) + + authenticator = cluster_opts.pop("authenticator", None) + if not authenticator: + raise InvalidArgumentException(message="Authenticator is mandatory.") + + # the cert_path _might_ be a part of the query options + cert_path = cluster_opts.pop('cert_path', None) + + # lets only pass in the authenticator, no kwargs + auth_kwargs = {k: v for k, v in cluster_opts.items() if k in authenticator.valid_keys()} + if isinstance(authenticator, CertificateAuthenticator) and 'trust_store_path' in auth_kwargs: + # the trust_store_path _should_ be in the cluster opts, however <= 3.x SDK allowed it in + # the CertificateAuthenticator, pop the trust_store_path from the auth_kwargs in case + if 'trust_store_path' not in authenticator.as_dict(): + auth_kwargs.pop('trust_store_path') + elif ( + isinstance(authenticator, PasswordAuthenticator) and + any(map(lambda mech: mech in auth_kwargs, ['sasl_mech_force', 'allowed_sasl_mechanisms'])) + ): + # 3.x SDK allowed sasl_mech_force in the query string, 4.x is going to allow allowed_sasl_mechanisms + # in the query string, if the PasswordAuthenticator's static ldap_compatible() method has been used + # it takes precendence. + # Set the PasswordAuthenticator's allowed_sasl_mechanisms, if not already set, to the value(s) found in the + # connection string. Note that allowed_sasl_mechanisms takes precendence over sasl_mech_force. + for sasl_key in ['sasl_mech_force', 'allowed_sasl_mechanisms']: + if sasl_key not in cluster_opts: + continue + # pop from auth_kwargs to satisfy future check + auth_kwargs.pop(sasl_key) + if authenticator._allowed_sasl_mechanisms is not None: + continue + if isinstance(cluster_opts[sasl_key], str): + authenticator._allowed_sasl_mechanisms = [cluster_opts[sasl_key]] + else: + authenticator._allowed_sasl_mechanisms = cluster_opts[sasl_key] + + if len(auth_kwargs.keys()) > 0: + raise InvalidArgumentException( + message="Authentication kwargs not allowed. Only provide the Authenticator.") + + self._auth = authenticator.as_dict() + # add the cert_path to the authenticator if found + if cert_path and 'cert_path' not in self._auth: + self._auth['cert_path'] = cert_path + + # after cluster options have been parsed (both from the query string and provided + # options/kwargs), separate into cluster options, timeout options and tracing options and txns config. + + self._transaction_config = cluster_opts.pop("transaction_config", TransactionConfig()) + self._transactions = None + + timeout_opts = {} + for key in ClusterTimeoutOptions.get_allowed_option_keys(use_transform_keys=True): + if key in cluster_opts: + timeout_opts[key] = cluster_opts.pop(key) + if timeout_opts: + cluster_opts['timeout_options'] = timeout_opts + + tracing_opts = {} + for key in ClusterTracingOptions.get_allowed_option_keys(use_transform_keys=True): + if key in cluster_opts: + tracing_opts[key] = cluster_opts.pop(key) + if tracing_opts: + cluster_opts['tracing_options'] = tracing_opts + + self._default_serializer = cluster_opts.pop("serializer", None) + if not self._default_serializer: + self._default_serializer = DefaultJsonSerializer() + + self._default_transcoder = cluster_opts.pop("transcoder", None) + if not self._default_transcoder: + self._default_transcoder = JSONTranscoder() + + cluster_opts['user_agent_extra'] = USER_AGENT_EXTRA + + self._cluster_opts = cluster_opts + self._streaming_timeouts = dict( + analytics_timeout=timeout_opts.get('analytics_timeout', None), + query_timeout=timeout_opts.get('query_timeout', None), + search_timeout=timeout_opts.get('search_timeout', None), + view_timeout=timeout_opts.get('view_timeout', None), + ) + self._connection = None + self._cluster_info = None + self._server_version = None + + def __del__(self): + self._destroy_connection() + + @property + def connection(self): + """ + **INTERNAL** + """ + if not hasattr(self, "_connection"): + self._connection = None + return self._connection + + @property + def default_transcoder(self) -> Optional[Transcoder]: + """ + **INTERNAL** not intended for use in public API. + """ + return self._default_transcoder + + @default_transcoder.setter + def default_transcoder(self, + value # type: Transcoder + ): + """ + **INTERNAL** not intended for use in public API. + """ + if not issubclass(value.__class__, Transcoder): + raise InvalidArgumentException('Cannot set default transcoder to non Transcoder type.') + + self._default_transcoder = value + + @property + def default_serializer(self) -> Optional[Serializer]: + return self._default_serializer + + @property + def serializer(self) -> Serializer: + return self._serializer + + @property + def connected(self) -> bool: + """ + bool: Indicator on if the cluster has been connected or not. + """ + return hasattr(self, "_connection") and self._connection is not None + + @property + def server_version(self) -> Optional[str]: + if self._cluster_info: + return self._cluster_info.server_version + + return None + + @property + def server_version_short(self) -> Optional[float]: + if self._cluster_info: + return self._cluster_info.server_version_short + + return None + + @property + def server_version_full(self) -> Optional[str]: + if self._cluster_info: + return self._cluster_info.server_version_full + + return None + + @property + def is_developer_preview(self) -> Optional[bool]: + if self._cluster_info: + return False + return None + + @property + def streaming_timeouts(self): + """ + **INTERNAL** + """ + return self._streaming_timeouts + + def _parse_connection_string(self, connection_str # type: str + ) -> Tuple[str, Dict[str, Any], Dict[str, Any]]: + """Parse the provided connection string + + The provided connection string will be parsed to split the connection string + and the query options. Query options will be split into legacy options + and 'current' options. + + Args: + connection_str (str): The connection string for the cluster. + + Returns: + Tuple[str, Dict[str, Any], Dict[str, Any]]: The parsed connection string, + current options and legacy options. + """ + # handle possible lack of URL scheme + if '//' not in connection_str: + warning_msg = 'Connection string has deprecated format. Start connection string with: couchbase://' + warnings.warn(warning_msg, DeprecationWarning, stacklevel=2) + connection_str = f'//{connection_str}' + + parsed_conn = urlparse(connection_str) + conn_str = '' + if parsed_conn.scheme: + conn_str = f'{parsed_conn.scheme}://{parsed_conn.netloc}{parsed_conn.path}' + else: + conn_str = f'{parsed_conn.netloc}{parsed_conn.path}' + query_str = parsed_conn.query + query_str_opts, legacy_query_str_opts = self._parse_query_string_options(query_str) + + return conn_str, query_str_opts, legacy_query_str_opts + + def _parse_query_string_options(self, + query_str + ) -> Tuple[Dict[str, Any], Dict[str, Any]]: + """Parse the query string options + + Query options will be split into legacy options and 'current' options. The values for the + 'current' options are cast to integers or booleans where applicable + + Args: + query_str (str): The query string. + + Returns: + Tuple[Dict[str, Any], Dict[str, Any]]: The parsed current options and legacy options. + """ + options = parse_qs(query_str) + + # @TODO: issue warning if it is overriding cluster options? + legacy_query_str_opts = {} + for k, v in options.items(): + if k not in self._LEGACY_CONNSTR_QUERY_ARGS.keys(): + continue + if len(v) > 1: + legacy_query_str_opts[k] = v + else: + legacy_query_str_opts[k] = v[0] + + query_str_opts = {} + for k, v in options.items(): + if k in self._LEGACY_CONNSTR_QUERY_ARGS.keys(): + continue + query_str_opts[k] = self._parse_query_string_value(v) + + return query_str_opts, legacy_query_str_opts + + def _parse_query_string_value(self, + value # type: List[str] + ) -> Union[List[str], str, bool, int]: + """Parse a query string value + + The provided value is a list of at least one element. Returns either a list of strings or a single element + which might be cast to an integer or a boolean if that's appropriate. + + Args: + value (List[str]): The query string value. + + Returns: + Union[List[str], str, bool, int]: The parsed current options and legacy options. + """ + + if len(value) > 1: + return value + v = value[0] + if v.isnumeric(): + return int(v) + elif v.lower() in ['true', 'false']: + return v.lower() == 'true' + return v + + def _parse_legacy_query_options(self, **query_opts # type: Dict[str, Any] + ) -> Dict[str, Any]: + """Parse legacy query string options + + See :attr:`~.ClusterLogic._LEGACY_CONNSTR_QUERY_ARGS` + + Returns: + Dict[str, Any]: Representation of parsed query string parameters. + """ + final_options = {} + for opt_key, opt_value in query_opts.items(): + if opt_key not in self._LEGACY_CONNSTR_QUERY_ARGS: + continue + for final_key, transform in self._LEGACY_CONNSTR_QUERY_ARGS[opt_key].items(): + converted = transform(opt_value) + if converted is not None: + final_options[final_key] = converted + return final_options + + def _get_connection_opts(self, auth_only=False, # type: Optional[bool] + conn_only=False # type: Optional[bool] + ) -> Union[Dict[str, Any], Dict[str, Any], Tuple[Dict[str, Any], Dict[str, Any]]]: + """Get connection related options + + **INTERNAL** not intended for use in public API. + + Args: + auth_only (bool, optional): Set to True to return only auth options. Defaults to False. + conn_only (bool, optional): Set to True to return only cluster options. Defaults to False. + + Returns: + Union[Dict[str, Any], Dict[str, Any], Tuple[Dict[str, Any], Dict[str, Any]]]: Either the + cluster auth, cluster options or a tuple of both the cluster auth and cluster options. + """ + if auth_only is True: + return self._auth + if conn_only is True: + return self._cluster_opts + return self._auth, self._cluster_opts + + def _get_client_connection_info(self) -> Dict[str, Any]: + """Get connection related options from the cxx client + + **INTERNAL** not intended for use in public API. + + Returns: + Dict[str, Any]: All connection related information. + """ + + return get_connection_info(self._connection) + + def _connect_cluster(self, **kwargs): + + connect_kwargs = { + 'auth': self._auth, + 'options': self._cluster_opts + } + + callback = kwargs.pop('callback', None) + if callback: + connect_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + connect_kwargs['errback'] = errback + + return create_connection( + self._connstr, **connect_kwargs, + ) + + def _close_cluster(self, **kwargs): + + # first close the transactions object, if any + log.debug("closing cluster") + if self._transactions: + self._transactions.close() + del self._transactions + + close_kwargs = {} + + callback = kwargs.pop('callback', None) + if callback: + close_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + close_kwargs['errback'] = errback + + return close_connection( + self._connection, **close_kwargs + ) + + def _set_connection(self, conn): + self._connection = conn + + def _destroy_connection(self): + if hasattr(self, '_connection'): + self._connection = None + + def _get_cluster_info(self, **kwargs) -> Optional[ClusterInfoResult]: + + cluster_info_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.CLUSTER.value, + "op_type": cluster_mgmt_operations.GET_CLUSTER_INFO.value + } + + callback = kwargs.pop('callback', None) + if callback: + cluster_info_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + cluster_info_kwargs['errback'] = errback + + return management_operation(**cluster_info_kwargs) + + def _enable_dp(self, **kwargs): + """ + @TODO(jc): cxx client: + libc++abi: terminating with uncaught exception of type + std::runtime_error: cannot map key: partition map is not available + """ + + enable_dp_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.CLUSTER.value, + "op_type": cluster_mgmt_operations.ENABLE_DP.value + } + + callback = kwargs.pop('callback', None) + if callback: + enable_dp_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + enable_dp_kwargs['errback'] = errback + + return management_operation(**enable_dp_kwargs) + + def ping(self, + *opts, # type: PingOptions + **kwargs # type: Any + ) -> Optional[PingResult]: + + ping_kwargs = { + 'conn': self._connection, + 'op_type': operations.PING.value + } + + callback = kwargs.pop('callback', None) + if callback: + ping_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + ping_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *opts) + service_types = final_args.get("service_types", None) + if not service_types: + service_types = list( + map(lambda st: st.value, [ServiceType(st.value) for st in ServiceType])) + + if not isinstance(service_types, list): + raise InvalidArgumentException("Service types must be a list/set.") + + service_types = list(map(lambda st: st.value if isinstance(st, ServiceType) else st, service_types)) + final_args["service_types"] = service_types + # TODO: tracing + # final_args.pop("span", None) + + ping_kwargs.update(final_args) + return diagnostics_operation(**ping_kwargs) + + def diagnostics(self, + *opts, # type: DiagnosticsOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[DiagnosticsResult]: + + diagnostics_kwargs = { + 'conn': self._connection, + 'op_type': operations.DIAGNOSTICS.value + } + + callback = kwargs.pop('callback', None) + if callback: + diagnostics_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + diagnostics_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *opts) + diagnostics_kwargs.update(final_args) + return diagnostics_operation(**diagnostics_kwargs) diff --git a/couchbase/logic/collection.py b/couchbase/logic/collection.py new file mode 100644 index 000000000..89b663f50 --- /dev/null +++ b/couchbase/logic/collection.py @@ -0,0 +1,630 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +from datetime import timedelta +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + Optional, + Union) + +from couchbase._utils import timedelta_as_microseconds +from couchbase.exceptions import InvalidArgumentException +from couchbase.kv_range_scan import (PrefixScan, + RangeScan, + SamplingScan) +from couchbase.logic.options import DeltaValueBase, SignedInt64Base +from couchbase.mutation_state import MutationState +from couchbase.options import forward_args +from couchbase.pycbc_core import (binary_operation, + kv_operation, + operations, + subdoc_operation) +from couchbase.result import (CounterResult, + ExistsResult, + GetReplicaResult, + GetResult, + LookupInReplicaResult, + LookupInResult, + MutateInResult, + MutationResult) +from couchbase.subdocument import (Spec, + StoreSemantics, + SubDocOp) +from couchbase.transcoder import Transcoder + +if TYPE_CHECKING: + from couchbase._utils import JSONType + from couchbase.options import (AppendOptions, + DecrementOptions, + ExistsOptions, + IncrementOptions, + InsertOptions, + MutateInOptions, + MutationOptions, + PrependOptions, + RemoveOptions, + ReplaceOptions, + TouchOptions, + UnlockOptions, + UpsertOptions) + + +class CollectionLogic: + def __init__(self, scope, name): + if not scope: + raise InvalidArgumentException(message="Collection must be given a scope") + # if not scope.connection: + # raise RuntimeError("No connection provided") + self._scope = scope + self._collection_name = name + self._connection = scope.connection + + @property + def connection(self): + """ + **INTERNAL** + """ + return self._connection + + @property + def default_transcoder(self) -> Optional[Transcoder]: + return self._scope.default_transcoder + + @property + def name(self) -> str: + """ + str: The name of this :class:`~.Collection` instance. + """ + return self._collection_name + + def _set_connection(self): + """ + **INTERNAL** + """ + self._connection = self._scope.connection + + def _get_connection_args(self) -> Dict[str, Any]: + return { + "conn": self._connection, + "bucket": self._scope.bucket_name, + "scope": self._scope.name, + "collection_name": self.name + } + + def _get_mutation_options(self, + *opts, # type: MutationOptions + **kwargs # type: Dict[str, Any] + ) -> Dict[str, Any]: + """**INTERNAL** + Parses the mutaiton operation options. If synchronous durability has been set and no timeout provided, the + default timeout will be set to the default KV durable timeout (10 seconds). + """ + args = forward_args(kwargs, *opts) + if 'durability' in args and isinstance(args['durability'], int) and 'timeout' not in args: + args['timeout'] = timedelta_as_microseconds(timedelta(seconds=10)) + + return args + + def get( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Optional[GetResult]: + """**INTERNAL** + + Key-Value *get* operation. Should only be called by classes that inherit from the base + class :class:`~couchbase.logic.CollectionLogic`. + + Args: + key (str): document key + kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + overrride provided :class:`~.options.GetOptions` + + Raises: + :class:`~.exceptions.DocumentNotFoundException`: If the provided document key does not exist. + """ + op_type = operations.GET.value + return kv_operation(**self._get_connection_args(), + key=key, + op_type=op_type, + op_args=kwargs) + + def get_any_replica( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Optional[GetReplicaResult]: + """**INTERNAL** + + Key-Value *get_any_replica* operation. Should only be called by classes that inherit from the base + class :class:`~couchbase.logic.CollectionLogic`. + + Args: + key (str): document key + kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + overrride provided :class:`~.options.GetAnyReplicaOptions` + + Raises: + :class:`~.exceptions.DocumentNotFoundException`: If the provided document key does not exist. + """ + op_type = operations.GET_ANY_REPLICA.value + return kv_operation(**self._get_connection_args(), + key=key, + op_type=op_type, + op_args=kwargs) + + def get_all_replicas( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Optional[Iterable[GetReplicaResult]]: + """**INTERNAL** + + Key-Value *get_all_replicas* operation. Should only be called by classes that inherit from the base + class :class:`~couchbase.logic.CollectionLogic`. + + Args: + key (str): document key + kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + overrride provided :class:`~.options.GetAllReplicasOptions` + + Raises: + :class:`~.exceptions.DocumentNotFoundException`: If the provided document key does not exist. + """ + op_type = operations.GET_ALL_REPLICAS.value + return kv_operation(**self._get_connection_args(), + key=key, + op_type=op_type, + op_args=kwargs) + + def exists( + self, + key, # type: str + *opts, # type: ExistsOptions + **kwargs, # type: Any + ) -> Optional[ExistsResult]: + op_type = operations.EXISTS.value + return kv_operation( + **self._get_connection_args(), key=key, op_type=op_type, op_args=forward_args(kwargs, *opts) + ) + + def insert( + self, + key, # type: str + value, # type: JSONType + *opts, # type: InsertOptions + **kwargs, # type: Any + ) -> Optional[MutationResult]: + final_args = self._get_mutation_options(*opts, **kwargs) + transcoder = final_args.pop('transcoder', self.default_transcoder) + transcoded_value = transcoder.encode_value(value) + op_type = operations.INSERT.value + return kv_operation( + **self._get_connection_args(), + key=key, + value=transcoded_value, + op_type=op_type, + op_args=final_args + ) + + def upsert( + self, + key, # type: str + value, # type: JSONType + *opts, # type: UpsertOptions + **kwargs, # type: Any + ) -> Optional[MutationResult]: + final_args = self._get_mutation_options(*opts, **kwargs) + transcoder = final_args.pop('transcoder', self.default_transcoder) + transcoded_value = transcoder.encode_value(value) + + op_type = operations.UPSERT.value + return kv_operation( + **self._get_connection_args(), + key=key, + value=transcoded_value, + op_type=op_type, + op_args=final_args + ) + + def replace(self, + key, # type: str + value, # type: JSONType + *opts, # type: ReplaceOptions + **kwargs, # type: Any + ) -> Optional[MutationResult]: + final_args = self._get_mutation_options(*opts, **kwargs) + expiry = final_args.get("expiry", None) + preserve_expiry = final_args.get("preserve_expiry", False) + if expiry and preserve_expiry is True: + raise InvalidArgumentException( + "The expiry and preserve_expiry options cannot both be set for replace operations." + ) + + transcoder = final_args.pop('transcoder', self.default_transcoder) + transcoded_value = transcoder.encode_value(value) + + op_type = operations.REPLACE.value + return kv_operation( + **self._get_connection_args(), + key=key, + value=transcoded_value, + op_type=op_type, + op_args=final_args + ) + + def remove(self, + key, # type: str + *opts, # type: RemoveOptions + **kwargs, # type: Any + ) -> Optional[MutationResult]: + final_args = self._get_mutation_options(*opts, **kwargs) + op_type = operations.REMOVE.value + return kv_operation( + **self._get_connection_args(), key=key, op_type=op_type, op_args=final_args + ) + + def touch(self, + key, # type: str + expiry, # type: timedelta + *opts, # type: TouchOptions + **kwargs, # type: Any + ) -> Optional[MutationResult]: + kwargs["expiry"] = expiry + op_type = operations.TOUCH.value + return kv_operation( + **self._get_connection_args(), key=key, op_type=op_type, op_args=forward_args(kwargs, *opts) + ) + + def get_and_touch(self, + key, # type: str + **kwargs, # type: Any + ) -> Optional[GetResult]: + op_type = operations.GET_AND_TOUCH.value + return kv_operation( + **self._get_connection_args(), key=key, op_type=op_type, op_args=kwargs + ) + + def get_and_lock(self, + key, # type: str + **kwargs, # type: Any + ) -> Optional[GetResult]: + op_type = operations.GET_AND_LOCK.value + return kv_operation( + **self._get_connection_args(), key=key, op_type=op_type, op_args=kwargs + ) + + def unlock(self, + key, # type: str + cas, # type: int + *opts, # type: UnlockOptions + **kwargs, # type: Any + ) -> None: + op_type = operations.UNLOCK.value + final_args = forward_args(kwargs, *opts) + final_args['cas'] = cas + return kv_operation( + **self._get_connection_args(), + key=key, + op_type=op_type, + op_args=final_args + ) + + def lookup_in(self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Any + ) -> Optional[LookupInResult]: + op_type = operations.LOOKUP_IN.value + return subdoc_operation( + **self._get_connection_args(), + key=key, + spec=spec, + op_type=op_type, + op_args=kwargs + ) + + def lookup_in_all_replicas(self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Any + ) -> Optional[Iterable[LookupInReplicaResult]]: + op_type = operations.LOOKUP_IN_ALL_REPLICAS.value + return subdoc_operation( + **self._get_connection_args(), + key=key, + spec=spec, + op_type=op_type, + op_args=kwargs + ) + + def lookup_in_any_replica(self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Any + ) -> Optional[LookupInReplicaResult]: + op_type = operations.LOOKUP_IN_ANY_REPLICA.value + return subdoc_operation( + **self._get_connection_args(), + key=key, + spec=spec, + op_type=op_type, + op_args=kwargs + ) + + def mutate_in( # noqa: C901 + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: MutateInOptions + **kwargs, # type: Any + ) -> Optional[MutateInResult]: # noqa: C901 + # no tc for sub-doc, use default JSON + final_args = self._get_mutation_options(*opts, **kwargs) + transcoder = final_args.pop('transcoder', self.default_transcoder) + + expiry = final_args.get('expiry', None) + preserve_expiry = final_args.get('preserve_expiry', False) + + spec_ops = [s[0] for s in spec] + if SubDocOp.DICT_ADD in spec_ops and preserve_expiry is True: + raise InvalidArgumentException( + 'The preserve_expiry option cannot be set for mutate_in with insert operations.') + + if SubDocOp.REPLACE in spec_ops and expiry and preserve_expiry is True: + raise InvalidArgumentException( + 'The expiry and preserve_expiry options cannot both be set for mutate_in with replace operations.') + + """ + @TODO(jc): document that the kwarg will override option: + await cb.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.INSERT), + upsert_doc=True) + + will set store_semantics to be UPSERT + """ + + insert_semantics = final_args.pop('insert_doc', None) + upsert_semantics = final_args.pop('upsert_doc', None) + replace_semantics = final_args.pop('replace_doc', None) + if insert_semantics is not None and (upsert_semantics is not None or replace_semantics is not None): + raise InvalidArgumentException("Cannot set multiple store semantics.") + if upsert_semantics is not None and (insert_semantics is not None or replace_semantics is not None): + raise InvalidArgumentException("Cannot set multiple store semantics.") + + if insert_semantics is not None: + final_args["store_semantics"] = StoreSemantics.INSERT + if upsert_semantics is not None: + final_args["store_semantics"] = StoreSemantics.UPSERT + if replace_semantics is not None: + final_args["store_semantics"] = StoreSemantics.REPLACE + + final_spec = [] + allowed_multi_ops = [SubDocOp.ARRAY_PUSH_FIRST, + SubDocOp.ARRAY_PUSH_LAST, + SubDocOp.ARRAY_INSERT] + + for s in spec: + if len(s) == 6: + tmp = list(s[:5]) + if s[0] in allowed_multi_ops: + new_value = json.dumps(s[5], ensure_ascii=False) + # this is an array, need to remove brackets + tmp.append(new_value[1:len(new_value)-1].encode('utf-8')) + else: + # no need to propagate the flags + tmp.append(transcoder.encode_value(s[5])[0]) + final_spec.append(tuple(tmp)) + else: + final_spec.append(s) + + op_type = operations.MUTATE_IN.value + return subdoc_operation( + **self._get_connection_args(), + key=key, + spec=final_spec, + op_type=op_type, + op_args=final_args + ) + + def _validate_delta_initial(self, delta=None, initial=None) -> None: + # @TODO: remove deprecation next .minor + # from couchbase.collection import DeltaValueDeprecated, SignedInt64Deprecated + if delta is not None: + if not DeltaValueBase.is_valid(delta): + raise InvalidArgumentException("Argument is not valid DeltaValue") + if initial is not None: + if not SignedInt64Base.is_valid(initial): + raise InvalidArgumentException("Argument is not valid SignedInt64") + + def _get_and_validate_delta_initial(self, final_args): + initial = final_args.get('initial', None) + delta = final_args.get('delta', None) + if not initial: + initial = SignedInt64Base(0) + if not delta: + delta = DeltaValueBase(1) + + self._validate_delta_initial(delta=delta, initial=initial) + + return delta, initial + + def increment( + self, + key, # type: str + *opts, # type: IncrementOptions + **kwargs, # type: Any + ) -> Optional[CounterResult]: + final_args = self._get_mutation_options(*opts, **kwargs) + if not final_args.get('initial', None): + final_args['initial'] = SignedInt64Base(0) + if not final_args.get('delta', None): + final_args['delta'] = DeltaValueBase(1) + + self._validate_delta_initial(delta=final_args['delta'], + initial=final_args['initial']) + + op_type = operations.INCREMENT.value + final_args['initial'] = int(final_args['initial']) + final_args['delta'] = int(final_args['delta']) + + if final_args['initial'] < 0: + # Negative 'initial' means no initial value + del final_args['initial'] + + return binary_operation(**self._get_connection_args(), + key=key, + op_type=op_type, + op_args=final_args) + + def decrement( + self, + key, # type: str + *opts, # type: DecrementOptions + **kwargs, # type: Any + ) -> Optional[CounterResult]: + final_args = self._get_mutation_options(*opts, **kwargs) + if not final_args.get('initial', None): + final_args['initial'] = SignedInt64Base(0) + if not final_args.get('delta', None): + final_args['delta'] = DeltaValueBase(1) + + self._validate_delta_initial(delta=final_args['delta'], + initial=final_args['initial']) + + op_type = operations.DECREMENT.value + final_args['initial'] = int(final_args['initial']) + final_args['delta'] = int(final_args['delta']) + + if final_args['initial'] < 0: + # Negative 'initial' means no initial value + del final_args['initial'] + + return binary_operation(**self._get_connection_args(), + key=key, + op_type=op_type, + op_args=final_args) + + def append( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: AppendOptions + **kwargs, # type: Any + ) -> Optional[MutationResult]: + final_args = self._get_mutation_options(*opts, **kwargs) + if isinstance(value, str): + value = value.encode("utf-8") + elif isinstance(value, bytearray): + value = bytes(value) + + if not isinstance(value, bytes): + raise ValueError( + "The value provided must of type str, bytes or bytearray.") + + op_type = operations.APPEND.value + return binary_operation(**self._get_connection_args(), + key=key, + op_type=op_type, + value=value, + op_args=final_args) + + def prepend( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: PrependOptions + **kwargs, # type: Any + ) -> Optional[MutationResult]: + final_args = self._get_mutation_options(*opts, **kwargs) + if isinstance(value, str): + value = value.encode("utf-8") + elif isinstance(value, bytearray): + value = bytes(value) + + if not isinstance(value, bytes): + raise ValueError( + "The value provided must of type str, bytes or bytearray.") + + op_type = operations.PREPEND.value + return binary_operation(**self._get_connection_args(), + key=key, + op_type=op_type, + value=value, + op_args=final_args) + + def build_scan_args(self, # noqa: C901 + scan_type, # type: Union[RangeScan, PrefixScan, SamplingScan] + **kwargs, # type: Dict[str, Any] + ) -> Dict[str, Any]: + """** INTERNAL ** + + Args: + scan_type (Union[RangeScan, PrefixScan, SamplingScan]): Either a :class:`~couchbase.kv_range_scan.RangeScan`, a + :class:`~couchbase.kv_range_scan.PrefixScan` or a :class:`~couchbase.kv_range_scan.SamplingScan` instance. + kwargs (Dict[str, Any]): Options for scan operation. + + Raises: + InvalidArgumentException: If scan_type is not either a RangeScan, PrefixScan or SamplingScan instance. + InvalidArgumentException: If sort option is provided and is incorrect type. + InvalidArgumentException: If consistent_with option is provided and is not a valid state + InvalidArgumentException: If concurrency is not positive + InvalidArgumentException: If sampling scan limit is not positive + + Returns: + Dict[str, Any]: Parsed and processed scan operation arguments. + """ # noqa: E501 + op_type = None + if 'concurrency' in kwargs and kwargs['concurrency'] < 1: + raise InvalidArgumentException('Concurrency option must be positive') + + if isinstance(scan_type, RangeScan): + op_type = operations.KV_RANGE_SCAN.value + if scan_type.start is not None: + kwargs['start'] = scan_type.start.to_dict() + if scan_type.end is not None: + kwargs['end'] = scan_type.end.to_dict() + elif isinstance(scan_type, PrefixScan): + op_type = operations.KV_PREFIX_SCAN.value + kwargs['prefix'] = scan_type.prefix + elif isinstance(scan_type, SamplingScan): + op_type = operations.KV_SAMPLING_SCAN.value + if scan_type.limit <= 0: + raise InvalidArgumentException('Sampling scan limit must be positive') + kwargs['limit'] = scan_type.limit + if scan_type.seed is not None: + kwargs['seed'] = scan_type.seed + else: + raise InvalidArgumentException('scan_type must be Union[RangeScan, PrefixScan, SamplingScan]') + + transcoder = kwargs.pop('transcoder', None) + + consistent_with = kwargs.pop('consistent_with', None) + if consistent_with: + if not (isinstance(consistent_with, MutationState) and len(consistent_with._sv) > 0): + raise InvalidArgumentException('Passed empty or invalid mutation state') + else: + kwargs['consistent_with'] = list(token.as_dict() for token in consistent_with._sv) + + return_args = { + 'transcoder': transcoder, + 'op_type': op_type, + 'op_args': kwargs, + } + return_args.update(**self._get_connection_args()) + return return_args diff --git a/couchbase/logic/kv_range_scan.py b/couchbase/logic/kv_range_scan.py new file mode 100644 index 000000000..ddc0bb622 --- /dev/null +++ b/couchbase/logic/kv_range_scan.py @@ -0,0 +1,167 @@ +# Copyright 2021, Couchbase, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from __future__ import annotations + +from abc import ABC +from typing import TYPE_CHECKING, Optional + +from couchbase.exceptions import ErrorMapper, InvalidArgumentException +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.wrappers import decode_value +from couchbase.pycbc_core import kv_range_scan_operation +from couchbase.result import ScanResult + +if TYPE_CHECKING: + from couchbase.transcoder import Transcoder + + +class ScanTerm: + """Represents a search term for a RangeScan + """ + + def __init__(self, term, # type: str + exclusive=None # type: Optional[bool] + ) -> None: + + self._term = None + if isinstance(term, str): + self._term = term + else: + raise InvalidArgumentException('Invalid term value provided. Expected str.') + + self._exclusive = exclusive + + def to_dict(self): + return { + 'term': self._term, + 'exclusive': self._exclusive + } + + +class ScanType(ABC): + """ + ** INTERNAL ** + """ + + +class RangeScan(ScanType): + """A RangeScan performs a scan on a range of keys with the range specified through a start and end ScanTerm. + """ + + def __init__(self, + start=None, # type: Optional[ScanTerm] + end=None, # type: Optional[ScanTerm] + ) -> None: + self._start = start + self._end = end + + @property + def start(self) -> ScanTerm: + return self._start + + @property + def end(self) -> ScanTerm: + return self._end + + +class PrefixScan(ScanType): + """A PrefixScan performs a scan on a given prefix + """ + + def __init__(self, + prefix, # type: str + ) -> None: + self._prefix = prefix + + @property + def prefix(self) -> str: + return self._prefix + + +class SamplingScan(ScanType): + """A SamplingScan performs a scan on a random sampling of keys with the sampling bounded by a limit. + """ + + def __init__(self, limit, # type: int + seed=None, # type: Optional[int] + ) -> None: + self._limit = limit + self._seed = seed + + @property + def limit(self) -> int: + return self._limit + + @property + def seed(self) -> Optional[int]: + return self._seed + + +class RangeScanRequestLogic: + """ + ** INTERNAL ** + """ + + def __init__(self, + **kwargs + ): + self._transcoder = kwargs.pop('transcoder', None) + self._ids_only = kwargs['op_args'].get('ids_only', False) + if not self._transcoder: + raise InvalidArgumentException('No transcoder provided.') + self._scan_args = kwargs + self._scan_iterator = None + self._started_streaming = False + self._done_streaming = False + + @property + def transcoder(self) -> Transcoder: + return self._transcoder + + @property + def started_streaming(self) -> bool: + return self._started_streaming + + @property + def done_streaming(self) -> bool: + return self._done_streaming + + def cancel_scan(self) -> None: + if self._scan_iterator.is_cancelled() is False: + self._scan_iterator.cancel_scan() + + def _submit_scan(self): + if self.done_streaming: + return + + self._started_streaming = True + self._scan_iterator = kv_range_scan_operation(**self._scan_args) + + def _get_next_row(self): + if self.done_streaming is True: + return + + resp = next(self._scan_iterator) + if isinstance(resp, CouchbaseBaseException): + raise ErrorMapper.build_exception(resp) + + value = resp.raw_result.get('value', None) + flags = resp.raw_result.get('flags', None) + if value: + resp.raw_result['value'] = decode_value(self.transcoder, value, flags) + + return ScanResult(resp, self._ids_only) diff --git a/couchbase/logic/n1ql.py b/couchbase/logic/n1ql.py new file mode 100644 index 000000000..c9ec0fe7b --- /dev/null +++ b/couchbase/logic/n1ql.py @@ -0,0 +1,810 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from datetime import timedelta +from enum import Enum +from typing import (TYPE_CHECKING, + Any, + Dict, + List, + Optional, + Union) + +from couchbase._utils import (JSONType, + timedelta_as_microseconds, + to_microseconds) +from couchbase.exceptions import ErrorMapper, InvalidArgumentException +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.options import QueryOptionsBase +from couchbase.options import QueryOptions, UnsignedInt64 +from couchbase.pycbc_core import n1ql_query +from couchbase.serializer import DefaultJsonSerializer, Serializer +from couchbase.tracing import CouchbaseSpan + +if TYPE_CHECKING: + from couchbase.mutation_state import MutationState # noqa: F401 + + +class QueryScanConsistency(Enum): + """ + Represents the various scan consistency options that are available when querying against the query service. + + .. warning:: + Importing :class:`.QueryScanConsistency` from ``couchbase.cluster`` is deprecated. + :class:`.QueryScanConsistency` should be imported from ``couchbase.n1ql``. + + """ + + NOT_BOUNDED = "not_bounded" + REQUEST_PLUS = "request_plus" + AT_PLUS = "at_plus" + + +class QueryProfile(Enum): + """ + Specifies the profiling mode for a query. + + .. warning:: + Importing :class:`.QueryProfile` from ``couchbase.cluster`` is deprecated. + :class:`.QueryProfile` should be imported from ``couchbase.n1ql``. + + """ + + OFF = "off" + PHASES = "phases" + TIMINGS = "timings" + + +class QueryStatus(Enum): + """ + Represents the status of a query. + """ + RUNNING = "running" + SUCCESS = "success" + ERRORS = "errors" + COMPLETED = "completed" + STOPPED = "stopped" + TIMEOUT = "timeout" + CLOSED = "closed" + FATAL = "fatal" + ABORTED = "aborted" + UNKNOWN = "unknown" + + @classmethod # noqa: C901 + def from_str(cls, value # type: str # noqa: C901 + ) -> str: + if isinstance(value, str): + if value == cls.RUNNING.value: + return cls.RUNNING + elif value == cls.SUCCESS.value: + return cls.SUCCESS + elif value == cls.ERRORS.value: + return cls.ERRORS + elif value == cls.COMPLETED.value: + return cls.COMPLETED + elif value == cls.STOPPED.value: + return cls.STOPPED + elif value == cls.TIMEOUT.value: + return cls.TIMEOUT + elif value == cls.CLOSED.value: + return cls.CLOSED + elif value == cls.FATAL.value: + return cls.FATAL + elif value == cls.ABORTED.value: + return cls.ABORTED + elif value == cls.UNKNOWN.value: + return cls.UNKNOWN + raise InvalidArgumentException(message=(f"{value} is not a valid QueryStatus option. " + "Excepted str representation of type QueryStatus.")) + + +class QueryProblem(object): + def __init__(self, raw): + self._raw = raw + + def code(self) -> int: + return self._raw.get("code", None) + + def message(self) -> str: + return self._raw.get("message", None) + + +class QueryWarning(QueryProblem): + def __init__(self, query_warning # type: QueryProblem + ): + super().__init__(query_warning) + + def __repr__(self): + return "QueryWarning:{}".format(super()._raw) + + +class QueryError(QueryProblem): + def __init__(self, query_error # type: QueryProblem + ): + super().__init__(query_error) + + def __repr__(self): + return "QueryError:{}".format(super()._raw) + + +class QueryMetrics(object): + def __init__(self, raw # type: Dict[str, Any] + ) -> None: + self._raw = raw + + def elapsed_time(self) -> timedelta: + """Get the total amount of time spent running the query. + + Returns: + timedelta: The total amount of time spent running the query. + """ + us = self._raw.get("elapsed_time") / 1000 + return timedelta(microseconds=us) + + def execution_time(self) -> timedelta: + """Get the total amount of time spent executing the query. + + Returns: + timedelta: The total amount of time spent executing the query. + """ + us = self._raw.get("execution_time") / 1000 + return timedelta(microseconds=us) + + def sort_count(self) -> UnsignedInt64: + """Get the total number of rows which were part of the sorting for the query. + + Returns: + :class:`~couchbase.options.UnsignedInt64`: The total number of rows which were part of the + sorting for the query. + """ + return UnsignedInt64(self._raw.get("sort_count", 0)) + + def result_count(self) -> UnsignedInt64: + """Get the total number of rows which were part of the result set. + + Returns: + :class:`~couchbase.options.UnsignedInt64`: The total number of rows which were part of the result set. + """ + return UnsignedInt64(self._raw.get("result_count", 0)) + + def result_size(self) -> UnsignedInt64: + """Get the total number of bytes which were generated as part of the result set. + + Returns: + :class:`~couchbase.options.UnsignedInt64`: The total number of bytes which were generated as + part of the result set. + """ + return UnsignedInt64(self._raw.get("result_size", 0)) + + def mutation_count(self) -> UnsignedInt64: + """Get the total number of rows which were altered by the query. + + Returns: + :class:`~couchbase.options.UnsignedInt64`: The total number of rows which were altered by the query. + """ + return UnsignedInt64(self._raw.get("mutation_count", 0)) + + def error_count(self) -> UnsignedInt64: + """Get the total number of errors which were encountered during the execution of the query. + + Returns: + :class:`~couchbase.options.UnsignedInt64`: The total number of errors which were encountered during + the execution of the query. + """ + return UnsignedInt64(self._raw.get("error_count", 0)) + + def warning_count(self) -> UnsignedInt64: + """Get the total number of warnings which were encountered during the execution of the query. + + Returns: + :class:`~couchbase.options.UnsignedInt64`: The total number of warnings which were encountered during + the execution of the query. + """ + return UnsignedInt64(self._raw.get("warning_count", 0)) + + def __repr__(self): + return "QueryMetrics:{}".format(self._raw) + + +class QueryMetaData: + def __init__(self, raw # type: Dict[str, Any] + ) -> None: + if raw is not None: + self._raw = raw.get('metadata', None) + sig = self._raw.get('signature', None) + if sig is not None: + self._raw['signature'] = json.loads(sig) + prof = self._raw.get('profile', None) + if prof is not None: + self._raw['profile'] = json.loads(prof) + else: + self._raw = None + + def request_id(self) -> str: + """Get the request ID which is associated with the executed query. + + Returns: + str: The request ID which is associated with the executed query. + """ + return self._raw.get("request_id", None) + + def client_context_id(self) -> str: + """Get the client context id which is assoicated with the executed query. + + Returns: + str: The client context id which is assoicated with the executed query. + """ + return self._raw.get("client_context_id", None) + + def status(self) -> QueryStatus: + """Get the status of the query at the time the query meta-data was generated. + + Returns: + :class:`.QueryStatus`: The status of the query at the time the query meta-data was generated. + """ + return QueryStatus.from_str(self._raw.get("status", "unknown")) + + def signature(self) -> Optional[JSONType]: + """Provides the signature of the query. + + Returns: + Optional[JSONType]: The signature of the query. + """ + return self._raw.get("signature", None) + + def warnings(self) -> List[QueryWarning]: + """Get warnings that occurred during the execution of the query. + + Returns: + List[:class:`.QueryWarning`]: Any warnings that occurred during the execution of the query. + """ + return list( + map(QueryWarning, self._raw.get("warnings", [])) + ) + + def errors(self) -> List[QueryError]: + """Get errors that occurred during the execution of the query. + + Returns: + List[:class:`.QueryWarning`]: Any errors that occurred during the execution of the query. + """ + return list( + map(QueryError, self._raw.get("errors", [])) + ) + + def metrics(self) -> Optional[QueryMetrics]: + """Get the various metrics which are made available by the query engine. + + Returns: + Optional[:class:`.QueryMetrics`]: A :class:`.QueryMetrics` instance. + """ + if "metrics" in self._raw: + return QueryMetrics(self._raw.get("metrics", {})) + return None + + def profile(self) -> Optional[JSONType]: + """Get the various profiling details that were generated during execution of the query. + + Returns: + Optional[JSONType]: Profiling details. + """ + return self._raw.get("profile", None) + + def __repr__(self): + return "QueryMetaData:{}".format(self._raw) + + +class N1QLQuery: + + # empty transform will skip updating the attribute when creating an + # N1QLQuery object + _VALID_OPTS = { + "timeout": {"timeout": lambda x: x}, + "read_only": {"readonly": lambda x: x}, + "scan_consistency": {"consistency": lambda x: x}, + "consistent_with": {"consistent_with": lambda x: x}, + "adhoc": {"adhoc": lambda x: x}, + "client_context_id": {"client_context_id": lambda x: x}, + "max_parallelism": {"max_parallelism": lambda x: x}, + "pipeline_batch": {"pipeline_batch": lambda x: x}, + "pipeline_cap": {"pipeline_cap": lambda x: x}, + "profile": {"profile": lambda x: x}, + "query_context": {"query_context": lambda x: x}, + "raw": {"raw": lambda x: x}, + "scan_cap": {"scan_cap": lambda x: x}, + "scan_wait": {"scan_wait": timedelta_as_microseconds}, + "metrics": {"metrics": lambda x: x}, + "flex_index": {"flex_index": lambda x: x}, + "preserve_expiry": {"preserve_expiry": lambda x: x}, + "use_replica": {"use_replica": lambda x: x}, + "serializer": {"serializer": lambda x: x}, + "positional_parameters": {}, + "named_parameters": {}, + "span": {"span": lambda x: x} + } + + def __init__(self, query, *args, **kwargs): + + self._params = {"statement": query} + self._serializer = DefaultJsonSerializer() + self._raw = None + if args: + self._add_pos_args(*args) + if kwargs: + self._set_named_args(**kwargs) + + def _set_named_args(self, **kv): + """ + Set a named parameter in the query. The named field must + exist in the query itself. + + :param kv: Key-Value pairs representing values within the + query. These values should be stripped of their leading + `$` identifier. + + """ + arg_dict = self._params.setdefault("named_parameters", {}) + arg_dict.update(kv) + + def _add_pos_args(self, *args): + """ + Set values for *positional* placeholders (``$1,$2,...``) + + :param args: Values to be used + """ + arg_array = self._params.setdefault("positional_parameters", []) + arg_array.extend(args) + + def set_option(self, name, value): + """ + Set a raw option in the query. This option is encoded + as part of the query parameters without any client-side + verification. Use this for settings not directly exposed + by the Python client. + + :param name: The name of the option + :param value: The value of the option + """ + self._params[name] = value + + @property + def params(self) -> Dict[str, Any]: + params = self._params + + # couchbase++ wants all args JSONified, + # For now encode to bytes to make couchbase::json_string <--> std::vector<std::byte> easier + raw = params.pop('raw', None) + if raw: + params['raw'] = {f'{k}': self._serializer.serialize(v) for k, v in raw.items()} + + positional_args = params.pop('positional_parameters', None) + if positional_args: + params['positional_parameters'] = [self._serializer.serialize(arg) for arg in positional_args] + + named_params = params.pop('named_parameters', None) + if named_params: + params['named_parameters'] = {f'${k}': self._serializer.serialize(v) for k, v in named_params.items()} + return params + + @property + def metrics(self) -> bool: + return self._params.get('metrics', False) + + @metrics.setter + def metrics(self, value # type: bool + ) -> None: + self.set_option('metrics', value) + + @property + def statement(self) -> str: + return self._params['statement'] + + @property + def timeout(self) -> Optional[float]: + value = self._params.get('timeout', None) + if not value: + return None + value = value[:-1] + return float(value) + + @timeout.setter + def timeout(self, value # type: Union[timedelta,float,int] + ) -> None: + if not value: + self._params.pop('timeout', 0) + else: + total_us = to_microseconds(value) + self.set_option('timeout', total_us) + + @property + def readonly(self) -> bool: + return self._params.get('readonly', False) + + @readonly.setter + def readonly(self, value # type: bool + ) -> None: + self._params['readonly'] = value + + @property + def consistency(self) -> QueryScanConsistency: + value = self._params.get( + 'scan_consistency', None + ) + if value is None and 'mutation_state' in self._params: + return QueryScanConsistency.AT_PLUS + if value is None: + return QueryScanConsistency.NOT_BOUNDED + if isinstance(value, str): + return QueryScanConsistency.REQUEST_PLUS if value == 'request_plus' else QueryScanConsistency.NOT_BOUNDED + + @consistency.setter + def consistency(self, value # type: Union[QueryScanConsistency, str] + ) -> None: + invalid_argument = False + if 'mutation_state' not in self._params: + if isinstance(value, QueryScanConsistency): + if value == QueryScanConsistency.AT_PLUS: + invalid_argument = True + else: + self.set_option('scan_consistency', value.value) + elif isinstance(value, str) and value in [sc.value for sc in QueryScanConsistency]: + if value == QueryScanConsistency.AT_PLUS.value: + invalid_argument = True + else: + self.set_option('scan_consistency', value) + else: + raise InvalidArgumentException(message=("Excepted consistency to be either of type " + "QueryScanConsistency or str representation " + "of QueryScanConsistency")) + + if invalid_argument: + raise InvalidArgumentException(message=("Cannot set consistency to AT_PLUS. Use " + "consistent_with instead or set consistency " + "to NOT_BOUNDED or REQUEST_PLUS")) + + @property + def consistent_with(self): + return { + 'consistency': self.consistency, + 'scan_vectors': self._params.get('mutation_state', None) + } + + @consistent_with.setter + def consistent_with(self, value # type: MutationState + ): + """ + Indicate that the query should be consistent with one or more + mutations. + + :param value: The state of the mutations it should be consistent + with. + :type state: :class:`~.couchbase.mutation_state.MutationState` + """ + if self.consistency != QueryScanConsistency.NOT_BOUNDED: + raise TypeError( + 'consistent_with not valid with other consistency options') + + # avoid circular import + from couchbase.mutation_state import MutationState # noqa: F811 + if not (isinstance(value, MutationState) and len(value._sv) > 0): + raise TypeError('Passed empty or invalid state') + # 3.x SDK had to set the consistency, couchbase++ will take care of that for us + self._params.pop('scan_consistency', None) + self.set_option('mutation_state', list(mt.as_dict() for mt in value._sv)) + + @property + def adhoc(self) -> bool: + return self._params.get('adhoc', True) + + @adhoc.setter + def adhoc(self, value # type: bool + ) -> None: + self.set_option('adhoc', value) + + @property + def client_context_id(self) -> Optional[str]: + return self._params.get('client_context_id', None) + + @client_context_id.setter + def client_context_id(self, value # type: str + ) -> None: + self.set_option('client_context_id', value) + + @property + def max_parallelism(self) -> Optional[int]: + return self._params.get('max_parallelism', None) + + @max_parallelism.setter + def max_parallelism(self, value # type: int + ) -> None: + self.set_option('max_parallelism', value) + + @property + def pipeline_batch(self) -> Optional[int]: + return self._params.get('pipeline_batch', None) + + @pipeline_batch.setter + def pipeline_batch(self, value # type: int + ) -> None: + self.set_option('pipeline_batch', value) + + @property + def pipeline_cap(self) -> Optional[int]: + return self._params.get('pipeline_cap', None) + + @pipeline_cap.setter + def pipeline_cap(self, value # type: int + ) -> None: + self.set_option('pipeline_cap', value) + + @property + def profile(self) -> QueryProfile: + value = self._params.get( + 'profile_mode', None + ) + + if value is None: + return QueryProfile.OFF + if isinstance(value, str): + if value == 'off': + return QueryProfile.OFF + elif value == 'phases': + return QueryProfile.PHASES + else: + return QueryProfile.TIMINGS + + @profile.setter + def profile(self, value # type: Union[QueryProfile, str] + ) -> None: + if isinstance(value, QueryProfile): + self.set_option('profile_mode', value.value) + elif isinstance(value, str) and value in [pm.value for pm in QueryProfile]: + self.set_option('profile_mode', value) + else: + raise InvalidArgumentException(message=("Excepted profile to be either of type " + "QueryProfile or str representation of QueryProfile")) + + @property + def query_context(self) -> Optional[str]: + return self._params.get('query_context', None) + + @query_context.setter + def query_context(self, value # type: str + ) -> None: + self.set_option('query_context', value) + + @property + def send_to_node(self) -> Optional[str]: + return self._params.get('send_to_node', None) + + @send_to_node.setter + def send_to_node(self, value # type: str + ) -> None: + self.set_option('send_to_node', value) + + @property + def scan_cap(self) -> Optional[int]: + return self._params.get('scan_cap', None) + + @scan_cap.setter + def scan_cap(self, value # type: int + ) -> None: + self.set_option('scan_cap', value) + + @property + def scan_wait(self) -> Optional[float]: + value = self._params.get('scan_wait', None) + if not value: + return None + value = value[:-1] + return float(value) + + @scan_wait.setter + def scan_wait(self, value # type: timedelta + ) -> None: + if not value: + self._params.pop('scan_wait', 0) + else: + # if using the setter, need to validate/transform timedelta, otherwise, just add the value + if 'scan_wait' in self._params: + value = timedelta_as_microseconds(value) + + self.set_option('scan_wait', value) + + @property + def flex_index(self) -> bool: + return self._params.get('flex_index', False) + + @flex_index.setter + def flex_index(self, value # type: bool + ) -> None: + self.set_option('flex_index', value) + + @property + def preserve_expiry(self) -> bool: + return self._params.get('preserve_expiry', False) + + @preserve_expiry.setter + def preserve_expiry(self, value # type: bool + ) -> None: + self.set_option('preserve_expiry', value) + + @property + def use_replica(self) -> Optional[bool]: + return self._params.get('use_replica', None) + + @use_replica.setter + def use_replica(self, value # type: Optional[bool] + ) -> None: + self.set_option('use_replica', value) + + @property + def raw(self) -> Optional[Dict[str, Any]]: + return self._params.get('raw', None) + + @raw.setter + def raw(self, value # type: Dict[str, Any] + ) -> None: + if not isinstance(value, dict): + raise TypeError("Raw option must be of type Dict[str, Any].") + for k in value.keys(): + if not isinstance(k, str): + raise TypeError("key for raw value must be str") + self.set_option('raw', value) + + @property + def span(self) -> Optional[CouchbaseSpan]: + return self._params.get('span', None) + + @span.setter + def span(self, value # type CouchbaseSpan + ) -> None: + if not issubclass(value.__class__, CouchbaseSpan): + raise InvalidArgumentException(message='Span should implement CouchbaseSpan interface') + self.set_option('span', value) + + @property + def serializer(self) -> Optional[Serializer]: + return self._params.get('serializer', None) + + @serializer.setter + def serializer(self, value # type: Serializer + ): + if not issubclass(value.__class__, Serializer): + raise InvalidArgumentException(message='Serializer should implement Serializer interface.') + self.set_option('serializer', value) + + @classmethod + def create_query_object(cls, statement, *options, **kwargs): + # lets make a copy of the options, and update with kwargs... + opt = QueryOptions() + # TODO: is it possible that we could have [QueryOptions, QueryOptions, ...]?? + # If so, why??? + opts = list(options) + for o in opts: + if isinstance(o, (QueryOptions, QueryOptionsBase)): + opt = o + opts.remove(o) + args = opt.copy() + args.update(kwargs) + + # now lets get positional parameters. Actual positional + # params OVERRIDE positional_parameters + positional_parameters = args.pop("positional_parameters", []) + if opts and len(opts) > 0: + positional_parameters = opts + + # now the named parameters. NOTE: all the kwargs that are + # not VALID_OPTS must be named parameters, and the kwargs + # OVERRIDE the list of named_parameters + new_keys = list(filter(lambda x: x not in cls._VALID_OPTS, args.keys())) + named_parameters = args.pop("named_parameters", {}) + for k in new_keys: + named_parameters[k] = args[k] + + query = cls(statement, *positional_parameters, **named_parameters) + # now lets try to setup the options. + # but for now we will use the existing N1QLQuery. Could be we can + # add to it, etc... + + # default to false on metrics + query.metrics = args.get("metrics", False) + + for k, v in ((k, args[k]) for k in (args.keys() & cls._VALID_OPTS)): + for target, transform in cls._VALID_OPTS[k].items(): + setattr(query, target, transform(v)) + return query + + +class QueryRequestLogic: + def __init__(self, + connection, + query_params, + row_factory=lambda x: x, + **kwargs + ): + + self._connection = connection + self._query_params = query_params + self.row_factory = row_factory + self._streaming_result = None + self._started_streaming = False + self._done_streaming = False + self._metadata = None + self._default_serializer = kwargs.pop('default_serializer', DefaultJsonSerializer()) + self._serializer = None + self._streaming_timeout = kwargs.pop('streaming_timeout', None) + + @property + def params(self) -> Dict[str, Any]: + return self._query_params + + @property + def serializer(self) -> Serializer: + if self._serializer: + return self._serializer + + serializer = self.params.get('serializer', None) + if not serializer: + serializer = self._default_serializer + + self._serializer = serializer + return self._serializer + + @property + def started_streaming(self) -> bool: + return self._started_streaming + + @property + def done_streaming(self) -> bool: + return self._done_streaming + + def metadata(self): + # @TODO: raise if query isn't complete? + return self._metadata + + def _set_metadata(self, query_response): + if isinstance(query_response, CouchbaseBaseException): + raise ErrorMapper.build_exception(query_response) + + self._metadata = QueryMetaData(query_response.raw_result.get('value', None)) + + def _submit_query(self, **kwargs): + if self.done_streaming: + return + + self._started_streaming = True + n1ql_kwargs = { + 'conn': self._connection, + 'query_args': self.params, + } + + streaming_timeout = self.params.get('timeout', self._streaming_timeout) + if streaming_timeout: + n1ql_kwargs['streaming_timeout'] = streaming_timeout + + # this is for txcouchbase... + callback = kwargs.pop('callback', None) + if callback: + n1ql_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + n1ql_kwargs['errback'] = errback + + self._streaming_result = n1ql_query(**n1ql_kwargs) diff --git a/couchbase/logic/options.py b/couchbase/logic/options.py new file mode 100644 index 000000000..bb40155cb --- /dev/null +++ b/couchbase/logic/options.py @@ -0,0 +1,1464 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import copy +import ctypes +from datetime import timedelta +from enum import Enum, IntEnum +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Union, + overload) + +from couchbase._utils import (timedelta_as_microseconds, + timedelta_as_timestamp, + validate_bool, + validate_int, + validate_str) +from couchbase.durability import DurabilityParser +from couchbase.exceptions import InvalidArgumentException + +if TYPE_CHECKING: + from couchbase._utils import JSONType + from couchbase.analytics import AnalyticsScanConsistency + from couchbase.auth import Authenticator + from couchbase.diagnostics import ClusterState, ServiceType + from couchbase.durability import DurabilityType + from couchbase.management.views import DesignDocumentNamespace + from couchbase.metrics import CouchbaseMeter + from couchbase.mutation_state import MutationState + from couchbase.n1ql import QueryProfile, QueryScanConsistency + from couchbase.replica_reads import ReadPreference + from couchbase.search import (Facet, + HighlightStyle, + SearchScanConsistency, + Sort) + from couchbase.serializer import Serializer + from couchbase.subdocument import StoreSemantics + from couchbase.tracing import CouchbaseTracer + from couchbase.transcoder import Transcoder + from couchbase.vector_search import VectorQueryCombination + from couchbase.views import (ViewErrorMode, + ViewOrdering, + ViewScanConsistency) + + +OptionsBase = dict + + +def _get_temp_opts( + arg_vars, # type: Optional[Dict[str,Any]] + *options # type: OptionsBase +) -> Dict[str, Any]: + arg_vars = copy.copy(arg_vars) if arg_vars else {} + temp_options = ( + copy.copy( + options[0]) if ( + options and options[0]) else dict()) + kwargs = arg_vars.pop("kwargs", {}) + temp_options.update(kwargs) + temp_options.update(arg_vars) + + return temp_options + + +def get_valid_args( + opt_type, # type: OptionsBase + arg_vars, # type: Optional[Dict[str,Any]] + *options # type: OptionsBase +) -> Dict[str, Any]: + + arg_vars = copy.copy(arg_vars) if arg_vars else {} + temp_options = ( + copy.copy( + options[0]) if ( + options and options[0]) else dict()) + kwargs = arg_vars.pop("kwargs", {}) + temp_options.update(kwargs) + temp_options.update(arg_vars) + + valid_opts = opt_type.get_valid_options() + final_options = {} + for opt_key, opt_value in temp_options.items(): + if opt_key not in valid_opts: + continue + for final_key, transform in valid_opts[opt_key].items(): + converted = transform(opt_value) + if converted is not None: + final_options[final_key] = converted + + return final_options + + +VALID_MULTI_OPTS = { + 'timeout': timedelta_as_microseconds, + 'expiry': timedelta_as_timestamp, + 'preserve_expiry': validate_bool, + 'with_expiry': validate_bool, + 'cas': validate_int, + 'durability': DurabilityParser.parse_durability, + 'transcoder': lambda x: x, + 'span': lambda x: x, + 'project': lambda x: x, + 'delta': lambda x: x, + 'initial': lambda x: x, + 'read_preference': lambda x: x.value, + 'per_key_options': lambda x: x, + 'return_exceptions': validate_bool +} + + +def _get_valid_global_multi_opts( + temp_options, # type: Dict[str, Any] + valid_opt_keys # type: List[str] +): + final_opts = {} + if not temp_options: + return final_opts + + for opt_key, opt_value in temp_options.items(): + if opt_key not in valid_opt_keys: + continue + transform = VALID_MULTI_OPTS.get(opt_key, None) + if transform: + final_opts[opt_key] = transform(opt_value) + + return final_opts + + +def _get_per_key_opts( + per_key_opts, # type: Dict[str, Any] + opt_type, # type: OptionsBase + valid_opt_keys # type: List[str] +) -> Dict[str, Any]: + final_key_opts = {} + for key, opts in per_key_opts.items(): + if not isinstance(opts, (opt_type, dict)): + raise InvalidArgumentException(message=f'Expected options to be of type Union[{opt_type.__name__}, dict]') + key_opts = {} + for opt_key, opt_value in opts.items(): + if opt_key not in valid_opt_keys: + continue + transform = VALID_MULTI_OPTS.get(opt_key, None) + if transform: + key_opts[opt_key] = transform(opt_value) + + final_key_opts[key] = key_opts + + return final_key_opts + + +def get_valid_multi_args( + opt_type, # type: OptionsBase + arg_vars, # type: Optional[Dict[str,Any]] + *options # type: OptionsBase +) -> Dict[str, Any]: + + temp_options = _get_temp_opts(arg_vars, *options) + valid_opt_keys = opt_type.get_valid_keys() + + final_opts = _get_valid_global_multi_opts(temp_options, valid_opt_keys) + + if not temp_options: + return final_opts + + per_key_opts = temp_options.pop('per_key_options', None) + if not per_key_opts: + return final_opts + + final_key_opts = _get_per_key_opts(per_key_opts, opt_type, valid_opt_keys) + + final_opts['per_key_options'] = final_key_opts + return final_opts + + +""" + +Couchbase Python SDK Options related Enumerations + +""" + + +class LockMode(IntEnum): + """ + **DEPRECATED** This Enum will be removed in a future version of the SDK. + + Enum kept for API compatibility with the 2.x/3.x series of the SDK. LockMode + is not used or needed with the 4.x series of the SDK. + """ + WAIT = 0 + EXC = 1 + NONE = 2 + + +class TLSVerifyMode(Enum): + NONE = 'none' + PEER = 'peer' + NO_VERIFY = 'no_verify' + + @classmethod + def from_str(cls, value # type: str + ) -> str: + if isinstance(value, str): + if value == cls.NONE.value: + return cls.NONE + elif value == cls.PEER.value: + return cls.PEER + elif value == cls.NO_VERIFY.value: + return cls.NONE + + raise InvalidArgumentException(message=(f"{value} is not a valid TLSVerifyMode option. " + "Excepted str representation of type TLSVerifyMode.")) + + @classmethod + def to_str(cls, value # type: Union[TLSVerifyMode, str] + ) -> str: + if isinstance(value, TLSVerifyMode): + if value == cls.NO_VERIFY: + return cls.NONE.value + return value.value + if isinstance(value, str): + if value == cls.NONE.value: + return cls.NONE.value + elif value == cls.PEER.value: + return cls.PEER.value + elif value == cls.NO_VERIFY.value: + return cls.NONE.value + + raise InvalidArgumentException(message=(f"{value} is not a valid TLSVerifyMode option. " + "Excepted TLS verify mode to be either of type " + "TLSVerifyMode or str representation " + "of TLSVerifyMode.")) + + +class IpProtocol(Enum): + Any = 'any' + ForceIPv4 = 'force_ipv4' + ForceIPv6 = 'force_ipv6' + + @classmethod + def from_str(cls, value # type: str + ) -> str: + if isinstance(value, str): + if value == cls.Any.value: + return cls.Any + elif value == cls.ForceIPv4.value: + return cls.ForceIPv4 + elif value == cls.ForceIPv6.value: + return cls.ForceIPv6 + + raise InvalidArgumentException(message=(f"{value} is not a valid IpProtocol option. " + "Excepted str representation of type IpProtocol.")) + + @classmethod + def to_str(cls, value # type: Union[IpProtocol, str] + ) -> str: + if isinstance(value, IpProtocol): + return value.value + if isinstance(value, str): + if value == cls.Any.value: + return cls.Any.value + elif value == cls.ForceIPv4.value: + return cls.ForceIPv4.value + elif value == cls.ForceIPv6.value: + return cls.ForceIPv6.value + + raise InvalidArgumentException(message=(f"{value} is not a valid IpProtocol option. " + "Excepted IP Protocol mode to be either of type " + "IpProtocol or str representation " + "of IpProtocol.")) + + +class Compression(Enum): + """ + Can be one of: + NONE: + The client will not compress or decompress the data. + IN: + The data coming back from the server will be decompressed, if it was compressed. + OUT: + The data coming into server will be compressed. + INOUT: + The data will be compressed on way in, decompressed on way out of server. + FORCE: + By default the library will send a HELLO command to the server to determine whether compression + is supported or not. Because commands may be + pipelined prior to the scheduing of the HELLO command it is possible that the first few commands + may not be compressed when schedule due to the library not yet having negotiated settings with the + server. Setting this flag will force the client to assume that all servers support compression + despite a HELLO not having been intially negotiated. + """ + + @classmethod + def from_int(cls, val): + if val == 0: + return cls.NONE + elif val == 1: + return cls.IN + elif val == 2: + return cls.OUT + elif val == 3: + return cls.INOUT + elif val == 7: + # note that the lcb flag is a 4, but when you set "force" in the connection + # string, it sets it as INOUT|FORCE. + return cls.FORCE + else: + raise InvalidArgumentException( + "cannot convert {} to a Compression".format(val) + ) + + NONE = "off" + IN = "inflate_only" + OUT = "deflate_only" + INOUT = "on" + FORCE = "force" + + +class KnownConfigProfiles(Enum): + """ + **VOLATILE** This API is subject to change at any time. + + Represents the name of a specific configuration profile that is associated with predetermined cluster options. + + """ + WanDevelopment = 'wan_development' + + @classmethod + def from_str(cls, value # type: str + ) -> str: + if isinstance(value, str): + if value == cls.WanDevelopment.value: + return cls.WanDevelopment + + raise InvalidArgumentException(message=(f"{value} is not a valid KnownConfigProfiles option. " + "Excepted str representation of type KnownConfigProfiles.")) + + @classmethod + def to_str(cls, value # type: Union[KnownConfigProfiles, str] + ) -> str: + if isinstance(value, KnownConfigProfiles): + return value.value + + # just retun the str to allow for future customer config profiles + if isinstance(value, str): + return value + + raise InvalidArgumentException(message=(f"{value} is not a valid KnownConfigProfiles option. " + "Excepted config profile to be either of type " + "KnownConfigProfiles or str representation " + "of KnownConfigProfiles.")) + + +""" + +Couchbase Python SDK Cluster related Options + +""" + + +class ClusterTimeoutOptionsBase(dict): + + _VALID_OPTS = { + "bootstrap_timeout": {"bootstrap_timeout": timedelta_as_microseconds}, + "resolve_timeout": {"resolve_timeout": timedelta_as_microseconds}, + "connect_timeout": {"connect_timeout": timedelta_as_microseconds}, + "kv_timeout": {"key_value_timeout": timedelta_as_microseconds}, + "kv_durable_timeout": {"key_value_durable_timeout": timedelta_as_microseconds}, + "views_timeout": {"view_timeout": timedelta_as_microseconds}, + "query_timeout": {"query_timeout": timedelta_as_microseconds}, + "analytics_timeout": {"analytics_timeout": timedelta_as_microseconds}, + "search_timeout": {"search_timeout": timedelta_as_microseconds}, + "management_timeout": {"management_timeout": timedelta_as_microseconds}, + "dns_srv_timeout": {"dns_srv_timeout": timedelta_as_microseconds}, + "idle_http_connection_timeout": {"idle_http_connection_timeout": timedelta_as_microseconds}, + "config_idle_redial_timeout": {"config_idle_redial_timeout": timedelta_as_microseconds} + } + + @overload + def __init__( + self, + bootstrap_timeout=None, # type: Optional[timedelta] + resolve_timeout=None, # type: Optional[timedelta] + connect_timeout=None, # type: Optional[timedelta] + kv_timeout=None, # type: Optional[timedelta] + kv_durable_timeout=None, # type: Optional[timedelta] + views_timeout=None, # type: Optional[timedelta] + query_timeout=None, # type: Optional[timedelta] + analytics_timeout=None, # type: Optional[timedelta] + search_timeout=None, # type: Optional[timedelta] + management_timeout=None, # type: Optional[timedelta] + dns_srv_timeout=None, # type: Optional[timedelta] + idle_http_connection_timeout=None, # type: Optional[timedelta] + config_idle_redial_timeout=None, # type: Optional[timedelta] + config_total_timeout=None # type: Optional[timedelta] + ): + """ClusterTimeoutOptions instance.""" + + def __init__(self, **kwargs): + # kv_timeout = kwargs.pop('kv_timeout', None) + # if kv_timeout: + # kwargs["key_value_timeout"] = kv_timeout + # kv_durable_timeout = kwargs.pop('kv_durable_timeout', None) + # if kv_durable_timeout: + # kwargs["key_value_durable_timeout"] = kv_durable_timeout + + # legacy... + kwargs.pop('config_total_timeout', None) + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + + super().__init__(**kwargs) + + def as_dict(self): + opts = {} + allowed_opts = ClusterTimeoutOptionsBase.get_allowed_option_keys() + for k, v in self.items(): + if k not in allowed_opts: + continue + if v is None: + continue + if isinstance(v, timedelta): + opts[k] = v.total_seconds() + elif isinstance(v, (int, float)): + opts[k] = v + return opts + + @staticmethod + def get_allowed_option_keys(use_transform_keys=False # type: Optional[bool] + ) -> List[str]: + if use_transform_keys is True: + keys = [] + for val in ClusterTimeoutOptionsBase._VALID_OPTS.values(): + keys.append(list(val.keys())[0]) + return keys + + return list(ClusterTimeoutOptionsBase._VALID_OPTS.keys()) + + +class ClusterTracingOptionsBase(dict): + + _VALID_OPTS = { + "tracing_threshold_kv": {"key_value_threshold": timedelta_as_microseconds}, + "tracing_threshold_view": {"view_threshold": timedelta_as_microseconds}, + "tracing_threshold_query": {"query_threshold": timedelta_as_microseconds}, + "tracing_threshold_search": {"search_threshold": timedelta_as_microseconds}, + "tracing_threshold_analytics": {"analytics_threshold": timedelta_as_microseconds}, + "tracing_threshold_eventing": {"eventing_threshold": timedelta_as_microseconds}, + "tracing_threshold_management": {"management_threshold": timedelta_as_microseconds}, + "tracing_threshold_queue_size": {"threshold_sample_size": validate_int}, + "tracing_threshold_queue_flush_interval": {"threshold_emit_interval": timedelta_as_microseconds}, + "tracing_orphaned_queue_size": {"orphaned_sample_size": validate_int}, + "tracing_orphaned_queue_flush_interval": {"orphaned_emit_interval": timedelta_as_microseconds} + } + + @overload + def __init__( + self, + tracing_threshold_kv=None, # type: Optional[timedelta] + tracing_threshold_view=None, # type: Optional[timedelta] + tracing_threshold_query=None, # type: Optional[timedelta] + tracing_threshold_search=None, # type: Optional[timedelta] + tracing_threshold_analytics=None, # type: Optional[timedelta] + tracing_threshold_eventing=None, # type: Optional[timedelta] + tracing_threshold_management=None, # type: Optional[timedelta] + tracing_threshold_queue_size=None, # type: Optional[int] + tracing_threshold_queue_flush_interval=None, # type: Optional[timedelta] + tracing_orphaned_queue_size=None, # type: Optional[int] + tracing_orphaned_queue_flush_interval=None, # type: Optional[timedelta] + ): + """ClusterTracingOptions instance.""" + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + def as_dict(self): + opts = {} + allowed_opts = ClusterTracingOptionsBase.get_allowed_option_keys() + for k, v in self.items(): + if k not in allowed_opts: + continue + if v is None: + continue + if isinstance(v, timedelta): + opts[k] = v.total_seconds() + elif isinstance(v, (int, float)): + opts[k] = v + return opts + + @staticmethod + def get_allowed_option_keys(use_transform_keys=False # type: Optional[bool] + ) -> List[str]: + if use_transform_keys is True: + keys = [] + for val in ClusterTracingOptionsBase._VALID_OPTS.values(): + keys.append(list(val.keys())[0]) + return keys + + return list(ClusterTracingOptionsBase._VALID_OPTS.keys()) + + +# @TODO: remove this when we remove all the deprecated 3.x imports +TransactionConfig = Union[Dict[str, Any], Any] + + +class ClusterOptionsBase(dict): + + _VALID_OPTS = { + 'allowed_sasl_mechanisms': {'allowed_sasl_mechanisms': lambda x: x.split(',') if isinstance(x, str) else x}, + "authenticator": {"authenticator": lambda x: x}, + "enable_tls": {"enable_tls": validate_bool}, + "enable_mutation_tokens": {"enable_mutation_tokens": validate_bool}, + "enable_tcp_keep_alive": {"enable_tcp_keep_alive": validate_bool}, + "ip_protocol": {"use_ip_protocol": IpProtocol.to_str}, + "enable_dns_srv": {"enable_dns_srv": validate_bool}, + "show_queries": {"show_queries": validate_bool}, + "enable_unordered_execution": {"enable_unordered_execution": validate_bool}, + "enable_clustermap_notification": {"enable_clustermap_notification": validate_bool}, + "enable_compression": {"enable_compression": validate_bool}, + "enable_tracing": {"enable_tracing": validate_bool}, + "enable_metrics": {"enable_metrics": validate_bool}, + "network": {"network": validate_str}, + "tls_verify": {"tls_verify": TLSVerifyMode.to_str}, + "serializer": {"serializer": lambda x: x}, + "transcoder": {"transcoder": lambda x: x}, + "span": {"span": lambda x: x}, + "tcp_keep_alive_interval": {"tcp_keep_alive_interval": timedelta_as_microseconds}, + "config_poll_interval": {"config_poll_interval": timedelta_as_microseconds}, + "config_poll_floor": {"config_poll_floor": timedelta_as_microseconds}, + "max_http_connections": {"max_http_connections": validate_int}, + "user_agent_extra": {"user_agent_extra": validate_str}, + "trust_store_path": {"trust_store_path": validate_str}, + "cert_path": {"cert_path": validate_str}, + "disable_mozilla_ca_certificates": {"disable_mozilla_ca_certificates": validate_bool}, + "logging_meter_emit_interval": {"emit_interval": timedelta_as_microseconds}, + "num_io_threads": {"num_io_threads": validate_int}, + "transaction_config": {"transaction_config": lambda x: x}, + "tracer": {"tracer": lambda x: x}, + "meter": {"meter": lambda x: x}, + "dns_nameserver": {"dns_nameserver": validate_str}, + "dns_port": {"dns_port": validate_int}, + "dump_configuration": {"dump_configuration": validate_bool}, + "preferred_server_group": {"preferred_server_group": validate_str}, + "enable_app_telemetry": {"enable_app_telemetry": validate_bool}, + "app_telemetry_endpoint": {"app_telemetry_endpoint": validate_str}, + "app_telemetry_backoff": {"app_telemetry_backoff": timedelta_as_microseconds}, + "app_telemetry_ping_interval": {"app_telemetry_ping_interval": timedelta_as_microseconds}, + "app_telemetry_ping_timeout": {"app_telemetry_ping_timeout": timedelta_as_microseconds}, + } + + @overload + def __init__( + self, + authenticator, # type: Authenticator + timeout_options=None, # type: Optional[ClusterTimeoutOptionsBase] + tracing_options=None, # type: Optional[ClusterTracingOptionsBase] + enable_tls=None, # type: Optional[bool] + enable_mutation_tokens=None, # type: Optional[bool] + enable_tcp_keep_alive=None, # type: Optional[bool] + ip_protocol=None, # type: Optional[Union[IpProtocol, str]] + enable_dns_srv=None, # type: Optional[bool] + show_queries=None, # type: Optional[bool] + enable_unordered_execution=None, # type: Optional[bool] + enable_clustermap_notification=None, # type: Optional[bool] + enable_compression=None, # type: Optional[bool] + enable_tracing=None, # type: Optional[bool] + enable_metrics=None, # type: Optional[bool] + network=None, # type: Optional[str] + tls_verify=None, # type: Optional[Union[TLSVerifyMode, str]] + serializer=None, # type: Optional[Serializer] + transcoder=None, # type: Optional[Transcoder] + tcp_keep_alive_interval=None, # type: Optional[timedelta] + config_poll_interval=None, # type: Optional[timedelta] + config_poll_floor=None, # type: Optional[timedelta] + max_http_connections=None, # type: Optional[int] + user_agent_extra=None, # type: Optional[str] + logging_meter_emit_interval=None, # type: Optional[timedelta] + transaction_config=None, # type: Optional[TransactionConfig] + log_redaction=None, # type: Optional[bool] + compression=None, # type: Optional[Compression] + compression_min_size=None, # type: Optional[int] + compression_min_ratio=None, # type: Optional[float] + lockmode=None, # type: Optional[LockMode] + tracer=None, # type: Optional[CouchbaseTracer] + meter=None, # type: Optional[CouchbaseMeter] + dns_nameserver=None, # type: Optional[str] + dns_port=None, # type: Optional[int] + disable_mozilla_ca_certificates=None, # type: Optional[bool] + dump_configuration=None, # type: Optional[bool] + preferred_server_group=None, # type: Optional[str] + enable_app_telemetry=None, # type: Optional[bool] + app_telemetry_endpoint=None, # type: Optional[str] + app_telemetry_backoff=None, # type: Optional[timedelta] + app_telemetry_ping_interval=None, # type: Optional[timedelta] + app_telemetry_ping_timeout=None, # type: Optional[timedelta] + ): + """ClusterOptions instance.""" + + def __init__(self, + authenticator, # type: Authenticator + **kwargs + ): + + if authenticator: + kwargs["authenticator"] = authenticator + + # flatten tracing and timeout options + tracing_opts = kwargs.pop('tracing_options', {}) + if tracing_opts: + for k, v in tracing_opts.items(): + if k not in kwargs: + kwargs[k] = v + + timeout_opts = kwargs.pop('timeout_options', {}) + if timeout_opts: + for k, v in timeout_opts.items(): + if k not in kwargs: + kwargs[k] = v + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @staticmethod + def get_allowed_option_keys(cluster_opts_only=False, # type: Optional[bool] + use_transform_keys=False # type: Optional[bool] + ) -> List[str]: + if use_transform_keys is True: + keys = [] + for val in ClusterOptionsBase._VALID_OPTS.values(): + keys.append(list(val.keys())[0]) + + if cluster_opts_only is True: + return keys + + keys.extend(ClusterTimeoutOptionsBase.get_allowed_option_keys(use_transform_keys=True)) + keys.extend(ClusterTracingOptionsBase.get_allowed_option_keys(use_transform_keys=True)) + + return keys + + if cluster_opts_only is True: + return list(ClusterOptionsBase._VALID_OPTS.keys()) + + valid_keys = ClusterTimeoutOptionsBase.get_allowed_option_keys() + valid_keys.extend(ClusterTracingOptionsBase.get_allowed_option_keys()) + valid_keys.extend(list(ClusterOptionsBase._VALID_OPTS.keys())) + + return valid_keys + + @staticmethod + def get_valid_options() -> Dict[str, Any]: + valid_opts = copy.copy(ClusterTimeoutOptionsBase._VALID_OPTS) + valid_opts.update(copy.copy(ClusterTracingOptionsBase._VALID_OPTS)) + valid_opts.update(copy.copy(ClusterOptionsBase._VALID_OPTS)) + return valid_opts + + +""" + +Couchbase Python SDK Key-Value related Options + +""" + + +class OptionsTimeoutBase(OptionsBase): + def __init__(self, + timeout=None, # type: Optional[timedelta] + span=None, # type: Optional[Any] + **kwargs # type: Dict[str, Any] + ) -> None: + """ + Base options with timeout and span options + :param timeout: Timeout for this operation + :param span: Parent tracing span to use for this operation + """ + if timeout: + kwargs["timeout"] = timeout + + if span: + kwargs["span"] = span + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + def timeout(self, + timeout, # type: timedelta + ) -> OptionsTimeoutBase: + self["timeout"] = timeout + return self + + def span(self, + span, # type: Any + ) -> OptionsTimeoutBase: + self["span"] = span + return self + + +# Diagnostic Operations + +class PingOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: timedelta + report_id=None, # type: str + service_types=None # type: Iterable[ServiceType] + ): + pass + + def __init__(self, + **kwargs + ): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DiagnosticsOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + report_id=None # type: str + ): + pass + + def __init__(self, + **kwargs + ): + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class WaitUntilReadyOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + desired_state=None, # type: ClusterState + service_types=None # type: Iterable[ServiceType] + ): + pass + + def __init__(self, + **kwargs + ): + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + +# Key-Value Operations + + +class DurabilityOptionBlockBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + expiry=None, # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @property + def expiry(self): + return self.get("expiry", None) + + +class InsertOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + expiry=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + transcoder=None # type: Optional[Transcoder] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class UpsertOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + expiry=None, # type: Optional[timedelta] + preserve_expiry=False, # type: Optional[bool] + durability=None, # type: Optional[DurabilityType] + transcoder=None # type: Optional[Transcoder] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class ScanOptionsBase(OptionsTimeoutBase): + @overload + def __init__( + self, + timeout=None, # type: Optional[timedelta] + ids_only=None, # type: Optional[bool] + consistent_with=None, # type: Optional[MutationState] + batch_byte_limit=None, # type: Optional[int] + batch_item_limit=None, # type: Optional[int] + batch_time_limit=None, # type: Optional[timedelta] + transcoder=None, # type: Optional[Transcoder] + concurrency=None, # type: Optional[int] + span=None, # type: Optional[Any] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', + 'ids_only', + 'consistent_with', + 'batch_byte_limit', + 'batch_item_limit', + 'concurrency', + 'transcoder', + 'span'] + + +class ReplaceOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + expiry=None, # type: Optional[timedelta] + cas=None, # type: Optional[int] + preserve_expiry=False, # type: Optional[bool] + durability=None, # type: Optional[DurabilityType] + transcoder=None # type: Optional[Transcoder] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class RemoveOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + cas=None, # type: Optional[int] + durability=None # type: Optional[DurabilityType] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetOptionsBase(OptionsTimeoutBase): + @overload + def __init__( + self, + timeout=None, # type: Optional[timedelta] + with_expiry=None, # type: Optional[bool] + project=None, # type: Optional[Iterable[str]] + transcoder=None # type: Optional[Transcoder] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @property + def with_expiry(self) -> bool: + return self.get("with_expiry", False) + + @property + def project(self) -> Iterable[str]: + return self.get("project", []) + + +class ExistsOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class TouchOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAllReplicasOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + transcoder=None, # type: Optional[Transcoder] + read_preference=None, # type: Optional[ReadPreference] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAndTouchOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + transcoder=None # type: Optional[Transcoder] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAndLockOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + transcoder=None # type: Optional[Transcoder] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAnyReplicaOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + transcoder=None, # type: Optional[Transcoder] + read_preference=None, # type: Optional[ReadPreference] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class UnlockOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +# Sub-document Operations + + +class LookupInOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + access_deleted=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class LookupInAllReplicasOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + span=None, # type: Optional[Any] + serializer=None, # type: Optional[Serializer] + read_preference=None, # type: Optional[ReadPreference] + ) -> None: + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class LookupInAnyReplicaOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + span=None, # type: Optional[Any] + serializer=None, # type: Optional[Serializer] + read_preference=None, # type: Optional[ReadPreference] + ) -> None: + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class MutateInOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + cas=0, # type: Optional[int] + durability=None, # type: Optional[DurabilityType] + store_semantics=None, # type: Optional[StoreSemantics] + access_deleted=None, # type: Optional[bool] + preserve_expiry=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +# Binary Operations + +class IncrementOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + expiry=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + delta=None, # type: Optional[DeltaValueBase] + initial=None, # type: Optional[SignedInt64Base] + span=None # type: Optional[Any] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DecrementOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + expiry=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + delta=None, # type: Optional[DeltaValueBase] + initial=None, # type: Optional[SignedInt64Base] + span=None # type: Optional[Any] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class AppendOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + cas=None, # type: Optional[int] + span=None # type: Optional[Any] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class PrependOptionsBase(DurabilityOptionBlockBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + cas=None, # type: Optional[int] + span=None # type: Optional[Any] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +""" + +Couchbase Python SDK N1QL related Options + +""" + + +class QueryOptionsBase(dict): + + # @TODO: span + @overload + def __init__( + self, + timeout=None, # type: Optional[timedelta] + read_only=None, # type: Optional[bool] + scan_consistency=None, # type: Optional[QueryScanConsistency] + adhoc=None, # type: Optional[bool] + client_context_id=None, # type: Optional[str] + max_parallelism=None, # type: Optional[int] + positional_parameters=None, # type: Optional[Iterable[JSONType]] + named_parameters=None, # type: Optional[Dict[str, JSONType]] + pipeline_batch=None, # type: Optional[int] + pipeline_cap=None, # type: Optional[int] + profile=None, # type: Optional[QueryProfile] + query_context=None, # type: Optional[str] + scan_cap=None, # type: Optional[int] + scan_wait=None, # type: Optional[timedelta] + metrics=None, # type: Optional[bool] + flex_index=None, # type: Optional[bool] + preserve_expiry=None, # type: Optional[bool] + use_replica=None, # type: Optional[bool] + consistent_with=None, # type: Optional[MutationState] + send_to_node=None, # type: Optional[str] + raw=None, # type: Optional[Dict[str,Any]] + span=None, # type: Optional[Any] + serializer=None # type: Optional[Serializer] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +""" + +Couchbase Python SDK Analytics related Options + +""" + + +class AnalyticsOptionsBase(OptionsTimeoutBase): + + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + read_only=None, # type: Optional[bool] + scan_consistency=None, # type: Optional[AnalyticsScanConsistency] + positional_parameters=None, # type: Optional[Iterable[JSONType]] + named_parameters=None, # type: Optional[Dict[str, JSONType]] + client_context_id=None, # type: Optional[str] + priority=None, # type: Optional[bool] + metrics=None, # type: Optional[bool] + query_context=None, # type: Optional[str] + raw=None, # type: Optional[Dict[str, Any]] + serializer=None # type: Optional[Serializer] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +""" + +Couchbase Python SDK Full Text Search (FTS) related Options + +""" + + +class SearchOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + limit=None, # type: Optional[int] + skip=None, # type: Optional[int] + explain=None, # type: Optional[bool] + fields=None, # type: Optional[List[str]] + highlight_style=None, # type: Optional[HighlightStyle] + highlight_fields=None, # type: Optional[List[str]] + scan_consistency=None, # type: Optional[SearchScanConsistency] + consistent_with=None, # type: Optional[MutationState] + facets=None, # type: Optional[Dict[str, Facet]] + raw=None, # type: Optional[Dict[str, Any]] + sort=None, # type: Optional[Union[List[str],List[Sort]]] + disable_scoring=None, # type: Optional[bool] + scope_name=None, # type: Optional[str] + collections=None, # type: Optional[List[str]] + include_locations=None, # type: Optional[bool] + client_context_id=None, # type: Optional[str] + serializer=None, # type: Optional[Serializer] + show_request=None, # type: Optional[bool] + log_request=None, # type: Optional[bool] + log_response=None, # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class VectorSearchOptionsBase(dict): + """ + **INTERNAL** + """ + @overload + def __init__(self, + vector_query_combination=None, # type: Optional[VectorQueryCombination] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +""" + +Couchbase Python SDK View related Options + +""" + + +class ViewOptionsBase(OptionsTimeoutBase): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + scan_consistency=None, # type: Optional[ViewScanConsistency] + skip=None, # type: Optional[int] + limit=None, # type: Optional[int] + startkey=None, # type: Optional[JSONType] + endkey=None, # type: Optional[JSONType] + startkey_docid=None, # type: Optional[str] + endkey_docid=None, # type: Optional[str] + inclusive_end=None, # type: Optional[bool] + group=None, # type: Optional[bool] + group_level=None, # type: Optional[int] + key=None, # type: Optional[JSONType] + keys=None, # type: Optional[List[JSONType]] + order=None, # type: Optional[ViewOrdering] + reduce=None, # type: Optional[bool] + on_error=None, # type: Optional[ViewErrorMode] + debug=None, # type: Optional[bool] + namespace=None, # type: Optional[DesignDocumentNamespace] + query_string=None, # type: Optional[List[str]] + client_context_id=None, # type: Optional[str] + raw=None, # type: Optional[Dict[str, str]] + full_set=None, # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + + # @TODO: do we need this?? Don't think so... + # val = kwargs.pop('scan_consistency', None) + # if val: + # kwargs['stale'] = val.value + # val = kwargs.pop('order', None) + # if val: + # kwargs['descending'] = val.value + # val = kwargs.pop('on_error', None) + # if val: + # kwargs['on_error'] = val.value + # val = kwargs.pop('raw', None) + # if val: + # kwargs[val[0]] = val[1] + # val = kwargs.pop('namespace', None) + # if val: + # kwargs['use_devmode'] = ( + # val == DesignDocumentNamespace.DEVELOPMENT) + + super().__init__(**kwargs) + + +""" + +Couchbase Python SDK constrained integer classes + +""" + + +AcceptableInts = Union[ctypes.c_int64, ctypes.c_uint64, int] + + +class ConstrainedIntBase(): + def __init__(self, value): + """ + A signed integer between cls.min() and cls.max() inclusive + + :param couchbase.options.AcceptableInts value: the value to initialise this with. + :raise: :exc:`~couchbase.exceptions.InvalidArgumentException` if not in range + """ + self.value = type(self).verify_value(value) + + @classmethod + def verify_value(cls, item # type: AcceptableInts + ): + # type: (...) -> int + value = getattr(item, 'value', item) + if not isinstance(value, int) or not (cls.min() <= value <= cls.max()): + raise InvalidArgumentException( + "Integer in range {} and {} inclusiverequired".format(cls.min(), cls.max())) + return value + + @classmethod + def is_valid(cls, + item # type: AcceptableInts + ): + return isinstance(item, cls) + + def __neg__(self): + return -self.value + + # Python 3.8 deprecated the implicit conversion to integers using __int__ + # use __index__ instead + # still needed for Python 3.7 + def __int__(self): + return self.value + + # __int__ falls back to __index__ + def __index__(self): + return self.value + + def __add__(self, other): + if not (self.min() <= (self.value + int(other)) <= self.max()): + raise InvalidArgumentException( + "{} + {} would be out of range {}-{}".format(self.value, other, self.min(), self.min())) + + @classmethod + def max(cls): + raise NotImplementedError() + + @classmethod + def min(cls): + raise NotImplementedError() + + def __str__(self): + return "{cls_name} with value {value}".format( + cls_name=type(self), value=self.value) + + def __repr__(self): + return str(self) + + def __eq__(self, other): + return isinstance(self, type(other)) and self.value == other.value + + def __gt__(self, other): + return self.value > other.value + + def __lt__(self, other): + return self.value < other.value + + +class SignedInt64Base(ConstrainedIntBase): + def __init__(self, value): + """ + A signed integer between -0x8000000000000000 and +0x7FFFFFFFFFFFFFFF inclusive. + + :param couchbase.options.AcceptableInts value: the value to initialise this with. + :raise: :exc:`~couchbase.exceptions.InvalidArgumentException` if not in range + """ + super().__init__(value) + + @classmethod + def max(cls): + return 0x7FFFFFFFFFFFFFFF + + @classmethod + def min(cls): + return -0x8000000000000000 + + +class UnsignedInt32Base(ConstrainedIntBase): + def __init__(self, value): + """ + An unsigned integer between 0x00000000 and +0x80000000 inclusive. + + :param couchbase.options.AcceptableInts value: the value to initialise this with. + :raise: :exc:`~couchbase.exceptions.ArgumentError` if not in range + """ + super().__init__(value) + + @classmethod + def max(cls): + return 0x00000000 + + @classmethod + def min(cls): + return 0x80000000 + + +class UnsignedInt64Base(ConstrainedIntBase): + def __init__(self, value): + """ + An unsigned integer between 0x0000000000000000 and +0x8000000000000000 inclusive. + + :param couchbase.options.AcceptableInts value: the value to initialise this with. + :raise: :exc:`~couchbase.exceptions.ArgumentError` if not in range + """ + super().__init__(value) + + @classmethod + def min(cls): + return 0x0000000000000000 + + @classmethod + def max(cls): + return 0x8000000000000000 + + +class DeltaValueBase(ConstrainedIntBase): + def __init__(self, + value # type: AcceptableInts + ): + """ + A non-negative integer between 0 and +0x7FFFFFFFFFFFFFFF inclusive. + Used as an argument for :meth:`Collection.increment` and :meth:`Collection.decrement` + + :param value: the value to initialise this with. + + :raise: :exc:`~couchbase.exceptions.InvalidArgumentException` if not in range + """ + super().__init__(value) + + @ classmethod + def max(cls): + return 0x7FFFFFFFFFFFFFFF + + @ classmethod + def min(cls): + return 0 diff --git a/couchbase/logic/scope.py b/couchbase/logic/scope.py new file mode 100644 index 000000000..ac1dac92f --- /dev/null +++ b/couchbase/logic/scope.py @@ -0,0 +1,503 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict, + Optional) + +from couchbase.analytics import AnalyticsQuery, AnalyticsRequest +from couchbase.collection import Collection +from couchbase.management.eventing import ScopeEventingFunctionManager +from couchbase.management.search import ScopeSearchIndexManager +from couchbase.n1ql import N1QLQuery, N1QLRequest +from couchbase.options import (AnalyticsOptions, + QueryOptions, + SearchOptions) +from couchbase.result import (AnalyticsResult, + QueryResult, + SearchResult) +from couchbase.search import (FullTextSearchRequest, + SearchQueryBuilder, + SearchRequest) +from couchbase.serializer import Serializer +from couchbase.transcoder import Transcoder + +if TYPE_CHECKING: + from couchbase.search import SearchQuery + + +class ScopeLogic: + def __init__(self, bucket, scope_name): + self._bucket = bucket + self._scope_name = scope_name + + @property + def connection(self): + """ + **INTERNAL** + """ + return self._bucket.connection + + @property + def streaming_timeouts(self): + """ + **INTERNAL** + """ + return self._bucket.streaming_timeouts + + @property + def default_serializer(self) -> Optional[Serializer]: + return self._bucket.default_serializer + + @property + def default_transcoder(self) -> Optional[Transcoder]: + return self._bucket.default_transcoder + + @property + def name(self) -> str: + """ + str: The name of this :class:`~.Scope` instance. + """ + return self._scope_name + + @property + def bucket_name(self) -> str: + """ + str: The name of the bucket in which this :class:`~.Scope` instance belongs. + """ + return self._bucket.name + + def collection(self, name # type: str + ) -> Collection: + """Creates a :class:`~.collection.Collection` instance of the specified collection. + + Args: + name (str): Name of the collection to reference. + + Returns: + :class:`~.collection.Collection`: A :class:`~.collection.Collection` instance of the specified collection. + """ + return Collection(self, name) + + def query(self, + statement, # type: str + *options, # type: QueryOptions + **kwargs # type: Dict[str, Any] + ) -> QueryResult: + """Executes a N1QL query against the scope. + + .. note:: + The query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.QueryResult` begins. + + .. note:: + Scope-level queries are only supported on Couchbase Server versions that support scopes and collections. + + .. seealso:: + * :class:`~couchbase.management.queries.QueryIndexManager`: For how to manage query indexes. + * :meth:`~couchbase.cluster.Cluster.query`: For how to execute cluster-level queries. + + Args: + statement (str): The N1QL statement to execute. + options (:class:`~couchbase.options.QueryOptions`): Optional parameters for the query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.QueryOptions` + + Returns: + :class:`~couchbase.result.QueryResult`: An instance of a :class:`~couchbase.result.QueryResult` which + provides access to iterate over the query results and access metadata and metrics about the query. + + Examples: + Simple query:: + + q_res = scope.query('SELECT * FROM `inventory` WHERE country LIKE 'United%' LIMIT 2;') + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple query with positional parameters:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `inventory` WHERE country LIKE $1 LIMIT $2;' + q_res = scope.query(q_str, QueryOptions(positional_parameters=['United%', 5])) + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple query with named parameters:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = scope.query(q_str, QueryOptions(named_parameters={'country': 'United%', 'lim':2})) + for row in q_res.rows(): + print(f'Found row: {row}') + + Retrieve metadata and/or metrics from query:: + + from couchbase.options import QueryOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = scope.query(q_str, QueryOptions(metrics=True)) + for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'Query metadata: {q_res.metadata()}') + print(f'Query metrics: {q_res.metadata().metrics()}') + + """ + opt = QueryOptions() + opts = list(options) + for o in opts: + if isinstance(o, QueryOptions): + opt = o + opts.remove(o) + + # set the query context as this bucket and scope if not provided + if not ('query_context' in opt or 'query_context' in kwargs): + kwargs['query_context'] = '`{}`.`{}`'.format(self.bucket_name, self.name) + + query = N1QLQuery.create_query_object(statement, opt, **kwargs) + # See cluster.query() for note on streaming timeout + streaming_timeout = self.streaming_timeouts.get('query_timeout', None) + return QueryResult(N1QLRequest.generate_n1ql_request(self.connection, + query.params, + streaming_timeout=streaming_timeout)) + + def analytics_query(self, + statement, # type: str + *options, # type: AnalyticsOptions + **kwargs # type: Dict[str, Any] + ) -> AnalyticsResult: + """Executes an analaytics query against the scope. + + .. note:: + The analytics query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.AnalyticsResult` begins. + + .. seealso:: + * :class:`~couchbase.management.analytics.AnalyticsIndexManager`: for how to manage analytics dataverses, datasets, indexes and links. + * :meth:`~couchbase.cluster.Cluster.analytics_query`: for how to execute cluster-level analytics queries + + Args: + statement (str): The analytics SQL++ statement to execute. + options (:class:`~couchbase.options.AnalyticsOptions`): Optional parameters for the analytics query + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.AnalyticsOptions` + + Returns: + :class:`~couchbase.result.AnalyticsResult`: An instance of a :class:`~couchbase.result.AnalyticsResult` which provides access to iterate over the analytics + query results and access metadata and metrics about the analytics query. + + Examples: + .. note:: + Be sure to setup the necessary dataverse(s), dataset(s) for your analytics queries. + See :analytics_intro:`Analytics Introduction <>` in Couchbase Server docs. + + Simple analytics query:: + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = scope.analytics_query(q_str) + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple analytics query with positional parameters:: + + from couchbase.options import AnalyticsOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $1 LIMIT $2;' + q_res = scope.analytics_query(q_str, AnalyticsOptions(positional_parameters=['United%', 5])) + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple analytics query with named parameters:: + + from couchbase.options import AnalyticsOptions + + # ... other code ... + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = scope.analytics_query(q_str, + AnalyticsOptions(named_parameters={'country': 'United%', 'lim':2})) + for row in q_res.rows(): + print(f'Found row: {row}') + + Retrieve metadata and/or metrics from analytics query:: + + q_str = 'SELECT * FROM `travel-sample` WHERE country LIKE $country LIMIT $lim;' + q_res = scope.analytics_query(q_str) + for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'Analytics query metadata: {q_res.metadata()}') + print(f'Analytics query metrics: {q_res.metadata().metrics()}') + + """ # noqa: E501 + opt = AnalyticsOptions() + opts = list(options) + for o in opts: + if isinstance(o, AnalyticsOptions): + opt = o + opts.remove(o) + + # set the query context as this bucket and scope if not provided + if not ('query_context' in opt or 'query_context' in kwargs): + kwargs['query_context'] = 'default:`{}`.`{}`'.format(self.bucket_name, self.name) + + query = AnalyticsQuery.create_query_object(statement, *options, **kwargs) + # See cluster.analytics_query() for note on streaming timeout + streaming_timeout = self.streaming_timeouts.get('analytics_timeout', None) + return AnalyticsResult(AnalyticsRequest.generate_analytics_request(self.connection, + query.params, + streaming_timeout=streaming_timeout)) + + def search_query(self, + index, # type: str + query, # type: SearchQuery + *options, # type: SearchOptions + **kwargs # type: Dict[str, Any] + ) -> SearchResult: + """Executes an search query against the scope. + + .. warning:: + This method is **DEPRECATED** and will be removed in a future release. + Use :meth:`~couchbase.scope.Scope.search` instead. + + .. note:: + The search query is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.SearchResult` begins. + + Args: + index (str): Name of the search query to use. + query (:class:`~couchbase.search.SearchQuery`): Type of search query to perform. + options (:class:`~couchbase.options.SearchOptions`): Optional parameters for the search query + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.SearchOptions` + + Returns: + :class:`~couchbase.result.SearchResult`: An instance of a + :class:`~couchbase.result.SearchResult` which provides access to iterate over the search + query results and access metadata and metrics about the search query. + + Examples: + + .. note:: + Be sure to create a search index prior to executing search queries. Also, if an application + desires to utilize search row locations, highlighting, etc. make sure the search index is + setup appropriately. See :search_create_idx:`Creating Indexes <>` in Couchbase Server docs. + + Simple search query:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + query = search.TermQuery('home') + q_res = scope.search_query('travel-sample-index', + query, + SearchOptions(limit=10)) + + for row in q_res.rows(): + print(f'Found row: {row}') + + + Simple search query with facets:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + facet_name = 'activity' + facet = search.TermFacet('activity') + query = search.TermQuery('home') + q_res = scope.search_query('travel-sample-index', + query, + SearchOptions(limit=10, facets={facet_name: facet})) + + for row in q_res.rows(): + print(f'Found row: {row}') + + print(f'facets: {q_res.facets()}') + + + Simple search query with fields and locations:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + search_fields = ['name', 'activity'] + query = search.TermQuery('home') + q_res = scope.search_query('travel-sample-index', + query, + SearchOptions(limit=10, + include_locations=True, + fields=search_fields)) + + for row in q_res.rows(): + print(f'Found row: {row}') + print(f'Fields: {row.fields}') + print(f'Locations: {row.locations}') + + """ + opt = SearchOptions() + opts = list(options) + for o in opts: + if isinstance(o, SearchOptions): + opt = o + opts.remove(o) + + # set the scope_name as this scope if not provided + if not ('scope_name' in opt or 'scope_name' in kwargs): + kwargs['scope_name'] = f'{self.name}' + # See cluster.search_query() for note on streaming timeout + streaming_timeout = self.streaming_timeouts.get('search_timeout', None) + query = SearchQueryBuilder.create_search_query_object(index, query, *options, **kwargs) + return SearchResult(FullTextSearchRequest.generate_search_request(self.connection, + query.as_encodable(), + streaming_timeout=streaming_timeout)) + + def search(self, + index, # type: str + request, # type: SearchRequest + *options, # type: SearchOptions + **kwargs, # type: Dict[str, Any] + ) -> SearchResult: + """Executes an search against the scope. + + .. note:: + The search is executed lazily in that it is executed once iteration over the + :class:`~couchbase.result.SearchResult` begins. + + .. seealso:: + * :class:`~couchbase.management.search.ScopeSearchIndexManager`: for how to manage search indexes. + * :meth:`~couchbase.cluster.Cluster.search`: for how to execute cluster-level search + + Args: + index (str): Name of the search index to use. + request (:class:`~couchbase.search.SearchRequest`): Type of search request to perform. + options (:class:`~couchbase.options.SearchOptions`): Optional parameters for the search query operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.SearchOptions` + + Returns: + :class:`~couchbase.result.SearchResult`: An instance of a + :class:`~couchbase.result.SearchResult` which provides access to iterate over the search + query results and access metadata and metrics about the search query. + + Examples: + + .. note:: + Be sure to create a search index prior to executing a search. Also, if an application + desires to utilize search row locations, highlighting, etc. make sure the search index is + setup appropriately. See :search_create_idx:`Creating Indexes <>` in Couchbase Server docs. + + Simple search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + + # ... other code ... + + request = search.SearchRequest.create(search.TermQuery('home')) + q_res = scope.search('travel-sample-index', request, SearchOptions(limit=10)) + + for row in q_res.rows(): + print(f'Found row: {row}') + + Simple vector search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + from couchbase.vector_search import VectorQuery, VectorSearch + + # ... other code ... + + # NOTE: the vector is expected to be type List[float], set the vector to the appropriate value, this is an example. + vector = [-0.014653487130999565, -0.008658270351588726, 0.017129190266132355, -0.015563474968075752] + request = search.SearchRequest.create(VectorSearch.from_vector_query(VectorQuery('vector_field', vector))) + q_res = scope.search('travel-sample-vector-index', request, SearchOptions(limit=10)) + + for row in q_res.rows(): + print(f'Found row: {row}') + + Combine search and vector search:: + + import couchbase.search as search + from couchbase.options import SearchOptions + from couchbase.vector_search import VectorQuery, VectorSearch + + # ... other code ... + + # NOTE: the vector is expected to be type List[float], set the vector to the appropriate value, this is an example. + vector_search = VectorSearch.from_vector_query(VectorQuery('vector_field', [-0.014653487130999565, + -0.008658270351588726, + 0.017129190266132355, + -0.015563474968075752])) + request = search.SearchRequest.create(search.MatchAllQuery()).with_vector_search(vector_search) + q_res = scope.search('travel-sample-vector-index', request, SearchOptions(limit=10)) + + for row in q_res.rows(): + print(f'Found row: {row}') + """ # noqa: E501 + # See cluster.search() for note on streaming timeout + streaming_timeout = self.streaming_timeouts.get('search_timeout', None) + query = SearchQueryBuilder.create_search_query_from_request(index, request, *options, **kwargs) + return SearchResult(FullTextSearchRequest.generate_search_request(self.connection, + query.as_encodable(), + default_serializer=self.default_serializer, + streaming_timeout=streaming_timeout, + bucket_name=self.bucket_name, + scope_name=self.name)) + + def search_indexes(self) -> ScopeSearchIndexManager: + """ + Get a :class:`~couchbase.management.search.ScopeSearchIndexManager` which can be used to manage the search + indexes of this scope. + + Returns: + :class:`~couchbase.management.search.ScopeSearchIndexManager`: A :class:`~couchbase.management.search.ScopeSearchIndexManager` instance. + + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return ScopeSearchIndexManager(self.connection, self.bucket_name, self.name) + + def eventing_functions(self) -> ScopeEventingFunctionManager: + """ + Get a :class:`~couchbase.management.search.ScopeEventingFunctionManager` which can be used to manage the eventing + functions of this scope. + + Returns: + :class:`~couchbase.management.search.ScopeEventingFunctionManager`: A :class:`~couchbase.management.search.ScopeSearchIndexManager` instance. + + """ # noqa: E501 + return ScopeEventingFunctionManager(self.connection, self.bucket_name, self.name) + + @staticmethod + def default_name(): + return "_default" diff --git a/couchbase/logic/search.py b/couchbase/logic/search.py new file mode 100644 index 000000000..bd7912bf3 --- /dev/null +++ b/couchbase/logic/search.py @@ -0,0 +1,1573 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from datetime import datetime, timedelta +from enum import Enum +from typing import (TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + Set, + Tuple, + Union) + +from couchbase._utils import is_null_or_empty, to_microseconds +from couchbase.exceptions import ErrorMapper, InvalidArgumentException +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.options import SearchOptionsBase +from couchbase.logic.supportability import Supportability +from couchbase.logic.vector_search import VectorQueryCombination +from couchbase.options import (SearchOptions, + UnsignedInt32, + UnsignedInt64) +from couchbase.pycbc_core import search_query +from couchbase.serializer import DefaultJsonSerializer, Serializer +from couchbase.tracing import CouchbaseSpan + +if TYPE_CHECKING: + from couchbase.logic.search_queries import SearchQuery + from couchbase.logic.search_request import SearchRequest + from couchbase.logic.vector_search import VectorSearch + from couchbase.mutation_state import MutationState # noqa: F401 + +""" + +_QueryBuilder class and _COMMON_FIELDS dict are strictly INTERNAL +and used to help implement the search functionality + +""" + + +class _QueryBuilder: + + @staticmethod # noqa: C901 + def _genprop(converter, *apipaths, **kwargs): # noqa: C901 + """ + This internal helper method returns a property (similar to the + @property decorator). In additional to a simple Python property, + this also adds a type validator (`converter`) and most importantly, + specifies the path within a dictionary where the value should be + stored. + + Any object using this property *must* have a dictionary called ``_json`` + as a property + + :param converter: The converter to be used, e.g. `int` or `lambda x: x` + for passthrough + :param apipaths: + Components of a path to store, e.g. `foo, bar, baz` for + `foo['bar']['baz']` + :return: A property object + """ + if not apipaths: + raise TypeError('Must have at least one API path') + + def fget(self): + d = self._json_ + try: + for x in apipaths: + d = d[x] + return d + except KeyError: + return None + + def fset(self, value): + value = converter(value) + d = self._json_ + for x in apipaths[:-1]: + d = d.setdefault(x, {}) + d[apipaths[-1]] = value + + def fdel(self): + d = self._json_ + try: + for x in apipaths[:-1]: + d = d[x] + del d[apipaths[-1]] + except KeyError: + pass + + doc = kwargs.pop( + 'doc', 'Corresponds to the ``{0}`` field'.format('.'.join(apipaths))) + return property(fget, fset, fdel, doc) + + @staticmethod + def _genprop_str(*apipaths, **kwargs): + """ + Convenience function to return a string property in which the value + is converted to a string + """ + return _QueryBuilder._genprop(str, *apipaths, **kwargs) + + @staticmethod + def _gen_location(value): + if len(value) != 2 or not all(map(lambda pt: isinstance(pt, (int, float)), value)): + raise InvalidArgumentException(message='Requires a tuple: (lon, lat)') + return [float(value[0]), float(value[1])] + + @staticmethod + def _gen_locations(value): + return [_QueryBuilder._gen_location(loc) for loc in value] + + @staticmethod + def _mk_range_bucket(name, # type: str + n1, # type: str + n2, # type: str + r1, # type: Optional[Union[str, int, float]] + r2 # type: Optional[Union[str, int, float]] + ) -> Dict[str, Any]: + """ + Create a named range specification for encoding. + + :param name: The name of the range as it should appear in the result + :param n1: The name of the lower bound of the range specifier + :param n2: The name of the upper bound of the range specified + :param r1: The value of the lower bound (user value) + :param r2: The value of the upper bound (user value) + :return: A dictionary containing the range bounds. The upper and lower + bounds are keyed under ``n1`` and ``n2``. + + More than just a simple wrapper, this will not include any range bound + which has a user value of `None`. Likewise it will raise an exception if + both range values are ``None``. + """ + d = {} + if r1 is not None: + d[n1] = r1 + if r2 is not None: + d[n2] = r2 + if not d: + raise InvalidArgumentException(message='Must specify at least one range boundary!') + d['name'] = name + return d + + @staticmethod + def _assign_kwargs(cls, kwargs): + """ + Assigns all keyword arguments to a given instance, raising an exception + if one of the keywords is not already the name of a property. + """ + for k in kwargs: + if not hasattr(cls, k): + raise AttributeError(k, 'Not valid for', cls.__class__.__name__) + setattr(cls, k, kwargs[k]) + + @staticmethod + def _validate_range_query(self, r1, r2, **kwargs): + _QueryBuilder._assign_kwargs(self, kwargs) + if r1 is None and r2 is None: + raise TypeError('At least one of {0} or {1} should be specified', + *self._MINMAX) + if r1 is not None: + setattr(self, self._MINMAX[0], r1) + if r2 is not None: + setattr(self, self._MINMAX[1], r2) + + @staticmethod + def _single_term_query(fields=None # type: Optional[List[str]] + ) -> Callable: + """ + **INTERNAL** + Class decorator to include common query fields + + :param fields: List of fields to include. These should be keys of the + `_COMMON_FIELDS` dict + """ + + def class_wrapper(cls): + if fields: + for f in fields: + field_prop = _COMMON_FIELDS[f] + setattr(cls, f, field_prop) + + def new_init(self, term, *args, **kwargs): + super(type(self), self).__init__() + if self._TERMPROP not in kwargs: + kwargs[self._TERMPROP] = term + _QueryBuilder._assign_kwargs(self, kwargs) + + cls.__init__ = new_init + + return cls + + return class_wrapper + + @staticmethod + def _with_fields(fields=None # type: Optional[List[str]] + ) -> Callable: + """ + **INTERNAL** + Class decorator to include common query fields + + :param fields: List of fields to include. These should be keys of the + `_COMMON_FIELDS` dict + """ + + def class_wrapper(cls): + if fields: + for f in fields: + field_prop = _COMMON_FIELDS[f] + setattr(cls, f, field_prop) + + return cls + + return class_wrapper + + +_COMMON_FIELDS = { + 'prefix_length': _QueryBuilder._genprop( + int, 'prefix_length', + doc=""" + When using :attr:`fuzziness`, this controls how much of the term + or phrase is *excluded* from fuzziness. This may help improve + performance at the expense of omitting potential fuzzy matches + at the beginning of the string. + """), + 'fuzziness': _QueryBuilder._genprop( + int, 'fuzziness', + doc=""" + Allow a given degree of fuzz in the match. Matches which are closer to + the original term will be scored higher. + + You can apply the fuzziness to only a portion of the string by + specifying :attr:`prefix_length` - indicating that only the part + of the field after the prefix length should be checked with fuzziness. + + This value is specified as a float + """), + 'field': _QueryBuilder._genprop( + str, 'field', + doc=""" + Restrict searching to a given document field + """), + 'analyzer': _QueryBuilder._genprop( + str, 'analyzer', + doc=""" + Use a defined server-side analyzer to process the input term prior to + executing the search + """ + ) +} + +""" + +Enum per the search RFC + +""" + + +class HighlightStyle(Enum): + """ + HighlightStyle + + Can be either: + + Ansi + Need Example + Html + Need Example + """ + Ansi = 'ansi' + Html = 'html' + + +class SearchScanConsistency(Enum): + """ + SearchScanConsistency + + This can be: + + NOT_BOUNDED + Which means we just return what is currently in the indexes. + """ + NOT_BOUNDED = 'not_bounded' + REQUEST_PLUS = 'request_plus' + AT_PLUS = 'at_plus' + + +class MatchOperator(Enum): + """ + **UNCOMMITTED** This API may change in the future. + Specifies how the individual match terms should be logically concatenated. + + Members: + OR (default): Specifies that individual match terms are concatenated with a logical OR. + AND: Specifies that individual match terms are concatenated with a logical AND. + """ + OR = "or" + AND = "and" + """ + +Search Metrics and Metadata per the RFC + +""" + + +class SearchMetrics: + def __init__(self, + raw # type: Dict[str, Any] + ): + self._raw = raw + + def success_partition_count(self) -> int: + return self._raw.get("success_partition_count", 0) + + def error_partition_count(self) -> int: + return self._raw.get("error_partition_count", 0) + + def took(self) -> timedelta: + us = self._raw.get("took") / 1000 + return timedelta(microseconds=us) + + def total_partition_count(self) -> int: + return self.success_partition_count() + self.error_partition_count() + + def max_score(self) -> float: + return self._raw.get("max_score", 0.0) + + def total_rows(self) -> int: + return self._raw.get("total_rows", 0) + + def __repr__(self): + return 'SearchMetrics:{}'.format(self._raw) + + +class SearchMetaData: + """Represents the meta-data returned along with a search query result.""" + + def __init__(self, + raw # type: Dict[str, Any] + ): + self._raw = raw + + def errors(self) -> Dict[str, str]: + return self._raw.get('errors', {}) + + def metrics(self) -> Optional[SearchMetrics]: + if 'metrics' in self._raw: + return SearchMetrics(self._raw.get('metrics', {})) + return None + + def client_context_id(self) -> Optional[str]: + return self._raw.get('client_context_id', None) + + def __repr__(self): + return 'SearchMetaData:{}'.format(self._raw) + + +""" + +Sorting Classes + +""" + + +class Sort: + def __init__(self, by, # type: str + **kwargs # type: Dict[str, Any] + ) -> None: + self._json_ = { + 'by': by + } + if 'descending' in kwargs: + kwargs['desc'] = kwargs.pop('descending') + _QueryBuilder._assign_kwargs(self, kwargs) + + def set_prop(self, key, # type: str + value # type: Any + ) -> None: + self._json_[key] = value + + # desc = _QueryBuilder._genprop(bool, 'desc', doc='Sort using descending order') + + @property + def desc(self) -> Optional[bool]: + return self._json_.get('desc', None) + + @desc.setter + def desc(self, value # type: bool + ) -> None: + self.set_prop('desc', value) + + @property + def descending(self) -> Optional[bool]: + return self._json_.get('desc', None) + + @descending.setter + def descending(self, value # type: bool + ) -> None: + self.set_prop('desc', value) + + def as_encodable(self): + return self._json_ + + +class SortString(Sort): + """ + Sorts by a list of fields. This is similar to specifying a list of + fields in :attr:`Params.sort` + """ + + def __init__(self, *fields # type: str + ) -> None: + self._json_ = list(fields) + + +class SortScore(Sort): + """ + Sorts by the score of each match. + """ + + def __init__(self, **kwargs # type: Dict[str, Any] + ) -> None: + super(SortScore, self).__init__('score', **kwargs) + + +class SortID(Sort): + """ + Sorts lexically by the document ID of each match + """ + + def __init__(self, **kwargs # type: Dict[str, Any] + ) -> None: + super(SortID, self).__init__('id', **kwargs) + + +class SortField(Sort): + """ + Sorts according to the properties of a given field + """ + + def __init__(self, field, # type: str + **kwargs # type: Dict[str, Any] + ) -> None: + kwargs['field'] = field + super(SortField, self).__init__('field', **kwargs) + + @property + def field(self) -> str: + return self._json_.get('field') + + @field.setter + def field(self, value # type: str + ) -> None: + self.set_prop('field', value) + + @property + def type(self) -> Optional[str]: + return self._json_.get('type', None) + + @type.setter + def type(self, value # type: str + ) -> None: + self.set_prop('type', value) + + @property + def mode(self) -> Optional[str]: + return self._json_.get('mode', None) + + @mode.setter + def mode(self, value # type: str + ) -> None: + self.set_prop('mode', value) + + @property + def missing(self) -> Optional[str]: + return self._json_.get('missing', None) + + @missing.setter + def missing(self, value # type: str + ) -> None: + self.set_prop('missing', value) + + # field = _QueryBuilder._genprop_str('field', doc='Field to sort by') + # type = _QueryBuilder._genprop_str('type', doc='Coerce field to this type') + # mode = _QueryBuilder._genprop_str('mode') + # missing = _QueryBuilder._genprop_str('missing') + + +class SortGeoDistance(Sort): + """ + Sorts matches based on their distance from a specific location + """ + + def __init__(self, location, # type: Tuple[float, float] + field, # type: str + **kwargs # type: Dict[str, Any] + ) -> None: + kwargs.update(location=location, field=field) + super().__init__('geo_distance', **kwargs) + + @property + def location(self) -> Tuple[float, float]: + return self._json_.get('location') + + @location.setter + def location(self, value # type: Tuple[float, float] + ) -> None: + location = _QueryBuilder._gen_location(value) + self.set_prop('location', location) + + @property + def field(self) -> str: + return self._json_.get('field') + + @field.setter + def field(self, value # type: str + ) -> None: + self.set_prop('field', value) + + @property + def unit(self) -> Optional[str]: + return self._json_.get('unit') + + @unit.setter + def unit(self, value # type: str + ) -> None: + self.set_prop('unit', value) + + # location = _genprop(_location_conv, 'location', + # doc='`(lon, lat)` of point of origin') + # field = _QueryBuilder._genprop_str('field', doc='Field that contains the distance') + # unit = _QueryBuilder._genprop_str('unit', doc='Distance unit used for measuring') + + +class SortRaw(Sort): + def __init__(self, raw # type: Dict[str, Any] + ) -> None: + self._json_ = raw + + +""" + +Facet Classes + +""" + + +class Facet(object): + """ + Base facet class. Each facet must have a field which it aggregates + """ + + def __init__(self, field, # type: str + limit=None # type: Optional[int] + ) -> None: + self._json_ = {'field': field} + if limit: + self._json_['size'] = limit + + def set_prop(self, key, # type: str + value # type: Any + ) -> None: + self._json_[key] = value + + @property + def encodable(self): + """ + Returns a reprentation of the object suitable for serialization + """ + return self._json_ + + @property + def field(self) -> str: + return self._json_.get('field', None) + + @field.setter + def field(self, value # type: str + ) -> None: + self.set_prop('field', value) + + @property + def limit(self) -> Optional[int]: + return self._json_.get('size', None) + + @limit.setter + def limit(self, value # type: int + ) -> None: + self.set_prop('size', value) + + """ + Field upon which the facet will aggregate + """ + # field = _QueryBuilder._genprop_str('field') + + # limit = _QueryBuilder._genprop(int, 'size', + # doc="Maximum number of facet results to return") + + +class TermFacet(Facet): + """ + Facet aggregating the most frequent terms used. + """ + + +class DateFacet(Facet): + """ + Facet to aggregate results based on a date range. + This facet must have at least one invocation of :meth:`add_range` before + it is added to :attr:`~Params.facets`. + """ + + def __init__(self, field, # type: str + limit=None # type: Optional[int] + ) -> None: + super().__init__(field, limit) + self.date_ranges = [] + + def add_range(self, name, # type: str + start=None, # type: Optional[str] + end=None # type: Optional[str] + ) -> DateFacet: + """ + Adds a date range to the given facet. + + :param str name: + The name by which the results within the range can be accessed + :param str start: Lower date range. Should be in RFC 3339 format + :param str end: Upper date range. + :return: The `DateFacet` object, so calls to this method may be + chained + """ + self.date_ranges.append(_QueryBuilder._mk_range_bucket(name, 'start', 'end', start, end)) + return self + + @property + def date_ranges(self) -> Optional[List[DateFacet]]: + return self._json_.get('date_ranges', None) + + @date_ranges.setter + def date_ranges(self, value # type: List[DateFacet] + ) -> None: + self.set_prop('date_ranges', value) + + +class NumericFacet(Facet): + """ + Facet to aggregate results based on a numeric range. + This facet must have at least one invocation of :meth:`add_range` + before it is added to :attr:`Params.facets` + """ + + def __init__(self, field, # type: str + limit=None # type: Optional[int] + ) -> None: + super().__init__(field, limit) + self.numeric_ranges = [] + + def add_range(self, name, # type: str + min=None, # type: Optional[Union[int, float]] + max=None # type: Optional[Union[int, float]] + ) -> NumericFacet: + """ + Add a numeric range. + + :param str name: + the name by which the range is accessed in the results + :param int | float min: Lower range bound + :param int | float max: Upper range bound + :return: This object; suitable for method chaining + """ + self.numeric_ranges.append(_QueryBuilder._mk_range_bucket(name, 'min', 'max', min, max)) + return self + + @property + def numeric_ranges(self) -> Optional[List[NumericFacet]]: + return self._json_.get('numeric_ranges', None) + + @numeric_ranges.setter + def numeric_ranges(self, value # type: List[NumericFacet] + ) -> None: + self.set_prop('numeric_ranges', value) + + # _ranges = _genprop(list, 'numeric_ranges') + + +class _FacetDict(dict): + """ + Internal dict subclass which ensures that facets added to this dictionary + have properly defined ranges. + """ + + # noinspection PyMissingConstructor + def __init__(self, *args, **kwargs): + self.update(*args, **kwargs) + + def __setitem__(self, key, value): + if not isinstance(value, Facet): + raise ValueError('Can only add facet') + # if hasattr(value, '_ranges') and not getattr(value, '_ranges'): + # raise ValueError('{} object must have at least one range. Use ' + # 'add_range'.format(value.__class__.__name__)) + if hasattr(value, 'date_ranges') and not getattr(value, 'date_ranges'): + raise InvalidArgumentException(message=(f'{value.__class__.__name__} object must have at ' + 'least one range. Use add_range()')) + if hasattr(value, 'numeric_ranges') and not getattr(value, 'numeric_ranges'): + raise InvalidArgumentException(message=(f'{value.__class__.__name__} object must have at ' + 'least one range. Use add_range()')) + + super().__setitem__(key, value) + + def update(self, *args, **kwargs): + if args: + if len(args) > 1: + raise TypeError('only one merge at a time!') + other = dict(args[0]) + for key in other: + self[key] = other[key] + for key in kwargs: + self[key] = kwargs[key] + + def setdefault(self, key, value=None): + if key not in self: + self[key] = value + return self[key] + + +""" + +Results Classes + +""" + + +@dataclass +class SearchTermFacet: + term: str = None + count: UnsignedInt64 = None + + +@dataclass +class SearchNumericRangeFacet: + name: str = None + count: UnsignedInt64 = None + min: Union[float, UnsignedInt64] = None + max: Union[float, UnsignedInt64] = None + + +@dataclass +class SearchDateRangeFacet: + name: str = None + count: UnsignedInt64 = None + start: Union[str, datetime] = None + end: Union[str, datetime] = None + + +@dataclass +class SearchFacetResult: + """ An individual facet result has both metadata and details, + as each facet can define ranges into which results are categorized.""" + name: str = None + field: str = None + total: UnsignedInt64 = None + missing: UnsignedInt64 = None + other: UnsignedInt64 = None + terms: List[SearchTermFacet] = None + numeric_ranges: List[SearchNumericRangeFacet] = None + date_ranges: List[SearchDateRangeFacet] = None + + +@dataclass +class SearchRowLocation: + field: str = None + term: str = None + position: UnsignedInt32 = None + start: UnsignedInt32 = None + end: UnsignedInt32 = None + array_positions: List[UnsignedInt32] = None + + +class SearchRowLocations: + def __init__(self, locations): + self._raw_locations = locations + + def get_all(self) -> List[SearchRowLocation]: + """list all locations (any field, any term)""" + locations = [] + for location in self._raw_locations: + locations.append(SearchRowLocation(**location)) + + # TODO: maybe needed when using couchbase++ streaming + # for loc_field, terms in self._raw_locations.items(): + # for term in terms.keys(): + # locations.extend(self.get(loc_field, term)) + + return locations + + def get(self, + field, # type: str + term # type: str + ) -> List[SearchRowLocation]: + """List all locations for a given field and term""" + if field not in self._raw_locations: + raise InvalidArgumentException(f'Cannot find "{field}" field in locations.') + if term not in self._raw_locations[field]: + raise InvalidArgumentException(f'Cannot find "{term}" within {field}\'s locations.') + + locations = [] + for loc in self._raw_locations[field][term]: + new_location = {'field': field, 'term': term, 'position': None} + new_location.update({k: v for k, v in loc.items() if k != 'pos'}) + new_location['position'] = loc.get('pos', None) + locations.append(new_location) + + return [SearchRowLocation(**loc) for loc in locations] + + def fields(self) -> List[str]: + """ + :return: the fields in this location + """ + return self._raw_locations.keys() + + def terms(self) -> Set[str]: + """ + List all terms in this locations, + considering all fields (so a set): + """ + result = set() + for field_terms in self._raw_locations.values(): + result.update(field_terms.keys()) + return result + + def terms_for(self, + field # type:str + ) -> List[str]: + """ list the terms for a given field """ + if field not in self._raw_locations: + raise InvalidArgumentException(f'Cannot find "{field}" field in locations.') + return list(self._raw_locations[field].keys()) + + def __repr__(self): + if self._raw_locations: + return f'SearchRowLocations({self._raw_locations})' + return str(None) + +# dunno why 3.x has this...seems sort of pointless + + +class SearchRowFields(dict): + def __init__(self, **kwargs): + super().__init__(**kwargs) + + +@dataclass +class SearchRow: + """A single entry of search results. The server calls them "hits", + and represents as a JSON object. The following interface describes + the contents of the result row.""" + index: str = None + id: str = None + score: float = None + fields: SearchRowFields = field(default_factory=SearchRowFields) + sort: list = field(default_factory=list) + locations: SearchRowLocations = field(default_factory=SearchRowLocations) + fragments: dict = field(default_factory=dict) + explanation: dict = field(default_factory=dict) + + +""" + +The SearchQueryBuild is the mechanism that holds and stores the +params related to a search query + +""" + + +class SearchQueryBuilder: + + # empty transform will skip updating the attribute when creating an + # SearchQueryBuilder object + _VALID_OPTS = { + "timeout": {"timeout": lambda x: x}, + "limit": {"limit": lambda x: x}, + "skip": {"skip": lambda x: x}, + "explain": {"explain": lambda x: x}, + "fields": {"fields": lambda x: x}, + "highlight_style": {"highlight_style": lambda x: x}, + "highlight_fields": {"highlight_fields": lambda x: x}, + "scan_consistency": {"consistency": lambda x: x}, + "consistent_with": {"consistent_with": lambda x: x}, + "raw": {"raw": lambda x: x}, + "disable_scoring": {"disable_scoring": lambda x: x}, + "scope_name": {"scope_name": lambda x: x}, + "collections": {"collections": lambda x: x}, + "include_locations": {"include_locations": lambda x: x}, + "client_context_id": {"client_context_id": lambda x: x}, + "serializer": {"serializer": lambda x: x}, + "facets": {}, + "sort": {}, + "show_request": {"show_request": lambda x: x}, + "span": {"span": lambda x: x}, + "vector_query_combination": {"vector_query_combination": lambda x: x}, + "log_request": {"log_request": lambda x: x}, + "log_response": {"log_response": lambda x: x} + } + + def __init__(self, + index_name, # type: str + query=None, # type: Optional[SearchQuery] + vector_search=None, # type: Optional[VectorSearch] + **kwargs # type: Dict[str, Any] + ): + + self._index_name = index_name + self._params = {} + self._query = query + self._vector_search = vector_search + # self._facets = {} + # facets = kwargs.pop('facets', {}) + # if facets: + self.facets = _FacetDict(**kwargs.pop('facets', {})) + self._sort = None + sort = kwargs.pop('sort', None) + if sort is not None: + self.sort = sort + + def set_option(self, name, value): + """ + Set a raw option in the query. This option is encoded + as part of the query parameters without any client-side + verification. Use this for settings not directly exposed + by the Python client. + + :param name: The name of the option + :param value: The value of the option + """ + self._params[name] = value + + def encode_vector_search(self) -> Optional[List[Dict[str, Any]]]: + if self._vector_search is None: + return None + + encoded_queries = [] + for query in self._vector_search.queries: + encoded_query = { + 'field': query.field_name, + 'k': query.num_candidates if query.num_candidates is not None else 3 + } + if query.vector is not None: + encoded_query['vector'] = query.vector + else: + encoded_query['vector_base64'] = query.vector_base64 + if query.boost is not None: + encoded_query['boost'] = query.boost + if query.prefilter is not None: + encoded_query['filter'] = query.prefilter.encodable + encoded_queries.append(encoded_query) + + return encoded_queries + + def as_encodable(self) -> Dict[str, Any]: + params = { + 'index_name': self._index_name + } + query = json.dumps(self._query.encodable) + params['query'] = query + vector_search = self.encode_vector_search() + if vector_search: + params['vector_search'] = json.dumps(vector_search) + + # deprecate the scope_name option, no need to pass it to the C++ client + # as the search API will not use + params.update({k: v for k, v in self._params.items() if k not in ['scope_name']}) + + if self.facets: + encoded_facets = {} + for name, facet in self.facets.items(): + encoded_facets[name] = json.dumps(facet.encodable) + params['facets'] = encoded_facets + + if self.sort: + sort_specs = [] + if all(map(lambda s: isinstance(s, str), self.sort)): + for s in self.sort: + encoded = json.dumps(s) + sort_specs.append(encoded) + else: + for s in self.sort: + encoded = json.dumps(s.as_encodable()) + sort_specs.append(encoded) + + params['sort_specs'] = sort_specs + + return params + + @property + def params(self) -> Dict[str, Any]: + return self._params + + @property + def timeout(self) -> Optional[float]: + value = self._params.get('timeout', None) + if not value: + return None + value = value[:-1] + return float(value) + + @timeout.setter + def timeout(self, value # type: Union[timedelta,float,int] + ) -> None: + if not value: + self._params.pop('timeout', 0) + else: + total_us = to_microseconds(value) + self.set_option('timeout', total_us) + + @property + def metrics(self) -> bool: + return self._params.get('metrics', True) + + @metrics.setter + def metrics(self, value # type: bool + ) -> None: + self.set_option('metrics', value) + + @property + def limit(self) -> Optional[int]: + return self._params.get('limit', None) + + @limit.setter + def limit(self, value # type: int + ) -> None: + self.set_option('limit', value) + + @property + def skip(self) -> Optional[int]: + return self._params.get('skip', None) + + @skip.setter + def skip(self, value # type: int + ) -> None: + self.set_option('skip', value) + + @property + def explain(self) -> bool: + return self._params.get('explain', False) + + @explain.setter + def explain(self, value # type: bool + ) -> None: + self.set_option('explain', value) + + @property + def disable_scoring(self) -> bool: + return self._params.get('disable_scoring', False) + + @disable_scoring.setter + def disable_scoring(self, value # type: bool + ) -> None: + self.set_option('disable_scoring', value) + + @property + def include_locations(self) -> bool: + return self._params.get('include_locations', False) + + @include_locations.setter + def include_locations(self, value # type: bool + ) -> None: + self.set_option('include_locations', value) + + @property + def fields(self) -> Optional[List[str]]: + return self._params.get('fields', None) + + @fields.setter + def fields(self, value # type: List[str] + ) -> None: + if not (isinstance(value, list) and all(map(lambda f: isinstance(f, str), value))): + raise InvalidArgumentException(message='Expected a list of strings') + self.set_option('fields', value) + + @property + def highlight_style(self) -> Optional[HighlightStyle]: + value = self._params.get('highlight_style', None) + if isinstance(value, HighlightStyle): + return value + if isinstance(value, str): + return HighlightStyle.Html if value == 'html' else HighlightStyle.Ansi + + @highlight_style.setter + def highlight_style(self, value # type: Union[HighlightStyle, str] + ) -> None: + if isinstance(value, HighlightStyle): + self.set_option('highlight_style', value.value) + elif isinstance(value, str): + if value in [HighlightStyle.Html.value, HighlightStyle.Ansi.value]: + self.set_option('highlight_style', value) + else: + raise InvalidArgumentException(message=("Excepted highlight_style to be either of type " + "HighlightStyle or str representation " + "of HighlightStyle")) + + @property + def highlight_fields(self) -> Optional[List[str]]: + return self._params.get('highlight_fields', None) + + @highlight_fields.setter + def highlight_fields(self, value # type: List[str] + ) -> None: + if not (isinstance(value, list) and all(map(lambda f: isinstance(f, str), value))): + raise InvalidArgumentException(message='Expected a list of strings') + self.set_option('highlight_fields', value) + + @property + def consistency(self) -> SearchScanConsistency: + value = self._params.get( + 'scan_consistency', None + ) + if value is None and 'mutation_state' in self._params: + return SearchScanConsistency.AT_PLUS + if value is None: + return SearchScanConsistency.NOT_BOUNDED + if isinstance(value, str): + return SearchScanConsistency.REQUEST_PLUS if value == 'request_plus' else SearchScanConsistency.NOT_BOUNDED + + @consistency.setter + def consistency(self, value # type: Union[SearchScanConsistency, str] + ) -> None: + invalid_argument = False + if 'mutation_state' not in self._params: + if isinstance(value, SearchScanConsistency): + if value == SearchScanConsistency.AT_PLUS: + invalid_argument = True + else: + self.set_option('scan_consistency', value.value) + elif isinstance(value, str) and value in [sc.value for sc in SearchScanConsistency]: + if value == SearchScanConsistency.AT_PLUS.value: + invalid_argument = True + else: + self.set_option('scan_consistency', value) + else: + raise InvalidArgumentException(message=("Excepted consistency to be either of type " + "SearchScanConsistency or str representation " + "of SearchScanConsistency")) + + if invalid_argument: + raise InvalidArgumentException(message=("Cannot set consistency to AT_PLUS. Use " + "consistent_with instead or set consistency " + "to NOT_BOUNDED")) + + @property + def consistent_with(self) -> Dict[str, Any]: + return { + 'consistency': self.consistency, + 'scan_vectors': self._params.get('mutation_state', None) + } + + @consistent_with.setter + def consistent_with(self, value # type: MutationState + ): + """ + Indicate that the search should be consistent with one or more + mutations. + + :param value: The state of the mutations it should be consistent + with. + :type state: :class:`~.couchbase.mutation_state.MutationState` + """ + if self.consistency != SearchScanConsistency.NOT_BOUNDED: + raise TypeError( + 'consistent_with not valid with other consistency options') + + # avoid circular import + from couchbase.mutation_state import MutationState # noqa: F811 + if not (isinstance(value, MutationState) and len(value._sv) > 0): + raise TypeError('Passed empty or invalid state') + # 3.x SDK had to set the consistency, couchbase++ will take care of that for us + self._params.pop('scan_consistency', None) + self.set_option('mutation_state', list(mt.as_dict() for mt in value._sv)) + + @property + def scope_name(self) -> Optional[str]: + Supportability.option_deprecated('scope_name', message='The scope_name option is not used by the search API.') + return self._params.get('scope_name', None) + + @scope_name.setter + def scope_name(self, value # type: str + ) -> None: + Supportability.option_deprecated('scope_name', message='The scope_name option is not used by the search API.') + self.set_option('scope_name', value) + + @property + def collections(self) -> Optional[List[str]]: + return self._params.get('collections', None) + + @collections.setter + def collections(self, value # type: List[str] + ) -> None: + if not (isinstance(value, list) and all(map(lambda f: isinstance(f, str), value))): + raise InvalidArgumentException(message='Expected a list of strings') + self.set_option('collections', value) + + @property + def client_context_id(self) -> Optional[str]: + return self._params.get('client_context_id', None) + + @client_context_id.setter + def client_context_id(self, value # type: str + ) -> None: + self.set_option('client_context_id', value) + + @property + def sort(self) -> Optional[Union[List[str], List[Sort]]]: + return self._sort + + @sort.setter + def sort(self, value # type: Union[List[str], List[Sort], List[Union[str, Sort]]] + ) -> None: + if all(map(lambda s: isinstance(s, str), value)): + self._sort = value + elif all(map(lambda s: isinstance(s, (Sort, str)), value)): + self._sort = value + else: + InvalidArgumentException(message='sort option must be either List[str] | List[Sort] | List[Sort | str]') + + @property + def raw(self) -> Optional[Dict[str, Any]]: + return self._params.get('raw', None) + + @raw.setter + def raw(self, value # type: Dict[str, Any] + ) -> None: + if not isinstance(value, dict): + raise TypeError("Raw option must be of type Dict[str, Any].") + for k in value.keys(): + if not isinstance(k, str): + raise TypeError("key for raw value must be str") + raw_params = {f'{k}': json.dumps(v) for k, v in value.items()} + self.set_option('raw', raw_params) + + @property + def serializer(self) -> Optional[Serializer]: + return self._params.get('serializer', None) + + @serializer.setter + def serializer(self, value # type: Serializer + ): + if not issubclass(value.__class__, Serializer): + raise InvalidArgumentException(message='Serializer should implement Serializer interface.') + self.set_option('serializer', value) + + @property + def show_request(self) -> bool: + return self._params.get('show_request', False) + + @show_request.setter + def show_request(self, + value # type: bool + ): + self.set_option('show_request', value) + + @property + def span(self) -> Optional[CouchbaseSpan]: + return self._params.get('span', None) + + @span.setter + def span(self, value # type: CouchbaseSpan + ): + if not issubclass(value.__class__, CouchbaseSpan): + raise InvalidArgumentException(message='Span should implement CouchbaseSpan interface') + self.set_option('span', value) + + @property + def vector_query_combination(self) -> Optional[VectorQueryCombination]: + value = self._params.get('highlight_style', None) + if isinstance(value, VectorQueryCombination): + return value + if isinstance(value, str): + return VectorQueryCombination.AND if value == 'and' else VectorQueryCombination.OR + + @vector_query_combination.setter + def vector_query_combination(self, + value # type: Union[VectorQueryCombination, str] + ) -> None: + if isinstance(value, VectorQueryCombination): + self.set_option('vector_query_combination', value.value) + elif isinstance(value, str): + if value.lower() in [VectorQueryCombination.AND.value, VectorQueryCombination.OR.value]: + self.set_option('vector_query_combination', value.lower()) + else: + raise InvalidArgumentException(message=("Excepted vector_query_combination to be either of type " + "VectorQueryCombination or str representation " + "of VectorQueryCombination")) + + @property + def log_request(self) -> bool: + return self._params.get('log_request', False) + + @log_request.setter + def log_request(self, value # type: bool + ) -> None: + self.set_option('log_request', value) + + @property + def log_response(self) -> bool: + return self._params.get('log_response', False) + + @log_response.setter + def log_response(self, value # type: bool + ) -> None: + self.set_option('log_response', value) + + @classmethod + def create_search_query_object(cls, + index_name, # type: str + search_query, # type: SearchQuery + *options, # type: Optional[SearchOptions] + **kwargs, # type: Dict[str, Any] + ) -> SearchQueryBuilder: + args = SearchQueryBuilder.get_search_query_args(*options, **kwargs) + + facets = args.pop('facets', {}) + sort = args.pop('sort', None) + + query = cls(index_name, query=search_query, facets=facets, sort=sort) + + # metrics defaults to True + query.metrics = args.get("metrics", True) + + for k, v in ((k, args[k]) for k in (args.keys() & cls._VALID_OPTS)): + for target, transform in cls._VALID_OPTS[k].items(): + setattr(query, target, transform(v)) + return query + + @classmethod + def create_search_query_from_request(cls, + index_name, # type: str + request, # type: SearchRequest + *options, # type: Optional[SearchOptions] + **kwargs, # type: Dict[str, Any] + ) -> SearchQueryBuilder: + args = SearchQueryBuilder.get_search_query_args(*options, **kwargs) + + facets = args.pop('facets', {}) + sort = args.pop('sort', None) + + # the search_query should default to MatchNoneQuery if SearchRequest only has vector_search + search_query = request.search_query + if search_query is None: + # avoid circular import + from couchbase.logic.search_queries import MatchNoneQuery + search_query = MatchNoneQuery() + + # only set query vector_query_combination if applicable + if request.vector_search and request.vector_search.options: + combo = request.vector_search.options.get('vector_query_combination', None) + if combo: + args['vector_query_combination'] = combo + + query = cls(index_name, + query=search_query, + vector_search=request.vector_search, + facets=facets, + sort=sort) + + # metrics defaults to True + query.metrics = args.get("metrics", True) + # show_request defaults to False + query.show_request = args.get("show_request", False) + + for k, v in ((k, args[k]) for k in (args.keys() & cls._VALID_OPTS)): + for target, transform in cls._VALID_OPTS[k].items(): + setattr(query, target, transform(v)) + return query + + @staticmethod + def get_search_query_args(*options, **kwargs): + # lets make a copy of the options, and update with kwargs... + opt = SearchOptions() + # TODO: is it possible that we could have [SearchOptions, SearchOptions, ...]?? + # If so, why??? + opts = list(options) + for o in opts: + if isinstance(o, (SearchOptions, SearchOptionsBase)): + opt = o + opts.remove(o) + args = opt.copy() + args.update(kwargs) + return args + + +class FullTextSearchRequestLogic: + def __init__(self, + connection, + encoded_query, + row_factory=SearchRow, + **kwargs + ): + + self._connection = connection + self._encoded_query = encoded_query + self.row_factory = row_factory + self._streaming_result = None + self._default_serializer = kwargs.pop('default_serializer', DefaultJsonSerializer()) + self._serializer = None + self._started_streaming = False + self._streaming_timeout = kwargs.pop('streaming_timeout', None) + self._done_streaming = False + self._metadata = None + self._result_rows = None + self._result_facets = None + self._bucket_name = kwargs.pop('bucket_name', None) + self._scope_name = kwargs.pop('scope_name', None) + + @property + def encoded_query(self) -> Dict[str, Any]: + return self._encoded_query + + @property + def serializer(self) -> Serializer: + if self._serializer: + return self._serializer + + serializer = self.encoded_query.get('serializer', None) + if not serializer: + serializer = self._default_serializer + + self._serializer = serializer + return self._serializer + + @property + def started_streaming(self) -> bool: + return self._started_streaming + + @property + def done_streaming(self) -> bool: + return self._done_streaming + + def metadata(self): + # @TODO: raise if query isn't complete? + return self._metadata + + def result_rows(self): + return self._result_rows + + def result_facets(self): + return self._result_facets + + def _set_facets(self, facets # type: List[Dict[str, Any]] + ) -> None: + if not facets: + return + + facet_keys = ['name', 'field', 'total', 'missing', 'other'] + if not self._result_facets: + self._result_facets = {} + for facet in facets: + facet_dict = {k: v for k, v in facet.items() if k in facet_keys} + new_facet = SearchFacetResult(**facet_dict) + terms = facet.pop('terms', None) + numeric_ranges = facet.pop('numeric_ranges', None) + date_ranges = facet.pop('date_ranges', None) + if terms: + new_facet.terms = [SearchTermFacet(**t) for t in terms] + # facet_results[k].terms = list( + # map(lambda t: SearchTermRange(**t), terms)) + if numeric_ranges: + new_facet.numeric_ranges = [SearchNumericRangeFacet(**nr) for nr in numeric_ranges] + # facet_results[k].numeric_ranges = list( + # map(lambda nr: SearchNumericRange(**nr), numeric_ranges)) + if date_ranges: + new_facet.date_ranges = [SearchDateRangeFacet(**dr) for dr in date_ranges] + # facet_results[k].date_ranges = list( + # map(lambda dr: SearchDateRange(**dr), date_ranges)) + + self._result_facets[new_facet.name] = new_facet + + def _set_metadata(self, search_response): + if isinstance(search_response, CouchbaseBaseException): + raise ErrorMapper.build_exception(search_response) + + result = search_response.raw_result.get('value', None) + if result: + self._metadata = SearchMetaData(result.get('metadata', None)) + self._set_facets(result.get('facets', None)) + + def _deserialize_row(self, row): + # TODO: until streaming, a dict is returned, no deserializing... + # deserialized_row = self.serializer.deserialize(row) + if not issubclass(self.row_factory, SearchRow): + return row + + deserialized_row = row + locations = deserialized_row.get('locations', None) + if locations: + locations = SearchRowLocations(locations) + deserialized_row['locations'] = locations + + fields = deserialized_row.get('fields', None) + if is_null_or_empty(fields): + fields = None + else: + fields = SearchRowFields(**json.loads(fields)) + deserialized_row['fields'] = fields + + explanation = deserialized_row.get('explanation', None) + if is_null_or_empty(explanation): + explanation = {} + else: + explanation = json.loads(explanation) + deserialized_row['explanation'] = explanation + + return self.row_factory(**deserialized_row) + + def _submit_query(self, **kwargs): + if self.done_streaming: + return + + self._started_streaming = True + span = self.encoded_query.pop('span', None) + op_args = self.encoded_query + if self._bucket_name is not None and self._scope_name is not None: + op_args['bucket_name'] = self._bucket_name + op_args['scope_name'] = self._scope_name + + search_kwargs = { + 'conn': self._connection, + 'op_args': op_args + } + if span: + search_kwargs['span'] = span + + streaming_timeout = self.encoded_query.get('timeout', self._streaming_timeout) + if streaming_timeout: + search_kwargs['streaming_timeout'] = streaming_timeout + + # this is for txcouchbase... + callback = kwargs.pop('callback', None) + if callback: + search_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + search_kwargs['errback'] = errback + + self._streaming_result = search_query(**search_kwargs) + + def __iter__(self): + raise NotImplementedError( + 'Cannot use synchronous iterator, are you using `async for`?' + ) + + def __aiter__(self): + raise NotImplementedError( + 'Cannot use asynchronous iterator.' + ) diff --git a/couchbase/logic/search_queries.py b/couchbase/logic/search_queries.py new file mode 100644 index 000000000..13d1eb4cc --- /dev/null +++ b/couchbase/logic/search_queries.py @@ -0,0 +1,1010 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (Any, + Dict, + List, + Optional, + Tuple, + Union) + +from couchbase.exceptions import InvalidArgumentException, NoChildrenException +from couchbase.logic.search import MatchOperator, _QueryBuilder +from couchbase.logic.supportability import Supportability + +# Query Types + + +class SearchQuery: + """ + Base query object. You probably want to use one of the subclasses. + + .. seealso:: :class:`MatchQuery`, :class:`BooleanQuery`, + :class:`RegexQuery`, :class:`PrefixQuery`, :class:`NumericRangeQuery`, + :class:`DateRangeQuery`, :class:`ConjunctionQuery`, + :class:`DisjunctionQuery`, and others in this module. + """ + + def __init__(self): + self._json_ = {} + + def set_prop(self, key, # type: str + value # type: Any + ) -> None: + self._json_[key] = value + + # boost = _genprop( + # float, 'boost', doc=""" + # When specifying multiple queries, you can give this query a + # higher or lower weight in order to prioritize it among sibling + # queries. See :class:`ConjunctionQuery` + # """) + + @property + def boost(self) -> Optional[float]: + return self._json_.get('boost', None) + + @boost.setter + def boost(self, value # type: float + ) -> None: + self.set_prop('boost', value) + + @property + def encodable(self): + """ + Returns an object suitable for serializing to JSON + + .. code-block:: python + + json.dumps(query.encodable) + """ + self.validate() + return self._json_ + + def validate(self): + """ + Optional validation function. Invoked before encoding + """ + pass + + +# Single Term Queries + + +@_QueryBuilder._single_term_query(fields=['fuzziness', 'prefix_length', 'field']) +class TermQuery(SearchQuery): + """ + Searches for a given term in documents. Unlike :class:`MatchQuery`, + the term is not analyzed. + + Example:: + + TermQuery('lcb_cntl_string') + """ + _TERMPROP = 'term' + + @property + def term(self) -> str: + return self._json_.get('term', None) + + @term.setter + def term(self, value # type: str + ) -> None: + self.set_prop('term', value) + # term = _QueryBuilder._genprop_str('term', doc='Exact term to search for') + + +@_QueryBuilder._single_term_query() +class QueryStringQuery(SearchQuery): + """ + Query which allows users to describe a query in a query language. + The server will then execute the appropriate query based on the contents + of the query string: + + .. seealso:: + + `Query Language <http://www.blevesearch.com/docs/Query-String-Query/>`_ + + Example:: + + QueryStringQuery('description:water and stuff') + """ + + _TERMPROP = 'query' + + @property + def query(self) -> str: + return self._json_.get('query', None) + + @query.setter + def query(self, value # type: str + ) -> None: + self.set_prop('query', value) + + +@_QueryBuilder._single_term_query(fields=['field']) +class WildcardQuery(SearchQuery): + """ + Query in which the characters `*` and `?` have special meaning, where + `?` matches 1 occurrence and `*` will match 0 or more occurrences of the + previous character + """ + _TERMPROP = 'wildcard' + + @property + def wildcard(self) -> str: + return self._json_.get('wildcard', None) + + @wildcard.setter + def wildcard(self, value # type: str + ) -> None: + self.set_prop('wildcard', value) + # wildcard = _genprop_str(_TERMPROP, doc='Wildcard pattern to use') + + +@_QueryBuilder._single_term_query() +class DocIdQuery(SearchQuery): + """ + Matches document IDs. This is must useful in a compound query + (for example, :class:`BooleanQuery`). When used as a criteria, only + documents with the specified IDs will be searched. + """ + _TERMPROP = 'ids' + + @property + def ids(self) -> str: + return self._json_.get('ids', None) + + @ids.setter + def ids(self, value # type: str + ) -> None: + self.set_prop('ids', value) + + # ids = _genprop(list, 'ids', doc=""" + # List of document IDs to use + # """) + + def validate(self): + super(DocIdQuery, self).validate() + if not self.ids: + raise NoChildrenException('`ids` must contain at least one ID') + + +@_QueryBuilder._single_term_query(fields=['prefix_length', 'fuzziness', 'field', 'analyzer']) +class MatchQuery(SearchQuery): + """ + Query which checks one or more fields for a match + """ + _TERMPROP = 'match' + + @property + def match(self) -> str: + return self._json_.get('match', None) + + @match.setter + def match(self, value # type: str + ) -> None: + self.set_prop('match', value) + + @property + def match_operator(self) -> Optional[MatchOperator]: + value = self._json_.get('operator', None) + if not value: + return value + + return MatchOperator.OR if value == 'or' else MatchOperator.AND + + @match_operator.setter + def match_operator(self, value # type: Union[MatchOperator, str] + ) -> None: + match_op = value + if isinstance(value, MatchOperator): + match_op = value.value + if match_op and match_op.lower() not in ('or', 'and'): + raise ValueError(("Excepted match_operator to be either of type " + "MatchOperator or str representation " + "of MatchOperator")) + self.set_prop('operator', match_op) + + # match = _genprop_str( + # 'match', doc=""" + # String to search for + # """) + # match_operator = _genprop( + # _match_operator, 'operator', doc='**UNCOMMITTED** This API may change in the future. + # Specifies how the individual match terms should be logically concatenated.') + + +@_QueryBuilder._single_term_query(fields=['field', 'analyzer']) +class MatchPhraseQuery(SearchQuery): + """ + Search documents which match a given phrase. The phrase is composed + of one or more terms. + + Example:: + + MatchPhraseQuery("Hello world!") + """ + _TERMPROP = 'match_phrase' + + @property + def match_phrase(self) -> str: + return self._json_.get('match_phrase', None) + + @match_phrase.setter + def match_phrase(self, value # type: str + ) -> None: + self.set_prop('match_phrase', value) + + # match_phrase = _genprop_str(_TERMPROP, doc="Phrase to search for") + + +@_QueryBuilder._with_fields(fields=['field']) +class PhraseQuery(SearchQuery): + _TERMPROP = 'terms' + + def __init__(self, *phrases, # type: str + **kwargs # type: Dict[str, Any] + ) -> None: + super().__init__() + if self._TERMPROP not in kwargs: + kwargs[self._TERMPROP] = phrases + _QueryBuilder._assign_kwargs(self, kwargs) + + @property + def terms(self) -> List[str]: + return self._json_.get('terms', None) + + @terms.setter + def terms(self, value # type: Union[List[str], Tuple[str]] + ) -> None: + if not (isinstance(value, (list, tuple)) and all(map(lambda f: isinstance(f, str), value))): + raise InvalidArgumentException(message='Expected a list of strings') + if isinstance(value, tuple): + self.set_prop('terms', list(value)) + else: + self.set_prop('terms', value) + + def validate(self): + super(PhraseQuery, self).validate() + if not self.terms: + raise NoChildrenException('Missing terms') + + # terms = _genprop(list, 'terms', doc='List of terms to search for') + + +@_QueryBuilder._single_term_query(fields=['field']) +class PrefixQuery(SearchQuery): + """ + Search documents for fields beginning with a certain prefix. This is + most useful for type-ahead or lookup queries. + """ + _TERMPROP = 'prefix' + + @property + def prefix(self) -> str: + return self._json_.get('prefix', None) + + @prefix.setter + def prefix(self, value # type: str + ) -> None: + self.set_prop('prefix', value) + + # prefix = _genprop_str('prefix', doc='The prefix to match') + + +@_QueryBuilder._single_term_query(fields=['field']) +class RegexQuery(SearchQuery): + """ + Search documents for fields matching a given regular expression + """ + _TERMPROP = 'regexp' + + @property + def regex(self) -> str: + return self._json_.get('regexp', None) + + @regex.setter + def regex(self, value # type: str + ) -> None: + self.set_prop('regexp', value) + + @property + def regexp(self) -> str: + return self._json_.get('regexp', None) + + @regexp.setter + def regexp(self, value # type: str + ) -> None: + self.set_prop('regexp', value) + + # regex = _genprop_str('regexp', doc="Regular expression to use") + + +RegexpQuery = RegexQuery + + +@_QueryBuilder._single_term_query(fields=['field']) +class BooleanFieldQuery(SearchQuery): + _TERMPROP = 'bool' + + @property + def bool(self) -> bool: + return self._json_.get('bool', None) + + @bool.setter + def bool(self, value # type: bool + ) -> None: + self.set_prop('bool', value) + # bool = _genprop(bool, 'bool', doc='Boolean value to search for') + + +# Geo Queries + + +@_QueryBuilder._with_fields(fields=['field']) +class GeoDistanceQuery(SearchQuery): + def __init__(self, distance, # type: str + location, # type: Tuple[float, float] + **kwargs # type: Dict[str, Any] + ) -> None: + """ + Search for items within a given radius + :param distance: The distance string specifying the radius + :param location: A tuple of `(lon, lat)` indicating point of origin + """ + super(GeoDistanceQuery, self).__init__() + kwargs['distance'] = distance + kwargs['location'] = location + _QueryBuilder._assign_kwargs(self, kwargs) + + @property + def location(self) -> Tuple[float, float]: + return self._json_.get('location') + + @location.setter + def location(self, value # type: Tuple[float, float] + ) -> None: + location = _QueryBuilder._gen_location(value) + self.set_prop('location', location) + + @property + def distance(self) -> str: + return self._json_.get('distance', None) + + @distance.setter + def distance(self, value # type: str + ) -> None: + self.set_prop('distance', value) + + # location = _genprop(_location_conv, 'location', doc='Location') + # distance = _genprop_str('distance') + + +@_QueryBuilder._with_fields(fields=['field']) +class GeoBoundingBoxQuery(SearchQuery): + def __init__(self, top_left, # type: Tuple[float, float] + bottom_right, # type: Tuple[float, float] + **kwargs # type: Dict[str, Any] + ) -> None: + super(GeoBoundingBoxQuery, self).__init__() + kwargs['top_left'] = top_left + kwargs['bottom_right'] = bottom_right + _QueryBuilder._assign_kwargs(self, kwargs) + + @property + def top_left(self) -> Tuple[float, float]: + return self._json_.get('top_left') + + @top_left.setter + def top_left(self, value # type: Tuple[float, float] + ) -> None: + top_left = _QueryBuilder._gen_location(value) + self.set_prop('top_left', top_left) + + @property + def bottom_right(self) -> Tuple[float, float]: + return self._json_.get('bottom_right') + + @bottom_right.setter + def bottom_right(self, value # type: Tuple[float, float] + ) -> None: + bottom_right = _QueryBuilder._gen_location(value) + self.set_prop('bottom_right', bottom_right) + + # top_left = _genprop( + # _location_conv, 'top_left', + # doc='Tuple of `(lon, lat)` for the top left corner of bounding box') + # bottom_right = _genprop( + # _location_conv, 'bottom_right', + # doc='Tuple of `(lon, lat`) for the bottom right corner of bounding box') + + +@_QueryBuilder._with_fields(fields=['field']) +class GeoPolygonQuery(SearchQuery): + def __init__(self, polygon_points, # type: List[Tuple[float, float]] + **kwargs # type: Dict[str, Any] + ) -> None: + super(GeoPolygonQuery, self).__init__() + kwargs['polygon_points'] = polygon_points + _QueryBuilder._assign_kwargs(self, kwargs) + + @property + def polygon_points(self) -> Tuple[float, float]: + return self._json_.get('polygon_points') + + @polygon_points.setter + def polygon_points(self, value # type: Tuple[float, float] + ) -> None: + polygon_points = _QueryBuilder._gen_locations(value) + self.set_prop('polygon_points', polygon_points) + + # polygon_points = _genprop( + # _locations_conv, 'polygon_points', + # doc='List of tuples `(lon, lat)` for the points of a polygon') + + +# Range Queries + + +@_QueryBuilder._with_fields(fields=['field']) +class NumericRangeQuery(SearchQuery): + """ + Search documents for fields containing a value within a given numerical + range. + + At least one of `min` or `max` must be specified. + """ + + def __init__(self, + min=None, # type: Optional[float] + max=None, # type: Optional[float] + **kwargs # type: Dict[str, Any] + ) -> None: + """ + :param float min: See :attr:`min` + :param float max: See :attr:`max` + """ + # super(NumericRangeQuery, self).__init__(min, max, **kwargs) + super().__init__() + _QueryBuilder._validate_range_query(self, min, max, **kwargs) + + _MINMAX = 'min', 'max' + + @property + def min(self) -> Optional[float]: + return self._json_.get('min', None) + + @min.setter + def min(self, + value # type: float + ) -> None: + self.set_prop('min', value) + + @property + def min_inclusive(self) -> Optional[bool]: + return self._json_.get('min_inclusive', None) + + @min_inclusive.setter + def min_inclusive(self, + value # type: bool + ) -> None: + Supportability.class_property_deprecated('min_inclusive', 'inclusive_min') + self.set_prop('inclusive_min', value) + + @property + def inclusive_min(self) -> Optional[bool]: + return self._json_.get('inclusive_min', None) + + @inclusive_min.setter + def inclusive_min(self, + value # type: bool + ) -> None: + self.set_prop('inclusive_min', value) + + @property + def max(self) -> Optional[float]: + return self._json_.get('max', None) + + @max.setter + def max(self, + value # type: float + ) -> None: + self.set_prop('max', value) + + @property + def max_inclusive(self) -> Optional[bool]: + return self._json_.get('max_inclusive', None) + + @max_inclusive.setter + def max_inclusive(self, + value # type: bool + ) -> None: + Supportability.class_property_deprecated('max_inclusive', 'inclusive_max') + self.set_prop('inclusive_max', value) + + @property + def inclusive_max(self) -> Optional[bool]: + return self._json_.get('inclusive_max', None) + + @inclusive_max.setter + def inclusive_max(self, + value # type: bool + ) -> None: + self.set_prop('inclusive_max', value) + + # min = _genprop( + # float, 'min', doc='Lower bound of range. See :attr:`min_inclusive`') + + # min_inclusive = _genprop( + # bool, 'inclusive_min', + # doc='Whether matches are inclusive of lower bound') + + # max = _genprop( + # float, 'max', + # doc='Upper bound of range. See :attr:`max_inclusive`') + + # max_inclusive = _genprop( + # bool, 'inclusive_max', + # doc='Whether matches are inclusive of upper bound') + + +@_QueryBuilder._with_fields(fields=['field']) +class DateRangeQuery(SearchQuery): + """ + Search documents for fields containing a value within a given date + range. + + The date ranges are parsed according to a given :attr:`datetime_parser`. + If no parser is specified, the RFC 3339 parser is used. See + `Generating an RFC 3339 Timestamp <http://goo.gl/LIkV7G>_`. + + The :attr:`start` and :attr:`end` parameters should be specified in the + constructor. Note that either `start` or `end` (but not both!) may be + omitted. + + .. code-block:: python + + DateRangeQuery(start='2014-12-25', end='2016-01-01') + """ + + def __init__(self, start=None, end=None, **kwargs): + """ + :param str start: Start of date range + :param str end: End of date range + :param kwargs: Additional options: :attr:`field`, :attr:`boost` + """ + super().__init__() + _QueryBuilder._validate_range_query(self, start, end, **kwargs) + + # super(DateRangeQuery, self).__init__(start, end, **kwargs) + + _MINMAX = 'start', 'end' + + @property + def start(self) -> Optional[str]: + return self._json_.get('start', None) + + @start.setter + def start(self, + value # type: str + ) -> None: + self.set_prop('start', value) + + @property + def start_inclusive(self) -> Optional[bool]: + return self._json_.get('start_inclusive', None) + + @start_inclusive.setter + def start_inclusive(self, + value # type: bool + ) -> None: + Supportability.class_property_deprecated('start_inclusive', 'inclusive_start') + self.set_prop('inclusive_start', value) + + @property + def inclusive_start(self) -> Optional[bool]: + return self._json_.get('inclusive_start', None) + + @inclusive_start.setter + def inclusive_start(self, + value # type: bool + ) -> None: + self.set_prop('inclusive_start', value) + + @property + def end(self) -> Optional[str]: + return self._json_.get('end', None) + + @end.setter + def end(self, + value # type: str + ) -> None: + self.set_prop('end', value) + + @property + def end_inclusive(self) -> Optional[bool]: + return self._json_.get('end_inclusive', None) + + @end_inclusive.setter + def end_inclusive(self, + value # type: bool + ) -> None: + Supportability.class_property_deprecated('end_inclusive', 'inclusive_end') + self.set_prop('inclusive_end', value) + + @property + def inclusive_end(self) -> Optional[bool]: + return self._json_.get('inclusive_end', None) + + @inclusive_end.setter + def inclusive_end(self, + value # type: bool + ) -> None: + self.set_prop('inclusive_end', value) + + @property + def datetime_parser(self) -> Optional[str]: + return self._json_.get('datetime_parser', None) + + @datetime_parser.setter + def datetime_parser(self, + value # type: str + ) -> None: + self.set_prop('datetime_parser', value) + + # start = _genprop_str('start', doc='Lower bound datetime') + # end = _genprop_str('end', doc='Upper bound datetime') + + # start_inclusive = _genprop( + # bool, 'inclusive_start', doc='If :attr:`start` is inclusive') + + # end_inclusive = _genprop( + # bool, 'inclusive_end', doc='If :attr:`end` is inclusive') + + # datetime_parser = _genprop_str( + # 'datetime_parser', + # doc=""" + # Parser to use when analyzing the :attr:`start` and :attr:`end` fields + # on the server. + + # If not specified, the RFC 3339 parser is used. + # Ensure to specify :attr:`start` and :attr:`end` in a format suitable + # for the given parser. + # """) + + +@_QueryBuilder._with_fields(fields=['field']) +class TermRangeQuery(SearchQuery): + """ + Search documents for fields containing a value within a given + lexical range. + """ + + _MINMAX = 'min', 'max' + + def __init__(self, + start=None, # type: Optional[str] + end=None, # type: Optional[str] + min=None, # type: Optional[str] + max=None, # type: Optional[str] + **kwargs # type: Dict[str, Any] + ) -> None: + """ + Args: + start (str): **DEPRECATED** Use min. + end (str): **DEPRECATED** Use max. + min (str): The lower end of the range. + max (str): The higher end of the range. + """ + super().__init__() + if start is not None and min is None: + Supportability.class_property_deprecated('start', 'min') + min = start + if end is not None and max is None: + Supportability.class_property_deprecated('end', 'max') + max = end + + _QueryBuilder._validate_range_query(self, min, max, **kwargs) + + @property + def start(self) -> Optional[str]: + return self._json_.get('start', None) + + @start.setter + def start(self, value # type: str + ) -> None: + Supportability.class_property_deprecated('start', 'min') + self.set_prop('min', value) + + @property + def min(self) -> Optional[str]: + return self._json_.get('min', None) + + @min.setter + def min(self, value # type: str + ) -> None: + self.set_prop('min', value) + + @property + def start_inclusive(self) -> Optional[bool]: + return self._json_.get('start_inclusive', None) + + @start_inclusive.setter + def start_inclusive(self, value # type: bool + ) -> None: + Supportability.class_property_deprecated('start_inclusive', 'inclusive_min') + self.set_prop('inclusive_min', value) + + @property + def inclusive_min(self) -> Optional[bool]: + return self._json_.get('start_inclusive', None) + + @inclusive_min.setter + def inclusive_min(self, value # type: bool + ) -> None: + self.set_prop('inclusive_min', value) + + @property + def end(self) -> Optional[str]: + return self._json_.get('end', None) + + @end.setter + def end(self, value # type: str + ) -> None: + Supportability.class_property_deprecated('end', 'max') + self.set_prop('max', value) + + @property + def max(self) -> Optional[str]: + return self._json_.get('max', None) + + @max.setter + def max(self, value # type: str + ) -> None: + self.set_prop('max', value) + + @property + def end_inclusive(self) -> Optional[bool]: + return self._json_.get('end_inclusive', None) + + @end_inclusive.setter + def end_inclusive(self, value # type: bool + ) -> None: + Supportability.class_property_deprecated('end_inclusive', 'inclusive_max') + self.set_prop('inclusive_max', value) + + @property + def inclusive_max(self) -> Optional[bool]: + return self._json_.get('inclusive_max', None) + + @inclusive_max.setter + def inclusive_max(self, value # type: bool + ) -> None: + self.set_prop('inclusive_max', value) + + # def __init__(self, start=None, end=None, **kwargs): + # super(TermRangeQuery, self).__init__(start=start, end=end, **kwargs) + + # start = _genprop_str('start', doc='Lower range of term') + + # end = _genprop_str('end', doc='Upper range of term') + + # start_inclusive = _genprop( + # bool, 'inclusive_start', doc='If :attr:`start` is inclusive') + + # end_inclusive = _genprop( + # bool, 'inclusive_end', doc='If :attr:`end` is inclusive') + + +# Compound Queries + +class ConjunctionQuery(SearchQuery): + """ + Compound query in which all sub-queries passed must be satisfied + """ + _COMPOUND_FIELDS = ('conjuncts', 'conjuncts'), + + def __init__(self, *queries, **kwargs): + super().__init__() + _QueryBuilder._assign_kwargs(self, kwargs) + self.conjuncts = list(queries) + + @property + def encodable(self): + self.validate() + + # Requires special handling since the compound queries in themselves + # cannot be JSON unless they are properly encoded. + # Note that the 'encodable' property also triggers validation + js = self._json_.copy() + for src, tgt in self._COMPOUND_FIELDS: + objs = getattr(self, src) + if not objs: + continue + js[tgt] = [q.encodable for q in objs] + return js + + def validate(self): + super(ConjunctionQuery, self).validate() + if not self.conjuncts: + raise NoChildrenException('No sub-queries') + + +class DisjunctionQuery(SearchQuery): + """ + Compound query in which at least :attr:`min` or more queries must be + satisfied + """ + _COMPOUND_FIELDS = ('disjuncts', 'disjuncts'), + + def __init__(self, *queries, **kwargs): + super().__init__() + _QueryBuilder._assign_kwargs(self, kwargs) + self.disjuncts = list(queries) + if 'min' not in self._json_: + self.min = 1 + + @property + def min(self) -> int: + return self._json_.get('min', None) + + @min.setter + def min(self, value # type: bool + ) -> None: + value = int(value) + if not value: + raise InvalidArgumentException(message='Must be > 0') + self.set_prop('min', value) + + @property + def encodable(self): + self.validate() + + # Requires special handling since the compound queries in themselves + # cannot be JSON unless they are properly encoded. + # Note that the 'encodable' property also triggers validation + js = self._json_.copy() + for src, tgt in self._COMPOUND_FIELDS: + objs = getattr(self, src) + if not objs: + continue + js[tgt] = [q.encodable for q in objs] + return js + + # min = _genprop( + # _convert_gt0, 'min', doc='Number of queries which must be satisfied') + + def validate(self): + super(DisjunctionQuery, self).validate() + if not self.disjuncts: + raise NoChildrenException('No queries specified') + if len(self.disjuncts) < self.min: + raise InvalidArgumentException(message='Specified min is larger than number of queries.') + + +CompoundQueryType = Union[SearchQuery, ConjunctionQuery, DisjunctionQuery, List[SearchQuery]] + + +class BooleanQuery(SearchQuery): + def __init__(self, must=None, should=None, must_not=None): + super().__init__() + self._subqueries = {} + self.must = must + self.should = should + self.must_not = must_not + + @property + def must(self) -> ConjunctionQuery: + return self._subqueries.get('must') + + @must.setter + def must(self, value # type: CompoundQueryType + ) -> None: + self._set_query('must', value, ConjunctionQuery) + + @property + def must_not(self) -> DisjunctionQuery: + return self._subqueries.get('must_not') + + @must_not.setter + def must_not(self, value # type: CompoundQueryType + ) -> None: + self._set_query('must_not', value, DisjunctionQuery) + + @property + def should(self) -> DisjunctionQuery: + return self._subqueries.get('should') + + @should.setter + def should(self, value # type: CompoundQueryType + ) -> None: + self._set_query('should', value, DisjunctionQuery) + + @property + def encodable(self) -> Dict[str, Any]: + # Overrides the default `encodable` implementation in order to + # serialize any sub-queries + for src, tgt in ((self.must, 'must'), + (self.must_not, 'must_not'), + (self.should, 'should')): + if src: + self._json_[tgt] = src.encodable + return super(BooleanQuery, self).encodable + + def validate(self) -> None: + super(BooleanQuery, self).validate() + if not self.must and not self.must_not and not self.should: + raise ValueError('No sub-queries specified', self) + + def _set_query(self, name, # type: str + value, # type: Optional[CompoundQueryType] + reqtype # type: Union[ConjunctionQuery, DisjunctionQuery] + ) -> None: + if value is None: + if name in self._subqueries: + del self._subqueries[name] + elif isinstance(value, reqtype): + self._subqueries[name] = value + elif isinstance(value, SearchQuery): + self._subqueries[name] = reqtype(value) + else: + try: + it = iter(value) + except ValueError: + raise TypeError('Value must be iterable') + + sub_query = [] + for query in it: + if not isinstance(query, SearchQuery): + raise TypeError('Item is not a query!', query) + sub_query.append(query) + self._subqueries[name] = reqtype(*sub_query) + + +# Special Queries + +class RawQuery(SearchQuery): + """ + This class is used to wrap a raw query payload. It should be used + for custom query parameters, or in cases where any of the other + query classes are insufficient. + """ + + def __init__(self, obj): + super(RawQuery, self).__init__() + self._json_ = obj + + +class MatchAllQuery(SearchQuery): + """ + Special query which matches all documents + """ + + def __init__(self, **kwargs): + super(MatchAllQuery, self).__init__() + self._json_['match_all'] = None + _QueryBuilder._assign_kwargs(self, kwargs) + + +class MatchNoneQuery(SearchQuery): + """ + Special query which matches no documents + """ + + def __init__(self, **kwargs): + super(MatchNoneQuery, self).__init__() + self._json_['match_none'] = None + _QueryBuilder._assign_kwargs(self, kwargs) diff --git a/couchbase/logic/search_request.py b/couchbase/logic/search_request.py new file mode 100644 index 000000000..935314ff5 --- /dev/null +++ b/couchbase/logic/search_request.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +from typing import Optional, Union + +from couchbase.exceptions import InvalidArgumentException +from couchbase.logic.search_queries import SearchQuery +from couchbase.logic.vector_search import VectorSearch + + +class SearchRequest: + """ Represents a search query and/or vector search to execute via the Couchbase Full Text Search (FTS) service. + + Args: + query (Union[:class:`~couchbase.search.SearchQuery`, :class:`~couchbase.vector_search.VectorSearch`]): A :class:`~couchbase.search.SearchQuery` or + :class:`~couchbase.vector_search.VectorSearch` to initialize the search request. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If neither a :class:`~couchbase.search.SearchQuery` or :class:`~couchbase.vector_search.VectorSearch` is provided. + + Returns: + :class:`~couchbase.search.SearchRequest`: The created search request. + """ # noqa: E501 + + def __init__(self, + query # type: Union[SearchQuery, VectorSearch] + ): + self._search_query = self._vector_search = None + if isinstance(query, SearchQuery): + self._search_query = query + elif isinstance(query, VectorSearch): + self._vector_search = query + + if self._search_query is None and self._vector_search is None: + raise InvalidArgumentException(('Must provide either a SearchQuery or VectorSearch ' + 'when creating a SearchRequest.')) + + @property + def search_query(self) -> Optional[SearchQuery]: + """ + Optional[:class:`~couchbase.search.SearchQuery`]: Returns the search request's :class:`~couchbase.search.SearchQuery`, if it exists. + """ # noqa: E501 + return self._search_query + + @property + def vector_search(self) -> Optional[VectorSearch]: + """ + Optional[:class:`~couchbase.vector_search.VectorSearch`]: Returns the search request's :class:`~couchbase.vector_search.VectorSearch`, if it exists. + """ # noqa: E501 + return self._vector_search + + def with_search_query(self, + query # type: SearchQuery + ) -> SearchRequest: + """ Add a :class:`~couchbase.search.SearchQuery` to the search request. + + Args: + query (:class:`~couchbase.search.SearchQuery`): The :class:`~couchbase.search.SearchQuery` to add to the search request. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the search request already contains a :class:`~couchbase.search.SearchQuery`. + :class:`~couchbase.exceptions.InvalidArgumentException`: If the provided query is not an instance of a :class:`~couchbase.search.SearchQuery`. + + Returns: + :class:`~couchbase.search.SearchRequest`: The search request in order to allow method chaining. + """ # noqa: E501 + is_search_query = isinstance(query, SearchQuery) + if not is_search_query: + raise InvalidArgumentException('Must provide a SearchQuery.') + if is_search_query and self._search_query is not None: + raise InvalidArgumentException('Request already has SearchQuery.') + + self._search_query = query + return self + + def with_vector_search(self, + vector_search # type: VectorSearch + ) -> SearchRequest: + """ Add a :class:`~couchbase.vector_search.VectorSearch` to the search request. + + Args: + vector_search (:class:`~couchbase.vector_search.VectorSearch`): The :class:`~couchbase.vector_search.VectorSearch` to add to the search request. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the search request already contains a :class:`~couchbase.vector_search.VectorSearch`. + :class:`~couchbase.exceptions.InvalidArgumentException`: If the provided query is not an instance of a :class:`~couchbase.vector_search.VectorSearch`. + + Returns: + :class:`~couchbase.search.SearchRequest`: The search request in order to allow method chaining. + """ # noqa: E501 + is_vector_search = isinstance(vector_search, VectorSearch) + if not is_vector_search: + raise InvalidArgumentException('Must provide a VectorSearch.') + if is_vector_search and self._vector_search is not None: + raise InvalidArgumentException('Request already has VectorSearch.') + + self._vector_search = vector_search + return self + + @classmethod + def create(cls, + query # type: Union[SearchQuery, VectorSearch] + ) -> SearchRequest: + """ Creates a :class:`~couchbase.search.SearchRequest`. + + Args: + query (Union[:class:`~couchbase.search.SearchQuery`, :class:`~couchbase.vector_search.VectorSearch`]): A :class:`~couchbase.search.SearchQuery` or + :class:`~couchbase.vector_search.VectorSearch` to initialize the search request. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If neither a :class:`~couchbase.search.SearchQuery` or :class:`~couchbase.vector_search.VectorSearch` is provided. + + Returns: + :class:`~couchbase.search.SearchRequest`: The created search request. + """ # noqa: E501 + return cls(query) diff --git a/couchbase/logic/supportability.py b/couchbase/logic/supportability.py new file mode 100644 index 000000000..75863170c --- /dev/null +++ b/couchbase/logic/supportability.py @@ -0,0 +1,170 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import warnings +from inspect import Signature +from typing import Optional + + +class CouchbaseDeprecationWarning(UserWarning): + """ + Couchbase Python SDK Warning Category + """ + + +class Supportability: + @classmethod + def import_deprecated(cls, old_path, new_path): + def decorator(cls): + old_init = cls.__init__ + + def new_init(self, *args, **kwargs): + msg = (f"Importing {cls.__name__} from {old_path} is deprecated " + "and will be removed in a future release. " + f" Import {cls.__name__} from {new_path} instead.") + warnings.warn(msg, CouchbaseDeprecationWarning, stacklevel=2) + old_init(self, *args, **kwargs) + + cls.__init__ = new_init + return cls + return decorator + + @classmethod + def class_deprecated(cls, use_instead): + def decorator(cls): + old_init = cls.__init__ + + def new_init(self, *args, **kwargs): + msg = (f"Class {cls.__name__} is deprecated " + "and will be removed in a future release. " + f"Use {use_instead} instead.") + warnings.warn(msg, CouchbaseDeprecationWarning, stacklevel=2) + old_init(self, *args, **kwargs) + + cls.__init__ = new_init + return cls + return decorator + + @staticmethod + def class_property_deprecated(property, # type: str + use_instead # type: str + ) -> None: + """Issue a `CouchbaseDeprecationWarning` indicating the provided class property is deprecated. + + Args: + property (str): The name of the deprecated property + use_instead (str): The name of the property to use instead of the deprecated property. + """ + message = (f"Class property {property} is deprecated and will be removed in a future release. " + f"Use {use_instead} instead.") + warnings.warn(message, CouchbaseDeprecationWarning, stacklevel=2) + + @staticmethod + def method_deprecated(method, # type: str + use_instead # type: str + ) -> None: + """Issue a `CouchbaseDeprecationWarning` indicating the provided method is deprecated. + + Args: + method (str): The name of the deprecated method + use_instead (str): The name of the method to use instead of the deprecated method. + """ + message = (f"Method {method} is deprecated and will be removed in a future release. " + f"Use {use_instead} instead.") + warnings.warn(message, CouchbaseDeprecationWarning, stacklevel=2) + + @staticmethod + def method_param_deprecated(param, # type: str + use_instead # type: str + ) -> None: + """Issue a `CouchbaseDeprecationWarning` indicating the provided param is deprecated. + + Args: + param (str): The name of the deprecated param + use_instead (str): The name of the param to use instead of the deprecated param. + """ + message = (f"Method parameter {param} is deprecated and will be removed in a future release. " + f"Use {use_instead} instead.") + warnings.warn(message, CouchbaseDeprecationWarning, stacklevel=2) + + @staticmethod + def method_kwarg_deprecated(kwarg, # type: str + use_instead # type: str + ) -> None: + """Issue a `CouchbaseDeprecationWarning` indicating the provided param is deprecated. + + Args: + kwarg (str): The name of the deprecated key-word argument. + use_instead (str): The name of the param to use instead of the deprecated param. + """ + message = (f"Method key-word (kwarg) parameter {kwarg} is deprecated and will be removed in a future release. " + f"Use {use_instead} instead.") + warnings.warn(message, CouchbaseDeprecationWarning, stacklevel=2) + + @staticmethod + def method_signature_deprecated(method_name, # type: str + signature, # type: Signature + use_instead, # type: Signature + ) -> None: + """Issue a `CouchbaseDeprecationWarning` indicating the method overload with the provided signature is + deprecated. + + Args + method_name (str): The name of the method + signature (Signature): The signature of the deprecated overload + use_instead (Signature): The signature of the overload that should be used instead. + + """ + message = (f"Method {method_name} with signature {signature} is deprecated and will be removed in " + f"a future release. Use {method_name} with signature {use_instead} instead.") + warnings.warn(message, CouchbaseDeprecationWarning, stacklevel=2) + + @staticmethod + def option_deprecated(param, # type: str + use_instead=None, # type: Optional[str] + message=None, # type: Optional[str] + ) -> None: + """Issue a `CouchbaseDeprecationWarning` indicating the provided param is deprecated. + + Args: + param (str): The name of the deprecated param + use_instead (Optional, str): The name of the param to use instead of the deprecated param. + message (Optional, str): A message to have in the warning to add context. + """ + msg = f"Option {param} is deprecated and will be removed in a future release. " + if use_instead: + msg += f"Use {use_instead} instead. " + if message: + msg += message + + warnings.warn(msg, CouchbaseDeprecationWarning, stacklevel=2) + + +class RemoveProperty: + """Used to override a get descriptor for a class property and raise an AttributeError to prevent access. + + This helper class should only be used in **rare** instances. Specifically, it allows for an inheritance + structure to remain intact while removing a property from a subclass. In an ideal scenario, the hierarchy + structure is revisted, but in some cases it is easier to keep the current structure. This class provides a + work-around. + """ + + def __init__(self, prop): + self._prop = prop + + def __get__(self, instance, cls): + raise AttributeError(f'Property "{self._prop}" is no longer a part of the {cls.__name__} class.') diff --git a/couchbase/logic/vector_search.py b/couchbase/logic/vector_search.py new file mode 100644 index 000000000..ebca81a33 --- /dev/null +++ b/couchbase/logic/vector_search.py @@ -0,0 +1,246 @@ +from __future__ import annotations + +from enum import Enum +from typing import (TYPE_CHECKING, + List, + Optional, + Union) + +from couchbase._utils import is_null_or_empty +from couchbase.exceptions import InvalidArgumentException +from couchbase.options import VectorSearchOptions + +if TYPE_CHECKING: + from couchbase.logic.search_queries import SearchQuery + + +class VectorQueryCombination(Enum): + """ Specifies how multiple vector searches are combined. + + This can be one of: + AND: Indicates that multiple vector queries should be combined with logical AND. + + OR: Indicates that multiple vector queries should be combined with logical OR. + + """ + AND = 'and' + OR = 'or' + + +class VectorQuery: + """ Represents a vector query. + + Args: + field_name (str): The name of the field in the search index that stores the vector. + vector (Union[List[float], str]): The vector to use in the query. + num_candidates (int, optional): Specifies the number of results returned. If provided, must be greater or equal to 1. + boost (float, optional): Add boost to query. + prefilter (`~couchbase.search.SearchQuery`, optional): Specifies a pre-filter to use for the vector query. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the vector is not provided. + :class:`~couchbase.exceptions.InvalidArgumentException`: If the vector is not a list or str. + :class:`~couchbase.exceptions.InvalidArgumentException`: If vector is a list and all values of the provided vector are not instances of float. + + Returns: + :class:`~couchbase.vector_search.VectorQuery`: The created vector query. + """ # noqa: E501 + + def __init__(self, + field_name, # type: str + vector, # type: Union[List[float], str] + num_candidates=None, # type: Optional[int] + boost=None, # type: Optional[float] + prefilter=None, # type: Optional[SearchQuery] + ): + if is_null_or_empty(field_name): + raise InvalidArgumentException('Must provide a field name.') + self._field_name = field_name + self._vector = None + self._vector_base64 = None + self._validate_and_set_vector(vector) + self._num_candidates = self._boost = self._prefilter = None + if num_candidates is not None: + self.num_candidates = num_candidates + if boost is not None: + self.boost = boost + if prefilter is not None: + self.prefilter = prefilter + + @property + def boost(self) -> Optional[float]: + """ + Optional[float]: Returns vector query's boost value, if it exists. + """ + return self._boost + + @boost.setter + def boost(self, + value # type: float + ): + if not isinstance(value, float): + raise InvalidArgumentException('boost must be a float.') + self._boost = value + + @property + def field_name(self) -> str: + """ + str: Returns vector query's field name + """ + return self._field_name + + @property + def num_candidates(self) -> Optional[int]: + """ + Optional[int]: Returns vector query's num candidates value, if it exists. + """ + return self._num_candidates + + @num_candidates.setter + def num_candidates(self, + value # type: int + ): + if not isinstance(value, int): + raise InvalidArgumentException('num_candidates must be an int.') + if value < 1: + raise InvalidArgumentException('num_candidates must be >= 1.') + self._num_candidates = value + + @property + def prefilter(self) -> Optional[SearchQuery]: + """ + Optional[SearchQuery]: Returns vector query's prefilter query, if it exists. + """ + return self._prefilter + + @prefilter.setter + def prefilter(self, + value # type: SearchQuery + ): + # avoid circular import + from couchbase.logic.search_queries import SearchQuery + if not isinstance(value, SearchQuery): + raise InvalidArgumentException('prefilter must be a SearchQuery.') + self._prefilter = value + + @property + def vector(self) -> Optional[List[float]]: + """ + Optional[List[float]]: Returns the vector query's vector. + """ + return self._vector + + @property + def vector_base64(self) -> Optional[str]: + """ + Optional[str]: Returns the vector query's base64 vector str. + """ + return self._vector_base64 + + def _validate_and_set_vector(self, + vector, # type: Union[List[float], str] + ) -> None: + if vector is None: + raise InvalidArgumentException('Provided vector cannot be empty.') + if isinstance(vector, list): + if len(vector) == 0: + raise InvalidArgumentException('Provided vector cannot be empty.') + if not all(map(lambda q: isinstance(q, float), vector)): + raise InvalidArgumentException('All vector values must be a float.') + self._vector = vector + return + elif not isinstance(vector, str): + raise InvalidArgumentException('Provided vector must be either a List[float] or base64 encoded str.') + + if len(vector) == 0: + raise InvalidArgumentException('Provided base64 encoded vector cannot be empty.') + + self._vector_base64 = vector + + @classmethod + def create(cls, + field_name, # type: str + vector, # type: Union[List[float], str] + num_candidates=None, # type: Optional[int] + boost=None, # type: Optional[float] + prefilter=None, # type: Optional[SearchQuery] + ) -> VectorQuery: + """ Creates a :class:`~couchbase.vector_search.VectorQuery`. + + Args: + field_name (str): The name of the field in the search index that stores the vector. + vector (Union[List[float], str]): The vector to use in the query. + num_candidates (int, optional): Specifies the number of results returned. If provided, must be greater or equal to 1. + boost (float, optional): Add boost to query. + prefilter (`~couchbase.search.SearchQuery`, optional): Specifies a pre-filter to use for the vector query. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the vector is not provided. + :class:`~couchbase.exceptions.InvalidArgumentException`: If the vector is not a list or str. + :class:`~couchbase.exceptions.InvalidArgumentException`: If vector is a list and all values of the provided vector are not instances of float. + + Returns: + :class:`~couchbase.vector_search.VectorQuery`: The created vector query. + """ # noqa: E501 + return cls(field_name, vector, num_candidates=num_candidates, boost=boost, prefilter=prefilter) + + +class VectorSearch: + """ Represents a vector search. + + Args: + queries (List[:class:`~couchbase.vector_search.VectorQuery`]): + The list of :class:`~couchbase.vector_search.VectorQuery`'s to use for the vector search. + options (:class:`~couchbase.options.VectorSearchOptions`, optional): Options to set for the vector search. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If a list of :class:`~couchbase.vector_search.VectorQuery` is not provided. + :class:`~couchbase.exceptions.InvalidArgumentException`: If all values of the provided queries are not instances of :class:`~couchbase.vector_search.VectorQuery`. + + Returns: + :class:`~couchbase.vector_search.VectorSearch`: The created vector search. + """ # noqa: E501 + + def __init__(self, + queries, # type: List[VectorQuery] + options=None # type: Optional[VectorSearchOptions] + ): + if queries is None or len(queries) == 0: + raise InvalidArgumentException('Provided queries cannot be empty.') + if not all(map(lambda q: isinstance(q, VectorQuery), queries)): + raise InvalidArgumentException('All queries must be a VectorQuery.') + + self._queries = queries + self._options = options if options is not None else None + + @property + def queries(self) -> List[VectorQuery]: + """ + **INTERNAL** + """ + return self._queries + + @property + def options(self) -> Optional[VectorSearchOptions]: + """ + **INTERNAL** + """ + return self._options + + @classmethod + def from_vector_query(cls, + query # type: VectorQuery + ) -> VectorSearch: + """ Creates a :class:`~couchbase.vector_search.VectorSearch` from a single :class:`~couchbase.vector_search.VectorQuery`. + + Args: + query (:class:`~couchbase.vector_search.VectorQuery`): + A :class:`~couchbase.vector_search.VectorQuery`'s to use for the vector search. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the provided query is not an instance of :class:`~couchbase.vector_search.VectorQuery`. + + Returns: + :class:`~couchbase.vector_search.VectorSearch`: The created vector search. + """ # noqa: E501 + return cls([query]) diff --git a/couchbase/logic/views.py b/couchbase/logic/views.py new file mode 100644 index 000000000..eb74d53e8 --- /dev/null +++ b/couchbase/logic/views.py @@ -0,0 +1,577 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import timedelta +from enum import Enum +from typing import (TYPE_CHECKING, + Any, + Dict, + List, + Optional, + Union) + +from couchbase._utils import to_microseconds +from couchbase.exceptions import ErrorMapper, InvalidArgumentException +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.options import ViewOptionsBase +from couchbase.management.views import DesignDocumentNamespace +from couchbase.options import UnsignedInt64, ViewOptions +from couchbase.pycbc_core import view_query +from couchbase.serializer import DefaultJsonSerializer, Serializer +from couchbase.tracing import CouchbaseSpan + +if TYPE_CHECKING: + from couchbase._utils import JSONType + + +class ViewScanConsistency(Enum): + NOT_BOUNDED = 'ok' + REQUEST_PLUS = 'false' + UPDATE_AFTER = 'update_after' + + +class ViewOrdering(Enum): + DESCENDING = 'true' + ASCENDING = 'false' + + +class ViewErrorMode(Enum): + CONTINUE = 'continue' + STOP = 'stop' + + +class ViewMetaData: + def __init__(self, raw # type: Dict[str, Any] + ) -> None: + if raw is not None: + self._raw = raw.get('metadata', None) + else: + self._raw = None + + def debug_info(self) -> Optional[str]: + return self._raw.get("debug_info", None) + + def total_rows(self) -> Optional[UnsignedInt64]: + return self._raw.get("total_rows", None) + + def __repr__(self): + return f'ViewMetaData({self._raw})' + + +@dataclass +class ViewRow(object): + key: str = None + id: str = None + value: object = None + document: object = None + + @classmethod + def from_json(cls, json_data: Dict[str, Any]) -> ViewRow: + output = cls(id=json_data.get('id', None)) + if 'key' in json_data: + output.key = json.loads(json_data['key']) + if 'value' in json_data: + output.value = json.loads(json_data['value']) + return output + + +class ViewQuery: + + # empty transform will skip updating the attribute when creating an + # N1QLQuery object + _VALID_OPTS = { + "timeout": {"timeout": lambda x: x}, + "skip": {"skip": lambda x: x}, + "limit": {"limit": lambda x: x}, + "scan_consistency": {"consistency": lambda x: x}, + "startkey": {"startkey": lambda x: x}, + "endkey": {"endkey": lambda x: x}, + "startkey_docid": {"startkey_docid": lambda x: x}, + "endkey_docid": {"endkey_docid": lambda x: x}, + "inclusive_end": {"inclusive_end": lambda x: x}, + "group": {"group": lambda x: x}, + "group_level": {"group_level": lambda x: x}, + "key": {"key": lambda x: x}, + "keys": {"keys": lambda x: x}, + "reduce": {"reduce": lambda x: x}, + "order": {"order": lambda x: x}, + "on_error": {"on_error": lambda x: x}, + "namespace": {"namespace": lambda x: x}, + "debug": {"debug": lambda x: x}, + "client_context_id": {"client_context_id": lambda x: x}, + "raw": {"raw": lambda x: x}, + "query_string": {"query_string": lambda x: x}, + "serializer": {"serializer": lambda x: x}, + "span": {"span": lambda x: x}, + "full_set": {"full_set": lambda x: x} + } + + def __init__(self, + bucket_name, # type: str + design_doc_name, # type: str + view_name, # type: str + *args, + **kwargs): + self._params = { + 'bucket_name': bucket_name, + 'document_name': design_doc_name, + 'view_name': view_name + } + + def set_option(self, name, value): + """ + Set a raw option in the query. This option is encoded + as part of the query parameters without any client-side + verification. Use this for settings not directly exposed + by the Python client. + + :param name: The name of the option + :param value: The value of the option + """ + self._params[name] = value + + # @TODO: I imagine some things might need to be jsonified... + + def as_encodable(self) -> Dict[str, Any]: + return self._params + + @property + def timeout(self) -> Optional[float]: + value = self._params.get('timeout', None) + if not value: + return None + value = value[:-1] + return float(value) + + @timeout.setter + def timeout(self, value # type: Union[timedelta,float,int] + ) -> None: + if not value: + self._params.pop('timeout', 0) + else: + total_us = to_microseconds(value) + self.set_option('timeout', total_us) + + @property + def limit(self) -> Optional[int]: + return self._params.get('limit', None) + + @limit.setter + def limit(self, value # type: int + ) -> None: + self.set_option('limit', value) + + @property + def skip(self) -> Optional[int]: + return self._params.get('skip', None) + + @skip.setter + def skip(self, value # type: int + ) -> None: + self.set_option('skip', value) + + @property + def consistency(self) -> ViewScanConsistency: + value = self._params.get( + 'scan_consistency', None + ) + if value is None: + return ViewScanConsistency.NOT_BOUNDED + if isinstance(value, str): + if value == 'ok': + return ViewScanConsistency.NOT_BOUNDED + elif value == 'false': + return ViewScanConsistency.REQUEST_PLUS + else: + return ViewScanConsistency.UPDATE_AFTER + + @consistency.setter + def consistency(self, value # type: Union[ViewScanConsistency, str] + ) -> None: + if isinstance(value, ViewScanConsistency): + self.set_option('scan_consistency', value.value) + elif isinstance(value, str) and value in [sc.value for sc in ViewScanConsistency]: + self.set_option('scan_consistency', value) + else: + raise InvalidArgumentException(message=("Excepted consistency to be either of type " + "ViewScanConsistency or str representation " + "of ViewScanConsistency")) + + @property + def startkey(self) -> Optional[str]: + return self._params.get('start_key', None) + + @startkey.setter + def startkey(self, value # type: JSONType + ) -> None: + self.set_option('start_key', json.dumps(value)) + + @property + def endkey(self) -> Optional[str]: + return self._params.get('end_key', None) + + @endkey.setter + def endkey(self, value # type: JSONType + ) -> None: + self.set_option('end_key', json.dumps(value)) + + @property + def startkey_docid(self) -> Optional[str]: + return self._params.get('start_key_doc_id', None) + + @startkey_docid.setter + def startkey_docid(self, value # type: str + ) -> None: + self.set_option('start_key_doc_id', value) + + @property + def endkey_docid(self) -> Optional[str]: + return self._params.get('end_key_doc_id', None) + + @endkey_docid.setter + def endkey_docid(self, value # type: str + ) -> None: + self.set_option('end_key_doc_id', value) + + @property + def inclusive_end(self) -> Optional[bool]: + return self._params.get('inclusive_end', None) + + @inclusive_end.setter + def inclusive_end(self, value # type: bool + ) -> None: + self.set_option('inclusive_end', value) + + @property + def group(self) -> Optional[bool]: + return self._params.get('group', None) + + @group.setter + def group(self, value # type: bool + ) -> None: + self.set_option('group', value) + + @property + def group_level(self) -> Optional[int]: + return self._params.get('group_level', None) + + @group_level.setter + def group_level(self, value # type: int + ) -> None: + self.set_option('group_level', value) + + @property + def key(self) -> Optional[str]: + return self._params.get('key', None) + + @key.setter + def key(self, value # type: JSONType + ) -> None: + self.set_option('key', json.dumps(value)) + + @property + def keys(self) -> Optional[List[str]]: + return self._params.get('keys', None) + + @keys.setter + def keys(self, value # type: JSONType + ) -> None: + if not isinstance(value, list): + raise InvalidArgumentException('keys must be a list.') + # couchbase++ wants a list of JSONified strings + self.set_option('keys', list(map(lambda k: json.dumps(k), value))) + + @property + def reduce(self) -> Optional[bool]: + return self._params.get('reduce', None) + + @reduce.setter + def reduce(self, value # type: bool + ) -> None: + self.set_option('reduce', value) + + @property + def order(self) -> ViewOrdering: + value = self._params.get( + 'order', None + ) + if value is None: + return ViewOrdering.DESCENDING + if isinstance(value, str): + if value == 'false': + return ViewOrdering.ASCENDING + else: + return ViewOrdering.DESCENDING + + @order.setter + def order(self, value # type: Union[ViewOrdering, str] + ) -> None: + if isinstance(value, ViewOrdering): + self.set_option('order', value.value) + elif isinstance(value, str) and value in [sc.value for sc in ViewOrdering]: + self.set_option('order', value) + else: + raise InvalidArgumentException(message=("Excepted order to be either of type " + "ViewOrdering or str representation " + "of ViewOrdering")) + + @property + def on_error(self) -> ViewErrorMode: + value = self._params.get( + 'on_error', None + ) + if value is None: + return ViewErrorMode.STOP + if isinstance(value, str): + if value == 'continue': + return ViewErrorMode.CONTINUE + else: + return ViewErrorMode.STOP + + @on_error.setter + def on_error(self, value # type: Union[ViewErrorMode, str] + ) -> None: + if isinstance(value, ViewErrorMode): + self.set_option('on_error', value.value) + elif isinstance(value, str) and value in [sc.value for sc in ViewErrorMode]: + self.set_option('on_error', value) + else: + raise InvalidArgumentException(message=("Excepted on_error to be either of type " + "ViewErrorMode or str representation " + "of ViewErrorMode")) + + @property + def namespace(self) -> DesignDocumentNamespace: + value = self._params.get( + 'namespace', None + ) + if value is None: + return DesignDocumentNamespace.DEVELOPMENT + + if isinstance(value, str): + if value == 'production': + return DesignDocumentNamespace.PRODUCTION + else: + return DesignDocumentNamespace.DEVELOPMENT + if isinstance(value, bool): + if not value: + return DesignDocumentNamespace.PRODUCTION + + return DesignDocumentNamespace.DEVELOPMENT + + @namespace.setter + def namespace(self, value # type: Union[DesignDocumentNamespace, str] + ) -> None: + if isinstance(value, DesignDocumentNamespace): + self.set_option('namespace', value.value) + elif isinstance(value, str) and value in [sc.value for sc in DesignDocumentNamespace]: + self.set_option('namespace', value) + else: + raise InvalidArgumentException(message=("Excepted namespace to be either of type " + "DesignDocumentNamespace or str representation " + "of DesignDocumentNamespace")) + + @property + def debug(self) -> Optional[bool]: + return self._params.get('debug', None) + + @debug.setter + def debug(self, value # type: bool + ) -> None: + self.set_option('debug', value) + + @property + def raw(self) -> Optional[Dict[str, str]]: + return self._params.get('raw', None) + + @raw.setter + def raw(self, value # type: Dict[str, str] + ) -> None: + self.set_option('raw', value) + + @property + def query_string(self) -> Optional[List[str]]: + return self._params.get('query_string', None) + + @query_string.setter + def query_string(self, value # type: str + ) -> None: + if not isinstance(value, list): + raise InvalidArgumentException('query_string must be a list.') + self.set_option('query_string', value) + + @property + def client_context_id(self) -> Optional[str]: + return self._params.get('client_context_id', None) + + @client_context_id.setter + def client_context_id(self, value # type: str + ) -> None: + self.set_option('client_context_id', value) + + @property + def serializer(self) -> Optional[Serializer]: + return self._params.get('serializer', None) + + @serializer.setter + def serializer(self, value # type: Serializer + ): + if not issubclass(value.__class__, Serializer): + raise InvalidArgumentException(message='Serializer should implement Serializer interface.') + self.set_option('serializer', value) + + @property + def span(self) -> Optional[CouchbaseSpan]: + return self._params.get('span', None) + + @span.setter + def span(self, value # type: CouchbaseSpan + ): + if not issubclass(value.__class__, CouchbaseSpan): + raise InvalidArgumentException(message='Span should implement CouchbaseSpan interface') + self.set_option('span', value) + + @property + def full_set(self) -> Optional[bool]: + return self._params.get('full_set', None) + + @full_set.setter + def full_set(self, value # type: bool + ) -> None: + self.set_option('full_set', value) + + @classmethod + def create_view_query_object(cls, + bucket_name, # type: str + design_doc_name, # type: str + view_name, # type: str + *options, # type: ViewOptions + **kwargs # type: Dict[str, Any] + ) -> ViewQuery: + + # lets make a copy of the options, and update with kwargs... + opt = ViewOptions() + # TODO: is it possible that we could have [ViewOptions, ViewOptions, ...]?? + # If so, why??? + opts = list(options) + for o in opts: + if isinstance(o, (ViewOptions, ViewOptionsBase)): + opt = o + opts.remove(o) + args = opt.copy() + args.update(kwargs) + + query = cls(bucket_name, design_doc_name, view_name) + + for k, v in ((k, args[k]) for k in (args.keys() & cls._VALID_OPTS)): + for target, transform in cls._VALID_OPTS[k].items(): + setattr(query, target, transform(v)) + return query + + +class ViewRequestLogic: + def __init__(self, + connection, + encoded_query, + row_factory=ViewRow, + **kwargs + ): + + self._connection = connection + self._encoded_query = encoded_query + self.row_factory = row_factory + self._streaming_result = None + self._default_serializer = kwargs.pop('default_serializer', DefaultJsonSerializer()) + self._serializer = None + self._started_streaming = False + self._streaming_timeout = kwargs.pop('streaming_timeout', None) + self._done_streaming = False + self._metadata = None + + @property + def encoded_query(self) -> Dict[str, Any]: + return self._encoded_query + + @property + def serializer(self) -> Serializer: + if self._serializer: + return self._serializer + + serializer = self.encoded_query.get('serializer', None) + if not serializer: + serializer = self._default_serializer + + self._serializer = serializer + return self._serializer + + @property + def started_streaming(self) -> bool: + return self._started_streaming + + @property + def done_streaming(self) -> bool: + return self._done_streaming + + def metadata(self) -> ViewMetaData: + # @TODO: raise if query isn't complete? + return self._metadata + + def _set_metadata(self, views_response): + if isinstance(views_response, CouchbaseBaseException): + raise ErrorMapper.build_exception(views_response) + + self._metadata = ViewMetaData(views_response.raw_result.get('value', None)) + + def _submit_query(self, **kwargs): + if self.done_streaming: + return + + self._started_streaming = True + span = self.encoded_query.pop('span', None) + view_kwargs = { + 'conn': self._connection, + 'op_args': self.encoded_query + } + if span: + view_kwargs['span'] = span + + streaming_timeout = self.encoded_query.get('timeout', self._streaming_timeout) + if streaming_timeout: + view_kwargs['streaming_timeout'] = streaming_timeout + + # this is for txcouchbase... + callback = kwargs.pop('callback', None) + if callback: + view_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + view_kwargs['errback'] = errback + + self._streaming_result = view_query(**view_kwargs) + + def __iter__(self): + raise NotImplementedError( + 'Cannot use synchronous iterator, are you using `async for`?' + ) + + def __aiter__(self): + raise NotImplementedError( + 'Cannot use asynchronous iterator.' + ) diff --git a/couchbase/logic/wrappers.py b/couchbase/logic/wrappers.py new file mode 100644 index 000000000..24e44d167 --- /dev/null +++ b/couchbase/logic/wrappers.py @@ -0,0 +1,195 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from copy import copy +from functools import wraps + +from couchbase.constants import FMT_JSON +from couchbase.exceptions import (PYCBC_ERROR_MAP, + CouchbaseException, + DocumentExistsException, + DocumentNotFoundException, + ErrorMapper, + ExceptionMap, + InternalSDKException, + PathNotFoundException, + ServiceUnavailableException, + UnAmbiguousTimeoutException) +from couchbase.exceptions import exception as BaseCouchbaseException +from couchbase.exceptions import exception as CouchbaseBaseException + + +def decode_value(transcoder, value, flags, is_subdoc=False): + if is_subdoc is False: + return transcoder.decode_value(value, flags) + + final_value = [] + for f in value: + if 'value' in f: + tmp = copy(f) + old = tmp.pop('value', None) + if old: + # no custom transcoder for subdoc ops, use JSON + tmp['value'] = transcoder.decode_value(old, FMT_JSON) + final_value.append(tmp) + else: + final_value.append(f) + + return final_value + + +def decode_replicas(transcoder, result, return_cls, is_subdoc=False): + while True: + try: + res = next(result) + except StopIteration: + # this is a timeout from pulling a result from the queue, kill the generator + raise UnAmbiguousTimeoutException('Timeout reached waiting for result in queue.') from None + else: + if isinstance(res, CouchbaseBaseException): + raise ErrorMapper.build_exception(res) + # should only be None once all replicas have been retrieved + if res is None: + return + + value = res.raw_result.get('value', None) + flags = res.raw_result.get('flags', None) + res.raw_result['value'] = decode_value(transcoder, value, flags, is_subdoc=is_subdoc) + yield return_cls(res) + + +class BlockingWrapper: + @classmethod # noqa: C901 + def block(cls, return_cls): # noqa: C901 + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + try: + ret = fn(self, *args, **kwargs) + if isinstance(ret, BaseCouchbaseException): + raise ErrorMapper.build_exception(ret) + if return_cls is None: + return None + elif return_cls is True: + retval = ret + else: + if ret is None: + raise InternalSDKException('Expected return value to be non-empty.') + retval = return_cls(ret) + return retval + except CouchbaseException as e: + if isinstance(e, ServiceUnavailableException) and fn.__name__ == '_get_cluster_info': + e._message = ('If using Couchbase Server < 6.6, ' + 'a bucket needs to be opened prior to cluster level operations') + raise e + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(message=str(ex)) + raise excptn from None + + return wrapped_fn + return decorator + + @classmethod + def block_and_decode(cls, return_cls): + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + try: + transcoder = kwargs.pop('transcoder') + ret = fn(self, *args, **kwargs) + if isinstance(ret, BaseCouchbaseException): + raise ErrorMapper.build_exception(ret) + + is_subdoc = fn.__name__ in [ + '_lookup_in_internal', '_lookup_in_any_replica_internal', '_lookup_in_all_replicas_internal' + ] + + # special case for get_all_replicas and lookup_in_all_replicas + if fn.__name__ in ['_get_all_replicas_internal', '_lookup_in_all_replicas_internal']: + return decode_replicas(transcoder, ret, return_cls, is_subdoc=is_subdoc) + + value = ret.raw_result.get('value', None) + flags = ret.raw_result.get('flags', None) + + ret.raw_result['value'] = decode_value(transcoder, value, flags, is_subdoc=is_subdoc) + if return_cls is None: + return None + elif return_cls is True: + retval = ret + else: + retval = return_cls(ret) + return retval + except CouchbaseException as e: + raise e + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(message=str(ex)) + raise excptn + + return wrapped_fn + return decorator + + @classmethod + def datastructure_op(cls, create_type=None): + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + try: + return fn(self, *args, **kwargs) + except DocumentNotFoundException: + if create_type is not None: + try: + self._collection.insert(self._key, create_type()) + except DocumentExistsException: + pass + return fn(self, *args, **kwargs) + else: + raise + + return wrapped_fn + return decorator + + @classmethod # noqa: C901 + def _dsop(cls, create_type=None, wrap_missing_path=True): # noqa: C901 + """ + ** DEPRECATED ** + """ + def real_decorator(fn): + @wraps(fn) + def newfn(self, key, *args, **kwargs): + try: + return fn(self, key, *args, **kwargs) + except DocumentNotFoundException: + if kwargs.get('create'): + try: + if create_type == 'list': + self.insert(key, list()) + elif create_type == 'dict': + self.insert(key, dict()) + except DocumentExistsException: + pass + return fn(self, key, *args, **kwargs) + else: + raise + except PathNotFoundException: + if wrap_missing_path: + raise IndexError(args[0]) + + return newfn + + return real_decorator diff --git a/couchbase/management/__init__.py b/couchbase/management/__init__.py new file mode 100644 index 000000000..fc6346764 --- /dev/null +++ b/couchbase/management/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/couchbase/management/analytics.py b/couchbase/management/analytics.py new file mode 100644 index 000000000..181039a4b --- /dev/null +++ b/couchbase/management/analytics.py @@ -0,0 +1,235 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import (Any, + Dict, + Iterable, + Optional) + +from couchbase.exceptions import InvalidArgumentException +from couchbase.management.logic.analytics_logic import AnalyticsEncryptionLevel # noqa: F401 +from couchbase.management.logic.analytics_logic import AnalyticsLinkType # noqa: F401 +from couchbase.management.logic.analytics_logic import AzureBlobExternalAnalyticsLink # noqa: F401 +from couchbase.management.logic.analytics_logic import CouchbaseAnalyticsEncryptionSettings # noqa: F401 +from couchbase.management.logic.analytics_logic import CouchbaseRemoteAnalyticsLink # noqa: F401 +from couchbase.management.logic.analytics_logic import S3ExternalAnalyticsLink # noqa: F401 +from couchbase.management.logic.analytics_logic import (AnalyticsDataset, + AnalyticsDataType, + AnalyticsIndex, + AnalyticsLink, + AnalyticsManagerLogic) +from couchbase.management.logic.wrappers import BlockingMgmtWrapper, ManagementType + +# @TODO: lets deprecate import of options from couchbase.management.analytics +from couchbase.management.options import (ConnectLinkOptions, + CreateAnalyticsIndexOptions, + CreateDatasetOptions, + CreateDataverseOptions, + CreateLinkAnalyticsOptions, + DisconnectLinkOptions, + DropAnalyticsIndexOptions, + DropDatasetOptions, + DropDataverseOptions, + DropLinkAnalyticsOptions, + GetAllAnalyticsIndexesOptions, + GetAllDatasetOptions, + GetLinksAnalyticsOptions, + GetPendingMutationsOptions, + ReplaceLinkAnalyticsOptions) + + +class AnalyticsIndexManager(AnalyticsManagerLogic): + + def __init__(self, connection): + super().__init__(connection) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def create_dataverse(self, + dataverse_name, # type: str + options=None, # type: Optional[CreateDataverseOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + + if not isinstance(dataverse_name, str): + raise InvalidArgumentException("dataverse_name must be provided when creating an analytics dataverse.") + + return super().create_dataverse(dataverse_name, options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def drop_dataverse(self, + dataverse_name, # type: str + options=None, # type: Optional[DropDataverseOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + + if not isinstance(dataverse_name, str): + raise InvalidArgumentException("dataverse_name must be provided when dropping an analytics dataverse.") + + return super().drop_dataverse(dataverse_name, options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def create_dataset(self, + dataset_name, # type: str + bucket_name, # type: str + options=None, # type: Optional[CreateDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + + if not isinstance(dataset_name, str): + raise InvalidArgumentException("dataset_name must be provided when creating an analytics dataset.") + + if not isinstance(bucket_name, str): + raise InvalidArgumentException("bucket_name must be provided when creating an analytics dataset.") + + return super().create_dataset(dataset_name, bucket_name, options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def drop_dataset(self, + dataset_name, # type: str + options=None, # type: Optional[DropDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + + if not isinstance(dataset_name, str): + raise InvalidArgumentException("dataset_name must be provided when dropping an analytics dataset.") + + return super().drop_dataset(dataset_name, options, **kwargs) + + @BlockingMgmtWrapper.block(AnalyticsDataset, ManagementType.AnalyticsIndexMgmt, + AnalyticsManagerLogic._ERROR_MAPPING) + def get_all_datasets(self, + options=None, # type: Optional[GetAllDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> Iterable[AnalyticsDataset]: + + return super().get_all_datasets(options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def create_index(self, + index_name, # type: str + dataset_name, # type: str + fields, # type: Dict[str, AnalyticsDataType] + options=None, # type: Optional[CreateAnalyticsIndexOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + + if not isinstance(index_name, str): + raise InvalidArgumentException("index_name must be provided when creating an analytics index.") + + if not isinstance(dataset_name, str): + raise InvalidArgumentException("dataset_name must be provided when creating an analytics index.") + + if fields is not None: + if not isinstance(fields, dict): + raise InvalidArgumentException("fields must be provided when creating an analytics index.") + + if not all(map(lambda v: isinstance(v, AnalyticsDataType), fields.values())): + raise InvalidArgumentException("fields must all be an AnalyticsDataType.") + + return super().create_index(index_name, dataset_name, fields, options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + dataset_name, # type: str + options=None, # type: Optional[DropAnalyticsIndexOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + + if not isinstance(index_name, str): + raise InvalidArgumentException("index_name must be provided when dropping an analytics index.") + + if not isinstance(dataset_name, str): + raise InvalidArgumentException("dataset_name must be provided when dropping an analytics index.") + + return super().drop_index(index_name, dataset_name, options, **kwargs) + + @BlockingMgmtWrapper.block(AnalyticsIndex, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + options=None, # type: Optional[GetAllAnalyticsIndexesOptions] + **kwargs # type: Dict[str, Any] + ) -> Iterable[AnalyticsIndex]: + + return super().get_all_indexes(options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def connect_link(self, + options=None, # type: Optional[ConnectLinkOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + return super().connect_link(options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def disconnect_link(self, + options=None, # type: Optional[DisconnectLinkOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + return super().disconnect_link(options, **kwargs) + + @BlockingMgmtWrapper.block(dict, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def get_pending_mutations(self, + options=None, # type: Optional[GetPendingMutationsOptions] + **kwargs # type: Dict[str, Any] + ) -> Dict[str, int]: + + return super().get_pending_mutations(options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def create_link( + self, + link, # type: AnalyticsLink + options=None, # type: Optional[CreateLinkAnalyticsOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + return super().create_link(link, options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def replace_link( + self, + link, # type: AnalyticsLink + options=None, # type: Optional[ReplaceLinkAnalyticsOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + return super().replace_link(link, options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def drop_link( + self, + link_name, # type: str + dataverse_name, # type: str + options=None, # type: Optional[DropLinkAnalyticsOptions] + **kwargs # type: Dict[str, Any] + ) -> None: + + if not isinstance(link_name, str): + raise ValueError("link_name must be provided when dropping an analytics link.") + + if not isinstance(dataverse_name, str): + raise ValueError("dataverse_name must be provided when dropping an analytics link.") + + return super().drop_link(link_name, dataverse_name, options, **kwargs) + + @BlockingMgmtWrapper.block((CouchbaseRemoteAnalyticsLink, + S3ExternalAnalyticsLink, + AzureBlobExternalAnalyticsLink), + ManagementType.AnalyticsIndexMgmt, AnalyticsManagerLogic._ERROR_MAPPING) + def get_links( + self, + options=None, # type: Optional[GetLinksAnalyticsOptions] + **kwargs # type: Dict[str, Any] + ) -> Iterable[AnalyticsLink]: + return super().get_links(options, **kwargs) diff --git a/couchbase/management/buckets.py b/couchbase/management/buckets.py new file mode 100644 index 000000000..161054094 --- /dev/null +++ b/couchbase/management/buckets.py @@ -0,0 +1,193 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (Any, + Dict, + List) + +from couchbase.management.logic.buckets_logic import BucketType # noqa: F401 +from couchbase.management.logic.buckets_logic import CompressionMode # noqa: F401 +from couchbase.management.logic.buckets_logic import ConflictResolutionType # noqa: F401 +from couchbase.management.logic.buckets_logic import EjectionMethod # noqa: F401 +from couchbase.management.logic.buckets_logic import EvictionPolicyType # noqa: F401 +from couchbase.management.logic.buckets_logic import StorageBackend # noqa: F401 +from couchbase.management.logic.buckets_logic import (BucketDescribeResult, + BucketManagerLogic, + BucketSettings, + CreateBucketSettings) +from couchbase.management.logic.wrappers import BlockingMgmtWrapper, ManagementType + +# @TODO: lets deprecate import of options from couchbase.management.buckets +from couchbase.management.options import (BucketDescribeOptions, + CreateBucketOptions, + DropBucketOptions, + FlushBucketOptions, + GetAllBucketOptions, + GetBucketOptions, + UpdateBucketOptions) + + +class BucketManager(BucketManagerLogic): + def __init__(self, connection): + super().__init__(connection) + + @BlockingMgmtWrapper.block(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def create_bucket(self, + settings, # type: CreateBucketSettings + *options, # type: CreateBucketOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Creates a new bucket. + + Args: + settings (:class:`.CreateBucketSettings`): The settings to use for the new bucket. + options (:class:`~couchbase.management.options.CreateBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.BucketAlreadyExistsException`: If the bucket already exists. + :class:`~couchbase.exceptions.InvalidArgumentsException`: If an invalid type or value is provided for the + settings argument. + """ + return super().create_bucket(settings, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def update_bucket(self, + settings, # type: BucketSettings + *options, # type: UpdateBucketOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Update the settings for an existing bucket. + + Args: + settings (:class:`.BucketSettings`): The settings to use for the new bucket. + options (:class:`~couchbase.management.options.UpdateBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentsException`: If an invalid type or value is provided for the + settings argument. + """ + return super().update_bucket(settings, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def drop_bucket(self, + bucket_name, # type: str + *options, # type: DropBucketOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Drops an existing bucket. + + Args: + bucket_name (str): The name of the bucket to drop. + options (:class:`~couchbase.management.options.DropBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.BucketDoesNotExistException`: If the bucket does not exist. + """ + return super().drop_bucket(bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(BucketSettings, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def get_bucket(self, + bucket_name, # type: str + *options, # type: GetBucketOptions + **kwargs # type: Dict[str, Any] + ) -> BucketSettings: + """Fetches the settings in use for a specified bucket. + + Args: + bucket_name (str): The name of the bucket to fetch. + options (:class:`~couchbase.management.options.GetBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`.BucketSettings`: The settings of the specified bucket. + + Raises: + :class:`~couchbase.exceptions.BucketDoesNotExistException`: If the bucket does not exist. + """ + return super().get_bucket(bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(BucketSettings, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def get_all_buckets(self, + *options, # type: GetAllBucketOptions + **kwargs # type: Dict[str, Any] + ) -> List[BucketSettings]: + """Returns a list of existing buckets in the cluster. + + Args: + options (:class:`~couchbase.management.options.GetAllBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + List[:class:`.BucketSettings`]: A list of existing buckets in the cluster. + """ + return super().get_all_buckets(*options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def flush_bucket(self, + bucket_name, # type: str + *options, # type: FlushBucketOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Flushes the bucket, deleting all the existing data that is stored in it. + + Args: + bucket_name (str): The name of the bucket to flush. + options (:class:`~couchbase.management.options.FlushBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.BucketDoesNotExistException`: If the bucket does not exist. + :class:`~couchbase.exceptions.BucketNotFlushableException`: If the bucket's settings have + flushing disabled. + """ + return super().flush_bucket(bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(BucketDescribeResult, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def bucket_describe(self, + bucket_name, # type: str + *options, # type: BucketDescribeOptions + **kwargs # type: Dict[str, Any] + ) -> BucketDescribeResult: + """Provides details on provided the bucket. + + Args: + bucket_name (str): The name of the bucket to flush. + options (:class:`~couchbase.management.options.BucketDescribeOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`.BucketDescribeResult`: Key-value pair details describing the bucket. + + Raises: + :class:`~couchbase.exceptions.BucketDoesNotExistException`: If the bucket does not exist. + """ + return super().bucket_describe(bucket_name, *options, **kwargs) diff --git a/couchbase/management/collections.py b/couchbase/management/collections.py new file mode 100644 index 000000000..aea8d4245 --- /dev/null +++ b/couchbase/management/collections.py @@ -0,0 +1,265 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from inspect import Parameter, Signature +from typing import (Any, + Dict, + Iterable, + Optional) + +from couchbase._utils import OverloadType +from couchbase.logic.supportability import Supportability +from couchbase.management.logic.collections_logic import (CollectionManagerLogic, + CollectionSpec, + CreateCollectionSettings, + ScopeSpec, + UpdateCollectionSettings) +from couchbase.management.logic.wrappers import BlockingMgmtWrapper, ManagementType + +# @TODO: lets deprecate import of options from couchbase.management.collections +from couchbase.management.options import (CreateCollectionOptions, + CreateScopeOptions, + DropCollectionOptions, + DropScopeOptions, + GetAllScopesOptions, + UpdateCollectionOptions) + + +class CollectionManager(CollectionManagerLogic): + + def __init__(self, connection, bucket_name): + super().__init__(connection, bucket_name) + + @BlockingMgmtWrapper.block(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def create_scope(self, + scope_name: str, + *options: CreateScopeOptions, + **kwargs: Dict[str, Any] + ) -> None: + """Creates a new scope. + + Args: + scope_name (str): The name of the scope. + options (:class:`~couchbase.management.options.CreateScopeOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters for this operation. + + Raises: + :class:`~couchbase.exceptions.ScopeAlreadyExistsException`: If the scope already exists. + """ # noqa: E501 + return super().create_scope(scope_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def drop_scope(self, + scope_name: str, + *options: DropScopeOptions, + **kwargs: Dict[str, Any] + ) -> None: + """Drops an existing scope. + + Args: + scope_name (str): The name of the scope. + options (:class:`~couchbase.management.options.DropScopeOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters for this operation. + + Raises: + :class:`~couchbase.exceptions.ScopeNotFoundException`: If the scope does not exist. + """ # noqa: E501 + return super().drop_scope(scope_name, *options, **kwargs) + + @BlockingMgmtWrapper.block((ScopeSpec, CollectionSpec), ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING) + def get_all_scopes(self, + *options: GetAllScopesOptions, + **kwargs: Dict[str, Any] + ) -> Iterable[ScopeSpec]: + """Returns all configured scopes along with their collections. + + Args: + scope_name (str): The name of the scope. + options (:class:`~couchbase.management.options.GetAllScopesOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters for this operation. + + Returns: + Iterable[:class:`.ScopeSpec`]: A list of all configured scopes. + """ # noqa: E501 + return super().get_all_scopes(*options, **kwargs) + + @BlockingMgmtWrapper.block(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.SECONDARY) + def create_collection(self, + collection: CollectionSpec, + *options: CreateCollectionOptions, + **kwargs: Dict[str, Any] + ) -> None: + """ + .. deprecated:: 4.1.9 + Use ``create_collection(scope_name, collection_name, settings=None, *options, **kwargs)`` instead. + + Creates a new collection in a specified scope. + + Args: + collection (:class:`.CollectionSpec`): The collection details. + options (:class:`~couchbase.management.options.CreateCollectionOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters for this operation. + + Raises: + :class:`~couchbase.exceptions.CollectionAlreadyExistsException`: If the collection already exists. + :class:`~couchbase.exceptions.ScopeNotFoundException`: If the scope does not exist. + """ # noqa: E501 + Supportability.method_signature_deprecated( + 'create_collection', + Signature( + parameters=[ + Parameter('collection', Parameter.POSITIONAL_OR_KEYWORD, annotation=CollectionSpec), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=CreateCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=None + ), + Signature( + parameters=[ + Parameter('scope_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('collection_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('settings', Parameter.POSITIONAL_OR_KEYWORD, + annotation=Optional[CreateCollectionSettings]), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=CreateCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=None + ) + ) + settings = None + if collection.max_expiry is not None: + settings = CreateCollectionSettings(max_expiry=collection.max_expiry) + + return super().create_collection(collection.scope_name, collection.name, settings, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.DEFAULT) + def create_collection(self, # noqa: F811 + scope_name: str, + collection_name: str, + settings: Optional[CreateCollectionSettings] = None, + *options: CreateCollectionOptions, + **kwargs: Dict[str, Any] + ) -> None: + """Creates a new collection in a specified scope. + + Args: + scope_name (str): The name of the scope the collection will be created in. + collection_name (str): The name of the collection to be created + settings (:class:`~.CreateCollectionSettings`, optional): Settings to apply for the collection + options (:class:`~couchbase.management.options.CreateCollectionOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters for this operation. + + Raises: + :class:`~couchbase.exceptions.CollectionAlreadyExistsException`: If the collection already exists. + :class:`~couchbase.exceptions.ScopeNotFoundException`: If the scope does not exist. + """ # noqa: E501 + return super().create_collection(scope_name, collection_name, settings, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.SECONDARY) + def drop_collection(self, + collection: CollectionSpec, + *options: DropCollectionOptions, + **kwargs: Dict[str, Any] + ) -> None: + """ + .. deprecated:: 4.1.9 + Use ``drop_collection(scope_name, collection_name, *options, **kwargs)`` instead. + + Drops a collection from the specified scope. + + Args: + collection (:class:`.CollectionSpec`): The collection details. + options (:class:`~couchbase.management.options.DropCollectionOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters for this operation. + + Raises: + :class:`~couchbase.exceptions.CollectionNotFoundException`: If the collection does not exist. + """ # noqa: E501 + Supportability.method_signature_deprecated( + 'drop_collection', + Signature( + parameters=[ + Parameter('collection', Parameter.POSITIONAL_OR_KEYWORD, annotation=CollectionSpec), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=DropCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=None, + ), + Signature( + parameters=[ + Parameter('scope_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('collection_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=DropCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=None + ) + ) + return super().drop_collection(collection.scope_name, collection.name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.DEFAULT) + def drop_collection(self, # noqa: F811 + scope_name: str, + collection_name: str, + *options: DropCollectionOptions, + **kwargs: Dict[str, Any]) -> None: + """Drops a collection from the specified scope. + + Args: + scope_name (str): The name of the scope the collection is in. + collection_name (str): The name of the collection to be dropped + options (:class:`~couchbase.management.options.DropCollectionOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters for this operation. + + Raises: + :class:`~couchbase.exceptions.CollectionNotFoundException`: If the collection does not exist. + """ # noqa: E501 + return super().drop_collection(scope_name, collection_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def update_collection(self, + scope_name: str, + collection_name: str, + settings: UpdateCollectionSettings, + *options: UpdateCollectionOptions, + **kwargs: Dict[str, Any] + ) -> None: + """Updates a collection in a specified scope. + + Args: + scope_name (str): The name of the scope the collection is in. + collection_name (str): The name of the collection that will be updated + settings (:class:`~.UpdateCollectionSettings`, optional): Settings to apply for the collection + options (:class:`~couchbase.management.options.UpdateCollectionOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters for this operation. + + Raises: + :class:`~couchbase.exceptions.CollectionNotFoundException`: If the collection does not exist. + :class:`~couchbase.exceptions.ScopeNotFoundException`: If the scope does not exist. + """ # noqa: E501 + return super().update_collection(scope_name, collection_name, settings, *options, **kwargs) diff --git a/couchbase/management/eventing.py b/couchbase/management/eventing.py new file mode 100644 index 000000000..47f357db1 --- /dev/null +++ b/couchbase/management/eventing.py @@ -0,0 +1,255 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import (Any, + Dict, + List) + +from couchbase.management.logic.eventing_logic import EventingFunctionBucketAccess # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionBucketBinding # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionConstantBinding # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionDcpBoundary # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionDeploymentStatus # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionKeyspace # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionLanguageCompatibility # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionLogLevel # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionProcessingStatus # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionSettings # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionState # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionStatus # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionUrlAuthBasic # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionUrlAuthBearer # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionUrlAuthDigest # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionUrlBinding # noqa: F401 +from couchbase.management.logic.eventing_logic import EventingFunctionUrlNoAuth # noqa: F401 +from couchbase.management.logic.eventing_logic import (EventingFunction, + EventingFunctionManagerLogic, + EventingFunctionsStatus) +from couchbase.management.logic.wrappers import BlockingMgmtWrapper, ManagementType + +# @TODO: lets deprecate import of options from couchbase.management.eventing +from couchbase.management.options import (DeployFunctionOptions, + DropFunctionOptions, + FunctionsStatusOptions, + GetAllFunctionOptions, + GetFunctionOptions, + PauseFunctionOptions, + ResumeFunctionOptions, + UndeployFunctionOptions, + UpsertFunctionOptions) + + +class EventingFunctionManager(EventingFunctionManagerLogic): + def __init__(self, connection): + super().__init__(connection) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def upsert_function( + self, + function, # type: EventingFunction + *options, # type: UpsertFunctionOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().upsert_function(function, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def drop_function( + self, + name, # type: str + *options, # type: DropFunctionOptions + **kwargs # type: Any + ) -> None: + return super().drop_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def deploy_function( + self, + name, # type: str + *options, # type: DeployFunctionOptions + **kwargs # type: Any + ) -> None: + return super().deploy_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(EventingFunction, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def get_all_functions( + self, + *options, # type: GetAllFunctionOptions + **kwargs # type: Any + ) -> List[EventingFunction]: + return super().get_all_functions(*options, **kwargs) + + @BlockingMgmtWrapper.block(EventingFunction, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def get_function( + self, + name, # type: str + *options, # type: GetFunctionOptions + **kwargs # type: Any + ) -> EventingFunction: + return super().get_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def pause_function( + self, + name, # type: str + *options, # type: PauseFunctionOptions + **kwargs # type: Any + ) -> None: + return super().pause_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def resume_function( + self, + name, # type: str + *options, # type: ResumeFunctionOptions + **kwargs # type: Any + ) -> None: + return super().resume_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def undeploy_function( + self, + name, # type: str + *options, # type: UndeployFunctionOptions + **kwargs # type: Any + ) -> None: + return super().undeploy_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(EventingFunctionsStatus, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def functions_status( + self, + *options, # type: FunctionsStatusOptions + **kwargs # type: Any + ) -> EventingFunctionsStatus: + return super().functions_status(*options, **kwargs) + + def _get_status( + self, + name, # type: str + ) -> EventingFunctionStatus: + + statuses = self.functions_status() + + if statuses.functions: + return next((f for f in statuses.functions if f.name == name), None) + + return None + + +class ScopeEventingFunctionManager(EventingFunctionManagerLogic): + def __init__(self, + connection, + bucket_name, # type: str + scope_name, # type: str + ): + super().__init__(connection, bucket_name=bucket_name, scope_name=scope_name) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def upsert_function( + self, + function, # type: EventingFunction + *options, # type: UpsertFunctionOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().upsert_function(function, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def drop_function( + self, + name, # type: str + *options, # type: DropFunctionOptions + **kwargs # type: Any + ) -> None: + return super().drop_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def deploy_function( + self, + name, # type: str + *options, # type: DeployFunctionOptions + **kwargs # type: Any + ) -> None: + return super().deploy_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(EventingFunction, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def get_all_functions( + self, + *options, # type: GetAllFunctionOptions + **kwargs # type: Any + ) -> List[EventingFunction]: + return super().get_all_functions(*options, **kwargs) + + @BlockingMgmtWrapper.block(EventingFunction, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def get_function( + self, + name, # type: str + *options, # type: GetFunctionOptions + **kwargs # type: Any + ) -> EventingFunction: + return super().get_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def pause_function( + self, + name, # type: str + *options, # type: PauseFunctionOptions + **kwargs # type: Any + ) -> None: + return super().pause_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def resume_function( + self, + name, # type: str + *options, # type: ResumeFunctionOptions + **kwargs # type: Any + ) -> None: + return super().resume_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.EventingFunctionMgmt, EventingFunctionManagerLogic._ERROR_MAPPING) + def undeploy_function( + self, + name, # type: str + *options, # type: UndeployFunctionOptions + **kwargs # type: Any + ) -> None: + return super().undeploy_function(name, *options, **kwargs) + + @BlockingMgmtWrapper.block(EventingFunctionsStatus, ManagementType.EventingFunctionMgmt, + EventingFunctionManagerLogic._ERROR_MAPPING) + def functions_status( + self, + *options, # type: FunctionsStatusOptions + **kwargs # type: Any + ) -> EventingFunctionsStatus: + return super().functions_status(*options, **kwargs) + + def _get_status( + self, + name, # type: str + ) -> EventingFunctionStatus: + + statuses = self.functions_status() + + if statuses.functions: + return next((f for f in statuses.functions if f.name == name), None) + + return None diff --git a/couchbase/management/logic/__init__.py b/couchbase/management/logic/__init__.py new file mode 100644 index 000000000..94a763efb --- /dev/null +++ b/couchbase/management/logic/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .wrappers import ManagementType # noqa: F401 +from .wrappers import handle_analytics_index_mgmt_response # noqa: F401 +from .wrappers import handle_bucket_mgmt_response # noqa: F401 +from .wrappers import handle_collection_mgmt_response # noqa: F401 +from .wrappers import handle_eventing_function_mgmt_response # noqa: F401 +from .wrappers import handle_query_index_mgmt_response # noqa: F401 +from .wrappers import handle_search_index_mgmt_response # noqa: F401 +from .wrappers import handle_user_mgmt_response # noqa: F401 +from .wrappers import handle_view_index_mgmt_response # noqa: F401 diff --git a/couchbase/management/logic/analytics_logic.py b/couchbase/management/logic/analytics_logic.py new file mode 100644 index 000000000..fd7f1211e --- /dev/null +++ b/couchbase/management/logic/analytics_logic.py @@ -0,0 +1,1103 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from enum import Enum +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + Optional, + Union) + +from couchbase._utils import is_null_or_empty, to_form_str +from couchbase.exceptions import (AnalyticsLinkExistsException, + AnalyticsLinkNotFoundException, + DatasetAlreadyExistsException, + DatasetNotFoundException, + DataverseAlreadyExistsException, + DataverseNotFoundException, + InvalidArgumentException) +from couchbase.options import forward_args +from couchbase.pycbc_core import (analytics_mgmt_operations, + management_operation, + mgmt_operations) + +if TYPE_CHECKING: + from couchbase.management.options import (ConnectLinkOptions, + CreateAnalyticsIndexOptions, + CreateDatasetOptions, + CreateDataverseOptions, + CreateLinkAnalyticsOptions, + DisconnectLinkOptions, + DropAnalyticsIndexOptions, + DropDatasetOptions, + DropDataverseOptions, + DropLinkAnalyticsOptions, + GetAllAnalyticsIndexesOptions, + GetAllDatasetOptions, + GetLinksAnalyticsOptions, + GetPendingMutationsOptions, + ReplaceLinkAnalyticsOptions) + + +class AnalyticsManagerLogic: + + _ERROR_MAPPING = {r'.*24055.*already exists': AnalyticsLinkExistsException, + r'.*24006.*does not exist': AnalyticsLinkNotFoundException, + r'.*24034.*Cannot find': DataverseNotFoundException, + r'.*24025.*Cannot find analytics (collection)*': DatasetNotFoundException, + r'.*24039.*An analytics\s+(scope)*.*already exists': DataverseAlreadyExistsException, + r'.*24040.*An analytics\s+(collection)*.*already exists': DatasetAlreadyExistsException} + + def __init__(self, connection): + self._connection = connection + + def create_dataverse(self, + dataverse_name, # type: str + *options, # type: CreateDataverseOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + + op_args = { + "dataverse_name": dataverse_name, + } + + if final_args.get("ignore_if_exists", False) is True: + op_args['ignore_if_exists'] = final_args.get('ignore_if_exists') + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.CREATE_DATAVERSE.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_dataverse(self, + dataverse_name, # type: str + *options, # type: DropDataverseOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + + op_args = { + "dataverse_name": dataverse_name, + } + + if final_args.get("ignore_if_not_exists", False) is True: + op_args['ignore_if_does_not_exist'] = final_args.get('ignore_if_not_exists') + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.DROP_DATAVERSE.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def create_dataset(self, + dataset_name, # type: str + bucket_name, # type: str + *options, # type: CreateDatasetOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + + op_args = { + "dataset_name": dataset_name, + "bucket_name": bucket_name, + } + + if final_args.get("dataverse_name", None) is not None: + op_args['dataverse_name'] = final_args.get('dataverse_name') + + if final_args.get("ignore_if_exists", False) is True: + op_args['ignore_if_exists'] = final_args.get('ignore_if_exists') + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.CREATE_DATASET.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_dataset(self, + dataset_name, # type: str + *options, # type: DropDatasetOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + + op_args = { + "dataset_name": dataset_name, + } + + if final_args.get("dataverse_name", None) is not None: + op_args['dataverse_name'] = final_args.get('dataverse_name') + + if final_args.get("ignore_if_not_exists", False) is True: + op_args['ignore_if_does_not_exist'] = final_args.get('ignore_if_not_exists') + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.DROP_DATASET.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_datasets(self, + *options, # type: GetAllDatasetOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[Iterable[AnalyticsDataset]]: + + final_args = forward_args(kwargs, *options) + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.GET_ALL_DATASETS.value, + "op_args": {} + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def create_index(self, + index_name, # type: str + dataset_name, # type: str + fields, # type: Dict[str, AnalyticsDataType] + *options, # type: CreateAnalyticsIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + + fields = {k: v.value for k, v in fields.items()} + + op_args = { + "index_name": index_name, + "dataset_name": dataset_name, + "fields": fields + } + + if final_args.get("dataverse_name", None) is not None: + op_args['dataverse_name'] = final_args.get('dataverse_name') + + if final_args.get("ignore_if_exists", False) is True: + op_args['ignore_if_exists'] = final_args.get('ignore_if_exists') + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.CREATE_INDEX.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_index(self, + index_name, # type: str + dataset_name, # type: str + *options, # type: DropAnalyticsIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + + op_args = { + "index_name": index_name, + "dataset_name": dataset_name + } + + if final_args.get("dataverse_name", None) is not None: + op_args['dataverse_name'] = final_args.get('dataverse_name') + + if final_args.get("ignore_if_not_exists", False) is True: + op_args['ignore_if_does_not_exist'] = final_args.get('ignore_if_not_exists') + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.DROP_INDEX.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_indexes(self, + *options, # type: GetAllAnalyticsIndexesOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[Iterable[AnalyticsIndex]]: + + final_args = forward_args(kwargs, *options) + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.GET_ALL_INDEXES.value, + "op_args": {} + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def connect_link(self, + *options, # type: ConnectLinkOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + + op_args = {} + + if final_args.get("dataverse_name", None) is not None: + op_args['dataverse_name'] = final_args.get('dataverse_name') + + if final_args.get("link_name", None) is not None: + op_args['link_name'] = final_args.get('link_name') + + if final_args.get("force", False) is True: + op_args['force'] = final_args.get('force') + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.LINK_CONNECT.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def disconnect_link(self, + *options, # type: DisconnectLinkOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + + op_args = {} + + if final_args.get("dataverse_name", None) is not None: + op_args['dataverse_name'] = final_args.get('dataverse_name') + + if final_args.get("link_name", None) is not None: + op_args['link_name'] = final_args.get('link_name') + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.LINK_DISCONNECT.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_pending_mutations(self, + *options, # type: GetPendingMutationsOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[Dict[str, int]]: + + final_args = forward_args(kwargs, *options) + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.GET_PENDING_MUTATIONS.value, + "op_args": {} + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def create_link( + self, + link, # type: AnalyticsLink + *options, # type: CreateLinkAnalyticsOptions + **kwargs + ) -> None: + link.validate() + final_args = forward_args(kwargs, *options) + + op_args = { + "link": link.as_dict(), + "link_type": link.link_type().value + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.LINK_CREATE.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def replace_link( + self, + link, # type: AnalyticsLink + *options, # type: ReplaceLinkAnalyticsOptions + **kwargs + ) -> None: + link.validate() + final_args = forward_args(kwargs, *options) + + op_args = { + "link": link.as_dict(), + "link_type": link.link_type().value + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.LINK_REPLACE.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_link( + self, + link_name, # type: str + dataverse_name, # type: str + *options, # type: DropLinkAnalyticsOptions + **kwargs + ) -> None: + final_args = forward_args(kwargs, *options) + + op_args = { + "link_name": link_name, + "dataverse_name": dataverse_name + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.DROP_LINK.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_links( + self, + *options, # type: GetLinksAnalyticsOptions + **kwargs + ) -> Optional[Iterable[AnalyticsLink]]: + + final_args = forward_args(kwargs, *options) + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.ANALYTICS.value, + "op_type": analytics_mgmt_operations.GET_ALL_LINKS.value, + "op_args": {} + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + +class AnalyticsDataType(Enum): + STRING = 'string' + INT64 = 'int64' + DOUBLE = 'double' + + +class AnalyticsLinkType(Enum): + S3External = 's3' + AzureBlobExternal = 'azureblob' + CouchbaseRemote = 'couchbase' + + +class AnalyticsEncryptionLevel(Enum): + NONE = 'none' + HALF = 'half' + FULL = 'full' + + +@dataclass +class AnalyticsDataset: + dataset_name: str = None + dataverse_name: str = None + link_name: str = None + bucket_name: str = None + + +@dataclass +class AnalyticsIndex: + name: str = None + dataset_name: str = None + dataverse_name: str = None + is_primary: bool = None + + +class AnalyticsLink(ABC): + """AnalytcsLinks are only available on Couchbase Server 7.0+ + """ + + @abstractmethod + def name( + self, # type: "AnalyticsLink" + ) -> str: + """Returns the name of the :class:`couchbase.analytics.AnalyticsLink` + + :return: The name of the :class:`couchbase.analytics.AnalyticsLink` + """ + pass + + @abstractmethod + def dataverse_name(self) -> str: + """Returns the name of the dataverse the :class:`couchbase.analytics.AnalyticsLink` belongs to + + :return: The name of the dataverse + """ + pass + + @abstractmethod + def form_encode(self) -> bytes: + """Encodes the :class:`couchbase.analytics.AnalyticsLink` into a form data representation, + to send as the body of a :func:`couchbase.management.analytics.CreateLink` or + :func:`couchbase.management.analytics.ReplaceLink` + + :return: A form encoded :class:`couchbase.analytics.AnalyticsLink` + """ + pass + + @abstractmethod + def validate(self) -> None: + """Ensures the :class:`couchbase.analytics.AnalyticsLink` is valid. + Raises a :class:`couchbase.exceptions.InvalidArgumentException` if link is invalid. + + :return: None + + :raises: :class:`couchbase.exceptions.InvalidArgumentException` + """ + pass + + @abstractmethod + def link_type(self) -> AnalyticsLinkType: + """Returns the :class:`couchbase.analytics.AnalyticsLinkType` of + the :class:`couchbase.analytics.AnalyticsLink` + + :return: The corresponding :class:`couchbase.analytics.AnalyticsLinkType` + of the :class:`couchbase.analytics.AnalyticsLink` + """ + pass + + +class CouchbaseAnalyticsEncryptionSettings(object): + + def __init__(self, + encryption_level=None, # type: Optional[AnalyticsEncryptionLevel] + certificate=None, # type: Optional[Union[bytes, bytearray]] + client_certificate=None, # type: Optional[Union[bytes, bytearray]] + client_key=None, # type: Optional[Union[bytes, bytearray]] + ): + self._encryption_level = encryption_level + if self._encryption_level is None: + self._encryption_level = AnalyticsEncryptionLevel.NONE + self._certificate = certificate + self._client_certificate = client_certificate + self._client_key = client_key + + @property + def encryption_level(self) -> AnalyticsEncryptionLevel: + return self._encryption_level + + @encryption_level.setter + def encryption_level(self, value # type: AnalyticsEncryptionLevel + ): + self._encryption_level = value + + @property + def certificate(self) -> Optional[bytes]: + return self._certificate + + @certificate.setter + def certificate(self, value # type: Union[bytes, bytearray] + ): + self._certificate = value + + @property + def client_certificate(self) -> Optional[bytes]: + return self._client_certificate + + @client_certificate.setter + def client_certificate(self, value # type: Union[bytes, bytearray] + ): + self._client_certificate = value + + @property + def client_key(self) -> Optional[bytes]: + return self._client_key + + @client_key.setter + def client_key(self, value # type: Union[bytes, bytearray] + ): + self._client_key = value + + def as_dict(self): + encryption_dict = { + 'encryption_level': self.encryption_level.value + } + if self.certificate: + encryption_dict['certificate'] = self.certificate.decode('utf-8') + if self.client_certificate: + encryption_dict['client_certificate'] = self.client_certificate.decode('utf-8') + if self.client_key: + encryption_dict['client_key'] = self.client_key.decode('utf-8') + return encryption_dict + + @classmethod + def from_server_json( + cls, + raw_data # type: dict + ) -> CouchbaseAnalyticsEncryptionSettings: + + encryption_settings = CouchbaseAnalyticsEncryptionSettings() + if raw_data['encryption_level'] == AnalyticsEncryptionLevel.NONE.value: + encryption_settings.encryption_level = AnalyticsEncryptionLevel.NONE + elif raw_data['encryption_level'] == AnalyticsEncryptionLevel.HALF.value: + encryption_settings.encryption_level = AnalyticsEncryptionLevel.HALF + elif raw_data['encryption_level'] == AnalyticsEncryptionLevel.FULL.value: + encryption_settings.encryption_level = AnalyticsEncryptionLevel.FULL + + if 'certificate' in raw_data and not is_null_or_empty(raw_data['certificate']): + encryption_settings.certificate = bytes( + raw_data['certificate'], 'utf-8') + + if 'client_certificate' in raw_data and not is_null_or_empty(raw_data['client_certificate']): + encryption_settings.certificate = bytes( + raw_data['client_certificate'], 'utf-8') + + return encryption_settings + + +class CouchbaseRemoteAnalyticsLink(AnalyticsLink): + def __init__(self, + dataverse, # type: str + link_name, # type: str + hostname, # type: str + encryption, # type: CouchbaseAnalyticsEncryptionSettings + username=None, # type: Optional[str] + password=None, # type: Optional[str] + + ): + super().__init__() + self._dataverse = dataverse + self._link_name = link_name + self._hostname = hostname + self._encryption = encryption + self._username = username + self._password = password + + def name(self,) -> str: + return self._link_name + + def dataverse_name(self) -> str: + return self._dataverse + + def form_encode(self) -> bytes: + params = {} + + if "/" not in self._dataverse: + params["dataverse"] = self._dataverse + params["name"] = self._link_name + + params["hostname"] = self._hostname + params["type"] = AnalyticsLinkType.CouchbaseRemote.value + params["encryption"] = self._encryption.encryption_level.value + + if not is_null_or_empty(self._username): + params["username"] = self._username + + if not is_null_or_empty(self._password): + params["password"] = self._password + + if self._encryption.certificate and len( + self._encryption.certificate) > 0: + params["certificate"] = self._encryption.certificate.decode( + "utf-8") + + if self._encryption.client_certificate and len( + self._encryption.client_certificate) > 0: + params["clientCertificate"] = self._encryption.client_certificate.decode( + "utf-8") + + if self._encryption.client_key and len( + self._encryption.client_key) > 0: + params["clientKey"] = self._encryption.client_key.decode("utf-8") + + return to_form_str(params).encode() + + def validate(self) -> None: + if is_null_or_empty(self._dataverse): + raise InvalidArgumentException(message="Dataverse must be set for couchbase analytics links.") + + if is_null_or_empty(self._link_name): + raise InvalidArgumentException(message="Link name must be set for couchbase analytics links.") + + if is_null_or_empty(self._hostname): + raise InvalidArgumentException(message="Hostname must be set for couchbase analytics links.") + + if self._encryption.encryption_level in [ + AnalyticsEncryptionLevel.NONE, AnalyticsEncryptionLevel.HALF]: + if is_null_or_empty( + self._username) or is_null_or_empty(self._password): + raise InvalidArgumentException( + message=("When encryption level is half or none, " + "username and password must be set for couchbase analytics links.")) + elif self._encryption.encryption_level == AnalyticsEncryptionLevel.FULL: + if not (self._encryption.certificate and len( + self._encryption.certificate) > 0): + raise InvalidArgumentException( + message="When encryption level is full a certificate must be set for couchbase analytics links.") + if not ((self._encryption.client_certificate and len(self._encryption.client_certificate) > 0) + and (self._encryption.client_key and len(self._encryption.client_key) > 0)): + raise InvalidArgumentException( + message=("When encryption level is full the client " + "certificate and key must be set for couchbase analytics links.")) + + def link_type(self) -> AnalyticsLinkType: + return AnalyticsLinkType.CouchbaseRemote + + def as_dict(self): + link_dict = { + 'link_name': self.name(), + 'dataverse': self.dataverse_name(), + 'hostname': self._hostname, + 'link_type': AnalyticsLinkType.CouchbaseRemote.value + } + if self._username: + link_dict['username'] = self._username + if self._password: + link_dict['password'] = self._password + if self._encryption: + link_dict['encryption'] = self._encryption.as_dict() + + return link_dict + + @classmethod + def link_from_server_json( + cls, + raw_data, # type: dict + ) -> CouchbaseRemoteAnalyticsLink: + + dataverse = raw_data['dataverse'] + link_name = raw_data['link_name'] + hostname = raw_data['hostname'] + encryption = CouchbaseAnalyticsEncryptionSettings.from_server_json( + raw_data['encryption_settings']) + username = raw_data.get('username', None) + + return CouchbaseRemoteAnalyticsLink( + dataverse, link_name, hostname, encryption, username) + + +class S3ExternalAnalyticsLink(AnalyticsLink): + def __init__( + self, + dataverse, # type: str + link_name, # type: str + access_key_id, # type: str + region, # type: str + secret_access_key=None, # type: Optional[str] + session_token=None, # type: Optional[str] + service_endpoint=None, # type: Optional[str] + + ): + super().__init__() + self._dataverse = dataverse + self._link_name = link_name + self._access_key_id = access_key_id + self._region = region + self._secret_access_key = secret_access_key + self._session_token = session_token + self._service_endpoint = service_endpoint + + def name(self) -> str: + return self._link_name + + def dataverse_name(self) -> str: + return self._dataverse + + def form_encode(self) -> bytes: + params = {} + + if "/" not in self._dataverse: + params["dataverse"] = self._dataverse + params["name"] = self._link_name + + params["type"] = AnalyticsLinkType.S3External.value + params["accessKeyId"] = self._access_key_id + params["secretAccessKey"] = self._secret_access_key + params["region"] = self._region + + if not is_null_or_empty(self._session_token): + params["sessionToken"] = self._session_token + + if not is_null_or_empty(self._service_endpoint): + params["serviceEndpoint"] = self._service_endpoint + + return to_form_str(params).encode() + + def validate(self) -> None: + if is_null_or_empty(self._dataverse): + raise InvalidArgumentException( + message="Dataverse must be set for S3 external analytics links.") + + if is_null_or_empty(self._link_name): + raise InvalidArgumentException( + message="Link name must be set for S3 external analytics links.") + + if is_null_or_empty(self._access_key_id): + raise InvalidArgumentException( + message="Access key id must be set for S3 external analytics links.") + + if is_null_or_empty(self._secret_access_key): + raise InvalidArgumentException( + message="Secret access key must be set for S3 external analytics links.") + + if is_null_or_empty(self._region): + raise InvalidArgumentException( + message="Region must be set for S3 external analytics links.") + + def link_type(self) -> AnalyticsLinkType: + return AnalyticsLinkType.S3External + + def as_dict(self): + link_dict = { + 'link_name': self.name(), + 'dataverse': self.dataverse_name(), + 'access_key_id': self._access_key_id, + 'secret_access_key': self._secret_access_key, + 'region': self._region, + 'link_type': AnalyticsLinkType.S3External.value + } + if self._session_token: + link_dict['session_token'] = self._session_token + if self._service_endpoint: + link_dict['service_endpoint'] = self._service_endpoint + + return link_dict + + @classmethod + def link_from_server_json(cls, + raw_data, # type: dict + ) -> S3ExternalAnalyticsLink: + + dataverse = raw_data['dataverse'] + link_name = raw_data['link_name'] + access_key_id = raw_data['access_key_id'] + region = raw_data['region'] + service_endpoint = raw_data.get('service_endpoint', None) + + return S3ExternalAnalyticsLink( + dataverse, link_name, access_key_id, region, service_endpoint=service_endpoint) + + +class AzureBlobExternalAnalyticsLink(AnalyticsLink): + def __init__( + self, + dataverse, # type: str + link_name, # type: str + connection_string=None, # type: Optional[str] + account_name=None, # type: Optional[str] + account_key=None, # type: Optional[str] + shared_access_signature=None, # type: Optional[str] + blob_endpoint=None, # type: Optional[str] + endpiont_suffix=None, # type: Optional[str] + + ): + super().__init__() + self._dataverse = dataverse + self._link_name = link_name + self._connection_string = connection_string + self._account_name = account_name + self._account_key = account_key + self._shared_access_signature = shared_access_signature + self._blob_endpoint = blob_endpoint + self._endpiont_suffix = endpiont_suffix + + def name(self) -> str: + return self._link_name + + def dataverse_name(self) -> str: + return self._dataverse + + def form_encode(self) -> bytes: + params = {} + + if "/" not in self._dataverse: + params["dataverse"] = self._dataverse + params["name"] = self._link_name + + params["type"] = AnalyticsLinkType.AzureBlobExternal.value + + if not is_null_or_empty(self._connection_string): + params["connectionString"] = self._connection_string + + if not is_null_or_empty(self._account_name): + params["accountName"] = self._account_name + + if not is_null_or_empty(self._account_key): + params["accountKey"] = self._account_key + + if not is_null_or_empty(self._shared_access_signature): + params["sharedAccessSignature"] = self._shared_access_signature + + if not is_null_or_empty(self._blob_endpoint): + params["blobEndpoint"] = self._blob_endpoint + + if not is_null_or_empty(self._endpiont_suffix): + params["endpointSuffix"] = self._endpiont_suffix + + return to_form_str(params).encode() + + def validate(self) -> None: + if is_null_or_empty(self._dataverse): + raise InvalidArgumentException( + "Dataverse must be set for Azure blob external analytics links.") + + if is_null_or_empty(self._link_name): + raise InvalidArgumentException( + "Link name must be set for Azure blob external analytics links.") + + if is_null_or_empty(self._connection_string): + acct_name_and_key = not (is_null_or_empty(self._account_name) + or is_null_or_empty(self._account_key)) + acct_name_and_sas = not (is_null_or_empty(self._account_name) + or is_null_or_empty(self._shared_access_signature)) + + if not (acct_name_and_key or acct_name_and_sas): + raise InvalidArgumentException( + "AccessKeyId must be set for Azure blob external analytics links.") + + def link_type(self) -> AnalyticsLinkType: + return AnalyticsLinkType.AzureBlobExternal + + def as_dict(self): + link_dict = { + 'link_name': self.name(), + 'dataverse': self.dataverse_name(), + 'link_type': AnalyticsLinkType.AzureBlobExternal.value + } + if self._connection_string: + link_dict['connection_string'] = self._connection_string + if self._account_name: + link_dict['account_name'] = self._account_name + if self._account_key: + link_dict['account_key'] = self._account_key + if self._shared_access_signature: + link_dict['shared_access_signature'] = self._shared_access_signature + if self._blob_endpoint: + link_dict['blob_endpoint'] = self._blob_endpoint + if self._endpiont_suffix: + link_dict['endpiont_suffix'] = self._endpiont_suffix + + return link_dict + + @classmethod + def link_from_server_json(cls, + raw_data, # type: dict + ) -> AzureBlobExternalAnalyticsLink: + + dataverse = raw_data['dataverse'] + link_name = raw_data['link_name'] + account_name = raw_data.get('account_name', None) + blob_endpoint = raw_data.get('blob_endpoint', None) + endpoint_suffix = raw_data.get('endpoint_suffix', None) + + return AzureBlobExternalAnalyticsLink(dataverse, + link_name, + account_name=account_name, + blob_endpoint=blob_endpoint, + endpiont_suffix=endpoint_suffix) diff --git a/couchbase/management/logic/buckets_logic.py b/couchbase/management/logic/buckets_logic.py new file mode 100644 index 000000000..f25c499ce --- /dev/null +++ b/couchbase/management/logic/buckets_logic.py @@ -0,0 +1,625 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import timedelta +from enum import Enum +from typing import (TYPE_CHECKING, + Any, + Dict, + List, + Optional, + Union, + overload) + +from couchbase._utils import (BidirectionalMapping, + BidirectionalTransform, + EnumToStr, + Identity, + ParamTransform, + SecondsToTimeDelta, + StrToEnum, + TimeDeltaToSeconds, + is_null_or_empty) +from couchbase.durability import DurabilityLevel +from couchbase.exceptions import (BucketAlreadyExistsException, + BucketDoesNotExistException, + BucketNotFlushableException, + FeatureUnavailableException, + InvalidArgumentException, + RateLimitedException) +from couchbase.options import forward_args +from couchbase.pycbc_core import (bucket_mgmt_operations, + management_operation, + mgmt_operations) + +if TYPE_CHECKING: + from couchbase.management.options import (BucketDescribeOptions, + CreateBucketOptions, + DropBucketOptions, + FlushBucketOptions, + GetAllBucketOptions, + GetBucketOptions, + UpdateBucketOptions) + + +class BucketManagerLogic: + + _ERROR_MAPPING = {'Bucket with given name (already|still) exists': BucketAlreadyExistsException, + 'Requested resource not found': BucketDoesNotExistException, + r'.*non existent bucket.*': BucketDoesNotExistException, + r'.*Flush is disabled for the bucket.*': BucketNotFlushableException, + r'.*Limit\(s\) exceeded\s+\[.*\].*': RateLimitedException, + r'.*is supported only with developer preview enabled.*': FeatureUnavailableException} + + def __init__(self, connection): + self._connection = connection + + def create_bucket(self, + settings, # type: CreateBucketSettings + *options, # type: CreateBucketOptions + **kwargs # type: Any + ) -> None: + """ + Creates a new bucket. + + :param: CreateBucketSettings settings: settings for the bucket. + :param: CreateBucketOptions options: options for setting the bucket. + :param: Any kwargs: override corresponding values in the options. + + :raises: BucketAlreadyExistsException + :raises: InvalidArgumentsException + """ + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.BUCKET.value, + "op_type": bucket_mgmt_operations.CREATE_BUCKET.value + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + params = settings.transform_to_dest() + + mgmt_kwargs["op_args"] = { + "bucket_settings": params + } + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def update_bucket(self, + settings, # type: BucketSettings + *options, # type: UpdateBucketOptions + **kwargs # type: Any + ) -> None: + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.BUCKET.value, + "op_type": bucket_mgmt_operations.UPDATE_BUCKET.value + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + params = settings.transform_to_dest() + + mgmt_kwargs["op_args"] = { + "bucket_settings": params + } + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_bucket(self, + bucket_name, # type: str + *options, # type: DropBucketOptions + **kwargs # type: Any + ) -> None: + + if is_null_or_empty(bucket_name): + raise InvalidArgumentException("Bucket name cannot be None or empty.") + + if not isinstance(bucket_name, str): + raise InvalidArgumentException("Bucket name must be a str.") + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.BUCKET.value, + "op_type": bucket_mgmt_operations.DROP_BUCKET.value + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + mgmt_kwargs["op_args"] = { + "bucket_name": bucket_name + } + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_bucket(self, + bucket_name, # type: str + *options, # type: GetBucketOptions + **kwargs # type: Any + ) -> BucketSettings: + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.BUCKET.value, + "op_type": bucket_mgmt_operations.GET_BUCKET.value + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + mgmt_kwargs["op_args"] = { + "bucket_name": bucket_name + } + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_buckets(self, + *options, # type: GetAllBucketOptions + **kwargs # type: Any + ) -> List[BucketSettings]: + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.BUCKET.value, + "op_type": bucket_mgmt_operations.GET_ALL_BUCKETS.value + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def flush_bucket(self, + bucket_name, # type: str + *options, # type: FlushBucketOptions + **kwargs # type: Any + ) -> None: + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.BUCKET.value, + "op_type": bucket_mgmt_operations.FLUSH_BUCKET.value + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + mgmt_kwargs["op_args"] = { + "bucket_name": bucket_name + } + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def bucket_describe(self, + bucket_name, # type: str + *options, # type: BucketDescribeOptions + **kwargs # type: Any + ) -> None: + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.BUCKET.value, + "op_type": bucket_mgmt_operations.BUCKET_DESCRIBE.value + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + mgmt_kwargs["op_args"] = { + "bucket_name": bucket_name + } + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + +class EvictionPolicyType(Enum): + NOT_RECENTLY_USED = "nruEviction" + NO_EVICTION = "noEviction" + FULL = "fullEviction" + VALUE_ONLY = "valueOnly" + + +class EjectionMethod(Enum): + FULL_EVICTION = "fullEviction" + VALUE_ONLY = "valueOnly" + + +class BucketType(Enum): + COUCHBASE = "membase" + MEMCACHED = "memcached" + EPHEMERAL = "ephemeral" + + +class CompressionMode(Enum): + OFF = "off" + PASSIVE = "passive" + ACTIVE = "active" + + +class ConflictResolutionType(Enum): + """ + Specifies the conflict resolution type to use for the bucket. + + Members: + TIMESTAMP: specifies to use timestamp conflict resolution + SEQUENCE_NUMBER: specifies to use sequence number conflict resolution + CUSTOM: specifies to use a custom conflict resolution + + """ + TIMESTAMP = "lww" + SEQUENCE_NUMBER = "seqno" + CUSTOM = "custom" + + +class StorageBackend(Enum): + """ + Specifies the storage type to use for the bucket. + """ + UNDEFINED = "undefined" + COUCHSTORE = "couchstore" + MAGMA = "magma" + + +class BucketSettings(dict): + mapping = BidirectionalMapping([ + BidirectionalTransform("flush_enabled", + ParamTransform("flushEnabled", Identity(bool)), + ParamTransform("flushEnabled", Identity(bool)), + default=False), + BidirectionalTransform("num_replicas", + ParamTransform("numReplicas", Identity(int)), + ParamTransform("numReplicas", Identity(int)), + default=0), + BidirectionalTransform("ram_quota_mb", + ParamTransform("ramQuotaMB", Identity(int)), + ParamTransform("ramQuotaMB", Identity(int))), + BidirectionalTransform("replica_index", + ParamTransform("replicaIndex", Identity(bool)), + ParamTransform("replicaIndex", Identity(bool)), + default=False), + BidirectionalTransform("bucket_type", + ParamTransform( + "bucketType", EnumToStr(BucketType)), + ParamTransform( + "bucketType", StrToEnum(BucketType)), + default=BucketType.COUCHBASE), + BidirectionalTransform("max_ttl", + ParamTransform("maxTTL", Identity(int)), + ParamTransform("maxTTL", Identity(int))), + BidirectionalTransform("max_expiry", + ParamTransform( + "maxExpiry", TimeDeltaToSeconds(int)), + ParamTransform("maxExpiry", SecondsToTimeDelta(timedelta))), + BidirectionalTransform("compression_mode", + ParamTransform("compressionMode", + EnumToStr(CompressionMode)), + ParamTransform("compressionMode", StrToEnum(CompressionMode, optional=True))), + BidirectionalTransform("conflict_resolution_type", + ParamTransform("conflictResolutionType", EnumToStr( + ConflictResolutionType)), + ParamTransform("conflictResolutionType", StrToEnum(ConflictResolutionType))), + BidirectionalTransform("eviction_policy", + ParamTransform("evictionPolicy", + EnumToStr(EvictionPolicyType)), + ParamTransform("evictionPolicy", StrToEnum(EvictionPolicyType))), + # BidirectionalTransform("ejection_method", + # ParamTransform("ejectionMethod", + # EnumToStr(EjectionMethod)), + # ParamTransform("ejectionMethod", StrToEnum(EjectionMethod))), + BidirectionalTransform("name", + ParamTransform(transform=Identity(str)), + ParamTransform(transform=Identity(str))), + BidirectionalTransform("minimum_durability_level", + ParamTransform("durabilityMinLevel", EnumToStr( + DurabilityLevel, DurabilityLevel.to_server_str)), + ParamTransform("durabilityMinLevel", StrToEnum( + DurabilityLevel, DurabilityLevel.from_server_str))), + BidirectionalTransform("storage_backend", + ParamTransform("storageBackend", + EnumToStr(StorageBackend)), + ParamTransform("storageBackend", + StrToEnum(StorageBackend))), + BidirectionalTransform("history_retention_collection_default", + ParamTransform("historyRetentionCollectionDefault", Identity(bool, optional=True)), + ParamTransform("historyRetentionCollectionDefault", Identity(bool, optional=True))), + BidirectionalTransform("history_retention_bytes", + ParamTransform("historyRetentionBytes", Identity(int)), + ParamTransform("historyRetentionBytes", Identity(int))), + BidirectionalTransform("history_retention_duration", + ParamTransform("historyRetentionDuration", TimeDeltaToSeconds(int)), + ParamTransform("historyRetentionDuration", SecondsToTimeDelta(timedelta))), + ]) + + @overload + def __init__(self, + name=None, # type: str + flush_enabled=False, # type: bool + ram_quota_mb=None, # type: int + num_replicas=0, # type: int + replica_index=None, # type: bool + bucket_type=None, # type: BucketType + eviction_policy=None, # type: EvictionPolicyType + max_ttl=None, # type: Union[timedelta,float,int] + max_expiry=None, # type: Union[timedelta,float,int] + compression_mode=None, # type: CompressionMode + minimum_durability_level=None, # type: DurabilityLevel + storage_backend=None, # type: StorageBackend + history_retention_collection_default=None, # type: Optional[bool] + history_retention_bytes=None, # type: int + history_retention_duration=None # type: timedelta + ): + # type: (...) -> None + """BucketSettings provides a means of mapping bucket settings into an object. + + """ + pass + + def __init__(self, **kwargs): + """BucketSettings provides a means of mapping bucket settings into an object. + + """ + if kwargs.get('bucket_type', None) == "couchbase": + kwargs['bucket_type'] = BucketType.COUCHBASE + super(BucketSettings, self).__init__(**kwargs) + + @property + def name(self) -> str: + """ Bucket name""" + return self.get("name") + + @property + def flush_enabled(self) -> bool: + """True if flush enabled on bucket, False otherwise""" + return self.get("flush_enabled") + + @property + def num_replicas(self): + # type: (...) -> int + """NumReplicas (int) - The number of replicas for documents.""" + return self.get('replica_number') + + @property + def replica_index(self): + # type: (...) -> bool + """ Whether replica indexes should be enabled for the bucket.""" + return self.get('replica_index') + + @property + def bucket_type(self): + # type: (...) -> BucketType + """BucketType {couchbase (sent on wire as membase), memcached, ephemeral} + The type of the bucket. Default to couchbase.""" + return self.get('bucket_type') + + @property + def eviction_policy(self): + # type: (...) -> EvictionPolicyType + """{fullEviction | valueOnly}. The eviction policy to use.""" + return self.get('eviction_policy') + + @property + def max_ttl(self) -> Optional[int]: + """ + **DEPRECATED** use max_expiry + Value for the maxTTL of new documents created without a ttl. + """ + return self.get('max_ttl', None) + + @property + def max_expiry(self) -> timedelta: + """ + Value for the max expiry of new documents created without an expiry. + """ + return self.get('max_expiry') + + @property + def compression_mode(self): + # type: (...) -> CompressionMode + """{off | passive | active} - The compression mode to use.""" + return self.get('compression_mode') + + @property + def conflict_resolution_type(self) -> Optional[ConflictResolutionType]: + """ + {TIMESTAMP | SEQUENCE_NUMBER | CUSTOM} - The conflict resolution type to use. + CUSTOM: **VOLATILE** This API is subject to change at any time. In Couchbase Server 7.1, + this feature is only available in "developer-preview" mode. See the UI XDCR settings. + """ + return self.get('conflict_resolution_type', None) + + @property + def storage_backend(self): + # type: (...) -> StorageBackend + """ + {couchstore | magma | undefined} - The storage backend to use. + """ + return self.get('storage_backend') + + @property + def history_retention_collection_default(self) -> Optional[bool]: + """ + Whether history retention on collections is enabled by default + """ + return self.get('history_retention_collection_default') + + @property + def history_retention_bytes(self) -> Optional[int]: + """ + The maximum size, in bytes, of the change history that is written to disk for all collections in this bucket + """ + return self.get('history_retention_bytes') + + @property + def history_retention_duration(self) -> Optional[timedelta]: + """ + The maximum duration to be covered by the change history that is written to disk for all collections in this + bucket + """ + return self.get('history_retention_duration') + + def transform_to_dest(self) -> Dict[str, Any]: + kwargs = {**self} + # needed? + kwargs["bucket_password"] = "" # nosec + params = self.mapping.transform_to_dest(kwargs) + params.update({ + 'authType': 'sasl', + 'saslPassword': kwargs['bucket_password'] + }) + return params + + @classmethod + def transform_from_dest(cls, data # type: Dict[str, Any] + ) -> BucketSettings: + params = cls.mapping.transform_from_dest(data) + return cls(**params) + + +class CreateBucketSettings(BucketSettings): + @overload # nosec + def __init__(self, # nosec + name=None, # type: str + flush_enabled=False, # type: bool + ram_quota_mb=None, # type: int + num_replicas=0, # type: int + replica_index=None, # type: bool + bucket_type=None, # type: BucketType + eviction_policy=None, # type: EvictionPolicyType + max_ttl=None, # type: Union[timedelta,float,int] + max_expiry=None, # type: Union[timedelta,float,int] + compression_mode=None, # type: CompressionMode + conflict_resolution_type=None, # type: ConflictResolutionType + bucket_password="", # type: str + ejection_method=None, # type: EjectionMethod + storage_backend=None, # type: StorageBackend + history_retention_collection_default=None, # type: Optional[bool] + history_retention_bytes=None, # type: int + history_retention_duration=None # type: timedelta + ): + """ + Bucket creation settings. + + :param name: name of the bucket + :param flush_enabled: whether flush is enabled + :param ram_quota_mb: raw quota in megabytes + :param num_replicas: number of replicas + :param replica_index: whether this is a replica index + :param bucket_type: type of bucket + :param eviction_policy: policy for eviction + :param max_ttl: **DEPRECATED** max time to live for bucket + :param max_expiry: max expiry time for bucket + :param compression_mode: compression mode + :param ejection_method: ejection method (deprecated, please use eviction_policy instead) + :param storage_backend: specifies the storage type to use for the bucket + :param history_retention_collection_default: specifies whether to enable history retention on collections + by default + :param history_retention_bytes: specifies the maximum size, in bytes, of the change history that is + written to disk for all collections in this bucket + :param history_retention_duration: specifies the maximum duration to be covered by the change history that + is written to disk for all collections in this bucket + + .. note:: + History retention settings are only supported for Magma buckets, the server will ignore retention settings + for other storage backends. + """ + + def __init__(self, **kwargs): + BucketSettings.__init__(self, **kwargs) + + @property + def conflict_resolution_type(self): + # type: (...) -> ConflictResolutionType + return self.get('conflict_resolution_type') + + +@dataclass +class BucketDescribeResult: + name: str = None + uuid: str = None + number_of_nodes: int = None + number_of_replicas: int = None + bucket_capabilities: List[str] = None + storage_backend: str = None diff --git a/couchbase/management/logic/collections_logic.py b/couchbase/management/logic/collections_logic.py new file mode 100644 index 000000000..38d573d6a --- /dev/null +++ b/couchbase/management/logic/collections_logic.py @@ -0,0 +1,399 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from datetime import timedelta +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional) + +from couchbase.exceptions import (CollectionAlreadyExistsException, + CollectionNotFoundException, + QuotaLimitedException, + RateLimitedException, + ScopeAlreadyExistsException, + ScopeNotFoundException) +from couchbase.logic.supportability import Supportability +from couchbase.options import forward_args +from couchbase.pycbc_core import (collection_mgmt_operations, + management_operation, + mgmt_operations) + +if TYPE_CHECKING: + from couchbase.management.options import (CreateCollectionOptions, + CreateScopeOptions, + DropCollectionOptions, + DropScopeOptions, + GetAllScopesOptions, + UpdateCollectionOptions) + + +class CollectionManagerLogic: + + _ERROR_MAPPING = {r'.*Scope with.*name.*already exists': ScopeAlreadyExistsException, + r'.*Scope with.*name.*not found': ScopeNotFoundException, + r'.*Collection with.*name.*not found': CollectionNotFoundException, + r'.*Collection with.*name.*already exists': CollectionAlreadyExistsException, + r'.*collection_not_found.*': CollectionNotFoundException, + r'.*scope_not_found.*': ScopeNotFoundException, + r'.*Maximum number of collections has been reached for scope.*': QuotaLimitedException, + r'.*Limit\(s\) exceeded\s+\[.*\].*': RateLimitedException} + + def __init__(self, connection, bucket_name): + self._connection = connection + self._bucket_name = bucket_name + + def create_scope(self, + scope_name, # type: str + *options, # type: CreateScopeOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + op_args = { + "bucket_name": self._bucket_name, + "scope_name": scope_name + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.COLLECTION.value, + "op_type": collection_mgmt_operations.CREATE_SCOPE.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_scope(self, + scope_name, # type: str + *options, # type: DropScopeOptions + **kwargs # type: Dict[str, Any] + ) -> None: + op_args = { + "bucket_name": self._bucket_name, + "scope_name": scope_name + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.COLLECTION.value, + "op_type": collection_mgmt_operations.DROP_SCOPE.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_scopes(self, + *options, # type: GetAllScopesOptions + **kwargs # type: Dict[str, Any] + ) -> List[ScopeSpec]: + op_args = { + "bucket_name": self._bucket_name + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.COLLECTION.value, + "op_type": collection_mgmt_operations.GET_ALL_SCOPES.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def create_collection(self, + scope_name: str, + collection_name: str, + settings: Optional[CreateCollectionSettings] = None, + *options: CreateCollectionOptions, + **kwargs: Dict[str, Any] + ) -> None: + op_args = { + "bucket_name": self._bucket_name, + "scope_name": scope_name, + "collection_name": collection_name, + } + + if settings is not None: + if settings.max_expiry is not None: + op_args["max_expiry"] = int(settings.max_expiry.total_seconds()) + if settings.history is not None: + op_args["history"] = settings.history + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.COLLECTION.value, + "op_type": collection_mgmt_operations.CREATE_COLLECTION.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_collection(self, + scope_name: str, + collection_name: str, + *options: DropCollectionOptions, + **kwargs: Dict[str, Any] + ) -> None: + op_args = { + "bucket_name": self._bucket_name, + "scope_name": scope_name, + "collection_name": collection_name, + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.COLLECTION.value, + "op_type": collection_mgmt_operations.DROP_COLLECTION.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def update_collection(self, + scope_name: str, + collection_name: str, + settings: UpdateCollectionSettings, + *options: UpdateCollectionOptions, + **kwargs: Dict[str, Any] + ) -> None: + op_args = { + "bucket_name": self._bucket_name, + "scope_name": scope_name, + "collection_name": collection_name, + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.COLLECTION.value, + "op_type": collection_mgmt_operations.UPDATE_COLLECTION.value, + "op_args": op_args + } + + if settings.max_expiry is not None: + op_args["max_expiry"] = int(settings.max_expiry.total_seconds()) + if settings.history is not None: + op_args["history"] = settings.history + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + +class CreateCollectionSettings: + """ + Specifies settings for a collection that will be created. + """ + + def __init__(self, + max_expiry=None, # type: Optional[timedelta] + history=None, # type: Optional[bool] + ): + self._max_expiry = max_expiry + self._history = history + + @property + def max_expiry(self) -> Optional[timedelta]: + """ + Optional[timedelta]: The maximum expiry for documents in the collection. + """ + return self._max_expiry + + @property + def history(self) -> Optional[bool]: + """ + Optional[bool]: Whether history retention override should be enabled in the collection. If not set, it will + default to the bucket-level setting + """ + return self._history + + +class UpdateCollectionSettings: + """ + Specifies settings for a collection that will be updated. + """ + + def __init__(self, + max_expiry=None, # type: Optional[timedelta] + history=None, # type: Optional[bool] + ): + self._max_expiry = max_expiry + self._history = history + + @property + def max_expiry(self) -> Optional[timedelta]: + """ + Optional[timedelta]: The maximum expiry for documents in the collection. + """ + return self._max_expiry + + @property + def history(self) -> Optional[bool]: + """ + Optional[bool]: Whether history retention override should be enabled in the collection. If not set, it will + default to the bucket-level setting + """ + return self._history + + +class CollectionSpec(object): + def __init__(self, + collection_name, # type: str + scope_name='_default', # type: Optional[str] + max_expiry=None, # type: Optional[timedelta] + max_ttl=None, # type: Optional[timedelta] + history=None # type: Optional[bool] + ): + self._name, self._scope_name = collection_name, scope_name + # in the event users utilize kwargs, set max_expiry if not set and max_ttl is set + if max_ttl is not None: + Supportability.class_property_deprecated('max_ttl', 'max_expiry') + if max_expiry is None: + max_expiry = max_ttl + self._max_expiry = max_expiry + self._history = history + + @property + def max_expiry(self) -> Optional[timedelta]: + """ + Optional[timedelta]: The expiry for documents in the collection. + """ + return self._max_expiry + + @property + def max_ttl(self) -> Optional[timedelta]: + """ + Optional[timedelta]: The expiry for documents in the collection. + + **DEPRECATED** Use :attr:`max_expiry` instead. + """ + Supportability.class_property_deprecated('max_ttl', 'max_expiry') + return self._max_expiry + + @property + def history(self) -> Optional[bool]: + """ + Optional[bool]: Whether history retention is enabled in the collection. None represents the bucket-level + setting. + """ + return self._history + + @property + def name(self) -> str: + """ + str: The name of the collection + """ + return self._name + + @property + def scope_name(self) -> str: + """ + str: The name of the collection's scope + """ + return self._scope_name + + +class ScopeSpec(object): + def __init__(self, + name, # type : str + collections, # type: Iterable[CollectionSpec] + ): + self._name, self._collections = name, collections + + @property + def name(self) -> str: + """ + str: The name of the scope + """ + return self._name + + @property + def collections(self) -> Iterable[CollectionSpec]: + """ + List[:class:`.CollectionSpec`]: A list of the scope's collections. + """ + return self._collections diff --git a/couchbase/management/logic/eventing_logic.py b/couchbase/management/logic/eventing_logic.py new file mode 100644 index 000000000..abc670ee5 --- /dev/null +++ b/couchbase/management/logic/eventing_logic.py @@ -0,0 +1,1602 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass, fields +from datetime import timedelta +from enum import Enum +from typing import (TYPE_CHECKING, + Any, + Dict, + List, + Optional) + +from couchbase._utils import is_null_or_empty, timedelta_as_microseconds +from couchbase.exceptions import (EventingFunctionAlreadyDeployedException, + EventingFunctionCollectionNotFoundException, + EventingFunctionCompilationFailureException, + EventingFunctionIdenticalKeyspaceException, + EventingFunctionNotBootstrappedException, + EventingFunctionNotDeployedException, + EventingFunctionNotFoundException, + EventingFunctionNotUnDeployedException, + InvalidArgumentException) +from couchbase.logic.n1ql import QueryScanConsistency +from couchbase.options import forward_args +from couchbase.pycbc_core import (eventing_function_mgmt_operations, + management_operation, + mgmt_operations) + +if TYPE_CHECKING: + from couchbase.management.options import (DeployFunctionOptions, + DropFunctionOptions, + FunctionsStatusOptions, + GetAllFunctionOptions, + GetFunctionOptions, + PauseFunctionOptions, + ResumeFunctionOptions, + UndeployFunctionOptions, + UpsertFunctionOptions) + + +class EventingFunctionManagerLogic: + + _ERROR_MAPPING = {r'.*ERR_APP_NOT_FOUND_TS': EventingFunctionNotFoundException, + r'.*ERR_HANDLER_COMPILATION': EventingFunctionCompilationFailureException, + r'.*ERR_APP_NOT_BOOTSTRAPPED': EventingFunctionNotBootstrappedException, + r'.*ERR_APP_NOT_DEPLOYED': EventingFunctionNotDeployedException, + r'.*ERR_APP_NOT_UNDEPLOYED': EventingFunctionNotUnDeployedException, + r'.*ERR_COLLECTION_MISSING': EventingFunctionCollectionNotFoundException, + r'.*ERR_APP_ALREADY_DEPLOYED': EventingFunctionAlreadyDeployedException, + r'.*ERR_SRC_MB_SAME': EventingFunctionIdenticalKeyspaceException} + + def __init__(self, + connection, + bucket_name=None, # type: Optional[str] + scope_name=None # type: Optional[str] + ): + self._connection = connection + self._bucket_name = bucket_name + self._scope_name = scope_name + + def upsert_function( + self, + function, # type: EventingFunction + *options, # type: UpsertFunctionOptions + **kwargs # type: Any + ) -> None: + """ + Upserts an eventing function + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param function: the EventingFunction to upsert + :param options: UpsertFunctionOptions to upsert an eventing function + :param kwargs: Override corresponding value in options + + :raises: EventingFunctionCompilationFailureException + :raises: EventingFunctionCollectionNotFoundException + :raises: InvalidArgumentException + Any exceptions raised by the underlying platform + """ + eventing_function = function.as_dict() + + op_args = { + 'eventing_function': eventing_function, + } + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.UPSERT_FUNCTION.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def drop_function( + self, + name, # type: str + *options, # type: DropFunctionOptions + **kwargs # type: Any + ) -> None: + """ + Drops an existing eventing function + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param name: the name of the eventing function to drop + :param options: DropFunctionOptions to drop an eventing function + :param kwargs: Override corresponding value in options + + :raises: EventingFunctionNotFoundException + :raises: EventingFunctionNotUndeployedException + Any exceptions raised by the underlying platform + """ + op_args = { + 'name': name, + } + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.DROP_FUNCTION.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def deploy_function( + self, + name, # type: str + *options, # type: DeployFunctionOptions + **kwargs # type: Any + ) -> None: + """ + Deploys an existing eventing function + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param name: the name of the eventing function to drop + :param options: DeployFunctionOptions to deploy an eventing function + :param kwargs: Override corresponding value in options + + :raises: EventingFunctionNotDeployedException + :raises: EventingFunctionAlreadyDeployedException + Any exceptions raised by the underlying platform + """ + op_args = { + 'name': name, + } + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.DEPLOY_FUNCTION.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def get_all_functions( + self, + *options, # type: GetAllFunctionOptions + **kwargs # type: Any + ) -> List[EventingFunction]: + """ + Returns a list of all eventing functions + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param options: GetAllFunctionOptions to get all eventing functions + :param kwargs: Override corresponding value in options + + :raises: + Any exceptions raised by the underlying platform + + :return: A list :class:`EventingFunction` objects + :rtype: list + """ + op_args = {} + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.GET_ALL_FUNCTIONS.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def get_function( + self, + name, # type: str + *options, # type: GetFunctionOptions + **kwargs # type: Any + ) -> EventingFunction: + """ + Returns specified eventing function + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param name: the name of the eventing function to retreive + :param options: GetFunctionOptions to get an eventing function + :param kwargs: Override corresponding value in options + + :raises: EventingFunctionNotFoundException + Any exceptions raised by the underlying platform + + :return: class:`EventingFunction` object + :rtype: class:`EventingFunction` + """ + op_args = { + 'name': name, + } + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.GET_FUNCTION.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def pause_function( + self, + name, # type: str + *options, # type: PauseFunctionOptions + **kwargs # type: Any + ) -> None: + """ + Pauses an existing eventing function + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param name: the name of the eventing function to pause + :param options: PauseFunctionOptions to pause an eventing function + :param kwargs: Override corresponding value in options + + :raises: EventingFunctionNotFoundException + :raises: EventingFunctionNotBootstrappedException + Any exceptions raised by the underlying platform + """ + op_args = { + 'name': name, + } + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.PAUSE_FUNCTION.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def resume_function( + self, + name, # type: str + *options, # type: ResumeFunctionOptions + **kwargs # type: Any + ) -> None: + """ + Resumes an existing eventing function + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param name: the name of the eventing function to resume + :param options: ResumeFunctionOptions to resume an eventing function + :param kwargs: Override corresponding value in options + + :raises: EventingFunctionNotFoundException + :raises: EventingFunctionNotDeployedException + Any exceptions raised by the underlying platform + """ + op_args = { + 'name': name, + } + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.RESUME_FUNCTION.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def undeploy_function( + self, + name, # type: str + *options, # type: UndeployFunctionOptions + **kwargs # type: Any + ) -> None: + """ + Undeploys an existing eventing function + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param name: the name of the eventing function to drop + :param options: UndeployFunctionOptions to undeploy an eventing function + :param kwargs: Override corresponding value in options + + :raises: EventingFunctionNotDeployedException + Any exceptions raised by the underlying platform + """ + op_args = { + 'name': name, + } + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.UNDEPLOY_FUNCTION.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def functions_status( + self, + *options, # type: FunctionsStatusOptions + **kwargs # type: Any + ) -> EventingFunctionsStatus: + """ + Returns the `EventingFunctionStatus` of all eventing functions + + **UNCOMMITTED** + This is an uncommitted API call that is unlikely to change, but may still change as + final consensus on its behavior has not yet been reached. + + :param options: UndeployFunctionOptions to undeploy an eventing function + :param kwargs: Override corresponding value in options + + :raises: + Any exceptions raised by the underlying platform + + :return: class:`EventingFunctionsStatus` object + :rtype: class:`EventingFunctionsStatus` + """ + op_args = {} + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.EVENTING_FUNCTION.value, + 'op_type': eventing_function_mgmt_operations.GET_STATUS.value, + 'op_args': op_args, + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + +class EventingFunctionBucketAccess(Enum): + """Indicates the bucket access of the eventing function. + + Values: + + ReadOnly + The eventing function has read-only bucket access + ReadWrite (default) + The eventing function has read and write bucket access. + """ + + ReadOnly = "r" + ReadWrite = "rw" + + def to_str(self): + if self.value == 'r': + return 'read_only' + elif self.value == 'rw': + return 'read_write' + + @classmethod + def from_server(cls, value # type: str + ) -> EventingFunctionBucketAccess: + if value in ['r', 'read_only']: + return cls.ReadOnly + elif value in ['rw', 'read_write']: + return cls.ReadWrite + else: + raise InvalidArgumentException( + "Invalid value for bucket access: {}".format(value) + ) + + +class EventingFunctionDcpBoundary(Enum): + """Indicates where to start DCP stream from. + + Values: + + Everything (default) + Start DCP stream from the beginning. + FromNow + Start DCP stream from present point. + """ + + Everything = "everything" + FromNow = "from_now" + + @classmethod + def from_server(cls, value): + if value == "everything": + return cls.Everything + elif value == "from_now": + return cls.FromNow + else: + raise InvalidArgumentException( + "Invalid value for DCP boundary: {}".format(value) + ) + + +class EventingFunctionDeploymentStatus(Enum): + """Indicates the eventing function's deployment status. + + Values: + + Undeployed (default) + Indicates the function is not deployed. + Deployed + Indicates the function is deployed. + """ + + Undeployed = False + Deployed = True + + def to_str(self): + if self.value: + return 'deployed' + + return 'undeployed' + + @classmethod + def from_server(cls, value): + if value == 'undeployed': + return cls.Undeployed + elif value == 'deployed': + return cls.Deployed + else: + raise InvalidArgumentException( + "Invalid value for deployment status: {}".format(value) + ) + + +class EventingFunctionProcessingStatus(Enum): + """Indicates if the eventing function is running + + Allowed values: + + Paused (default) + Indicates the eventing function is not running. + Running + Indicates the eventing function is running. + """ + + Paused = False + Running = True + + def to_str(self): + if self.value: + return 'running' + + return 'paused' + + @classmethod + def from_server(cls, value): + if value == 'paused': + return cls.Paused + elif value == 'running': + return cls.Running + else: + raise InvalidArgumentException( + "Invalid value for processing status: {}".format(value) + ) + + +class EventingFunctionLogLevel(Enum): + """Indicates the system logs level of detail. + + Allowed values: + + Info (default) + Error + Warning + Debug + Trace + """ + + Info = "INFO" + Error = "ERROR" + Warning = "WARNING" + Debug = "DEBUG" + Trace = "TRACE" + + @classmethod + def from_server(cls, value): + if value.upper() == "INFO": + return cls.Info + elif value.upper() == "ERROR": + return cls.Error + elif value.upper() == "WARNING": + return cls.Warning + elif value.upper() == "DEBUG": + return cls.Debug + elif value.upper() == "TRACE": + return cls.Trace + else: + raise InvalidArgumentException( + "Invalid value for log level: {}".format(value) + ) + + +class EventingFunctionLanguageCompatibility(Enum): + """Eventing language version the eventing function assumes with respect to syntax and behavior. + + Allowed values: + + Version_6_0_0 + Version_6_5_0 + Version_6_6_2 + Version_7_2_0 + """ + + Version_6_0_0 = "6.0.0" + Version_6_5_0 = "6.5.0" + Version_6_6_2 = "6.6.2" + Version_7_2_0 = "7.2.0" + + def to_str(self): + if self.value == '6.0.0': + return 'version_6_0_0' + if self.value == '6.5.0': + return 'version_6_5_0' + if self.value == '6.6.2': + return 'version_6_6_2' + if self.value == '7.2.0': + return 'version_7_2_0' + + @classmethod + def from_server(cls, value): + if value == "version_6_0_0": + return cls.Version_6_0_0 + elif value == "version_6_5_0": + return cls.Version_6_5_0 + elif value == "version_6_6_2": + return cls.Version_6_6_2 + elif value == "version_7_2_0": + return cls.Version_7_2_0 + else: + raise InvalidArgumentException( + "Invalid value for language compatibility: {}".format(value) + ) + + +class EventingFunctionState(Enum): + """Indicates the eventing function's composite status. + + Values: + + Undeployed (default) + Indicates the function is not deployed. + Deploying + Indicates the function is deploying. + Deployed + Indicates the function is deployed. + Undeploying + Indicates the function is undeploying. + Paused + Indicates the function is pausedd. + Pausing + Indicates the function is pausing. + """ + + Undeployed = "undeployed" + Deployed = "deployed" + Deploying = "deploying" + Undeploying = "undeploying" + Paused = "paused" + Pausing = "pausing" + + @classmethod + def from_server(cls, value): + if value == "undeployed": + return cls.Undeployed + elif value == "deployed": + return cls.Deployed + elif value == "deploying": + return cls.Deploying + elif value == "undeploying": + return cls.Undeploying + elif value == "paused": + return cls.Paused + elif value == "pausing": + return cls.Pausing + else: + raise InvalidArgumentException( + "Invalid value for state: {}".format(value)) + + +@dataclass +class EventingFunctionKeyspace: + """Object representation for a Couchbase Server eventing keyspace + + A keyspace is a triple consisting of the following + components: bucket (required), scope (optional) and + collection (optional) + + :param bucket: bucket name + :type bucket: str + :param scope: scope name + :type scope: str, optional + :param collection: collection name + :type collection: str, optional + """ + bucket: str = None + scope: str = None + collection: str = None + + def as_dict(self) -> Dict[str, Any]: + output = { + 'bucket': self.bucket, + } + if self.scope: + output['scope'] = self.scope + + if self.collection: + output['collection'] = self.collection + + return output + + @classmethod + def from_server( + cls, + bucket, # type: str + scope=None, # type: Optional[str] + collection=None, # type: Optional[str] + ) -> EventingFunctionKeyspace: + """Returns a new `EventingFunctionKeyspace` object based + on the JSON response received from Couchbase Server + + :param bucket: bucket name + :type bucket: str + :param scope: scope name + :type scope: str, optional + :param collection: collection name + :type collection: str, optional + + :return: new `EventingFunctionKeyspace` object + :rtype: `EventingFunctionKeyspace` + """ + keyspace = cls(bucket) + if not is_null_or_empty(scope) and scope != "_default": + keyspace.scope = scope + + if not is_null_or_empty(collection) and collection != "_default": + keyspace.collection = collection + + return keyspace + + +@dataclass +class EventingFunctionBucketBinding: + """Object representation for an eventing function bucket binding + + :param alias: binding's alias + :type alias: str + :param name: binding's keyspace which consists of the bucket name, scope (optional) and collection (optional) + :type name: class:`couchbase.management.EventingFunctionKeyspace` + :param access: binding's bucket access + :type access: class:`EventingFunctionBucketAccess` + """ + + alias: str = None + name: EventingFunctionKeyspace = None + access: EventingFunctionBucketAccess = None + + def as_dict(self) -> Dict[str, Any]: + """Returns a representation of the `EventingFunctionBucketBinding` that + aligns with what Couchbase Server eventing spec. + + :return: dict representation of the `EventingFunctionBucketBinding` + :rtype: Dict[str, Any] + """ + output = {'alias': self.alias, + 'access': self.access.to_str(), + 'name': self.name.as_dict() + } + + return output + + @classmethod + def from_server( + cls, + bucket_binding # type: Dict[str, Any] + ) -> EventingFunctionBucketBinding: + alias = bucket_binding.get('alias', None) + name = bucket_binding.get('name', {}) + keyspace = EventingFunctionKeyspace.from_server(**name) + access = EventingFunctionBucketAccess.from_server(bucket_binding.get('access', None)) + return cls(alias=alias, name=keyspace, access=access) + + +@dataclass +class EventingFunctionUrlAuth: + """Base class for all object representation of eventing function URL binding + authorization types + """ + pass + + +@dataclass +class EventingFunctionUrlNoAuth(EventingFunctionUrlAuth): + """Object representation for an eventing function URL binding + with no authorization + """ + pass + + +@dataclass +class EventingFunctionUrlAuthBasic(EventingFunctionUrlAuth): + """Object representation for an eventing function URL binding + basic authorization + + :param username: auth username + :type username: str + :param password: auth password + :type password: str + """ + + username: str = None + password: str = None + + +@dataclass +class EventingFunctionUrlAuthDigest(EventingFunctionUrlAuth): + """Object representation for an eventing function URL binding + digest authorization + + :param username: auth digest username + :type username: str + :param password: auth digest password + :type password: str + """ + + username: str = None + password: str = None + + +@dataclass +class EventingFunctionUrlAuthBearer(EventingFunctionUrlAuth): + """Object representation for an eventing function URL binding + bearer authorization + + :param key: bearer key + :type key: str + """ + + key: str = None + + +@dataclass +class EventingFunctionUrlBinding: + """Object representation for an eventing function URL binding + + :param hostname: binding's hostname + :type hostname: str + :param alias: binding's alias + :type alias: str + :param allow_cookies: If the binding should allow cookies + :type allow_cookies: bool + :param validate_ssl_certificate: If the binding should validate SSL cert + :type validate_ssl_certificate: bool + :param auth: binding's authorization type + :type auth: class:`EventingFunctionUrlAuth` + """ + + hostname: str = None + alias: str = None + allow_cookies: bool = None + validate_ssl_certificate: bool = None + auth: EventingFunctionUrlAuth = None + + def as_dict(self) -> Dict[str, Any]: + """Returns a representation of the `EventingFunctionUrlBinding` that + aligns with what Couchbase Server eventing spec. + + :return: dict representation of the `EventingFunctionUrlBinding` + :rtype: Dict[str, Any] + """ + output = { + "hostname": self.hostname, + "alias": self.alias, + "allow_cookies": self.allow_cookies, + "validate_ssl_certificate": self.validate_ssl_certificate, + } + + if isinstance(self.auth, EventingFunctionUrlNoAuth): + output["auth_type"] = "no-auth" + elif isinstance(self.auth, EventingFunctionUrlAuthBasic): + output["auth_type"] = "basic" + output["username"] = self.auth.username + output["password"] = self.auth.password + elif isinstance(self.auth, EventingFunctionUrlAuthDigest): + output["auth_type"] = "digest" + output["username"] = self.auth.username + output["password"] = self.auth.password + elif isinstance(self.auth, EventingFunctionUrlAuthBearer): + output["auth_type"] = "bearer" + output["bearer_key"] = self.auth.key + + return output + + @classmethod + def from_server( + cls, + url_binding # type: Dict[str, Any] + ) -> EventingFunctionUrlBinding: + input = { + 'alias': url_binding.get('alias', None), + 'hostname': url_binding.get('hostname', None), + 'allow_cookies': url_binding.get('allow_cookies', None), + 'validate_ssl_certificate': url_binding.get('validate_ssl_certificate', None) + } + auth_type = url_binding.get('auth_type', 'no-auth') + if auth_type == 'no-auth': + input['auth'] = EventingFunctionUrlNoAuth() + elif auth_type == 'basic': + username = url_binding.get('username', None) + input['auth'] = EventingFunctionUrlAuthBasic(username=username) + elif auth_type == 'digest': + username = url_binding.get('username', None) + input['auth'] = EventingFunctionUrlAuthDigest(username=username) + elif auth_type == 'bearer': + key = url_binding.get('key', None) + input['auth'] = EventingFunctionUrlAuthBearer(key=key) + return cls(**input) + + +@dataclass +class EventingFunctionConstantBinding: + """Object representation for an eventing function constant binding + + :param alias: binding's alias + :type alias: str + :param literal: binding's value + :type literal: str + """ + + alias: str = None + literal: str = None + + def as_dict(self) -> Dict[str, Any]: + """Returns a representation of the `EventingFunctionConstantBinding` that + aligns with what Couchbase Server eventing spec. + + :return: dict representation of the `EventingFunctionConstantBinding` + :rtype: Dict[str, Any] + """ + + return {"alias": self.alias, "literal": self.literal} + + @classmethod + def from_server( + cls, + constant_binding # type: Dict[str, Any] + ) -> EventingFunctionConstantBinding: + input = { + 'alias': constant_binding.get('alias', None), + 'literal': constant_binding.get('literal', None) + } + return cls(**input) + + +@dataclass +class EventingFunctionSettings: + """Object representation for an settings relevant to an eventing function + + :param cpp_worker_thread_count: Number of threads each worker utilizes + :type cpp_worker_thread_count: int + :param description: Free form text for user to describe the eventing function + :type description: str + :param execution_timeout: Maximum time the eventing function can run before it is forcefully terminated (in seconds) + :type execution_timeout: timedelta + :param lcb_inst_capacity: Maximum number of libcouchbase connections that may be opened and pooled + :type lcb_inst_capacity: int + :param lcb_retry_count: Number of retries of retriable libcouchbase failures, 0 keeps trying till execution_timeout + :type lcb_retry_count: int + :param lcb_timeout: Maximum time the lcb command is waited until completion before we terminate the request + (in seconds) + :type lcb_timeout: timedelta + :param num_timer_partitions: Number of timer shards, defaults to number of vbuckets + :type num_timer_partitions: int + :param sock_batch_size: Batch size for messages from producer to consumer, normally not specified + :type sock_batch_size: int + :param tick_duration: Duration to log stats from this eventing function, in milliseconds + :type tick_duration: timedelta + :param timer_context_size: Size limit of timer context object + :type timer_context_size: int + :param user_prefix: Key prefix for all data stored in metadata by this eventing function + :type user_prefix: str + :param bucket_cache_size: Maximum size in bytes the bucket cache can grow to + :type bucket_cache_size: int + :param bucket_cache_age: Time in milliseconds after which a cached bucket object is considered stale + :type bucket_cache_age: timedelta + :param curl_max_allowed_resp_size: maximum allowable curl call response in MegaBytes + :type curl_max_allowed_resp_size: int + :param worker_count: Number of worker processes eventing function utilizes on each eventing node + :type worker_count: int + :param query_prepare_all: Automatically prepare all N1QL statements in the eventing function + :type query_prepare_all: bool + :param enable_applog_rotation: Enable rotating this eventing function's log() message files + :type enable_applog_rotation: bool + :param app_log_dir: Directory to write content of log() message files + :type app_log_dir: str + :param app_log_max_size: Rotate logs when file grows to this size in bytes approximately + :type app_log_max_size: int + :param app_log_max_files: Number of log() message files to retain when rotating + :type app_log_max_files: int + :param checkpoint_interval: Number of seconds before writing a progress checkpoint + :type checkpoint_interval: timedelta + :param dcp_stream_boundary: indicates where to start dcp stream from (beginning of time or present point) + :type dcp_stream_boundary: `EventingFunctionDcpBoundary` + :param deployment_status: Indicates if the function is deployed + :type deployment_status: `EventingFunctionDeploymentStatus` + :param processing_status: Indicates if the function is running + :type processing_status: `EventingFunctionProcessingStatus` + :param language_compatibility: Eventing language version this eventing function assumes in terms of syntax + and behavior + :type language_compatibility: `EventingFunctionLanguageCompatibility` + :param log_level: Level of detail in system logging + :type log_level: `EventingFunctionLogLevel` + :param query_consistency: Consistency level used by n1ql statements in the eventing function + :type query_consistency: `couchbase.cluster.QueryScanConsistency` + :param handler_headers: Code to automatically prepend to top of eventing function code + :type handler_headers: List[str] + :param handler_footers: Code to automatically prepend to top of eventing function code + :type handler_footers: List[str] + """ + + cpp_worker_thread_count: int = None + description: str = None + execution_timeout: timedelta = None + lcb_inst_capacity: int = None + lcb_retry_count: int = None + lcb_timeout: timedelta = None + num_timer_partitions: int = None + sock_batch_size: int = None + tick_duration: timedelta = None + timer_context_size: int = None + user_prefix: str = None + bucket_cache_size: int = None + bucket_cache_age: timedelta = None + curl_max_allowed_resp_size: int = None + query_prepare_all: bool = None + worker_count: int = None + enable_applog_rotation: bool = None + app_log_dir: str = None + app_log_max_size: int = None + app_log_max_files: int = None + checkpoint_interval: timedelta = None + dcp_stream_boundary: EventingFunctionDcpBoundary = None + deployment_status: EventingFunctionDeploymentStatus = None + processing_status: EventingFunctionProcessingStatus = None + language_compatibility: EventingFunctionLanguageCompatibility = None + log_level: EventingFunctionLogLevel = None + query_consistency: QueryScanConsistency = None + handler_headers: List[str] = None + handler_footers: List[str] = None + + def as_dict(self) -> Dict[str, Any]: # noqa: C901 + """Returns a representation of the `EventingFunctionSettings` that + aligns with what Couchbase Server eventing spec. + + :return: dict representation of the `EventingFunctionSettings` + :rtype: Dict[str, Any] + """ + + output = {} + + if self.cpp_worker_thread_count: + output['cpp_worker_count'] = self.cpp_worker_thread_count + + if self.dcp_stream_boundary: + output['dcp_stream_boundary'] = self.dcp_stream_boundary.value + + if self.description: + output['description'] = self.description + + if self.deployment_status: + output['deployment_status'] = self.deployment_status.to_str() + + if self.processing_status: + output['processing_status'] = self.processing_status.to_str() + + if self.log_level: + output['log_level'] = self.log_level.value.lower() + + if self.language_compatibility: + output['language_compatibility'] = self.language_compatibility.to_str() + + if self.execution_timeout: + if not isinstance(self.execution_timeout, timedelta): + raise InvalidArgumentException( + 'EventingFunctionSettings execution timeout should be a timedelta' + ) + output['execution_timeout'] = int(self.execution_timeout.total_seconds()) + + if self.lcb_inst_capacity: + output['lcb_inst_capacity'] = self.lcb_inst_capacity + + if self.lcb_retry_count: + output['lcb_retry_count'] = self.lcb_retry_count + + if self.query_consistency: + if self.query_consistency.value in ['not_bounded', 'request_plus']: + output['query_consistency'] = self.query_consistency.value + else: + raise InvalidArgumentException('Only not_bounded and request_plus allowed for query consistency.') + + if self.num_timer_partitions: + output['num_timer_partitions'] = self.num_timer_partitions + + if self.sock_batch_size: + output['sock_batch_size'] = self.sock_batch_size + + if self.tick_duration: + output['tick_duration'] = timedelta_as_microseconds(self.tick_duration) + + if self.timer_context_size: + output['timer_context_size'] = self.timer_context_size + + if self.user_prefix: + output['user_prefix'] = self.user_prefix + + if self.bucket_cache_size: + output['bucket_cache_size'] = self.bucket_cache_size + + if self.bucket_cache_age: + output['bucket_cache_age'] = timedelta_as_microseconds(self.bucket_cache_age) + + if self.curl_max_allowed_resp_size: + output['curl_max_allowed_resp_size'] = self.curl_max_allowed_resp_size + + if self.query_prepare_all: + output['query_prepare_all'] = self.query_prepare_all + + if self.worker_count: + output['worker_count'] = self.worker_count + + if self.handler_headers: + output['handler_headers'] = self.handler_headers + + if self.handler_footers: + output['handler_footers'] = self.handler_footers + + if self.enable_applog_rotation: + output['enable_app_log_rotation'] = self.enable_applog_rotation + + if self.app_log_dir: + output['app_log_dir'] = self.app_log_dir + + if self.app_log_max_size: + output['app_log_max_size'] = self.app_log_max_size + + if self.app_log_max_files: + output['app_log_max_files'] = self.app_log_max_files + + if self.checkpoint_interval: + if not isinstance(self.checkpoint_interval, timedelta): + raise InvalidArgumentException( + 'EventingFunctionSettings checkpoint interval should be a timedelta' + ) + output['checkpoint_interval'] = int(self.checkpoint_interval.total_seconds()) + + return output + + @classmethod + def new_settings(cls, **kwargs): + """Returns a new `EventingFunctionSettings` object + + :param kwargs: Keyword arguments to populate `EventingFunctionSettings` object + :type kwargs: dict + + :return: new `EventingFunctionSettings` object + :rtype: `EventingFunctionSettings` + """ + if "dcp_stream_boundary" not in kwargs: + kwargs["dcp_stream_boundary"] = None + + if "deployment_status" not in kwargs: + kwargs["deployment_status"] = EventingFunctionDeploymentStatus.Undeployed + + if "processing_status" not in kwargs: + kwargs["processing_status"] = EventingFunctionProcessingStatus.Paused + + if "language_compatibility" not in kwargs: + kwargs["language_compatibility"] = None + + if "log_level" not in kwargs: + kwargs["log_level"] = None + + if "query_consistency" not in kwargs: + kwargs["query_consistency"] = None + + settings_fields = [f.name for f in fields(cls)] + final_json = {k: v for k, v in kwargs.items() if k in settings_fields} + return cls(**final_json) + + @classmethod # noqa: C901 + def from_server( # noqa: C901 + cls, + settings # type: Dict[str, Any] + ) -> EventingFunctionSettings: + input = { + 'cpp_worker_thread_count': settings.get('cpp_worker_count', None), + 'description': settings.get('description', None), + 'lcb_inst_capacity': settings.get('lcb_inst_capacity', None), + 'lcb_retry_count': settings.get('lcb_retry_count', None), + 'num_timer_partitions': settings.get('num_timer_partitions', None), + 'sock_batch_size': settings.get('sock_batch_size', None), + 'timer_context_size': settings.get('timer_context_size', None), + 'user_prefix': settings.get('user_prefix', None), + 'bucket_cache_size': settings.get('bucket_cache_size', None), + 'curl_max_allowed_resp_size': settings.get('curl_max_allowed_resp_size', None), + 'query_prepare_all': settings.get('query_prepare_all', None), + 'worker_count': settings.get('worker_count', None), + 'handler_headers': settings.get('handler_headers', None), + 'handler_footers': settings.get('handler_footers', None), + 'enable_applog_rotation': settings.get('enable_app_log_rotation', None), + 'app_log_dir': settings.get('app_log_dir', None), + 'app_log_max_size': settings.get('app_log_max_size', None), + 'app_log_max_files': settings.get('app_log_max_files', None) + } + + dcp_stream_boundary = settings.get('dcp_stream_boundary', None) + if dcp_stream_boundary: + input['dcp_stream_boundary'] = EventingFunctionDcpBoundary.from_server(dcp_stream_boundary) + + deployment_status = settings.get('deployment_status', None) + if deployment_status: + input['deployment_status'] = EventingFunctionDeploymentStatus.from_server(deployment_status) + + processing_status = settings.get('processing_status', None) + if processing_status: + input['processing_status'] = EventingFunctionProcessingStatus.from_server(processing_status) + + log_level = settings.get('log_level', None) + if log_level: + input['log_level'] = EventingFunctionLogLevel.from_server(log_level) + + language_compatibility = settings.get('language_compatibility', None) + if language_compatibility: + input['language_compatibility'] = EventingFunctionLanguageCompatibility.from_server(language_compatibility) + + query_consistency = settings.get('query_consistency', None) + if query_consistency: + if query_consistency == 'not_bounded': + input['query_consistency'] = QueryScanConsistency.NOT_BOUNDED + elif query_consistency == 'request_plus': + input['query_consistency'] = QueryScanConsistency.REQUEST_PLUS + + execution_timeout = settings.get('execution_timeout', None) + if execution_timeout: + input['execution_timeout'] = timedelta(seconds=execution_timeout) + + lcb_timeout = settings.get('lcb_timeout', None) + if lcb_timeout: + input['lcb_timeout'] = timedelta(seconds=lcb_timeout) + + tick_duration = settings.get('tick_duration', None) + if tick_duration: + input['tick_duration'] = timedelta(milliseconds=tick_duration) + + bucket_cache_age = settings.get('bucket_cache_age', None) + if bucket_cache_age: + input['bucket_cache_age'] = timedelta(milliseconds=bucket_cache_age) + + checkpoint_interval = settings.get('checkpoint_interval', None) + if checkpoint_interval: + input['checkpoint_interval'] = timedelta(seconds=checkpoint_interval) + + return cls(**input) + + +@dataclass +class EventingFunction: + """Object representation for settings relevant to an eventing function + + :param name: + :type name: str + :param code: + :type code: str + :param version: + :type version: str + :param enforce_schema: + :type enforce_schema: bool + :param handler_uuid: + :type handler_uuid: int + :param function_instance_id: + :type function_instance_id: str + :param metadata_keyspace: + :type metadata_keyspace: `.generic.EventingFunctionKeyspace` + :param source_keyspace: + :type source_keyspace: `.generic.EventingFunctionKeyspace` + :param bucket_bindings: + :type bucket_bindings: List[`EventingFunctionBucketBinding`] + :param url_bindings: + :type url_bindings: List[`EventingFunctionUrlBinding`] + :param constant_bindings: + :type constant_bindings: List[`EventingFunctionConstantBinding`] + :param settings: + :type settings: `EventingFunctionSettings` + """ + + name: str = None + code: str = None + version: str = None + enforce_schema: bool = None + handler_uuid: int = None + function_instance_id: str = None + metadata_keyspace: EventingFunctionKeyspace = None + source_keyspace: EventingFunctionKeyspace = None + bucket_bindings: List[EventingFunctionBucketBinding] = None + url_bindings: List[EventingFunctionUrlBinding] = None + constant_bindings: List[EventingFunctionConstantBinding] = None + settings: EventingFunctionSettings = None + + def as_dict(self) -> Dict[str, Any]: # noqa: C901 + """Returns a representation of the `EventingFunction` that + aligns with what Couchbase Server eventing spec. + + :return: dict representation of the `EventingFunction` + :rtype: Dict[str, Any] + """ + + output = {} + + if self.name: + output['name'] = self.name + + if self.code: + output['code'] = self.code + + if self.version: + output['version'] = self.version + + if self.enforce_schema: + output['enforce_schema'] = self.enforce_schema + + if self.handler_uuid: + output['handler_uuid'] = self.handler_uuid + + if self.function_instance_id: + output['function_instance_id'] = self.function_instance_id + + if self.metadata_keyspace: + output['metadata_keyspace'] = self.metadata_keyspace.as_dict() + + if self.source_keyspace: + output['source_keyspace'] = self.source_keyspace.as_dict() + + if self.bucket_bindings: + output['bucket_bindings'] = [b.as_dict() for b in self.bucket_bindings] + + if self.url_bindings: + output['url_bindings'] = [b.as_dict() for b in self.url_bindings] + + if self.constant_bindings: + output['constant_bindings'] = [b.as_dict() for b in self.constant_bindings] + + if self.settings: + output['settings'] = self.settings.as_dict() + + return output + + @classmethod + def from_server( + cls, + eventing_function # type: Dict[str, Any] + ) -> EventingFunction: + input = { + 'name': eventing_function.get('name', None), + 'code': eventing_function.get('code', None), + 'version': eventing_function.get('version', None), + 'enforce_schema': eventing_function.get('enforce_schema', None), + 'handler_uuid': eventing_function.get('handler_uuid', None) + } + + metadata_keyspace = eventing_function.get('metadata_keyspace', None) + if metadata_keyspace: + input['metadata_keyspace'] = EventingFunctionKeyspace.from_server(metadata_keyspace) + + source_keyspace = eventing_function.get('source_keyspace', None) + if source_keyspace: + input['source_keyspace'] = EventingFunctionKeyspace.from_server(source_keyspace) + + bucket_bindings = eventing_function.get('bucket_bindings', None) + if bucket_bindings: + input['bucket_bindings'] = [EventingFunctionBucketBinding.from_server(b) for b in bucket_bindings] + + url_bindings = eventing_function.get('url_bindings', None) + if url_bindings: + input['url_bindings'] = [EventingFunctionUrlBinding.from_server(b) for b in url_bindings] + + constant_bindings = eventing_function.get('constant_bindings', None) + if constant_bindings: + input['constant_bindings'] = [EventingFunctionConstantBinding.from_server(b) for b in constant_bindings] + + settings = eventing_function.get('settings', None) + if settings: + input['settings'] = EventingFunctionSettings.from_server(settings) + + return cls(**input) + + +@dataclass +class EventingFunctionStatus: + """Object representation for the status of an eventing function + + Particularly useful when determining an interim state of an evening function (i.e, + deploying, pausing, undeploying). + + :param name: eventing function's name + :type name: str + :param num_bootstrapping_nodes: number of nodes for bootstrapping + :type num_bootstrapping_nodes: int + :param num_deployed_nodes: number of nodes eventing function is deployed on + :type num_deployed_nodes: int + :param state: composite status of eventing function + :type state: `EventingFunctionState` + :param deployment_status: Indicates if the function is deployed + :type deployment_status: `EventingFunctionDeploymentStatus` + :param processing_status: Indicates if the function is running + :type processing_status: `EventingFunctionProcessingStatus` + :param redeploy_required: Indicates if function needs to be redeployed + :type redeploy_required: bool + :param function_scope: Indicates the eventing function's scope + :type function_scope: Dict[str, Any] + """ + + name: str = None + num_bootstrapping_nodes: int = None + num_deployed_nodes: int = None + state: EventingFunctionState = None + deployment_status: EventingFunctionDeploymentStatus = None + processing_status: EventingFunctionProcessingStatus = None + redeploy_required: bool = None + function_scope: Dict[str, Any] = None + + @classmethod + def from_server( + cls, + function_status # type: Dict[str, Any] + ) -> EventingFunctionStatus: + input = { + 'name': function_status.get('name', None), + 'num_bootstrapping_nodes': function_status.get('num_bootstrapping_nodes', None), + 'num_deployed_nodes': function_status.get('num_deployed_nodes', None), + 'redeploy_required': function_status.get('redeploy_required', None), + 'function_scope': function_status.get('function_scope', None) + } + + state = function_status.get('status', None) + if state: + input['state'] = EventingFunctionState.from_server(state) + + deployment_status = function_status.get('deployment_status', None) + if deployment_status: + input['deployment_status'] = EventingFunctionDeploymentStatus.from_server(deployment_status) + + processing_status = function_status.get('processing_status', None) + if processing_status: + input['processing_status'] = EventingFunctionProcessingStatus.from_server(processing_status) + + return cls(**input) + + +@dataclass +class EventingFunctionsStatus: + """Object representation for statuses for all eventing functions + + :param num_eventing_nodes: + :type num_eventing_nodes: int + :param functions: + :type functions: List[`EventingFunctionStatus`] + """ + + num_eventing_nodes: int = None + functions: List[EventingFunctionStatus] = None + + @classmethod + def from_server( + cls, # type: EventingFunctionsStatus + function_status, # type: Dict[str, Any] + ) -> EventingFunctionsStatus: + """Returns a new `EventingFunctionsStatus` object based + on the Dict[str, Any] response received from Couchbase Server + + :param function_status: Dict[str, Any] + :type function_status: dict + + :return: new `EventingFunctionsStatus` object + :rtype: `EventingFunctionsStatus` + """ + input = { + 'num_eventing_nodes': function_status.get('num_eventing_nodes', None) + } + functions = function_status.get('functions', None) + if functions: + input['functions'] = [EventingFunctionStatus.from_server(f) for f in functions] + + return cls(**input) diff --git a/couchbase/management/logic/query_index_logic.py b/couchbase/management/logic/query_index_logic.py new file mode 100644 index 000000000..5b30c686a --- /dev/null +++ b/couchbase/management/logic/query_index_logic.py @@ -0,0 +1,372 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + Optional) + +from couchbase.exceptions import QueryIndexAlreadyExistsException, QueryIndexNotFoundException +from couchbase.management.options import GetAllQueryIndexOptions +from couchbase.options import forward_args +from couchbase.pycbc_core import (management_operation, + mgmt_operations, + query_index_mgmt_operations) + +if TYPE_CHECKING: + from couchbase.management.options import (BuildDeferredQueryIndexOptions, + CreatePrimaryQueryIndexOptions, + CreateQueryIndexOptions, + DropPrimaryQueryIndexOptions, + DropQueryIndexOptions) + + +class QueryIndexManagerLogic: + + _ERROR_MAPPING = {r'.*[iI]ndex.*already exists.*': QueryIndexAlreadyExistsException, + r'.*[iI]ndex.*[nN]ot [fF]ound.*': QueryIndexNotFoundException} + + def __init__(self, connection): + self._connection = connection + + def _create_index(self, + bucket_name, # type: str + keys, # type: Iterable[str] + index_name=None, # type: Optional[str] + **kwargs # type: Dict[str, Any] + ) -> None: + + op_args = self._parse_create_index_args(bucket_name, + keys, + index_name=index_name, + **kwargs) + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.QUERY_INDEX.value, + 'op_type': query_index_mgmt_operations.CREATE_INDEX.value, + 'op_args': op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if kwargs.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = kwargs.get('timeout') + + return management_operation(**mgmt_kwargs) + + def _parse_create_index_args(self, # noqa: C901 + bucket_name, # type: str + keys, # type: Iterable[str] + index_name=None, # type: Optional[str] + **kwargs # type: Dict[str, Any] + ) -> Dict[str, Any]: + + # @TODO: validate scope/collection? + + primary = kwargs.get('primary', False) + condition = kwargs.get('condition', None) + + if primary and keys: + raise TypeError('Cannot create primary index with explicit keys') + elif not primary and not keys: + raise ValueError('Keys required for non-primary index') + + if condition and primary: + raise ValueError('cannot specify condition for primary index') + + op_args = { + 'bucket_name': bucket_name, + } + if index_name: + op_args['index_name'] = index_name + if primary is True: + op_args['is_primary'] = primary + if condition: + op_args['condition'] = condition + if keys and len(keys) > 0: + if isinstance(keys, list): + op_args['keys'] = keys + else: + op_args['keys'] = list(keys) + if kwargs.get('ignore_if_exists', False) is True: + op_args['ignore_if_exists'] = kwargs.get('ignore_if_exists') + if kwargs.get('scope_name', None) is not None: + op_args['scope_name'] = kwargs.get('scope_name') + if kwargs.get('collection_name', None) is not None: + op_args['collection_name'] = kwargs.get('collection_name') + if kwargs.get('deferred', None) is not None: + op_args['deferred'] = kwargs.get('deferred') + if kwargs.get('num_replicas', None) is not None: + op_args['num_replicas'] = kwargs.get('num_replicas') + + return op_args + + def _drop_index(self, + bucket_name, # type: str + index_name=None, # type: Optional[str] + **kwargs # type: Dict[str, Any] + ) -> None: + + # @TODO: validate scope/collection? + + ignore_if_not_exists = kwargs.pop('ignore_if_not_exists', False) + # previous ignore_missing was a variable kwarg - should only have ignore_if_not_exists + ignore_missing = kwargs.pop('ignore_missing', False) + + op_args = { + 'bucket_name': bucket_name, + } + if index_name: + op_args['index_name'] = index_name + + if kwargs.get('scope_name', None) is not None: + op_args['scope_name'] = kwargs.get('scope_name') + + if kwargs.get('collection_name', None) is not None: + op_args['collection_name'] = kwargs.get('collection_name') + + if ignore_if_not_exists is True or ignore_missing is True: + op_args['ignore_if_does_not_exist'] = True + + if kwargs.get('primary', False) is True: + op_args['is_primary'] = kwargs.get('primary') + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.QUERY_INDEX.value, + 'op_type': query_index_mgmt_operations.DROP_INDEX.value, + 'op_args': op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if kwargs.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = kwargs.get('timeout') + + return management_operation(**mgmt_kwargs) + + def create_index(self, + bucket_name, # type: str + index_name, # type: str + keys, # type: Iterable[str] + *options, # type: CreateQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> None: + + final_args = forward_args(kwargs, *options) + return self._create_index(bucket_name, keys, index_name, **final_args) + + def create_primary_index(self, + bucket_name, # type: str + *options, # type: CreatePrimaryQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> None: + """ + Creates a new primary index. + + :param str bucket_name: name of the bucket. + :param str index_name: name of the index. + :param CreatePrimaryQueryIndexOptions options: Options to use when creating primary index + :param Any kwargs: Override corresponding values in options. + :raises: QueryIndexAlreadyExistsException + :raises: InvalidArgumentsException + """ + # CREATE INDEX index_name ON bucket_name WITH { "num_replica": 2 } + # https://docs.couchbase.com/server/current/n1ql/n1ql-language-reference/createindex.html + # + + kwargs['primary'] = True + final_args = forward_args(kwargs, *options) + index_name = final_args.pop('index_name', None) + return self._create_index(bucket_name, [], index_name, **final_args) + + def drop_index(self, + bucket_name, # type: str + index_name, # type: str + *options, # type: DropQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> None: + """ + Drops an index. + + :param str bucket_name: name of the bucket. + :param str index_name: name of the index. + :param DropQueryIndexOptions options: Options for dropping index. + :param Any kwargs: Override corresponding value in options. + :raises: QueryIndexNotFoundException + :raises: InvalidArgumentsException + """ + final_args = forward_args(kwargs, *options) + return self._drop_index(bucket_name, index_name, **final_args) + + def drop_primary_index(self, + bucket_name, # type: str + *options, # type: DropPrimaryQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> None: + """ + Drops a primary index. + + :param bucket_name: name of the bucket. + :param index_name: name of the index. + :param ignore_if_not_exists: Don't error/throw if the index does not exist. + :param timeout: the time allowed for the operation to be terminated. This is controlled by the client. + + :raises: QueryIndexNotFoundException + :raises: InvalidArgumentsException + """ + kwargs['primary'] = True + final_args = forward_args(kwargs, *options) + index_name = final_args.pop('index_name', None) + return self._drop_index(bucket_name, index_name, **final_args) + + def get_all_indexes(self, + bucket_name, # type: str + *options, # type: GetAllQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> Optional[Iterable[QueryIndex]]: + + op_args = { + 'bucket_name': bucket_name, + } + final_args = forward_args(kwargs, *options) + + if final_args.get('scope_name', None) is not None: + op_args['scope_name'] = final_args.get('scope_name') + + if final_args.get('collection_name', None) is not None: + op_args['collection_name'] = final_args.get('collection_name') + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.QUERY_INDEX.value, + 'op_type': query_index_mgmt_operations.GET_ALL_INDEXES.value, + 'op_args': op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + def build_deferred_indexes(self, + bucket_name, # type: str + *options, # type: BuildDeferredQueryIndexOptions + **kwargs + ) -> None: + """ + Build Deferred builds all indexes which are currently in deferred state. + + :param str bucket_name: name of the bucket. + :param BuildDeferredQueryIndexOptions options: Options for building deferred indexes. + :param Any kwargs: Override corresponding value in options. + :raise: InvalidArgumentsException + + """ + op_args = { + 'bucket_name': bucket_name, + } + final_args = forward_args(kwargs, *options) + + if final_args.get('scope_name', None) is not None: + op_args['scope_name'] = final_args.get('scope_name') + + if final_args.get('collection_name', None) is not None: + op_args['collection_name'] = final_args.get('collection_name') + + mgmt_kwargs = { + 'conn': self._connection, + 'mgmt_op': mgmt_operations.QUERY_INDEX.value, + 'op_type': query_index_mgmt_operations.BUILD_DEFERRED_INDEXES.value, + 'op_args': op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get('timeout', None) is not None: + mgmt_kwargs['timeout'] = final_args.get('timeout') + + return management_operation(**mgmt_kwargs) + + +@dataclass +class QueryIndex: + name: str + is_primary: bool + type: str + state: str + namespace: str + datastore_id: str + keyspace: str + index_key: list = field(default_factory=list) + condition: str = None + bucket_name: str = None + scope_name: str = None + collection_name: str = None + partition: str = None + + @classmethod + def from_server(cls, + json_data # type: Dict[str, Any] + ): + + bucket_name = json_data.get('bucket_name', None) + if bucket_name is None: + bucket_name = json_data.get('keyspace_id', None) + return cls(json_data.get('name'), + bool(json_data.get('is_primary')), + json_data.get('type'), + json_data.get('state'), + json_data.get('namespace_id'), + json_data.get('datastore_id'), + json_data.get('keyspace_id'), + json_data.get('index_key', []), + json_data.get('condition', None), + bucket_name, + json_data.get('scope_name', None), + json_data.get('collection_name', None), + json_data.get('partition', None) + ) diff --git a/couchbase/management/logic/search_index_logic.py b/couchbase/management/logic/search_index_logic.py new file mode 100644 index 000000000..c9ddbe882 --- /dev/null +++ b/couchbase/management/logic/search_index_logic.py @@ -0,0 +1,799 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from dataclasses import dataclass, field +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + Optional, + Union) + +from couchbase._utils import is_null_or_empty +from couchbase.exceptions import (FeatureUnavailableException, + InvalidArgumentException, + QuotaLimitedException, + SearchIndexNotFoundException) +from couchbase.options import forward_args +from couchbase.pycbc_core import (management_operation, + mgmt_operations, + search_index_mgmt_operations) + +if TYPE_CHECKING: + from couchbase.management.options import (AllowQueryingSearchIndexOptions, + AnalyzeDocumentSearchIndexOptions, + DisallowQueryingSearchIndexOptions, + DropSearchIndexOptions, + FreezePlanSearchIndexOptions, + GetAllSearchIndexesOptions, + GetAllSearchIndexStatsOptions, + GetSearchIndexedDocumentsCountOptions, + GetSearchIndexOptions, + GetSearchIndexStatsOptions, + PauseIngestSearchIndexOptions, + ResumeIngestSearchIndexOptions, + UnfreezePlanSearchIndexOptions, + UpsertSearchIndexOptions) + + +class SearchIndexManagerLogic: + + _ERROR_MAPPING = {r'.*[iI]ndex not found.*': SearchIndexNotFoundException, + r'.*[Cc]reate[Ii]ndex, [Pp]repare failed, err: [Ee]xceeds indexes limit': QuotaLimitedException} + + def __init__(self, + connection, + bucket_name=None, # type: Optional[str] + scope_name=None # type: Optional[str] + ): + self._connection = connection + self._bucket_name = bucket_name + self._scope_name = scope_name + + def upsert_index(self, + index, # type: SearchIndex + *options, # type: UpsertSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + if not index: + raise InvalidArgumentException("expected index to not be None") + else: + if not index.is_valid(): + raise InvalidArgumentException( + "Index must have name, source set") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index': index.as_dict() + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.UPSERT_INDEX.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_index(self, + index_name, # type: str + *options, # type: DropSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + if is_null_or_empty(index_name): + raise InvalidArgumentException("expected an index_name") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.DROP_INDEX.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_index(self, + index_name, # type: str + *options, # type: GetSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[SearchIndex]: + + if is_null_or_empty(index_name): + raise InvalidArgumentException( + "expected index_name to not be empty") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.GET_INDEX.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_indexes(self, + *options, # type: GetAllSearchIndexesOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[Iterable[SearchIndex]]: + + final_args = forward_args(kwargs, *options) + op_args = {} + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.GET_ALL_INDEXES.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_indexed_documents_count(self, + index_name, # type: str + *options, # type: GetSearchIndexedDocumentsCountOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[int]: + + if is_null_or_empty(index_name): + raise InvalidArgumentException( + "expected index_name to not be empty") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.GET_INDEX_DOCUMENT_COUNT.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def pause_ingest(self, + index_name, # type: str + *options, # type: PauseIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + if is_null_or_empty(index_name): + raise InvalidArgumentException( + "expected index_name to not be empty") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name, + 'pause': True + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.CONTROL_INGEST.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def resume_ingest(self, + index_name, # type: str + *options, # type: ResumeIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + if is_null_or_empty(index_name): + raise InvalidArgumentException( + "expected index_name to not be empty") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name, + 'pause': False + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.CONTROL_INGEST.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def allow_querying(self, + index_name, # type: str + *options, # type: AllowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + if is_null_or_empty(index_name): + raise InvalidArgumentException( + "expected index_name to not be empty") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name, + 'allow': True + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.CONTROL_QUERY.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def disallow_querying(self, + index_name, # type: str + *options, # type: DisallowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + if is_null_or_empty(index_name): + raise InvalidArgumentException( + "expected index_name to not be empty") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name, + 'allow': False + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.CONTROL_QUERY.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def freeze_plan(self, + index_name, # type: str + *options, # type: FreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + if is_null_or_empty(index_name): + raise InvalidArgumentException( + "expected index_name to not be empty") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name, + 'freeze': True + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.FREEZE_PLAN.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def unfreeze_plan(self, + index_name, # type: str + *options, # type: UnfreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + if is_null_or_empty(index_name): + raise InvalidArgumentException( + "expected index_name to not be empty") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name, + 'freeze': False + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.FREEZE_PLAN.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def analyze_document(self, # noqa: C901 + index_name, # type: str + document, # type: Any + *options, # type: AnalyzeDocumentSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + + if is_null_or_empty(index_name): + raise InvalidArgumentException("expected an index_name") + if not document: + raise InvalidArgumentException("expected a document to analyze") + json_doc = None + try: + json_doc = json.dumps(document) + except Exception: + raise InvalidArgumentException( + "cannot convert doc to json to analyze") + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name, + 'encoded_document': json_doc + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + if self._bucket_name is not None: + op_args['bucket_name'] = self._bucket_name + + if self._scope_name is not None: + op_args['scope_name'] = self._scope_name + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.ANALYZE_DOCUMENT.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_index_stats(self, + index_name, # type: str + *options, # type: GetSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + + if is_null_or_empty(index_name): + raise InvalidArgumentException("expected an index_name") + + if self._bucket_name is not None and self._scope_name is not None: + raise FeatureUnavailableException(('get_index_stats unavailable at scope level. ' + 'Use cluster.searchIndexes().get_index_stats(...) instead.')) + + final_args = forward_args(kwargs, *options) + op_args = { + 'index_name': index_name + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.GET_INDEX_STATS.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_index_stats(self, + *options, # type: GetAllSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + + final_args = forward_args(kwargs, *options) + op_args = {} + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.SEARCH_INDEX.value, + "op_type": search_index_mgmt_operations.GET_ALL_STATS.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + +@dataclass +class SearchIndex: + name: str + source_type: str = 'couchbase' + idx_type: str = 'fulltext-index' + source_name: str = None + uuid: str = None + params: dict = field(default_factory=dict) + source_uuid: str = None + source_params: dict = field(default_factory=dict) + plan_params: dict = field(default_factory=dict) + + def is_valid(self): + return not (is_null_or_empty(self.name) + or is_null_or_empty(self.idx_type) + or is_null_or_empty(self.source_type)) + + def as_dict(self): + output = { + 'name': self.name, + 'type': self.idx_type, + 'source_type': self.source_type + } + if self.uuid: + output['uuid'] = self.uuid + if len(self.params) > 0: + output['params_json'] = json.dumps(self.params) + if self.source_uuid: + output['source_uuid'] = self.source_uuid + if self.source_name: + output['source_name'] = self.source_name + if len(self.source_params) > 0: + output['source_params_json'] = json.dumps(self.source_params) + if len(self.plan_params) > 0: + output['plan_params_json'] = json.dumps(self.plan_params) + return output + + @classmethod + def from_server(cls, + json_data # type: Dict[str, Any] + ): + params = {} + if 'params_json' in json_data: + params = json.loads(json_data.get('params_json')) + + source_params = {} + if 'source_params_json' in json_data: + source_params = json.loads(json_data.get('source_params_json')) + + plan_params = {} + if 'plan_params_json' in json_data: + plan_params = json.loads(json_data.get('plan_params_json')) + + return cls(json_data.get('name'), + json_data.get('source_type'), + json_data.get('type'), + json_data.get('source_name', None), + json_data.get('uuid', None), + params, + json_data.get('source_uuid', None), + source_params, + plan_params + ) + + @classmethod + def from_json(cls, + json_input # type: Union[str, Dict[str, Any]] + ) -> SearchIndex: + """ Creates a `.SearchIndex` from a provided JSON str or Python dict derived from JSON. + + Args: + json_input (Union[str, Dict[str, Any]]): JSON representation of the search index. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the provided JSON is not str or dict. + :class:`~couchbase.exceptions.InvalidArgumentException`: If the provided JSON does not include an index name. + """ # noqa: E501 + json_data = json_input + if isinstance(json_input, str): + json_data = json.loads(json_input) + + if not isinstance(json_data, dict): + msg = 'Provided JSON input is either not a Python dict or a JSON str that produces a Python dict.' + raise InvalidArgumentException(msg) + + name = json_data.get('name', None) + if name is None: + raise InvalidArgumentException('Provided JSON input does not contain an index name.') + if not isinstance(name, str): + raise InvalidArgumentException('Index name must be a str.') + + name_tokens = name.split('.') + if len(name_tokens) > 1: + name = name_tokens[len(name_tokens)-1] + + source_type = json_data.get('sourceType', 'couchbase') + idx_type = json_data.get('type', 'fulltext-index') + source_name = json_data.get('sourceName', None) + # cannot pass in the UUID or sourceUUID + uuid = None + source_uuid = None + params = json_data.get('params', None) + if params is None: + params = {} + + source_params = json_data.get('sourceParams', None) + if source_params is None: + source_params = {} + plan_params = json_data.get('planParams', None) + if plan_params is None: + plan_params = {} + + return cls(name, + source_type, + idx_type, + source_name, + uuid, + params, + source_uuid, + source_params, + plan_params) diff --git a/couchbase/management/logic/users_logic.py b/couchbase/management/logic/users_logic.py new file mode 100644 index 000000000..d39c3728e --- /dev/null +++ b/couchbase/management/logic/users_logic.py @@ -0,0 +1,859 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from copy import deepcopy +from datetime import datetime +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Set, + Union) + +from couchbase.auth import AuthDomain +from couchbase.exceptions import (FeatureUnavailableException, + GroupNotFoundException, + InvalidArgumentException, + RateLimitedException, + UserNotFoundException) +from couchbase.options import forward_args +from couchbase.pycbc_core import (management_operation, + mgmt_operations, + user_mgmt_operations) + +if TYPE_CHECKING: + from couchbase.management.options import (ChangePasswordOptions, + DropGroupOptions, + DropUserOptions, + GetAllGroupsOptions, + GetAllUsersOptions, + GetGroupOptions, + GetRolesOptions, + GetUserOptions, + UpsertGroupOptions, + UpsertUserOptions) + + +class UserManagerLogic: + + _ERROR_MAPPING = {r'Unknown group.*': GroupNotFoundException, + r'Unknown user.*': UserNotFoundException, + r'Not found.*': FeatureUnavailableException, + r'Method Not Allowed.*': FeatureUnavailableException, + r'.*Limit\(s\) exceeded\s+\[.*\].*': RateLimitedException} + + def __init__(self, connection): + self._connection = connection + + def _get_valid_domain(self, auth_domain # type: Union[AuthDomain,str] + ) -> str: + if isinstance(auth_domain, str) and auth_domain in [ + "local", "external"]: + return auth_domain + elif isinstance(auth_domain, AuthDomain): + return AuthDomain.to_str(auth_domain) + else: + raise InvalidArgumentException(message="Unknown Authentication Domain") + + def get_user(self, + username, # type: str + *options, # type: GetUserOptions + **kwargs # type: Any + ) -> Optional[UserAndMetadata]: + + final_args = forward_args(kwargs, *options) + domain = final_args.pop("domain_name", "local") + + domain = self._get_valid_domain(domain) + + op_args = { + "domain": domain, + "username": username + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.GET_USER.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_users(self, + *options, # type: GetAllUsersOptions + **kwargs # type: Any + ) -> Optional[Iterable[UserAndMetadata]]: + final_args = forward_args(kwargs, *options) + domain = final_args.pop("domain_name", "local") + domain = self._get_valid_domain(domain) + + op_args = { + "domain": domain + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.GET_ALL_USERS.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def upsert_user(self, + user, # type: User + *options, # type: UpsertUserOptions + **kwargs # type: Any + ) -> None: + + final_args = forward_args(kwargs, *options) + domain = final_args.pop("domain_name", "local") + domain = self._get_valid_domain(domain) + + if not user.groups and (not user.roles or not isinstance(user.roles, set)): + raise InvalidArgumentException("Roles must be a non-empty list") + + user_dict = {k: v for k, v in user.as_dict().items() if k in { + "password", "name", "username", "groups"}} + + if user_dict["password"] and domain == "external": + raise InvalidArgumentException( + "External domains must not have passwords") + + if user.roles: + user_dict["roles"] = list(map(lambda r: r.as_dict(), user.roles)) + + op_args = { + "domain": domain, + "user": user_dict + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.UPSERT_USER.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_user(self, + username, # type: str + *options, # type: DropUserOptions + **kwargs # type: Any + ) -> None: + + final_args = forward_args(kwargs, *options) + domain = final_args.pop("domain_name", "local") + domain = self._get_valid_domain(domain) + + op_args = { + "domain": domain, + "username": username + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.DROP_USER.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def change_password(self, + new_password, # type: str + *options, # type: ChangePasswordOptions + **kwargs # type: Any + ) -> None: + + final_args = forward_args(kwargs, *options) + + op_args = { + "password": new_password + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.CHANGE_PASSWORD.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_roles(self, + *options, # type: GetRolesOptions + **kwargs # type: Any + ) -> Optional[Iterable[RoleAndDescription]]: + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.GET_ROLES.value + } + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_group(self, + group_name, # type: str + *options, # type: GetGroupOptions + **kwargs # type: Any + ) -> Optional[Group]: + + op_args = { + "name": group_name + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.GET_GROUP.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_groups(self, + *options, # type: GetAllGroupsOptions + **kwargs # type: Any + ) -> Optional[Iterable[Group]]: + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.GET_ALL_GROUPS.value + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def upsert_group(self, + group, # type: Group + *options, # type: UpsertGroupOptions + **kwargs # type: Any + ) -> None: + + group_dict = {k: v for k, v in group.as_dict().items() if k in { + "name", "description", "ldap_group_reference"}} + + if group.roles: + group_dict["roles"] = list(map(lambda r: r.as_dict(), group.roles)) + + op_args = { + "group": group_dict + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.UPSERT_GROUP.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_group(self, + group_name, # type: str + *options, # type: DropGroupOptions + **kwargs # type: Any + ) -> None: + + op_args = { + "name": group_name + } + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.USER.value, + "op_type": user_mgmt_operations.DROP_GROUP.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + final_args = forward_args(kwargs, *options) + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + +class UserManagementUtils(object): + """ + ** INTERNAL ** + """ + + @classmethod + def to_set(cls, value, valid_type, display_name): + + if not value: + return value + elif isinstance(value, set): + cls.validate_all_set_types(value, valid_type, display_name) + return value + elif isinstance(value, list): + cls.validate_all_set_types(value, valid_type, display_name) + return set(value) + elif isinstance(value, tuple): + cls.validate_all_set_types(value, valid_type, display_name) + return set(value) + elif isinstance(value, valid_type): + return set([value]) + else: + raise InvalidArgumentException( + '{} must be of type {}.'.format(display_name, + valid_type.__name__)) + + @classmethod + def validate_all_set_types(cls, value, valid_type, display_name): + + if all(map(lambda r: isinstance(r, type), value)): + raise InvalidArgumentException( + '{} must contain only objects of type {}.'.format(display_name, + valid_type.__name__)) + + +class Role: + + def __init__(self, + name=None, # type: str + bucket=None, # type: str + scope=None, # type: str + collection=None, # type: str + ): + + if not name: + raise InvalidArgumentException('A role must have a name') + + self._name = name + self._bucket = bucket + self._scope = scope + self._collection = collection + + @property + def name(self) -> str: + return self._name + + @property + def bucket(self) -> str: + return self._bucket + + @property + def scope(self) -> str: + return self._scope + + @property + def collection(self) -> str: + return self._collection + + def as_dict(self): + return { + 'name': self._name, + 'bucket': self._bucket, + 'scope': self._scope, + 'collection': self._collection + } + + def __eq__(self, other): + if not isinstance(other, Role): + return False + return (self.name == other.name + and self.bucket == other.bucket + and self.scope == other.scope + and self.collection == other.collection) + + def __hash__(self): + return hash((self.name, self.bucket, self.scope, self.collection)) + + @classmethod + def create_role(cls, raw_data): + return cls( + name=raw_data.get("name", None), + bucket=raw_data.get("bucket_name", None), + scope=raw_data.get("scope_name", None), + collection=raw_data.get("collection_name", None) + ) + + +class RoleAndDescription: + + def __init__(self, + role=None, # type: Role + display_name=None, # type: str + description=None, # type: str + ce=None, # type: bool + ): + + self._role = role + self._display_name = display_name + self._description = description + self._ce = ce + + @property + def role(self) -> Role: + return self._role + + @property + def display_name(self) -> str: + return self._display_name + + @property + def description(self) -> str: + return self._description + + @property + def ce(self) -> bool: + return self._ce + + @classmethod + def create_role_and_description(cls, raw_data): + return cls( + role=Role.create_role(raw_data), + display_name=raw_data.get('display_name', None), + description=raw_data.get('description', None), + ce=raw_data.get('ce', None) + ) + + +class Origin: + """ + Indicates why the user has a specific role. + If the type is "user" it means the role is assigned + directly to the user. If the type is "group" it means + the role is inherited from the group identified by + the "name" field. + """ + + def __init__(self, + type=None, # type: str + name=None # type: str + ): + + self._type = type + self._name = name + + @property + def type(self) -> str: + return self._type + + @property + def name(self) -> str: + return self._name + + +class RoleAndOrigins: + + def __init__(self, + role=None, # type: Role + origins=[] # type: List[Origin] + ): + + self._role = role + self._origins = origins + + @property + def role(self) -> Role: + return self._role + + @property + def origins(self) -> List[Origin]: + return self._origins + + @classmethod + def create_role_and_origins(cls, raw_data): + + # RBAC prior to v6.5 does not have origins + origin_data = raw_data.get("origins", None) + + return cls( + role=Role.create_role(raw_data.get("role")), + origins=list(map(lambda o: Origin(**o), origin_data)) + if origin_data else [] + ) + + +class User: + + def __init__(self, + username=None, # type: str + display_name=None, # type: str + groups=None, # type: Set[str] + roles=None, # type: Set[Role] + password=None # type: str + ): + + if not username: + raise InvalidArgumentException('A user must have a username') + + self._username = username + self._display_name = display_name + self._groups = UserManagementUtils.to_set(groups, str, 'Groups') + self._roles = UserManagementUtils.to_set(roles, Role, 'Roles') + self._password = password + + @property + def username(self) -> str: + return self._username + + @property + def display_name(self) -> str: + return self._display_name + + @display_name.setter + def display_name(self, + value # type: str + ): + self._display_name = value + + @property + def groups(self) -> Set[str]: + """names of the groups""" + return self._groups + + @groups.setter + def groups(self, + value # type: Set[str] + ): + self._groups = UserManagementUtils.to_set(value, str, 'Groups') + + @property + def roles(self) -> Set[Role]: + """only roles assigned directly to the user (not inherited from groups)""" + return self._roles + + @roles.setter + def roles(self, + value # type: Set[Role] + ): + self._roles = UserManagementUtils.to_set(value, Role, 'Roles') + + def password(self, value): + self._password = value + + password = property(None, password) + + def as_dict(self): + output = { + "username": self.username, + "name": self.display_name, + "password": self._password + } + + if self.roles: + output["roles"] = list(self.roles) + + if self.groups: + output["groups"] = list(self.groups) + + return output + + @classmethod + def create_user(cls, raw_data, roles=None): + + user_roles = roles + if not user_roles: + set(map(lambda r: Role.create_role(r), + raw_data.get("roles"))) + + # RBAC prior to v6.5 does not have groups + group_data = raw_data.get("groups", None) + + return cls( + username=raw_data.get("username"), + display_name=raw_data.get("display_name"), + roles=user_roles, + groups=set(group_data) if group_data else None + ) + + +class UserAndMetadata: + """ + Models the "get user" / "get all users" response. + + Associates the mutable properties of a user with + derived properties such as the effective roles + inherited from groups. + """ + + def __init__(self, + domain=None, # type: AuthDomain + user=None, # type: User + effective_roles=[], # type: List[RoleAndOrigins] + password_changed=None, # type: datetime + external_groups=None, # type: Set[str] + **kwargs # type: Dict[str, Any] + ): + + self._domain = domain + self._user = user + self._effective_roles = effective_roles + self._password_changed = password_changed + self._external_groups = external_groups + self._raw_data = kwargs.get("raw_data", None) + + @property + def domain(self) -> AuthDomain: + """ AuthDomain is an enumeration with values "local" and "external". + It MAY alternatively be represented as String.""" + return self._domain + + @property + def user(self) -> User: + """returns a new mutable User object each time this method is called. + Modifying the fields of the returned User MUST have no effect on the + UserAndMetadata object it came from.""" + return deepcopy(self._user) + + @property + def effective_roles(self) -> List[RoleAndOrigins]: + """all roles, regardless of origin.""" + return self._effective_roles + + @property + def password_changed(self) -> Optional[datetime]: + return self._password_changed + + @property + def external_groups(self) -> Set[str]: + return self._external_groups + + @property + def raw_data(self) -> Dict[str, Any]: + return self._raw_data + + @classmethod + def create_user_and_metadata(cls, raw_data): + + effective_roles = list(map(lambda r: RoleAndOrigins.create_role_and_origins(r), + raw_data.get("effective_roles"))) + + user_roles = set(r.role for r in effective_roles + if any(map(lambda o: o.type == "user", r.origins)) or len(r.origins) == 0) + + # RBAC prior to v6.5 does not have groups + ext_group_data = raw_data.get("external_groups", None) + + # password_change_date is optional + pw_data = raw_data.get("password_changed", None) + pw_changed = None + formats = ['%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ'] + for f in formats: + if pw_changed: + break + + try: + pw_changed = datetime.strptime(pw_data, f) + except Exception: # nosec + pass + + return cls( + domain=AuthDomain.from_str(raw_data.get("domain")), + effective_roles=effective_roles, + user=User.create_user(raw_data.get("user"), roles=user_roles), + password_changed=pw_changed, + external_groups=set(ext_group_data) if ext_group_data else None, + raw_data=raw_data + ) + + +class Group: + def __init__(self, + name=None, # type: str + description=None, # type: str + roles=None, # type: Set[Role] + ldap_group_reference=None, # type: str + **kwargs # type: Any + ): + + if not name: + raise InvalidArgumentException('A group must have a name') + + self._name = name + self._description = description + self._roles = UserManagementUtils.to_set(roles, Role, 'Roles') + self._ldap_group_reference = ldap_group_reference + self._raw_data = kwargs.get('raw_data', None) + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return self._description + + @description.setter + def description(self, + value # type: str + ): + self._description = value + + @property + def roles(self) -> Set[Role]: + return self._roles + + @roles.setter + def roles(self, + value # type: Set[Role] + ): + self._roles = UserManagementUtils.to_set(value, Role, 'Roles') + + @property + def ldap_group_reference(self) -> str: + return self._ldap_group_reference + + @ldap_group_reference.setter + def ldap_group_reference(self, + value # type: str + ): + self._ldap_group_reference = value + + @property + def raw_data(self) -> Dict[str, Any]: + return self._raw_data + + def as_dict(self): + rs = list(map(lambda r: r.as_dict(), self.roles)) + for r in self.roles: + r.as_dict() + return { + 'name': self.name, + 'description': self.description, + 'roles': rs, + 'ldap_group_reference': self.ldap_group_reference + } + + @classmethod + def create_group(cls, raw_data): + return cls( + raw_data.get('name'), + description=raw_data.get('description', None), + roles=set(map(lambda r: Role.create_role( + r), raw_data.get('roles'))), + ldap_group_referenc=raw_data.get('ldap_group_ref', None) + ) diff --git a/couchbase/management/logic/view_index_logic.py b/couchbase/management/logic/view_index_logic.py new file mode 100644 index 000000000..1974e9e7d --- /dev/null +++ b/couchbase/management/logic/view_index_logic.py @@ -0,0 +1,331 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from enum import Enum +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + Optional) + +from couchbase.exceptions import (DesignDocumentNotFoundException, + InvalidArgumentException, + RateLimitedException) +from couchbase.options import forward_args +from couchbase.pycbc_core import (management_operation, + mgmt_operations, + view_index_mgmt_operations) + +if TYPE_CHECKING: + from couchbase.management.options import (DropDesignDocumentOptions, + GetAllDesignDocumentsOptions, + GetDesignDocumentOptions, + UpsertDesignDocumentOptions) + + +class ViewIndexManagerLogic: + + _ERROR_MAPPING = {r'not_found': DesignDocumentNotFoundException, + r'.*Limit\(s\) exceeded\s+\[.*\].*': RateLimitedException} + + def __init__(self, connection, bucket_name): + self._connection = connection + self._bucket_name = bucket_name + + def get_design_document(self, + design_doc_name, # type: str + namespace, # type: DesignDocumentNamespace + *options, # type: GetDesignDocumentOptions + **kwargs + ) -> Optional[DesignDocument]: + if not design_doc_name: + raise InvalidArgumentException("Expected design document name to not be None") + + if not namespace: + raise InvalidArgumentException("Expected design document namespace to not be None") + + final_args = forward_args(kwargs, *options) + op_args = { + 'bucket_name': self._bucket_name, + 'document_name': design_doc_name, + 'namespace': namespace.to_str() + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.VIEW_INDEX.value, + "op_type": view_index_mgmt_operations.GET_INDEX.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def get_all_design_documents(self, + namespace, # type: DesignDocumentNamespace + *options, # type: GetAllDesignDocumentsOptions + **kwargs) -> Optional[Iterable[DesignDocument]]: + if not namespace: + raise InvalidArgumentException("Expected design document namespace to not be None") + + final_args = forward_args(kwargs, *options) + op_args = { + 'bucket_name': self._bucket_name, + 'namespace': namespace.to_str() + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.VIEW_INDEX.value, + "op_type": view_index_mgmt_operations.GET_ALL_INDEXES.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def upsert_design_document(self, + design_doc_data, # type: DesignDocument + namespace, # type: DesignDocumentNamespace + *options, # type: UpsertDesignDocumentOptions + **kwargs) -> None: + if not design_doc_data: + raise InvalidArgumentException("Expected design document to not be None") + + if not namespace: + raise InvalidArgumentException("Expected design document namespace to not be None") + + final_args = forward_args(kwargs, *options) + op_args = { + 'bucket_name': self._bucket_name, + 'design_docucment': design_doc_data.as_dict(namespace) + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.VIEW_INDEX.value, + "op_type": view_index_mgmt_operations.UPSERT_INDEX.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + def drop_design_document(self, + design_doc_name, # type: str + namespace, # type: DesignDocumentNamespace + *options, # type: DropDesignDocumentOptions + **kwargs) -> None: + if not design_doc_name: + raise InvalidArgumentException("Expected design document name to not be None") + + if not namespace: + raise InvalidArgumentException("Expected design document namespace to not be None") + + final_args = forward_args(kwargs, *options) + op_args = { + 'bucket_name': self._bucket_name, + 'document_name': design_doc_name, + 'namespace': namespace.to_str() + } + + if final_args.get("client_context_id", None) is not None: + op_args["client_context_id"] = final_args.get("client_context_id") + + mgmt_kwargs = { + "conn": self._connection, + "mgmt_op": mgmt_operations.VIEW_INDEX.value, + "op_type": view_index_mgmt_operations.DROP_INDEX.value, + "op_args": op_args + } + + callback = kwargs.pop('callback', None) + if callback: + mgmt_kwargs['callback'] = callback + + errback = kwargs.pop('errback', None) + if errback: + mgmt_kwargs['errback'] = errback + + if final_args.get("timeout", None) is not None: + mgmt_kwargs["timeout"] = final_args.get("timeout") + + return management_operation(**mgmt_kwargs) + + +class DesignDocumentNamespace(Enum): + PRODUCTION = False + DEVELOPMENT = True + + # def prefix(self, ddocname): + # if ddocname.startswith('dev_') or not self.value: + # return ddocname + # return 'dev_' + ddocname + + def to_str(self): + return 'development' if self.value else 'production' + + @classmethod + def unprefix(cls, name): + for prefix in ('_design/', 'dev_'): + name = name[name.startswith(prefix) and len(prefix):] + return name + + @classmethod + def from_str(cls, value): + if value == 'production': + return cls.PRODUCTION + else: + return cls.DEVELOPMENT + + +class View: + def __init__(self, + map, # type: str + reduce=None # type: Optional[str] + ) -> None: + self._map = map + self._reduce = reduce + + @property + def map(self) -> str: + return self._map + + @property + def reduce(self) -> Optional[str]: + return self._reduce + + def as_dict(self) -> Dict[str, Any]: + return {k: v for k, v in {"map": self._map, + "reduce": self._reduce}.items() if v} + + def to_json(self) -> str: + return json.dumps(self.as_dict()) + + @classmethod + def from_json(cls, json_view) -> View: + return cls(json.loads(json_view)) + + +class DesignDocument(object): + def __init__(self, + name, # type: str + views, # Dict[str, View] + namespace=None, # type: Optional[DesignDocumentNamespace] + rev=None # type: Optional[str] + ) -> None: + self._name = DesignDocumentNamespace.unprefix(name) + self._views = views + self._rev = rev + self._namespace = namespace + + @property + def name(self) -> str: + return self._name + + @property + def views(self) -> Dict[str, View]: + return self._views + + @property + def rev(self) -> Optional[str]: + return self._rev + + @property + def namespace(self) -> Optional[DesignDocumentNamespace]: + return self._namespace + + def as_dict(self, + namespace # type: DesignDocumentNamespace + ) -> Dict[str, Any]: + output = { + 'name': self._name + } + if namespace is not None: + output['namespace'] = namespace.to_str() + output['views'] = dict({key: value.as_dict() for key, value in self.views.items()}) + + if self.rev: + output['rev'] = self.rev + + return output + + def add_view(self, + name, # type: str + view # type: View + ) -> DesignDocument: + self.views[name] = view + return self + + def get_view(self, + name # type: str + ) -> View: + return self._views.get(name, None) + + @classmethod + def from_json(cls, raw_json # type: Dict[str, Any] + ) -> DesignDocument: + name = raw_json.get('name') + rev = raw_json.get('rev', None) + ns = DesignDocumentNamespace.from_str(raw_json.get('namespace', None)) + views = raw_json.get('views', dict()) + views = dict({key: View(**value) for key, value in views.items()}) + return cls(name, views, namespace=ns, rev=rev) + + def __repr__(self): + output = self.as_dict(self.namespace) + return f'DesignDocument({output})' diff --git a/couchbase/management/logic/wrappers.py b/couchbase/management/logic/wrappers.py new file mode 100644 index 000000000..8b4b9d49a --- /dev/null +++ b/couchbase/management/logic/wrappers.py @@ -0,0 +1,508 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from datetime import timedelta +from enum import Enum +from functools import wraps + +from couchbase._utils import Overload, OverloadType +from couchbase.exceptions import (PYCBC_ERROR_MAP, + CouchbaseException, + ErrorMapper, + ExceptionMap, + HTTPException) +from couchbase.exceptions import exception as BaseCouchbaseException +from couchbase.logic.supportability import Supportability + + +class ManagementType(Enum): + BucketMgmt = 'bucket_mgmt' + CollectionMgmt = 'collection_mgmt' + UserMgmt = 'user_mgmt' + QueryIndexMgmt = 'query_index_mgmt' + SearchIndexMgmt = 'search_index_mgmt' + AnalyticsIndexMgmt = 'analytics_index_mgmt' + ViewIndexMgmt = 'view_index_mgmt' + EventingFunctionMgmt = 'eventing_function_mgmt' + + +""" + +Bucket mgmt helpers for parsing returned results + +""" + + +def get_bucket_settings(res, return_cls): + raw_settings = res.raw_result.get('bucket_settings', None) + bucket_settings = None + if raw_settings: + bucket_settings = return_cls.transform_from_dest(raw_settings) + + return bucket_settings + + +def get_all_bucket_settings(res, return_cls): + raw_buckets = res.raw_result.get('buckets', None) + buckets = [] + if raw_buckets: + for b in raw_buckets: + bucket_settings = return_cls.transform_from_dest(b) + buckets.append(bucket_settings) + + return buckets + + +def get_bucket_describe_result(res, return_cls): + bucket_info = res.raw_result.get('bucket_info', None) + if bucket_info: + return return_cls(**bucket_info) + return None + + +""" + +Collection mgmt helpers for parsing returned results + +""" + + +def get_all_scopes(res, return_cls): + scopes = [] + raw_scopes = res.raw_result.get('scopes', None) + # TODO: better exception? + if raw_scopes: + for s in raw_scopes: + scope = return_cls[0](s['name'], list()) + for c in s['collections']: + scope.collections.append( + return_cls[1](c['name'], + c['scope_name'], + timedelta(seconds=c['max_expiry']), + history=c.get('history'))) + scopes.append(scope) + + return scopes + + +""" + +User mgmt helpers for parsing returned results + +""" + + +def get_user(res, return_cls): + raw_user = res.raw_result.get('user_and_metadata', None) + user = None + if raw_user: + user = return_cls.create_user_and_metadata(raw_user) + + return user + + +def get_all_users(res, return_cls): + users = [] + raw_users = res.raw_result.get('users', None) + if raw_users: + for u in raw_users: + user = return_cls.create_user_and_metadata(u) + users.append(user) + + return users + + +def get_roles(res, return_cls): + roles = [] + raw_roles = res.raw_result.get('roles', None) + if raw_roles: + for r in raw_roles: + role = return_cls.create_role_and_description(r) + roles.append(role) + + return roles + + +def get_group(res, return_cls): + raw_group = res.raw_result.get('group', None) + group = None + if raw_group: + group = return_cls.create_group(raw_group) + + return group + + +def get_all_groups(res, return_cls): + groups = [] + raw_groups = res.raw_result.get('groups', None) + if raw_groups: + for g in raw_groups: + group = return_cls.create_group(g) + groups.append(group) + + return groups + + +""" + +Analytics mgmt helpers for parsing returned results + +""" + + +def get_all_datasets(res, return_cls): + datasets = [] + raw_datasets = res.raw_result.get('datasets', None) + if raw_datasets: + datasets = [return_cls(**ds) for ds in raw_datasets] + + return datasets + + +def get_all_analytics_indexes(res, return_cls): + indexes = [] + raw_indexes = res.raw_result.get('indexes', None) + if raw_indexes: + indexes = [return_cls(**ds) for ds in raw_indexes] + + return indexes + + +def get_links(res, return_cls): + analytics_links = [] + cb_links = res.raw_result.get('couchbase_links', None) + if cb_links and len(cb_links) > 0: + analytics_links.extend(map(lambda l: return_cls[0].link_from_server_json(l), cb_links)) + s3_links = res.raw_result.get('s3_links', None) + if s3_links and len(s3_links) > 0: + analytics_links.extend(map(lambda l: return_cls[1].link_from_server_json(l), s3_links)) + azure_blob_links = res.raw_result.get('azure_blob_links', None) + if azure_blob_links and len(azure_blob_links) > 0: + analytics_links.extend( + map(lambda l: return_cls[2].link_from_server_json(l), azure_blob_links)) + + return analytics_links + + +def get_all_query_indexes(res, return_cls): + indexes = [] + raw_indexes = res.raw_result.get('indexes', None) + if raw_indexes: + indexes = [return_cls.from_server(idx) for idx in raw_indexes] + + return indexes + + +""" + +Search index mgmt helpers for parsing returned results + +""" + + +def get_search_index(res, return_cls): + raw_index = res.raw_result.get('index', None) + index = None + if raw_index: + index = return_cls.from_server(raw_index) + + return index + + +def get_all_search_indexes(res, return_cls): + indexes = [] + raw_indexes = res.raw_result.get('indexes', None) + if raw_indexes: + indexes = [return_cls.from_server(idx) for idx in raw_indexes] + + return indexes + + +def analyze_search_index_document(res): + output = {} + analysis = res.raw_result.get('analysis', None) + if analysis: + output['analysis'] = json.loads(analysis) + status = res.raw_result.get('status', None) + if status: + output['status'] = status + + return output + + +def get_search_index_stats(res): + raw_stats = res.raw_result.get('stats', None) + stats = None + if raw_stats: + stats = json.loads(raw_stats) + + return stats + + +def get_all_search_index_stats(res): + raw_stats = res.raw_result.get('stats', None) + stats = None + if raw_stats: + stats = json.loads(raw_stats) + + return stats + + +""" + +View index mgmt helpers for parsing returned results + +""" + + +def get_design_document(res, return_cls): + raw_ddoc = res.raw_result.get('design_document', None) + ddoc = None + if raw_ddoc: + ddoc = return_cls.from_json(raw_ddoc) + + return ddoc + + +def get_all_design_documents(res, return_cls): + ddocs = [] + raw_ddocs = res.raw_result.get('design_documents', None) + if raw_ddocs: + ddocs = [return_cls.from_json(ddoc) for ddoc in raw_ddocs] + + return ddocs + + +""" + +Eventing mgmt helpers for parsing returned results + +""" + + +def get_eventing_function(res, return_cls): + raw_func = res.raw_result.get('function', None) + func = None + if raw_func: + func = return_cls.from_server(raw_func) + + return func + + +def get_all_eventing_functions(res, return_cls): + functions = [] + raw_functions = res.raw_result.get('functions', None) + if raw_functions: + functions = [return_cls.from_server(f) for f in raw_functions] + + return functions + + +def get_eventing_functions_status(res, return_cls): + raw_status = res.raw_result.get('status', None) + status = None + if raw_status: + status = return_cls.from_server(raw_status) + + return status + + +""" + +""" + + +def handle_mgmt_exception(exc, mgmt_type, error_map): + raise ErrorMapper.build_exception(exc, mapping=error_map) + + +def handle_bucket_mgmt_response(ret, fn_name, return_cls): + if fn_name == 'get_bucket': + retval = get_bucket_settings(ret, return_cls) + elif fn_name == 'get_all_buckets': + retval = get_all_bucket_settings(ret, return_cls) + elif fn_name == 'bucket_describe': + retval = get_bucket_describe_result(ret, return_cls) + else: + retval = return_cls(ret) + + return retval + + +def handle_collection_mgmt_response(ret, fn_name, return_cls): + if fn_name == 'get_all_scopes': + retval = get_all_scopes(ret, return_cls) + else: + retval = return_cls(ret) + + return retval + + +def handle_user_mgmt_response(ret, fn_name, return_cls): + if fn_name == 'get_user': + retval = get_user(ret, return_cls) + elif fn_name == 'get_all_users': + retval = get_all_users(ret, return_cls) + elif fn_name == 'get_roles': + retval = get_roles(ret, return_cls) + elif fn_name == 'get_group': + retval = get_group(ret, return_cls) + elif fn_name == 'get_all_groups': + retval = get_all_groups(ret, return_cls) + else: + retval = return_cls(ret) + + return retval + + +def handle_query_index_mgmt_response(ret, fn_name, return_cls): + if fn_name == 'get_all_indexes': + retval = get_all_query_indexes(ret, return_cls) + else: + retval = return_cls(ret) + + return retval + + +def handle_analytics_index_mgmt_response(ret, fn_name, return_cls): + if fn_name == 'get_all_datasets': + retval = get_all_datasets(ret, return_cls) + elif fn_name == 'get_all_indexes': + retval = get_all_analytics_indexes(ret, return_cls) + elif fn_name == 'get_pending_mutations': + retval = ret.raw_result.get('stats', None) + elif fn_name == 'get_links': + retval = get_links(ret, return_cls) + else: + retval = return_cls(ret) + + return retval + + +def handle_search_index_mgmt_response(ret, fn_name, return_cls): + if fn_name == 'get_index': + retval = get_search_index(ret, return_cls) + elif fn_name == 'get_all_indexes': + retval = get_all_search_indexes(ret, return_cls) + elif fn_name == 'get_indexed_documents_count': + retval = ret.raw_result.get('count', 0) + elif fn_name == 'analyze_document': + retval = analyze_search_index_document(ret) + elif fn_name == 'get_index_stats': + retval = get_search_index_stats(ret) + elif fn_name == 'get_all_index_stats': + retval = get_all_search_index_stats(ret) + else: + retval = return_cls(ret) + + return retval + + +def handle_view_index_mgmt_response(ret, fn_name, return_cls): + if fn_name == 'get_design_document': + retval = get_design_document(ret, return_cls) + elif fn_name == 'get_all_design_documents': + retval = get_all_design_documents(ret, return_cls) + else: + retval = return_cls(ret) + + return retval + + +def handle_eventing_function_mgmt_response(ret, fn_name, return_cls): + if fn_name == 'get_function': + retval = get_eventing_function(ret, return_cls) + elif fn_name == 'get_all_functions': + retval = get_all_eventing_functions(ret, return_cls) + elif fn_name == 'functions_status': + retval = get_eventing_functions_status(ret, return_cls) + else: + retval = return_cls(ret) + + return retval + + +mgmt_overload_registry = {} + + +class BlockingMgmtWrapper: + + @classmethod # noqa: C901 + def block(cls, return_cls, mgmt_type, error_map, overload_type=None): # noqa: C901 + def decorator(fn): + if overload_type is not None: + mgmt_overload = mgmt_overload_registry.get(fn.__qualname__) + if mgmt_overload is None: + mgmt_overload = mgmt_overload_registry[fn.__qualname__] = Overload(fn.__qualname__) + if overload_type is OverloadType.DEFAULT: + mgmt_overload.register_default(fn) + else: + mgmt_overload.register(fn) + + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + try: + func = mgmt_overload_registry.get(fn.__qualname__, fn) + # work-around for PYCBC-1375, I doubt users are calling the index mgmt method + # using fields=[...], but in the event they do (as we do in the tests) this corrects + # the kwarg name. + if ('QueryIndexManager' in fn.__qualname__ + and fn.__qualname__.endswith('create_index') + and 'fields' in kwargs): + kwargs['keys'] = kwargs.pop('fields') + Supportability.method_kwarg_deprecated('fields', 'keys') + ret = func(self, *args, **kwargs) + if isinstance(ret, BaseCouchbaseException): + handle_mgmt_exception(ret, mgmt_type, error_map) + if return_cls is None: + return None + elif return_cls is True: + retval = ret + else: + if mgmt_type == ManagementType.BucketMgmt: + retval = handle_bucket_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.CollectionMgmt: + retval = handle_collection_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.UserMgmt: + retval = handle_user_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.QueryIndexMgmt: + retval = handle_query_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.AnalyticsIndexMgmt: + retval = handle_analytics_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.SearchIndexMgmt: + retval = handle_search_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.ViewIndexMgmt: + retval = handle_view_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.EventingFunctionMgmt: + retval = handle_eventing_function_mgmt_response(ret, fn.__name__, return_cls) + else: + retval = None + return retval + except HTTPException as e: + raise e + except CouchbaseException as e: + raise e + except Exception as ex: + if isinstance(ex, (TypeError, ValueError)): + raise ex + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(message=str(ex)) + raise excptn + + return wrapped_fn + return decorator diff --git a/couchbase/management/options.py b/couchbase/management/options.py new file mode 100644 index 000000000..01c590638 --- /dev/null +++ b/couchbase/management/options.py @@ -0,0 +1,1385 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Optional, + overload) + +from couchbase.management.logic.analytics_logic import AnalyticsLinkType + +if TYPE_CHECKING: + from datetime import timedelta + + +class CreateBucketOptions(dict): + """Available options for a :class:`~couchbase.management.buckets.BucketManager`'s create bucket operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class UpdateBucketOptions(dict): + """Available options to for a :class:`~couchbase.management.buckets.BucketManager`'s update bucket operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class DropBucketOptions(dict): + """Available options to for a :class:`~couchbase.management.buckets.BucketManager`'s update bucket operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class GetAllBucketOptions(dict): + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class GetBucketOptions(dict): + """Available options to for a :class:`~couchbase.management.buckets.BucketManager`'s get bucket operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class FlushBucketOptions(dict): + """Available options to for a :class:`~couchbase.management.buckets.BucketManager`'s flush bucket operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class BucketDescribeOptions(dict): + """Available options to for a :class:`~couchbase.management.buckets.BucketManager`'s bucket describe operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + +# Collection Management API + + +class GetAllScopesOptions(dict): + """Available options for a :class:`~couchbase.management.collections.CollectionManager`'s get all + scopes operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: timedelta + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class CreateCollectionOptions(dict): + """Available options for a :class:`~couchbase.management.collections.CollectionManager`'s create collection + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: timedelta + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class DropCollectionOptions(dict): + """Available options for a :class:`~couchbase.management.collections.CollectionManager`'s drop collection + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: timedelta + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class CreateScopeOptions(dict): + """Available options for a :class:`~couchbase.management.collections.CollectionManager`'s create scope + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: timedelta + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class DropScopeOptions(dict): + """Available options for a :class:`~couchbase.management.collections.CollectionManager`'s drop scope + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: timedelta + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class UpdateCollectionOptions(dict): + """Available options for a :class:`~couchbase.management.collections.CollectionManager`'s update collection + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: timedelta + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +# User Management API + + +class UserOptions(dict): + """ + Base class User Management API options + """ + + def __init__(self, + domain_name="local", # type: Optional[str] + timeout=None # type: Optional[timedelta] + ): + + kwargs = {"domain_name": domain_name} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class GetUserOptions(UserOptions): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s get user + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + domain_name (str, optional): The user's domain name (either ``local`` or ``external``). Defaults + to ``local``. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAllUsersOptions(UserOptions): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s get all users + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + domain_name (str, optional): The user's domain name (either ``local`` or ``external``). Defaults + to ``local``. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + +class UpsertUserOptions(UserOptions): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s upsert user + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + domain_name (str, optional): The user's domain name (either ``local`` or ``external``). Defaults + to ``local``. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + +class ChangePasswordOptions(UserOptions): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s change password user + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + domain_name (str, optional): The user's domain name (either ``local`` or ``external``). Defaults + to ``local``. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + +class DropUserOptions(UserOptions): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s drop user + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + domain_name (str, optional): The user's domain name (either ``local`` or ``external``). Defaults + to ``local``. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + +class GetRolesOptions(dict): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s get roles + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class DropGroupOptions(dict): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s drop group + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class GetGroupOptions(dict): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s get group + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class GetAllGroupsOptions(dict): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s get all + groups operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +class UpsertGroupOptions(dict): + """Available options to for a :class:`~couchbase.management.users.UserManager`'s upsert group + operation. + + .. note:: + All management options should be imported from ``couchbase.management.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ + + def __init__( + self, + timeout=None # type: Optional[timedelta] + ): + kwargs = {} + if timeout: + kwargs["timeout"] = timeout + + super().__init__(**kwargs) + + +# Query Index Management + +class GetAllQueryIndexOptions(dict): + """Available options to for a :class:`~couchbase.management.queries.QueryIndexManager`'s get + all indexes operation. + + .. note:: + All management options should be imported from `couchbase.management.options`. + + Args: + collection_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the collection of the indexes. + scope_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the scope of the indexes. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ # noqa: E501 + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + scope_name=None, # type: Optional[str] + collection_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class CreateQueryIndexOptions(dict): + """Available options to for a :class:`~couchbase.management.queries.QueryIndexManager`'s create + index operation. + + .. note:: + All management options should be imported from `couchbase.management.options`. + + Args: + collection_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the collection of the index. + condition (str, optional): Specifies the 'where' condition for partial index creation. + deferred (bool, optional): Specifies whether this index creation should be deferred until + a later point in time when they can be explicitly built together. + ignore_if_exists (bool, optional): Whether or not the call should ignore the + index already existing when determining whether the call was successful. + num_replicas (int, optional): The number of replicas of this index that should be created. + scope_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the scope of the index. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ # noqa: E501 + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + ignore_if_exists=None, # type: Optional[bool] + num_replicas=None, # type: Optional[int] + deferred=None, # type: Optional[bool] + condition=None, # type: Optional[str] + scope_name=None, # type: Optional[str] + collection_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + if 'ignore_if_exists' not in kwargs: + kwargs['ignore_if_exists'] = False + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class CreatePrimaryQueryIndexOptions(dict): + """Available options to for a :class:`~couchbase.management.queries.QueryIndexManager`'s create + primary index operation. + + .. note:: + All management options should be imported from `couchbase.management.options`. + + Args: + collection_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the collection of the index. + condition (str, optional): Specifies the 'where' condition for partial index creation. + deferred (bool, optional): Specifies whether this index creation should be deferred until + a later point in time when they can be explicitly built together. + ignore_if_exists (bool, optional): Whether or not the call should ignore the + index already existing when determining whether the call was successful. + index_name (str, optional): Specifies the name of the primary index. + num_replicas (int, optional): The number of replicas of this index that should be created. + scope_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the scope of the index. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ # noqa: E501 + @overload + def __init__(self, + index_name=None, # type: Optional[str] + timeout=None, # type: Optional[timedelta] + ignore_if_exists=None, # type: Optional[bool] + num_replicas=None, # type: Optional[int] + deferred=None, # type: Optional[bool] + condition=None, # type: Optional[str] + scope_name=None, # type: Optional[str] + collection_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + if 'ignore_if_exists' not in kwargs: + kwargs['ignore_if_exists'] = False + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DropQueryIndexOptions(dict): + """Available options to for a :class:`~couchbase.management.queries.QueryIndexManager`'s drop + index operation. + + .. note:: + All management options should be imported from `couchbase.management.options`. + + Args: + collection_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the collection of the index. + ignore_if_not_exists (bool, optional): Whether or not the call should ignore the + index not existing when determining whether the call was successful. + scope_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the scope of the index. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ # noqa: E501 + @overload + def __init__(self, + ignore_if_not_exists=None, # type: Optional[bool] + timeout=None, # type: Optional[timedelta] + scope_name=None, # type: Optional[str] + collection_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super(DropQueryIndexOptions, self).__init__(**kwargs) + + +class DropPrimaryQueryIndexOptions(dict): + """Available options to for a :class:`~couchbase.management.queries.QueryIndexManager`'s drop + primary index operation. + + .. note:: + All management options should be imported from `couchbase.management.options`. + + Args: + collection_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the collection of the index. + ignore_if_not_exists (bool, optional): Whether or not the call should ignore the + index not existing when determining whether the call was successful. + index_name (str, optional): Specifies the name of the primary index. + scope_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the scope of the index. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ # noqa: E501 + @overload + def __init__(self, + index_name=None, # type: Optional[str] + ignore_if_not_exists=None, # type: Optional[bool] + timeout=None, # type: Optional[timedelta] + scope_name=None, # type: Optional[str] + collection_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super(DropPrimaryQueryIndexOptions, self).__init__(**kwargs) + + +class WatchQueryIndexOptions(dict): + """Available options to for a :class:`~couchbase.management.queries.QueryIndexManager`'s watch + indexes operation. + + .. note:: + All management options should be imported from `couchbase.management.options`. + + Args: + collection_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the collection of the index. + scope_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the scope of the index. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + watch_primary (bool, optional): Specifies whether the primary indexes should + be watched as well. + """ # noqa: E501 + @overload + def __init__(self, + watch_primary=None, # type: Optional[bool] + timeout=None, # type: Optional[timedelta] + scope_name=None, # type: Optional[str] + collection_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class BuildDeferredQueryIndexOptions(dict): + """Available options to for a :class:`~couchbase.management.queries.QueryIndexManager`'s build + deferred indexes operation. + + .. note:: + All management options should be imported from `couchbase.management.options`. + + Args: + collection_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the collection of the index. + scope_name (str, optional): ** DEPRECATED ** - use `~couchbase.management.queries.CollectionQueryIndexManager`. + Specifies the scope of the index. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + management operation timeout. + """ # noqa: E501 + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + scope_name=None, # type: Optional[str] + collection_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super(BuildDeferredQueryIndexOptions, self).__init__(**kwargs) + +# Analytics Management Options + + +class CreateDataverseOptions(dict): + @overload + def __init__(self, + timeout=None, # type: timedelta + ignore_if_exists=False # type: bool + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @property + def ignore_if_exists(self): + return self.get('ignore_if_exists', False) + + +class DropDataverseOptions(dict): + @overload + def __init__(self, + timeout=None, # type: timedelta + ignore_if_not_exists=False # type: bool + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @property + def ignore_if_not_exists(self): + return self.get('ignore_if_not_exists', False) + + +class CreateDatasetOptions(dict): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + ignore_if_exists=None, # type: Optional[bool] + condition=None, # type: Optional[str] + dataverse_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @property + def ignore_if_exists(self) -> bool: + return self.get('ignore_if_exists', False) + + @property + def condition(self) -> Optional[str]: + return self.get('condition', None) + + @property + def dataverse_name(self) -> str: + return self.get('dataverse_name', 'Default') + + +class DropDatasetOptions(dict): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + ignore_if_not_exists=None, # type: Optional[bool] + dataverse_name=None, # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @property + def ignore_if_not_exists(self) -> bool: + return self.get('ignore_if_not_exists', False) + + @property + def dataverse_name(self) -> str: + return self.get('dataverse_name', 'Default') + + +class GetAllDatasetOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class CreateAnalyticsIndexOptions(dict): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + ignore_if_exists=None, # type: Optional[bool] + dataverse_name=None, # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @property + def ignore_if_exists(self) -> bool: + return self.get('ignore_if_exists', False) + + @property + def dataverse_name(self) -> str: + return self.get('dataverse_name', 'Default') + + +class DropAnalyticsIndexOptions(dict): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + ignore_if_not_exists=None, # type: Optional[bool] + dataverse_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @property + def dataverse_name(self) -> str: + return self.get('dataverse_name', 'Default') + + @property + def ignore_if_not_exists(self) -> bool: + return self.get('ignore_if_not_exists', False) + + +class GetAllAnalyticsIndexesOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class ConnectLinkOptions(dict): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + dataverse_name=None, # type: Optional[str] + link_name=None, # type: Optional[str] + force=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + super(ConnectLinkOptions, self).__init__(**kwargs) + + @property + def dataverse_name(self) -> str: + return self.get('dataverse_name', 'Default') + + @property + def link_name(self) -> str: + return self.get('link_name', 'Local') + + @property + def force(self) -> bool: + return self.get('force', False) + + +class DisconnectLinkOptions(dict): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + dataverse_name=None, # type: Optional[str] + link_name=None # type: Optional[str] + ): + pass + + def __init__(self, **kwargs): + super(DisconnectLinkOptions, self).__init__(**kwargs) + + @property + def dataverse_name(self) -> str: + return self.get('dataverse_name', 'Default') + + @property + def link_name(self) -> str: + return self.get('link_name', 'Local') + + +class GetPendingMutationsOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class CreateLinkAnalyticsOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class ReplaceLinkAnalyticsOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DropLinkAnalyticsOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetLinksAnalyticsOptions(dict): + @overload + def __init__(self, + timeout=None, # type: Optional[timedelta] + dataverse_name=None, # type: Optional[str] + name=None, # type: Optional[str] + link_type=None, # type: Optional[AnalyticsLinkType] + ): + pass + + def __init__(self, **kwargs): + super(GetLinksAnalyticsOptions, self).__init__(**kwargs) + + @property + def dataverse_name(self) -> str: + return self.get('dataverse_name', 'Default') + + @property + def name(self) -> Optional[str]: + return self.get('name', None) + + @property + def link_type(self) -> Optional[AnalyticsLinkType]: + return self.get('link_type', None) + +# Search Index Management Options + + +class UpsertSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DropSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAllSearchIndexesOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetSearchIndexedDocumentsCountOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class PauseIngestSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class ResumeIngestSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class AllowQueryingSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DisallowQueryingSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class FreezePlanSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class UnfreezePlanSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class AnalyzeDocumentSearchIndexOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetSearchIndexStatsOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAllSearchIndexStatsOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + +# Views + + +class GetDesignDocumentOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAllDesignDocumentsOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class UpsertDesignDocumentOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DropDesignDocumentOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class PublishDesignDocumentOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +# Eventing Function Management API + +class EventingFunctionOptions(dict): + def __init__(self, **kwargs): + """ + EventingFunctionOptions + Various options for eventing function management API + + :param timeout: + Uses this timeout value, rather than the default for the cluster. + :type timeout: timedelta + """ + super(EventingFunctionOptions, self).__init__(**kwargs) + + +class UpsertFunctionOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DropFunctionOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class DeployFunctionOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetAllFunctionOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class GetFunctionOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class PauseFunctionOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class ResumeFunctionOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class UndeployFunctionOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class FunctionsStatusOptions(dict): + @overload + def __init__(self, + timeout=None # type: Optional[timedelta] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) diff --git a/couchbase/management/queries.py b/couchbase/management/queries.py new file mode 100644 index 000000000..cad9b8b10 --- /dev/null +++ b/couchbase/management/queries.py @@ -0,0 +1,571 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from time import perf_counter, sleep +from typing import (Any, + Dict, + Iterable) + +from couchbase.exceptions import (AmbiguousTimeoutException, + InvalidArgumentException, + QueryIndexNotFoundException, + WatchQueryIndexTimeoutException) +from couchbase.management.logic.query_index_logic import QueryIndex, QueryIndexManagerLogic +from couchbase.management.logic.wrappers import BlockingMgmtWrapper, ManagementType + +# @TODO: lets deprecate import of options from couchbase.management.queries +from couchbase.management.options import (BuildDeferredQueryIndexOptions, + CreatePrimaryQueryIndexOptions, + CreateQueryIndexOptions, + DropPrimaryQueryIndexOptions, + DropQueryIndexOptions, + GetAllQueryIndexOptions, + WatchQueryIndexOptions) +from couchbase.options import forward_args + + +class QueryIndexManager(QueryIndexManagerLogic): + def __init__(self, connection): + super().__init__(connection) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def create_index(self, + bucket_name, # type: str + index_name, # type: str + keys, # type: Iterable[str] + *options, # type: CreateQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Creates a new query index. + + Args: + bucket_name (str): The name of the bucket this index is for. + index_name (str): The name of the index. + keys (Iterable[str]): The keys which this index should cover. + options (:class:`~couchbase.management.options.CreateQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name, index_name or keys + are invalid types. + :class:`~couchbase.exceptions.QueryIndexAlreadyExistsException`: If the index already exists. + """ + + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when creating a secondary index.') + if not isinstance(index_name, str): + raise InvalidArgumentException('The index_name must be provided when creating a secondary index.') + if not isinstance(keys, (list, tuple)): + raise InvalidArgumentException('Index keys must be provided when creating a secondary index.') + + return super().create_index(bucket_name, index_name, keys, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def create_primary_index(self, + bucket_name, # type: str + *options, # type: CreatePrimaryQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Creates a new primary query index. + + Args: + bucket_name (str): The name of the bucket this index is for. + options (:class:`~couchbase.management.options.CreatePrimaryQueryIndexOptions`): Optional parameters for + this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name is an invalid type. + :class:`~couchbase.exceptions.QueryIndexAlreadyExistsException`: If the index already exists. + """ + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when creating a primary index.') + + return super().create_primary_index(bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + bucket_name, # type: str + index_name, # type: str + *options, # type: DropQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Drops an existing query index. + + Args: + bucket_name (str): The name of the bucket containing the index to drop. + index_name (str): The name of the index to drop. + options (:class:`~couchbase.management.options.DropQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name or index_name are + invalid types. + :class:`~couchbase.exceptions.QueryIndexNotFoundException`: If the index does not exists. + """ + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when dropping a secondary index.') + if not isinstance(index_name, str): + raise InvalidArgumentException('The index_name must be provided when dropping a secondary index.') + + return super().drop_index(bucket_name, index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def drop_primary_index(self, + bucket_name, # type: str + *options, # type: DropPrimaryQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Drops an existing primary query index. + + Args: + bucket_name (str): The name of the bucket this index to drop. + options (:class:`~couchbase.management.options.DropPrimaryQueryIndexOptions`): Optional parameters for + this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name is an invalid type. + :class:`~couchbase.exceptions.QueryIndexNotFoundException`: If the index does not exists. + """ + + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when dropping a primary index.') + + return super().drop_primary_index(bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(QueryIndex, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + bucket_name, # type: str + *options, # type: GetAllQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> Iterable[QueryIndex]: + """Returns a list of indexes for a specific bucket. + + Args: + bucket_name (str): The name of the bucket to fetch indexes for. + options (:class:`~couchbase.management.options.GetAllQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Iterable[:class:`.QueryIndex`]: A list of indexes for a specific bucket. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name is an invalid type. + """ + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when retrieving all indexes.') + + return super().get_all_indexes(bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def build_deferred_indexes(self, + bucket_name, # type: str + *options, # type: BuildDeferredQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Starts building any indexes which were previously created with ``deferred=True``. + + Args: + bucket_name (str): The name of the bucket to perform build on. + options (:class:`~couchbase.management.options.BuildDeferredQueryIndexOptions`): Optional parameters + for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name is an invalid type. + """ + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when building deferred indexes.') + + return super().build_deferred_indexes(bucket_name, *options, **kwargs) + + def watch_indexes(self, # noqa: C901 + bucket_name, # type: str + index_names, # type: Iterable[str] + *options, # type: WatchQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> None: + """Waits for a number of indexes to finish creation and be ready to use. + + Args: + bucket_name (str): The name of the bucket to watch for indexes on. + index_names (Iterable[str]): The names of the indexes to watch. + options (:class:`~couchbase.management.options.WatchQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the bucket_name or index_names are + invalid types. + :class:`~couchbase.exceptions.WatchQueryIndexTimeoutException`: If the specified timeout is reached + before all the specified indexes are ready to use. + """ + + if not isinstance(bucket_name, str): + raise InvalidArgumentException('The bucket_name must be provided when watching indexes.') + if not isinstance(index_names, (list, tuple)): + raise InvalidArgumentException('One or more index_names must be provided when watching indexes.') + + final_args = forward_args(kwargs, *options) + + scope_name = final_args.get('scope_name', None) + collection_name = final_args.get('collection_name', None) + + if final_args.get('watch_primary', False): + index_names.append('#primary') + + timeout = final_args.get("timeout", None) + if not timeout: + raise InvalidArgumentException('Must specify a timeout condition for watch indexes') + + def check_indexes(index_names, indexes): + for idx_name in index_names: + match = next((i for i in indexes if i.name == idx_name), None) + if not match: + raise QueryIndexNotFoundException(f'Cannot find index with name: {idx_name}') + + return all(map(lambda i: i.state == 'online', indexes)) + + # timeout is converted to microsecs via final_args() + timeout_millis = timeout / 1000 + + interval_millis = float(50) + start = perf_counter() + time_left = timeout_millis + while True: + + opts = GetAllQueryIndexOptions( + timeout=timedelta(milliseconds=time_left)) + if scope_name: + opts['scope_name'] = scope_name + opts['collection_name'] = collection_name + + try: + indexes = self.get_all_indexes(bucket_name, opts) + except AmbiguousTimeoutException: + pass # go ahead and move on, raise WatchQueryIndexTimeoutException later if needed + + all_online = check_indexes(index_names, indexes) + if all_online: + break + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise WatchQueryIndexTimeoutException('Failed to find all indexes online within the alloted time.') + + sleep(interval_millis / 1000) + + +class CollectionQueryIndexManager(QueryIndexManagerLogic): + def __init__(self, connection, bucket_name, scope_name, collection_name): + self._bucket_name = bucket_name + self._scope_name = scope_name + self._collection_name = collection_name + super().__init__(connection) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def create_index(self, + index_name, # type: str + keys, # type: Iterable[str] + *options, # type: CreateQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Creates a new query index. + + Args: + index_name (str): The name of the index. + keys (Iterable[str]): The keys which this index should cover. + options (:class:`~couchbase.management.options.CreateQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the index_name or keys are invalid types. + :class:`~couchbase.exceptions.QueryIndexAlreadyExistsException`: If the index already exists. + """ + + if not isinstance(index_name, str): + raise InvalidArgumentException('The index_name must be provided when creating a secondary index.') + if not isinstance(keys, (list, tuple)): + raise InvalidArgumentException('Index keys must be provided when creating a secondary index.') + + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + return super().create_index(self._bucket_name, index_name, keys, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def create_primary_index(self, + *options, # type: CreatePrimaryQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Creates a new primary query index. + + Args: + options (:class:`~couchbase.management.options.CreatePrimaryQueryIndexOptions`): Optional parameters for + this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.QueryIndexAlreadyExistsException`: If the index already exists. + """ + + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + return super().create_primary_index(self._bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + *options, # type: DropQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Drops an existing query index. + + Args: + index_name (str): The name of the index to drop. + options (:class:`~couchbase.management.options.DropQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the index_name is an invalid types. + :class:`~couchbase.exceptions.QueryIndexNotFoundException`: If the index does not exists. + """ + if not isinstance(index_name, str): + raise InvalidArgumentException('The index_name must be provided when dropping a secondary index.') + + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + return super().drop_index(self._bucket_name, index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def drop_primary_index(self, + *options, # type: DropPrimaryQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Drops an existing primary query index. + + Args: + options (:class:`~couchbase.management.options.DropPrimaryQueryIndexOptions`): Optional parameters for + this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.QueryIndexNotFoundException`: If the index does not exists. + """ + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + return super().drop_primary_index(self._bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(QueryIndex, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + *options, # type: GetAllQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> Iterable[QueryIndex]: + """Returns a list of indexes for a specific collection. + + Args: + options (:class:`~couchbase.management.options.GetAllQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Iterable[:class:`.QueryIndex`]: A list of indexes. + + """ + + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + return super().get_all_indexes(self._bucket_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.QueryIndexMgmt, QueryIndexManagerLogic._ERROR_MAPPING) + def build_deferred_indexes(self, + *options, # type: BuildDeferredQueryIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + """Starts building any indexes which were previously created with ``deferred=True``. + + Args: + options (:class:`~couchbase.management.options.BuildDeferredQueryIndexOptions`): Optional parameters + for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + """ + + if not kwargs: + kwargs = {} + + if kwargs.get('scope_name') or (options and options[0].get('scope_name')): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if kwargs.get('collection_name') or (options and options[0].get('collection_name')): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + kwargs['scope_name'] = self._scope_name + kwargs['collection_name'] = self._collection_name + return super().build_deferred_indexes(self._bucket_name, *options, **kwargs) + + def watch_indexes(self, # noqa: C901 + index_names, # type: Iterable[str] + *options, # type: WatchQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> None: + """Waits for a number of indexes to finish creation and be ready to use. + + Args: + index_names (Iterable[str]): The names of the indexes to watch. + options (:class:`~couchbase.management.options.WatchQueryIndexOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the index_names are invalid types. + :class:`~couchbase.exceptions.WatchQueryIndexTimeoutException`: If the specified timeout is reached + before all the specified indexes are ready to use. + """ + + if not isinstance(index_names, (list, tuple)): + raise InvalidArgumentException('One or more index_names must be provided when watching indexes.') + + final_args = forward_args(kwargs, *options) + if final_args.get('watch_primary', False): + index_names.append('#primary') + + if final_args.get('scope_name'): + raise InvalidArgumentException('scope_name cannot be set in the options when using the collection-level ' + 'query index manager') + if final_args.get('collection_name'): + raise InvalidArgumentException('collection_name cannot be set in the options when using the ' + 'collection-level query index manager') + + timeout = final_args.get('timeout', None) + if not timeout: + raise InvalidArgumentException('Must specify a timeout condition for watch indexes') + + def check_indexes(index_names, indexes): + for idx_name in index_names: + match = next((i for i in indexes if i.name == idx_name), None) + if not match: + raise QueryIndexNotFoundException(f'Cannot find index with name: {idx_name}') + + return all(map(lambda i: i.state == 'online', indexes)) + + # timeout is converted to microsecs via final_args() + timeout_millis = timeout / 1000 + + interval_millis = float(50) + start = perf_counter() + time_left = timeout_millis + while True: + opts = GetAllQueryIndexOptions(timeout=timedelta(milliseconds=time_left)) + + try: + indexes = self.get_all_indexes(opts) + except AmbiguousTimeoutException: + pass # go ahead and move on, raise WatchQueryIndexTimeoutException later if needed + + all_online = check_indexes(index_names, indexes) + if all_online: + break + + interval_millis += 500 + if interval_millis > 1000: + interval_millis = 1000 + + time_left = timeout_millis - ((perf_counter() - start) * 1000) + if interval_millis > time_left: + interval_millis = time_left + + if time_left <= 0: + raise WatchQueryIndexTimeoutException('Failed to find all indexes online within the alloted time.') + + sleep(interval_millis / 1000) diff --git a/couchbase/management/search.py b/couchbase/management/search.py new file mode 100644 index 000000000..2a4c84364 --- /dev/null +++ b/couchbase/management/search.py @@ -0,0 +1,282 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (Any, + Dict, + Iterable) + +from couchbase.management.logic.search_index_logic import SearchIndex, SearchIndexManagerLogic +from couchbase.management.logic.wrappers import BlockingMgmtWrapper, ManagementType + +# @TODO: lets deprecate import of options from couchbase.management.search +from couchbase.management.options import (AllowQueryingSearchIndexOptions, + AnalyzeDocumentSearchIndexOptions, + DisallowQueryingSearchIndexOptions, + DropSearchIndexOptions, + FreezePlanSearchIndexOptions, + GetAllSearchIndexesOptions, + GetAllSearchIndexStatsOptions, + GetSearchIndexedDocumentsCountOptions, + GetSearchIndexOptions, + GetSearchIndexStatsOptions, + PauseIngestSearchIndexOptions, + ResumeIngestSearchIndexOptions, + UnfreezePlanSearchIndexOptions, + UpsertSearchIndexOptions) + + +class SearchIndexManager(SearchIndexManagerLogic): + def __init__(self, + connection + ): + super().__init__(connection) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def upsert_index(self, + index, # type: SearchIndex + *options, # type: UpsertSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + return super().upsert_index(index, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + *options, # type: DropSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + return super().drop_index(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(SearchIndex, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index(self, + index_name, # type: str + *options, # type: GetSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> SearchIndex: + + return super().get_index(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(SearchIndex, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + *options, # type: GetAllSearchIndexesOptions + **kwargs # type: Dict[str, Any] + ) -> Iterable[SearchIndex]: + return super().get_all_indexes(*options, **kwargs) + + @BlockingMgmtWrapper.block(int, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_indexed_documents_count(self, + index_name, # type: str + *options, # type: GetSearchIndexedDocumentsCountOptions + **kwargs # type: Dict[str, Any] + ) -> int: + return super().get_indexed_documents_count(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def pause_ingest(self, + index_name, # type: str + *options, # type: PauseIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().pause_ingest(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def resume_ingest(self, + index_name, # type: str + *options, # type: ResumeIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().resume_ingest(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def allow_querying(self, + index_name, # type: str + *options, # type: AllowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().allow_querying(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def disallow_querying(self, + index_name, # type: str + *options, # type: DisallowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().disallow_querying(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def freeze_plan(self, + index_name, # type: str + *options, # type: FreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().freeze_plan(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def unfreeze_plan(self, + index_name, # type: str + *options, # type: UnfreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().unfreeze_plan(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def analyze_document(self, + index_name, # type: str + document, # type: Any + *options, # type: AnalyzeDocumentSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Dict[str, Any]: + return super().analyze_document(index_name, document, *options, **kwargs) + + @BlockingMgmtWrapper.block(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index_stats(self, + index_name, # type: str + *options, # type: GetSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Dict[str, Any]: + return super().get_index_stats(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_index_stats(self, + *options, # type: GetAllSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Dict[str, Any]: + return super().get_all_index_stats(*options, **kwargs) + + +class ScopeSearchIndexManager(SearchIndexManagerLogic): + + def __init__(self, + connection, + bucket_name, # type: str + scope_name # type: str + ): + super().__init__(connection, bucket_name=bucket_name, scope_name=scope_name) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def upsert_index(self, + index, # type: SearchIndex + *options, # type: UpsertSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + return super().upsert_index(index, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + *options, # type: DropSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + + return super().drop_index(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(SearchIndex, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index(self, + index_name, # type: str + *options, # type: GetSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> SearchIndex: + + return super().get_index(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(SearchIndex, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_indexes(self, + *options, # type: GetAllSearchIndexesOptions + **kwargs # type: Dict[str, Any] + ) -> Iterable[SearchIndex]: + return super().get_all_indexes(*options, **kwargs) + + @BlockingMgmtWrapper.block(int, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_indexed_documents_count(self, + index_name, # type: str + *options, # type: GetSearchIndexedDocumentsCountOptions + **kwargs # type: Dict[str, Any] + ) -> int: + return super().get_indexed_documents_count(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def pause_ingest(self, + index_name, # type: str + *options, # type: PauseIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().pause_ingest(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def resume_ingest(self, + index_name, # type: str + *options, # type: ResumeIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().resume_ingest(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def allow_querying(self, + index_name, # type: str + *options, # type: AllowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().allow_querying(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def disallow_querying(self, + index_name, # type: str + *options, # type: DisallowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().disallow_querying(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def freeze_plan(self, + index_name, # type: str + *options, # type: FreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().freeze_plan(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def unfreeze_plan(self, + index_name, # type: str + *options, # type: UnfreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().unfreeze_plan(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def analyze_document(self, + index_name, # type: str + document, # type: Any + *options, # type: AnalyzeDocumentSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Dict[str, Any]: + return super().analyze_document(index_name, document, *options, **kwargs) + + @BlockingMgmtWrapper.block(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index_stats(self, + index_name, # type: str + *options, # type: GetSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Dict[str, Any]: + return super().get_index_stats(index_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_index_stats(self, + *options, # type: GetAllSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Dict[str, Any]: + return super().get_all_index_stats(*options, **kwargs) diff --git a/couchbase/management/users.py b/couchbase/management/users.py new file mode 100644 index 000000000..d976afeab --- /dev/null +++ b/couchbase/management/users.py @@ -0,0 +1,249 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Iterable + +from couchbase.management.logic.users_logic import Origin # noqa: F401 +from couchbase.management.logic.users_logic import Role # noqa: F401 +from couchbase.management.logic.users_logic import RoleAndOrigins # noqa: F401 +from couchbase.management.logic.users_logic import (Group, + RoleAndDescription, + User, + UserAndMetadata, + UserManagerLogic) +from couchbase.management.logic.wrappers import BlockingMgmtWrapper, ManagementType + +# @TODO: lets deprecate import of options from couchbase.management.users +from couchbase.management.options import (ChangePasswordOptions, + DropGroupOptions, + DropUserOptions, + GetAllGroupsOptions, + GetAllUsersOptions, + GetGroupOptions, + GetRolesOptions, + GetUserOptions, + UpsertGroupOptions, + UpsertUserOptions) + + +class UserManager(UserManagerLogic): + + def __init__(self, connection): + super().__init__(connection) + + @BlockingMgmtWrapper.block(UserAndMetadata, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_user(self, + username, # type: str + *options, # type: GetUserOptions + **kwargs # type: Any + ) -> UserAndMetadata: + """Returns a user by its username. + + Args: + username (str): The name of the user to retrieve. + options (:class:`~couchbase.management.options.GetUserOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`UserAndMetadata`: A :class:`UserAndMetadata` instance. + + Raises: + :class:`~couchbase.exceptions.UserNotFoundException`: If the user does not exist. + """ + return super().get_user(username, *options, **kwargs) + + @BlockingMgmtWrapper.block(UserAndMetadata, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_all_users(self, + *options, # type: GetAllUsersOptions + **kwargs # type: Any + ) -> Iterable[UserAndMetadata]: + """Returns a list of all existing users. + + Args: + options (:class:`~couchbase.management.options.GetAllUsersOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Iterable[:class:`UserAndMetadata`]: A list of existing users. + """ + return super().get_all_users(*options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def upsert_user(self, + user, # type: User + *options, # type: UpsertUserOptions + **kwargs # type: Any + ) -> None: + """Creates a new user or updates an existing user. + + Args: + user (:class:`.User`): The user to create or update. + options (:class:`~couchbase.management.options.UpsertUserOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the provided user argument contains an + invalid value or type. + """ + return super().upsert_user(user, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def drop_user(self, + username, # type: str + *options, # type: DropUserOptions + **kwargs # type: Any + ) -> None: + """Drops an existing user. + + Args: + username (str): The name of the user to drop. + options (:class:`~couchbase.management.options.DropUserOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.UserNotFoundException`: If the user does not exist. + """ + return super().drop_user(username, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def change_password(self, + new_password, # type: str + *options, # type: ChangePasswordOptions + **kwargs # type: Any + ) -> None: + """Changes the password of the currently authenticated user. SDK must be re-started and a new connection + established after running, as the previous credentials will no longer be valid. + + Args: + new_password (str): The new password for the user + options (:class:`~couchbase.management.options.ChangePasswordOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the provided group argument contains an + invalid value or type. + + """ + return super().change_password(new_password, *options, **kwargs) + + @BlockingMgmtWrapper.block(RoleAndDescription, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_roles(self, + *options, # type: GetRolesOptions + **kwargs # type: Any + ) -> Iterable[RoleAndDescription]: + """Returns a list of roles available on the server. + + Args: + options (:class:`~couchbase.management.options.GetRolesOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Iterable[:class:`RoleAndDescription`]: A list of roles available on the server. + """ + return super().get_roles(*options, **kwargs) + + @BlockingMgmtWrapper.block(Group, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_group(self, + group_name, # type: str + *options, # type: GetGroupOptions + **kwargs # type: Any + ) -> Group: + """Returns a group by it's name. + + Args: + group_name (str): The name of the group to retrieve. + options (:class:`~couchbase.management.options.GetGroupOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + :class:`Group`: A :class:`Group` instance. + + Raises: + :class:`~couchbase.exceptions.GroupNotFoundException`: If the group does not exist. + """ + return super().get_group(group_name, *options, **kwargs) + + @BlockingMgmtWrapper.block(Group, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def get_all_groups(self, + *options, # type: GetAllGroupsOptions + **kwargs # type: Any + ) -> Iterable[Group]: + """Returns a list of all existing groups. + + Args: + options (:class:`~couchbase.management.options.GetAllGroupsOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Iterable[:class:`Group`]: A list of existing groups. + """ + return super().get_all_groups(*options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def upsert_group(self, + group, # type: Group + *options, # type: UpsertGroupOptions + **kwargs # type: Any + ) -> None: + """Creates a new group or updates an existing group. + + Args: + group (:class:`.Group`): The group to create or update. + options (:class:`~couchbase.management.options.UpsertGroupOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the provided group argument contains an + invalid value or type. + """ + return super().upsert_group(group, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.UserMgmt, UserManagerLogic._ERROR_MAPPING) + def drop_group(self, + group_name, # type: str + *options, # type: DropGroupOptions + **kwargs # type: Any + ) -> None: + """Drops an existing group. + + Args: + group_name (str): The name of the group to drop. + options (:class:`~couchbase.management.options.DropGroupOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Raises: + :class:`~couchbase.exceptions.GroupNotFoundException`: If the group does not exist. + """ + return super().drop_group(group_name, *options, **kwargs) diff --git a/couchbase/management/views.py b/couchbase/management/views.py new file mode 100644 index 000000000..15c94c18b --- /dev/null +++ b/couchbase/management/views.py @@ -0,0 +1,82 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (Any, + Dict, + Iterable) + +from couchbase.management.logic.view_index_logic import View # noqa: F401 +from couchbase.management.logic.view_index_logic import (DesignDocument, + DesignDocumentNamespace, + ViewIndexManagerLogic) +from couchbase.management.logic.wrappers import BlockingMgmtWrapper, ManagementType + +# @TODO: lets deprecate import of options from couchbase.management.views +from couchbase.management.options import (DropDesignDocumentOptions, + GetAllDesignDocumentsOptions, + GetDesignDocumentOptions, + PublishDesignDocumentOptions, + UpsertDesignDocumentOptions) + + +class ViewIndexManager(ViewIndexManagerLogic): + def __init__(self, connection, bucket_name): + super().__init__(connection, bucket_name) + + @BlockingMgmtWrapper.block(DesignDocument, ManagementType.ViewIndexMgmt, ViewIndexManagerLogic._ERROR_MAPPING) + def get_design_document(self, + design_doc_name, # type: str + namespace, # type: DesignDocumentNamespace + *options, # type: GetDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> DesignDocument: + return super().get_design_document(design_doc_name, namespace, *options, **kwargs) + + @BlockingMgmtWrapper.block(DesignDocument, ManagementType.ViewIndexMgmt, ViewIndexManagerLogic._ERROR_MAPPING) + def get_all_design_documents(self, + namespace, # type: DesignDocumentNamespace + *options, # type: GetAllDesignDocumentsOptions + **kwargs # type: Dict[str, Any] + ) -> Iterable[DesignDocument]: + return super().get_all_design_documents(namespace, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.ViewIndexMgmt, ViewIndexManagerLogic._ERROR_MAPPING) + def upsert_design_document(self, + design_doc_data, # type: DesignDocument + namespace, # type: DesignDocumentNamespace + *options, # type: UpsertDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().upsert_design_document(design_doc_data, namespace, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.ViewIndexMgmt, ViewIndexManagerLogic._ERROR_MAPPING) + def drop_design_document(self, + design_doc_name, # type: str + namespace, # type: DesignDocumentNamespace + *options, # type: DropDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> None: + return super().drop_design_document(design_doc_name, namespace, *options, **kwargs) + + @BlockingMgmtWrapper.block(None, ManagementType.ViewIndexMgmt, ViewIndexManagerLogic._ERROR_MAPPING) + def publish_design_document(self, + design_doc_name, # type: str + *options, # type: PublishDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> None: + doc = self.get_design_document( + design_doc_name, DesignDocumentNamespace.DEVELOPMENT, *options, **kwargs) + return self.upsert_design_document( + doc, DesignDocumentNamespace.PRODUCTION, *options, **kwargs) diff --git a/couchbase/metrics.py b/couchbase/metrics.py new file mode 100644 index 000000000..36c93a221 --- /dev/null +++ b/couchbase/metrics.py @@ -0,0 +1,84 @@ +# Copyright 2021, Couchbase, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from abc import ABC, abstractmethod +from typing import (Any, + Dict, + Optional) + + +class CouchbaseValueRecorder(ABC): + """ + This is an abstract base class intended to wrap an external recorder implementation, to use withing the + python client. You will want to create a class deriving from this, which implements the :meth:`record_value` + method. + """ + + def __init__(self, + recorder=None # type: Optional[Any] + ): + self._recorder = recorder + super().__init__() + + @property + def recorder(self) -> Optional[Any]: + """ + Optional[Any]: The underlying recorder object this class wrapped. + """ + return self._recorder + + @abstractmethod + def record_value(self, + value, # type: int + ) -> None: + """ + This method will need to be implemented in derived classes. + + Args: + value (int): The value to record. + + """ + pass + + +class CouchbaseMeter(ABC): + """ + This is an abstract base class intended to wrap an external meter implementation, to use withing the + python client. You will want to create a class deriving from this, which implements the :meth:`value_recorder` + method. + """ + + def __init__(self, + external_meter=None # type: Optional[Any] + ): + self._external_meter = external_meter + super().__init__() + + @abstractmethod + def value_recorder(self, + name, # type: str + tags # type: Dict[str, str] + ) -> CouchbaseValueRecorder: + """ + This method will need to be implemented in derived classes. + + Args: + name (int): The name of the recorder. + tags (Dict[str, str]): The tags associated with the recorder. + + Returns: + :class:`~couchbase.metrics.CouchbaseValueRecorder`: + """ + pass diff --git a/couchbase/mockserver.py b/couchbase/mockserver.py deleted file mode 100644 index 5bc3327c4..000000000 --- a/couchbase/mockserver.py +++ /dev/null @@ -1,188 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from subprocess import Popen, PIPE -from couchbase._pyport import urlopen, ulp, basestring -import socket -import json -import os.path -import select - -class BucketSpec(object): - def __init__(self, name='default', bucket_type='couchbase', password=''): - self.name = name - self.bucket_type = bucket_type - self.password = password - - def __str__(self): - return ':'.join([self.name, self.password, self.bucket_type]) - -class MockControlClient(object): - def __init__(self, mockport=18091, host='127.0.0.1'): - self.urlbase = "http://{0}:{1}/mock/".format(host, mockport) - - def _get_common(self, command, params): - qparams = {} - for k, v in params.items(): - qparams[k] = json.dumps(v) - qparams = ulp.urlencode(qparams) - url = self.urlbase + "{0}?{1}".format(command, qparams) - data = urlopen(url).read() - if not isinstance(data, basestring): - data = str(data, "utf-8") - ret = json.loads(data) - return ret - - def _params_common(self, key, - bucket=None, - on_master=False, - replica_count=None, - replicas=None, - cas=None, - value=None): - r = { - 'Key' : key, - 'OnMaster' : on_master - } - if bucket: - r['Bucket'] = bucket - if replica_count is not None: - r['OnReplicas'] = replica_count - else: - r['OnReplicas'] = replicas - if cas is not None: - r['CAS'] = cas - - if value is not None: - r['Value'] = value - - return r - - def _do_request(self, cmd, *args, **kwargs): - params = self._params_common(*args, **kwargs) - return self._get_common(cmd, params) - - def keyinfo(self, *args, **kwargs): - return self._do_request("keyinfo", *args, **kwargs) - - def persist(self, *args, **kwargs): - return self._do_request("persist", *args, **kwargs) - - def endure(self, *args, **kwargs): - return self._do_request("endure", *args, **kwargs) - - def cache(self, *args, **kwargs): - return self._do_request("cache", *args, **kwargs) - - def uncache(self, *args, **kwargs): - return self._do_request("uncache", *args, **kwargs) - - def unpersist(self, *args, **kwargs): - return self._do_request("unpersist", *args, **kwargs) - - def purge(self, *args, **kwargs): - return self._do_request("purge", *args, **kwargs) - - -class CouchbaseMock(object): - def _setup_listener(self): - sock = socket.socket() - sock.bind( ('', 0) ) - sock.listen(5) - - addr, port = sock.getsockname() - self.listen = sock - self.port = port - - def _invoke(self): - self._setup_listener() - args = [ - "java", "-client", "-jar", self.runpath, - "--port", "0", "--with-beer-sample", - "--harakiri-monitor", "127.0.0.1:" + str(self.port), - "--nodes", str(self.nodes) - ] - - if self.vbuckets is not None: - args += ["--vbuckets", str(self.vbuckets)] - - if self.replicas is not None: - args += ["--replicas", str(self.replicas)] - - bspec = ",".join([str(x) for x in self.buckets]) - args += ["--buckets", bspec] - - self.po = Popen(args) - - # Sometimes we get an invalid JAR file. Unfortunately there is no - # way to determine or "wait for completion". The next best thing - # is to set a maximum of 15 seconds for the process to start (and - # connect to the listening socket); - - rlist, w_, x_ = select.select([self.listen], [], [], 15) - if not rlist: - raise Exception('Mock server was not ready in time') - - self.harakiri_sock, addr = self.listen.accept() - self.ctlfp = self.harakiri_sock.makefile() - - sbuf = "" - while True: - c = self.ctlfp.read(1) - if c == '\0': - break - sbuf += c - self.rest_port = int(sbuf) - - def __init__(self, buckets, runpath, - url=None, - replicas=None, - vbuckets=None, - nodes=4): - """ - Creates a new instance of the mock server. You must actually call - 'start()' for it to be invoked. - :param list buckets: A list of BucketSpec items - :param string runpath: a string pointing to the location of the mock - :param string url: The URL to use to download the mock. This is only - used if runpath does not exist - :param int replicas: How many replicas should each bucket have - :param int vbuckets: How many vbuckets should each bucket have - :param int nodes: How many total nodes in the cluster - - Note that you must have ``java`` in your `PATH` - """ - - - self.runpath = runpath - self.buckets = buckets - self.nodes = nodes - self.vbuckets = vbuckets - self.replicas = replicas - - if not os.path.exists(runpath): - if not url: - raise Exception(runpath + " Does not exist and no URL specified") - fp = open(runpath, "wb") - ulp = urlopen(url) - jarblob = ulp.read() - fp.write(jarblob) - fp.close() - - def start(self): - self._invoke() - - def stop(self): - self.po.kill() diff --git a/couchbase/mutation_state.py b/couchbase/mutation_state.py new file mode 100644 index 000000000..7188c3612 --- /dev/null +++ b/couchbase/mutation_state.py @@ -0,0 +1,108 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict, + List) + +from couchbase.exceptions import MissingTokenException +from couchbase.result import MutationToken + +if TYPE_CHECKING: + from couchbase.result import MutationResult + + +class MutationState: + def __init__(self, *docs, # type: List[MutationResult] + **kwargs # type: Dict[str, Any] + ): + self._sv = set() + if docs: + self.add_results(*docs, **kwargs) + + def add_mutation_token(self, mut_token # type: MutationToken + ) -> None: + if isinstance(mut_token, MutationToken): + self._sv.add(mut_token) + + def _add_scanvec(self, mut_token # type: MutationToken + ) -> bool: + """ + Internal method used to specify a scan vector. + :param mut_token: A tuple in the form of + `(vbucket id, vbucket uuid, mutation sequence)` + """ + if isinstance(mut_token, MutationToken): + self._sv.add(mut_token) + return True + + return False + + def add_results(self, *rvs, # type: List[MutationResult] + **kwargs # type: Dict[str, Any] + ) -> bool: + """ + Changes the state to reflect the mutation which yielded the given + result. + + In order to use the result, the `enable_mutation_tokens` option must + have been specified in the connection string, _and_ the result + must have been successful. + + :param rvs: One or more :class:`~.OperationResult` which have been + returned from mutations + :param quiet: Suppress errors if one of the results does not + contain a convertible state. + :return: `True` if the result was valid and added, `False` if not + added (and `quiet` was specified + :raise: :exc:`~.MissingTokenException` if `result` does not contain + a valid token + """ + if not rvs: + raise MissingTokenException(message='No results passed') + for rv in rvs: + mut_token = rv.mutation_token() + if not isinstance(mut_token, MutationToken): + if kwargs.get('quiet', False) is True: + return False + raise MissingTokenException( + message='Result does not contain token') + if not self._add_scanvec(mut_token): + return False + return True + + def add_all(self, bucket, quiet=False): + """ + Ensures the query result is consistent with all prior + mutations performed by a given bucket. + + Using this function is equivalent to keeping track of all + mutations performed by the given bucket, and passing them to + :meth:`~add_result` + + :param bucket: A :class:`~couchbase_core.client.Client` object + used for the mutations + :param quiet: If the bucket contains no valid mutations, this + option suppresses throwing exceptions. + :return: `True` if at least one mutation was added, `False` if none + were added (and `quiet` was specified) + :raise: :exc:`~.MissingTokenException` if no mutations were added and + `quiet` was not specified + """ + raise NotImplementedError("Feature currently not implemented in 4.x series of the Python SDK") + + def __repr__(self): + return "MutationState:{}".format(self._sv) diff --git a/couchbase/n1ql.py b/couchbase/n1ql.py index 68cc1f61e..28e7a6e4f 100644 --- a/couchbase/n1ql.py +++ b/couchbase/n1ql.py @@ -1,345 +1,100 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. # -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json - -from couchbase._pyport import basestring -from couchbase.views.iterator import AlreadyQueriedError -from couchbase.exceptions import CouchbaseError, NotSupportedError - - -class N1QLError(CouchbaseError): - @property - def n1ql_errcode(self): - return self.objextra['code'] - - -CONSISTENCY_REQUEST = 'request_plus' -""" -For use with :attr:`~.N1QLQuery.consistency`, will ensure that query -results always reflect the latest data in the server -""" - -CONSISTENCY_NONE = 'none' -""" -For use with :attr:`~.N1QLQuery.consistency`, will allow cached -values to be returned. This will improve performance but may not -reflect the latest data in the server. -""" - +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.n1ql import N1QLQuery # noqa: F401 +from couchbase.logic.n1ql import QueryError # noqa: F401 +from couchbase.logic.n1ql import QueryMetaData # noqa: F401 +from couchbase.logic.n1ql import QueryMetrics # noqa: F401 +from couchbase.logic.n1ql import QueryProfile # noqa: F401 +from couchbase.logic.n1ql import QueryScanConsistency # noqa: F401 +from couchbase.logic.n1ql import QueryStatus # noqa: F401 +from couchbase.logic.n1ql import QueryWarning # noqa: F401 +from couchbase.logic.n1ql import QueryRequestLogic + + +class N1QLRequest(QueryRequestLogic): + def __init__(self, + connection, + query_params, + row_factory=lambda x: x, + **kwargs + ): + super().__init__(connection, query_params, row_factory=row_factory, **kwargs) + + @classmethod + def generate_n1ql_request(cls, connection, query_params, row_factory=lambda x: x, **kwargs): + return cls(connection, query_params, row_factory=row_factory, **kwargs) -class N1QLQuery(object): - def __init__(self, query, *args, **kwargs): - """ - Create an N1QL Query object. This may be passed as the - `params` argument to :class:`N1QLRequest`. - - :param query: The query string to execute - :param args: Positional placeholder arguments. - :param kwargs: Named placeholder arguments. - (The client will automatically prepend a leading ``$`` - to each named parameter). - - Use positional parameters:: - - q = N1QLQuery('SELECT * FROM `travel-sample` ' - 'WHERE type=$1 AND id=$2', - 'airline', 0) - - Use named parameters:: - - q = N1QLQuery('SELECT * FROM `travel-sample` ' - 'WHERE type=$type AND id=$id', - type='airline', id=0) - - """ - - self._adhoc = True - self._body = {'statement': query} - if args: - self._add_pos_args(*args) - if kwargs: - self._set_named_args(**kwargs) + def execute(self): + return [r for r in list(self)] + + def _get_metadata(self): + try: + # @TODO: PYCBC-1524 + query_response = next(self._streaming_result) + self._set_metadata(query_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn - def _set_named_args(self, **kv): - """ - Set a named parameter in the query. The named field must - exist in the query itself. + def __iter__(self): + if self.done_streaming: + raise AlreadyQueriedException() - :param kv: Key-Value pairs representing values within the - query. These values should be stripped of their leading - `$` identifier. + if not self.started_streaming: + self._submit_query() - """ - for k in kv: - self._body['${0}'.format(k)] = kv[k] return self - def _add_pos_args(self, *args): - """ - Set values for *positional* placeholders (``$1,$2,...``) - - :param args: Values to be used - """ - arg_array = self._body.setdefault('args', []) - arg_array.extend(args) - - def set_option(self, name, value): - """ - Set a raw option in the query. This option is encoded - as part of the query parameters without any client-side - verification. Use this for settings not directly exposed - by the Python client. - - :param name: The name of the option - :param value: The value of the option - """ - self._body[name] = value - - @property - def statement(self): - return self._body['statement'] - - @property - def consistency(self): - """ - Sets the consistency level. - - :see: :data:`CONSISTENCY_NONE`, :data:`CONSISTENCY_REQUEST` - """ - return self._body.get('scan_consistency', CONSISTENCY_NONE) - - @consistency.setter - def consistency(self, value): - self._body['scan_consistency'] = value - - def _add_scanvec(self, mutinfo): - """ - Internal method used to specify a scan vector. - :param mutinfo: A tuple in the form of - `(vbucket id, vbucket uuid, mutation sequence)` - """ - vb, uuid, seq = mutinfo - self._body.setdefault('scan_vector', {})[vb] = { - 'value': seq, - 'guard': str(uuid) - } - self.consistency = 'at_plus' - - def consistent_with_ops(self, *ops, **kwargs): - """ - Ensure results reflect consistency of one or more mutations. - This is similar to setting :attr:`consistency` to - :data:`CONSISTENCY_REQUEST`, but is more optimal as the query - will use cached data, *except* when the given mutation(s) are - concerned. This option is useful for use patterns when an - application has just performed a mutation, and wishes to - perform a query in which the newly-performed mutation - should reflect on the query results. - - :param ops: One or more :class:`~.OperationResult` objects - :param bool quiet: Whether to suppress throwing an exception - if one or more operations are missing mutation information. - :raise: :exc:`~.NotSupportedError` if one of the mutations - - .. note:: - - This feature requires Couchbase Server 4.0 or greater, - and also requires that `fetch_mutation_tokens=true` - be specified in the connection string when creating - a :class:`~couchbase.bucket.Bucket` - - - .. code-block:: python - - cb = Bucket('couchbase://localhost/default?fetch_mutation_tokens=true') - - rvs = cb.upsert_multi({ - 'foo': {'type': 'user', 'value': 'a foo value'}, - 'bar': {'type': 'user', 'value': 'a bar value'} - }) - - nq = N1QLQuery('SELECT type, value FROM default WHERE type="user"') - nq.consistent_with_ops(*rvs.values()) - for row in cb.n1ql_query(nq): - # ... - - - .. seealso:: :meth:`~consistent_with_all` - """ - for op in ops: - if not op._mutinfo: - if kwargs.get('quiet'): - continue - raise NotSupportedError.pyexc( - "OperationResult object missing mutation information. " - "Ensure that `fetch_mutation_tokens=true` was specified " - "in the connection string") - self._add_scanvec(op._mutinfo) - - def consistent_with_all(self, bucket): - """ - Ensures the query result is consistent with all prior - mutations performed by a given bucket. - - Using this function is equivalent to keeping track of all - mutations performed by the given bucket, and passing them to - :meth:`~consistent_with_ops` - - :param bucket: A :class:`~couchbase.bucket.Bucket` object - used for the mutations - """ - for mt in bucket._mutinfo(): - self._add_scanvec(mt) - - # TODO: I really wish Sphinx were able to automatically - # document instance vars - @property - def adhoc(self): - """ - A non-`adhoc` query can be internally optimized so that repeated - executions of the same query can be quicker. If this query is issued - repeatedly in your application, then you should set this property to - `False`. - - Note that this optimization involves an up-front "preparation" - cost, and should only be used for queries that are issued multiple - times. - """ - return self._adhoc - - @adhoc.setter - def adhoc(self, arg): - self._adhoc = arg - - @property - def encoded(self): - """ - Get an encoded representation of the query. - - This is used internally by the client, and can be useful - to debug queries. - """ - return json.dumps(self._body) - - def __repr__(self): - return ('<{cls} stmt={stmt} at {oid}>'.format( - cls=self.__class__.__name__, - stmt=repr(self._body), - oid=id(self))) - - -class N1QLRequest(object): - def __init__(self, params, parent, row_factory=lambda x: x): - """ - Object representing the execution of the request on the - server. - - .. warning:: - - You should typically not call this constructor by - yourself, rather use the :meth:`~.Bucket.n1ql_query` - method (or one of its async derivatives). - - :param params: An :class:`N1QLQuery` object. - :param parent: The parent :class:`.Bucket` object - :param row_factory: Callable which accepts the raw dictionary - of each row, and can wrap them in a customized class. - The default is simply to return the dictionary itself. - - To actually receive results of the query, iterate over this - object. - """ - if isinstance(params, basestring): - params = N1QLQuery(params) - - self._params = params - self._parent = parent - self.row_factory = row_factory - self.errors = [] - self._mres = None - self._do_iter = True - self.__raw = False - - def _start(self): - if self._mres: + def _get_next_row(self): + if self.done_streaming is True: return - self._mres = self._parent._n1ql_query(self._params.encoded, - not self._params.adhoc) - self.__raw = self._mres[None] - - @property - def raw(self): - return self.__raw - - def _clear(self): - del self._parent - del self._mres - - def _handle_meta(self, value): - if not isinstance(value, dict): - return - if 'errors' in value: - for err in value['errors']: - raise N1QLError.pyexc('N1QL Execution failed', err) - - def _process_payload(self, rows): - if rows: - return [self.row_factory(row) for row in rows] - - elif self.raw.done: - self._handle_meta(self.raw.value) - self._do_iter = False - return [] - else: - # We can only get here if another concurrent query broke out the - # event loop before we did. - return [] - - def execute(self): - """ - Execute the statement and raise an exception on failure. - - This method is useful for statements which modify data or - indexes, where the application does not need to extract any - data, but merely determine success or failure. - """ - for _ in self: - pass - - def get_single_result(self): - """ - Execute the statement and return its single result. - - This should only be used on statements which are intended to - return only a single result. - - :return: The single result, as encapsulated by the - `row_factory` - """ - for r in self: - return r - - def __iter__(self): - if not self._do_iter: - raise AlreadyQueriedError() - - self._start() - while self._do_iter: - raw_rows = self.raw.fetch(self._mres) - for row in self._process_payload(raw_rows): - yield row + try: + row = next(self._streaming_result) + except StopIteration: + # @TODO: PYCBC-1524 + row = next(self._streaming_result) + + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None once query request is complete and _no_ errors found + if row is None: + raise StopIteration + + return self.serializer.deserialize(row) + + def __next__(self): + try: + return self._get_next_row() + except StopIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/couchbase/options.py b/couchbase/options.py new file mode 100644 index 000000000..cbb6ab2df --- /dev/null +++ b/couchbase/options.py @@ -0,0 +1,2049 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# used to allow for unquoted (i.e. forward reference, Python >= 3.7, PEP563) +from __future__ import annotations + +import copy +from abc import (ABC, + ABCMeta, + abstractmethod) +from datetime import timedelta +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + Optional, + Tuple, + TypeVar, + Union, + overload) + +from couchbase._utils import (timedelta_as_microseconds, + timedelta_as_timestamp, + validate_int) +from couchbase.durability import DurabilityParser +from couchbase.exceptions import InvalidArgumentException +from couchbase.logic.options import AcceptableInts # noqa: F401 +from couchbase.logic.options import Compression # noqa: F401 +from couchbase.logic.options import IpProtocol # noqa: F401 +from couchbase.logic.options import KnownConfigProfiles # noqa: F401 +from couchbase.logic.options import LockMode # noqa: F401 +from couchbase.logic.options import TLSVerifyMode # noqa: F401 +from couchbase.logic.options import get_valid_args # noqa: F401 +from couchbase.logic.options import get_valid_multi_args # noqa: F401 +from couchbase.logic.options import (AnalyticsOptionsBase, + AppendOptionsBase, + ClusterOptionsBase, + ClusterTimeoutOptionsBase, + ClusterTracingOptionsBase, + ConstrainedIntBase, + DecrementOptionsBase, + DeltaValueBase, + DiagnosticsOptionsBase, + DurabilityOptionBlockBase, + ExistsOptionsBase, + GetAllReplicasOptionsBase, + GetAndLockOptionsBase, + GetAndTouchOptionsBase, + GetAnyReplicaOptionsBase, + GetOptionsBase, + IncrementOptionsBase, + InsertOptionsBase, + LookupInAllReplicasOptionsBase, + LookupInAnyReplicaOptionsBase, + LookupInOptionsBase, + MutateInOptionsBase, + OptionsTimeoutBase, + PingOptionsBase, + PrependOptionsBase, + QueryOptionsBase, + RemoveOptionsBase, + ReplaceOptionsBase, + ScanOptionsBase, + SearchOptionsBase, + SignedInt64Base, + TouchOptionsBase, + UnlockOptionsBase, + UnsignedInt32Base, + UnsignedInt64Base, + UpsertOptionsBase, + VectorSearchOptionsBase, + ViewOptionsBase, + WaitUntilReadyOptionsBase) +from couchbase.logic.supportability import Supportability +from couchbase.pycbc_core import (transaction_config, + transaction_options, + transaction_query_options) +from couchbase.serializer import DefaultJsonSerializer + +# allows for imports only during type checking and not during runtime -- :) +if TYPE_CHECKING: + from couchbase._utils import JSONType + from couchbase.auth import Authenticator + from couchbase.collection import Collection + from couchbase.durability import DurabilityType, ServerDurability + from couchbase.n1ql import QueryScanConsistency + from couchbase.replica_reads import ReadPreference + from couchbase.transactions import TransactionKeyspace + from couchbase.transcoder import Transcoder + + +class ClusterTracingOptions(ClusterTracingOptionsBase): + """Available tracing options to set when creating a cluster. + + .. warning:: + Importing options from ``couchbase.cluster`` is deprecated. + All options should be imported from ``couchbase.options``. + + These will be the default timeouts for operations for the entire cluster + + Args: + tracing_threshold_kv (timedelta, optional): KV operations threshold. Defaults to None. + tracing_threshold_view (timedelta, optional): Views operations threshold. Defaults to None. + tracing_threshold_query (timedelta, optional): Query operations threshold. Defaults to None. + tracing_threshold_search (timedelta, optional): Search operations threshold.. Defaults to None. + tracing_threshold_analytics (timedelta, optional): Analytics operations threshold. Defaults to None. + tracing_threshold_eventing (timedelta, optional): Eventing operations threshold. Defaults to None. + tracing_threshold_management (timedelta, optional): Management operations threshold. Defaults to None. + tracing_threshold_queue_size (int, optional): Size of tracing operations queue. Defaults to None. + tracing_threshold_queue_flush_interval (timedelta, optional): Interveral to flush tracing operations queue. + Defaults to None. + tracing_orphaned_queue_size (int, optional): Size of tracing orphaned operations queue. Defaults to None. + tracing_orphaned_queue_flush_interval (timedelta, optional): Interveral to flush tracing orphaned operations + queue. Defaults to None. + """ + + +class ClusterTimeoutOptions(ClusterTimeoutOptionsBase): + """Available timeout options to set when creating a cluster. + + .. warning:: + Importing options from ``couchbase.cluster`` is deprecated. + All options should be imported from ``couchbase.options``. + + These will set the default timeouts for operations for the cluster. Some operations allow the timeout to + be overridden on a per operation basis. + + Args: + bootstrap_timeout (timedelta, optional): Overall bootstrap timeout. Defaults to None. + resolve_timeout (timedelta, optional): Time to resolve hostnames. Defaults to None. + connect_timeout (timedelta, optional): connect timeout. Defaults to None. + kv_timeout (timedelta, optional): KV operations timeout. Defaults to None. + kv_durable_timeout (timedelta, optional): KV durability operations timeout. Defaults to None. + views_timeout (timedelta, optional): views operations timeout. Defaults to None. + query_timeout (timedelta, optional): query operations timeout. Defaults to None. + analytics_timeout (timedelta, optional): analytics operations timeout. Defaults to None. + search_timeout (timedelta, optional): search operations timeout. Defaults to None. + management_timeout (timedelta, optional): management operations timeout. Defaults to None. + dns_srv_timeout (timedelta, optional): Time to make DNS-SRV query. Defaults to None. + idle_http_connection_timeout (timedelta, optional): Idle HTTP connection timeout. Defaults to None. + config_idle_redial_timeout (timedelta, optional): Idle redial timeout. Defaults to None. + config_total_timeout (timedelta, optional): **DEPRECATED** complete bootstrap timeout. Defaults to None. + """ + + +class ConfigProfile(ABC): + """ + **VOLATILE** This API is subject to change at any time. + + This is an abstract base class intended to use with creating Configuration Profiles. Any derived class + will need to implement the :meth:`apply` method. + """ + + def __init__(self): + super().__init__() + + @abstractmethod + def apply(self, + options # type: ClusterOptions + ) -> None: + """ + **VOLATILE** This API is subject to change at any time. + + Apply the provided options to ClusterOptions. This method will need to be implemented in derived classes. + + Args: + options (:class:`~couchbase.options.ClusterOptions`): The options the profile will apply toward. + """ + pass + + +class WanDevelopmentProfile(ConfigProfile): + """ + **VOLATILE** This API is subject to change at any time. + + The WAN Development profile sets various timeout options that are useful when develoption in a WAN environment. + """ + + def __init__(self): + super().__init__() + + def apply(self, + options # type: ClusterOptions + ) -> None: + # Need to use keys in couchbase.logic.ClusterTimeoutOptionsBase._VALID_OPTS + options['kv_timeout'] = timedelta(seconds=20) + options['kv_durable_timeout'] = timedelta(seconds=20) + options['connect_timeout'] = timedelta(seconds=20) + options['analytics_timeout'] = timedelta(seconds=120) + options['query_timeout'] = timedelta(seconds=120) + options['search_timeout'] = timedelta(seconds=120) + options['management_timeout'] = timedelta(seconds=120) + options['views_timeout'] = timedelta(seconds=120) + options['dns_srv_timeout'] = timedelta(seconds=20) # time to make DNS-SRV query + options['resolve_timeout'] = timedelta(seconds=20) # time to resolve hostnames + options['bootstrap_timeout'] = timedelta(seconds=120) # overall bootstrap timeout + + +class ConfigProfiles(): + """ + **VOLATILE** This API is subject to change at any time. + + The `ConfigProfiles` class is responsible for keeping track of registered/known Configuration + Profiles. + """ + + def __init__(self): + self._profiles = {} + self.register_profile(KnownConfigProfiles.WanDevelopment.value, WanDevelopmentProfile()) + + def apply_profile(self, + profile_name, # type: str + options # type: ClusterOptions + ) -> None: + """ + **VOLATILE** This API is subject to change at any time. + + Apply the provided ConfigProfile options. + + Args: + profile_name (str): The name of the profile to apply. + options (:class:`~couchbase.options.ClusterOptions`): The options to apply the ConfigProfile options + toward. The ConfigProfile options will override any matching option(s) previously set. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the specified profile is not registered. + """ + if profile_name not in self._profiles: + raise InvalidArgumentException(f'{profile_name} is not a registered profile.') + + self._profiles[profile_name].apply(options) + + def register_profile(self, + profile_name, # type: str + profile, # type: ConfigProfile + ) -> None: + """ + **VOLATILE** This API is subject to change at any time. + + Register a :class:`~couchbase.options.ConfigProfile`. + + Args: + profile_name (str): The name of the :class:`~couchbase.options.ConfigProfile` to register. + profile (:class:`~couchbase.options.ConfigProfile`): The :class:`~couchbase.options.ConfigProfile` + to register. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the specified profile is not derived + from :class:`~couchbase.options.ConfigProfile`. + + """ + if not issubclass(profile.__class__, ConfigProfile): + raise InvalidArgumentException('A Configuration Profile must be derived from ConfigProfile') + + self._profiles[profile_name] = profile + + def unregister_profile(self, + profile_name # type: str + ) -> Optional[ConfigProfile]: + """ + **VOLATILE** This API is subject to change at any time. + + Unregister a :class:`~couchbase.options.ConfigProfile`. + + Args: + profile_name (str): The name of the :class:`~couchbase.options.ConfigProfile` to unregister. + + Returns + Optional(:class:`~couchbase.options.ConfigProfile`): The unregistered :class:`~couchbase.options.ConfigProfile` + """ # noqa: E501 + + return self._profiles.pop(profile_name, None) + + +""" +**VOLATILE** The ConfigProfiles API is subject to change at any time. +""" +CONFIG_PROFILES = ConfigProfiles() + + +class ClusterOptions(ClusterOptionsBase): + """Available options to set when creating a cluster. + + .. warning:: + Importing options from ``couchbase.cluster`` is deprecated. + All options should be imported from ``couchbase.options``. + + Cluster options enable the configuration of various global cluster settings. + Some options can be set globally for the cluster, but overridden for specific + operations (i.e. ClusterTimeoutOptions) + + .. note:: + + The authenticator is mandatory, all the other cluster options are optional. + + Args: + authenticator (Union[:class:`~.PasswordAuthenticator`, :class:`~.CertificateAuthenticator`]): An + authenticator instance + timeout_options (:class:`~.ClusterTimeoutOptions`): Timeout options for + various SDK operations. See :class:`~.options.ClusterTimeoutOptions` for details. + tracing_options (:class:`~.options.ClusterTimeoutOptions`): Tracing options for SDK tracing bevavior. + See :class:`~.options.ClusterTracingOptions` for details. These are ignored if an external tracer + is specified. + enable_tls (bool, optional): Set to True to enable tls. Defaults to False (disabled). + enable_mutation_tokens (bool, optional): Set to False to disable mutation tokens in mutation results. + Defaults to True (enabled). + enable_tcp_keep_alive (bool, optional): Set to False to disable tcp keep alive. Defaults to True (enabled). + ip_protocol (Union[str, :class:`.IpProtocol`): Set IP protocol. Defaults to IpProtocol.Any. + enable_dns_srv (bool, optional): Set to False to disable DNS SRV. Defaults to True (enabled). + show_queries (bool, optional): Set to True to enabled showing queries. Defaults to False (disabled). + enable_unordered_execution (bool, optional): Set to False to disable unordered query execution. + Defaults to True (enabled). + enable_clustermap_notification (bool, optional): Set to False to disable cluster map notification. + Defaults to True (enabled). + enable_compression (bool, optional): Set to False to disable compression. Defaults to True (enabled). + enable_tracing (bool, optional): Set to False to disable tracing (enables no-op tracer). + Defaults to True (enabled). + enable_metrics (bool, optional): Set to False to disable metrics (enables no-op meter). + Defaults to True (enabled). + network (str, optional): Set network resolution method. Can be set to 'default' (if the client is running on the + same network as the server) or 'external' (if the client is running on a different network). Defaults to + 'auto'. + tls_verify (Union[str, :class:`.TLSVerifyMode`], optional): Set tls verify mode. Defaults to + TLSVerifyMode.PEER. + disable_mozilla_ca_certificates (bool, optional): Set to True to disable loading Mozilla's list of CA + certificates for TLS verification. Defaults to False (enabled). + serializer (:class:`~.serializer.Serializer`, optional): Global serializer to translate JSON to Python objects. + Defaults to :class:`~.serializer.DefaultJsonSerializer`. + transcoder (:class:`~.transcoder.Transcoder`, optional): Global transcoder to use for kv-operations. + Defaults to :class:`~.transcoder.JsonTranscoder`. + tcp_keep_alive_interval (timedelta, optional): TCP keep-alive interval. Defaults to None. + config_poll_interval (timedelta, optional): Config polling floor interval. + Defaults to None. + config_poll_floor (timedelta, optional): Config polling floor interval. + Defaults to None. + max_http_connections (int, optional): Maximum number of HTTP connections. Defaults to None. + logging_meter_emit_interval (timedelta, optional): Logging meter emit interval. Defaults to 10 minutes. + transaction_config (:class:`.TransactionConfig`, optional): Global configuration for transactions. + Defaults to None. + log_redaction (bool, optional): Set to True to enable log redaction. Defaults to False (disabled). + compression (:class:`~.Compression`, optional): Set compression mode. Defaults to None. + compression_min_size (int, optional): Set compression min size. Defaults to None. + compression_min_ratio (float, optional): Set compression min size. Defaults to None. + lockmode (:class:`~.LockMode`, optional): **DEPRECATED** This option will be removed in a future version of the SDK. + Set LockMode mode. Defaults to None. + tracer (:class:`~couchbase.tracing.CouchbaseTracer`, optional): Set an external tracer. Defaults to None, + enabling the `threshold_logging_tracer`. Note when this is set, all tracing_options + (see :class:`~.ClusterTracingOptions`) and then `enable_tracing` option are ignored. + meter (:class:`~couchbase.metrics.CouchbaseMeter`, optional): Set an external meter. Defaults to None, + enabling the `logging_meter`. Note when this is set, the `logging_meter_emit_interval` option is ignored. + dns_nameserver (str, optional): **VOLATILE** This API is subject to change at any time. Set to configure custom DNS nameserver. Defaults to None. + dns_port (int, optional): **VOLATILE** This API is subject to change at any time. Set to configure custom DNS port. Defaults to None. + dump_configuration (bool, optional): Set to True to dump every new configuration when TRACE level logging. Defaults to False (disabled). + preferred_server_group (str, optional): Specifies the preferred server group to be used for replica reads with 'selected server group' read preference. + enable_app_telemetry (bool, optional): Specifies if application telemetry feature should be enabled. Defaults to True (enabled). + app_telemetry_endpoint (str, optional): Specifies an endpoint to override the application metrics endpoint discovered during configuration. Defaults to None. + app_telemetry_backoff (timedelta, optional): Specifies the time to wait before attempting a websocket reconnection. Defaults to 5 seconds. + app_telemetry_ping_interval (timedelta, optional): Specifies the time to wait between sending consecutive websocket PING commands to the server. Defaults to 30 seconds. + app_telemetry_ping_timeout (timedelta, optional): Specifies the time allowed for the server to respond to websocket PING command. Defaults to 2 seconds. + """ # noqa: E501 + + def apply_profile(self, + profile_name # type: Union[KnownConfigProfiles, str] + ) -> None: + """ + **VOLATILE** This API is subject to change at any time. + + Apply the provided ConfigProfile options. + + Args: + profile_name ([:class:`~couchbase.options.KnownConfigProfiles`, str]): The name of the profile to apply + toward ClusterOptions. + authenticator (Union[:class:`~couchbase.auth.PasswordAuthenticator`, :class:`~couchbaes.auth.CertificateAuthenticator`]): An authenticator instance. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the specified profile is not registered. + + """ # noqa: E501 + prof_name = profile_name.value if isinstance(profile_name, KnownConfigProfiles) else profile_name + CONFIG_PROFILES.apply_profile(prof_name, self) + + @classmethod + def create_options_with_profile(cls, + authenticator, # type: Optional[Authenticator] + profile_name # type: Union[KnownConfigProfiles, str] + ) -> ClusterOptions: + """ + **VOLATILE** This API is subject to change at any time. + + Create a ClusterOptions instance and apply the provided ConfigProfile options. + + Args: + authenticator (Union[:class:`~couchbase.auth.PasswordAuthenticator`, :class:`~couchbaes.auth.CertificateAuthenticator`]): An authenticator instance. + profile_name ([:class:`~couchbase.options.KnownConfigProfiles`, str]): The name of the profile to apply + toward ClusterOptions. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the specified profile is not registered. + + """ # noqa: E501 + opts = cls(authenticator) + prof_name = profile_name.value if isinstance(profile_name, KnownConfigProfiles) else profile_name + CONFIG_PROFILES.apply_profile(prof_name, opts) + return opts + +# Diagnostics Operations + + +class PingOptions(PingOptionsBase): + """Available options to for a ping operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + report_id (str, optional): A unique identifier for the report generated by this operation. + service_types (Iterable[class:`~couchbase.diagnostics.ServiceType`]): The services which should be pinged. + """ + + +class DiagnosticsOptions(DiagnosticsOptionsBase): + """Available options to for a diagnostics operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + report_id (str, optional): A unique identifier for the report generated by this operation. + """ + + +class WaitUntilReadyOptions(WaitUntilReadyOptionsBase): + """Available options to for a wait until ready operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + desired_state (:class:`~.couchbase.diagnostics.ClusterState`, optional): The desired state to wait for in + order to determine the cluster or bucket is ready. Defaults to `Online`. + service_types (Iterable[class:`~couchbase.diagnostics.ServiceType`]): The services which should be pinged. + """ + + +# Key-Value Operations + +class OptionsTimeout(OptionsTimeoutBase): + pass + + +class DurabilityOptionBlock(DurabilityOptionBlockBase): + pass + + +class ExistsOptions(ExistsOptionsBase): + """Available options to for a key-value exists operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + """ + + +class GetOptions(GetOptionsBase): + """Available options to for a key-value get operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + with_expiry (bool, optional): Indicates that the expiry of the document should be + fetched alongside the data itself. Defaults to False. + project (Iterable[str], optional): Specifies a list of fields within the document which should be fetched. + This allows for easy retrieval of select fields without incurring the overhead of fetching the + whole document. + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ + + +class GetAllReplicasOptions(GetAllReplicasOptionsBase): + """Available options to for a key-value get and touch operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + read_preference(:class:`~couchbase.replica_reads.ReadPreference`, optional): Specifies how the replica nodes + will be selected. Defaults to no preference. + """ + + +class GetAndLockOptions(GetAndLockOptionsBase): + """Available options to for a key-value get and lock operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ + + +class GetAndTouchOptions(GetAndTouchOptionsBase): + """Available options to for a key-value get and touch operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ + + +class GetAnyReplicaOptions(GetAnyReplicaOptionsBase): + """Available options to for a key-value get and touch operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + read_preference(:class:`~couchbase.replica_reads.ReadPreference`, optional): Specifies how the replica nodes + will be selected. Defaults to no preference. + """ + + +class InsertOptions(InsertOptionsBase): + """Available options to for a key-value insert operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + expiry (timedelta, optional): Specifies the expiry time for this document. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ + + +class RemoveOptions(RemoveOptionsBase): + """Available options to for a key-value remove operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + """ + + +class ReplaceOptions(ReplaceOptionsBase): + """Available options to for a key-value replace operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + expiry (timedelta, optional): Specifies the expiry time for this document. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + preserve_expiry (bool, optional): Specifies that any existing expiry on the document should be preserved. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ + + +class TouchOptions(TouchOptionsBase): + """Available options to for a key-value exists operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + """ + + +class UnlockOptions(UnlockOptionsBase): + """Available options to for a key-value exists operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + """ + + +class UpsertOptions(UpsertOptionsBase): + """Available options to for a key-value upsert operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + expiry (timedelta, optional): Specifies the expiry time for this document. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + preserve_expiry (bool, optional): Specifies that any existing expiry on the document should be preserved. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ + + +class ScanOptions(ScanOptionsBase): + """Available options to for a key-value scan operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + range scan operation timeout. + ids_only (bool, optional): Specifies that scan should only return document ids. Defaults to False. + consistent_with (:class:`~couchbase.mutation_state.MutationState`, optional): Specifies a + :class:`~couchbase.mutation_state.MutationState` which the scan should be consistent with. Defaults to None. + batch_byte_limit (int, optional): The limit applied to the number of bytes returned from the server + for each partition batch. Defaults to 15k. + batch_item_limit (int, optional): The limit applied to the number of items returned from the server + for each partition batch. Defaults to 50. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~couchbase.transcoder.JsonTranscoder`. + concurrency (int, optional): The upper bound on the number of vbuckets that should be scanned in parallel. + Defaults to 1. + """ # noqa: E501 + + +# Sub-document Operations + + +class LookupInOptions(LookupInOptionsBase): + """Available options to for a subdocument lookup-in operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + subdocument operation timeout. + """ + + +class LookupInAnyReplicaOptions(LookupInAnyReplicaOptionsBase): + """Available options to for a subdocument lookup-in operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + subdocument operation timeout. + read_preference(:class:`~couchbase.replica_reads.ReadPreference`, optional): Specifies how the replica nodes + will be selected. Defaults to no preference. + """ + + +class LookupInAllReplicasOptions(LookupInAllReplicasOptionsBase): + """Available options to for a subdocument lookup-in operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + subdocument operation timeout. + read_preference(:class:`~couchbase.replica_reads.ReadPreference`, optional): Specifies how the replica nodes + will be selected. Defaults to no preference. + """ + + +class MutateInOptions(MutateInOptionsBase): + """Available options to for a subdocument mutate-in operation. + + .. warning:: + Importing options from ``couchbase.subdocument`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + subdocument operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + preserve_expiry (bool, optional): Specifies that any existing expiry on the document should be preserved. + store_semantics (:class:`~couchbase.subdocument.StoreSemantics`, optional): Specifies the store semantics + to use for this operation. + """ + +# Binary Operations + + +class AppendOptions(AppendOptionsBase): + """Available options to for a binary append operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + """ + + +class PrependOptions(PrependOptionsBase): + """Available options to for a binary prepend operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + """ + + +class IncrementOptions(IncrementOptionsBase): + """Available options to for a binary increment operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + delta (:class:`.DeltaValue`, optional): The amount to increment the key. Defaults to 1. + initial (:class:`.SignedInt64`, optional): The initial value to use for the document if it does not already + exist. Setting it to a negative value means that the document will not be created if it does not exist. + Defaults to 0. + """ + + +class DecrementOptions(DecrementOptionsBase): + """Available options to for a decrement append operation. + + .. warning:: + Importing options from ``couchbase.collection`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + delta (:class:`.DeltaValue`, optional): The amount to increment the key. Defaults to 1. + initial (:class:`.SignedInt64`, optional): The initial value to use for the document if it does not already + exist. Setting it to a negative value means that the document will not be created if it does not exist. + Defaults to 0. + """ + + +""" + +Multi-operations Options + +""" + + +class GetAllReplicasMultiOptions(dict): + """Available options to for a key-value multi-get-all-replicas operation. + + Options can be set at a global level (i.e. for all get operations handled with this multi-get-all-replicas + operation). Use *per_key_options* to set specific :class:`.GetAllReplicasOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + read_preference(:class:`~couchbase.replica_reads.ReadPreference`, optional): Specifies how the replica nodes + will be selected. Defaults to no preference. + per_key_options (Dict[str, :class:`.GetAllReplicasOptions`], optional): Specify + :class:`.GetAllReplicasOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + transcoder=None, # type: Optional[Transcoder] + read_preference=None, # type: Optional[ReadPreference] + per_key_options=None, # type: Dict[str, GetAllReplicasOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'transcoder', 'read_preference', 'per_key_options', 'return_exceptions'] + + +class GetAnyReplicaMultiOptions(dict): + """Available options to for a key-value multi-get-any-replica operation. + + Options can be set at a global level (i.e. for all get operations handled with this multi-get-any-replica + operation). Use *per_key_options* to set specific :class:`.GetAnyReplicaOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + read_preference(:class:`~couchbase.replica_reads.ReadPreference`, optional): Specifies how the replica nodes + will be selected. Defaults to no preference. + per_key_options (Dict[str, :class:`.GetAnyReplicaOptions`], optional): Specify + :class:`.GetAnyReplicaOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + transcoder=None, # type: Optional[Transcoder] + read_preference=None, # type: Optional[ReadPreference] + per_key_options=None, # type: Dict[str, GetAnyReplicaOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'transcoder', 'read_preference', 'per_key_options', 'return_exceptions'] + + +class GetMultiOptions(dict): + """Available options to for a key-value multi-get operation. + + Options can be set at a global level (i.e. for all get operations handled with this multi-get operation). + Use *per_key_options* to set specific :class:`.GetOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + with_expiry (bool, optional): Indicates that the expiry of the document should be + fetched alongside the data itself. Defaults to False. + project (Iterable[str], optional): Specifies a list of fields within the document which should be fetched. + This allows for easy retrieval of select fields without incurring the overhead of fetching the + whole document. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + per_key_options (Dict[str, :class:`.GetOptions`], optional): Specify :class:`.GetOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + with_expiry=None, # type: bool + project=None, # type: Iterable[str] + transcoder=None, # type: Transcoder + per_key_options=None, # type: Dict[str, GetOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'with_expiry', 'project', 'transcoder', + 'per_key_options', 'return_exceptions'] + + +class ExistsMultiOptions(dict): + """Available options to for a key-value multi-exists operation. + + Options can be set at a global level (i.e. for all exists operations handled with this multi-exists operation). + Use *per_key_options* to set specific :class:`.ExistsOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + per_key_options (Dict[str, :class:`.ExistsOptions`], optional): Specify :class:`.ExistsOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + per_key_options=None, # type: Dict[str, ExistsOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'per_key_options', 'return_exceptions'] + + +class UpsertMultiOptions(dict): + """Available options to for a key-value multi-upsert operation. + + Options can be set at a global level (i.e. for all upsert operations handled with this multi-upsert operation). + Use *per_key_options* to set specific :class:`.UpsertOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + expiry (timedelta, optional): Specifies the expiry time for this document. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + preserve_expiry (bool, optional): Specifies that any existing expiry on the document should be preserved. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + per_key_options (Dict[str, :class:`.UpsertOptions`], optional): Specify :class:`.UpsertOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + expiry=None, # type: timedelta + preserve_expiry=False, # type: bool + durability=None, # type: DurabilityType + transcoder=None, # type: Transcoder + per_key_options=None, # type: Dict[str, UpsertOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'expiry', 'preserve_expiry', 'durability', + 'transcoder', 'per_key_options', 'return_exceptions'] + + +class InsertMultiOptions(dict): + """Available options to for a key-value multi-insert operation. + + Options can be set at a global level (i.e. for all insert operations handled with this multi-insert operation). + Use *per_key_options* to set specific :class:`.InsertOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + expiry (timedelta, optional): Specifies the expiry time for this document. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + per_key_options (Dict[str, :class:`.InsertOptions`], optional): Specify :class:`.InsertOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + expiry=None, # type: timedelta + durability=None, # type: DurabilityType + transcoder=None, # type: Transcoder + per_key_options=None, # type: Dict[str, InsertOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'expiry', 'durability', 'transcoder', 'per_key_options', 'return_exceptions'] + + +class ReplaceMultiOptions(dict): + """Available options to for a key-value multi-replace operation. + + Options can be set at a global level (i.e. for all replace operations handled with this multi-replace operation). + Use *per_key_options* to set specific :class:`.ReplaceOptions` for specific keys. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + expiry (timedelta, optional): Specifies the expiry time for this document. + preserve_expiry (bool, optional): Specifies that any existing expiry on the document should be preserved. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + per_key_options (Dict[str, :class:`.ReplaceOptions`], optional): Specify :class:`.ReplaceOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + expiry=None, # type: timedelta + cas=0, # type: int + preserve_expiry=False, # type: bool + durability=None, # type: DurabilityType + transcoder=None, # type: Transcoder + per_key_options=None, # type: Dict[str, ReplaceOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'expiry', 'cas', 'preserve_expiry', + 'durability', 'transcoder', 'per_key_options', 'return_exceptions'] + + +class RemoveMultiOptions(dict): + """Available options to for a key-value multi-remove operation. + + Options can be set at a global level (i.e. for all remove operations handled with this multi-remove operation). + Use *per_key_options* to set specific :class:`.RemoveOptions` for specific keys. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + transcoder (:class:`~couchbase.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + per_key_options (Dict[str, :class:`.RemoveOptions`], optional): Specify :class:`.RemoveOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + cas=0, # type: int + durability=None, # type: DurabilityType + transcoder=None, # type: Transcoder + per_key_options=None, # type: Dict[str, RemoveOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'cas', 'durability', 'transcoder', 'per_key_options', 'return_exceptions'] + + +class TouchMultiOptions(dict): + """Available options to for a key-value multi-touch operation. + + Options can be set at a global level (i.e. for all touch operations handled with this multi-touch operation). + Use *per_key_options* to set specific :class:`.TouchOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + per_key_options (Dict[str, :class:`.TouchOptions`], optional): Specify :class:`.TouchOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + per_key_options=None, # type: Dict[str, TouchOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'expiry', 'per_key_options', 'return_exceptions'] + + +class GetAndLockMultiOptions(dict): + """Available options to for a key-value multi-lock operation. + + Options can be set at a global level (i.e. for all lock operations handled with this multi-lock operation). + Use *per_key_options* to set specific :class:`.GetAndLockOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + per_key_options (Dict[str, :class:`.GetAndLockOptions`], optional): Specify :class:`.GetAndLockOptions` per + key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + transcoder=None, # type: Transcoder + per_key_options=None, # type: Dict[str, GetAndLockOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'transcoder', 'per_key_options', 'return_exceptions'] + + +LockMultiOptions = GetAndLockMultiOptions + + +class UnlockMultiOptions(dict): + """Available options to for a key-value multi-unlock operation. + + Options can be set at a global level (i.e. for all unlock operations handled with this multi-unlock operation). + Use *per_key_options* to set specific :class:`.UnlockOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + per_key_options (Dict[str, :class:`.UnlockOptions`], optional): Specify :class:`.UnlockOptions` per + key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ + @overload + def __init__( + self, + timeout=None, # type: timedelta + per_key_options=None, # type: Dict[str, UnlockOptions] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'per_key_options', 'return_exceptions'] + + +class IncrementMultiOptions(dict): + """Available options to for a binary multi-increment operation. + + Options can be set at a global level (i.e. for all increment operations handled with this multi-increment operation). + Use *per_key_options* to set specific :class:`.IncrementOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + delta (:class:`.DeltaValue`, optional): The amount to increment the key. Defaults to 1. + initial (:class:`.SignedInt64`, optional): The initial value to use for the document if it does not already + exist. Defaults to 0. + per_key_options (Dict[str, :class:`.IncrementOptions`], optional): Specify :class:`.IncrementOptions` per + key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ # noqa: E501 + @overload + def __init__( + self, + timeout=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + delta=None, # type: Optional[DeltaValue] + initial=None, # type: Optional[SignedInt64] + span=None, # type: Optional[Any] + per_key_options=None, # type: Optional[Dict[str, IncrementOptions]] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'durability', 'delta', + 'initial', 'span', 'per_key_options', 'return_exceptions'] + + +class DecrementMultiOptions(dict): + """Available options to for a binary multi-decrement operation. + + Options can be set at a global level (i.e. for all decrement operations handled with this multi-decrement operation). + Use *per_key_options* to set specific :class:`.DecrementOptions` for specific keys. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + delta (:class:`.DeltaValue`, optional): The amount to decrement the key. Defaults to 1. + initial (:class:`.SignedInt64`, optional): The initial value to use for the document if it does not already + exist. Defaults to 0. + per_key_options (Dict[str, :class:`.DecrementOptions`], optional): Specify :class:`.DecrementOptions` per + key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ # noqa: E501 + @overload + def __init__( + self, + timeout=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + delta=None, # type: Optional[DeltaValue] + initial=None, # type: Optional[SignedInt64] + span=None, # type: Optional[Any] + per_key_options=None, # type: Optional[Dict[str, DecrementOptions]] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'durability', 'delta', + 'initial', 'span', 'per_key_options', 'return_exceptions'] + + +class AppendMultiOptions(dict): + """Available options to for a binary multi-append operation. + + Options can be set at a global level (i.e. for all append operations handled with this multi-append operation). + Use *per_key_options* to set specific :class:`.AppendOptions` for specific keys. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + per_key_options (Dict[str, :class:`.AppendOptions`], optional): Specify :class:`.AppendOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ # noqa: E501 + @overload + def __init__( + self, + timeout=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + cas=None, # type: Optional[int] + span=None, # type: Optional[Any] + per_key_options=None, # type: Optional[Dict[str, AppendOptions]] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'durability', 'cas', + 'span', 'per_key_options', 'return_exceptions'] + + +class PrependMultiOptions(dict): + """Available options to for a binary multi-prepend operation. + + Options can be set at a global level (i.e. for all prepend operations handled with this multi-prepend operation). + Use *per_key_options* to set specific :class:`.PrependOptions` for specific keys. + + Args: + cas (int, optional): If specified, indicates that operation should be failed if the CAS has changed from + this value, indicating that the document has changed. + timeout (timedelta, optional): The timeout for this operation. Defaults to global + key-value operation timeout. + durability (:class:`~couchbase.durability.DurabilityType`, optional): Specifies the level of durability + for this operation. + per_key_options (Dict[str, :class:`.PrependOptions`], optional): Specify :class:`.PrependOptions` per key. + return_exceptions(bool, optional): If False, raise an Exception when encountered. If True return the + Exception without raising. Defaults to True. + """ # noqa: E501 + @overload + def __init__( + self, + timeout=None, # type: Optional[timedelta] + durability=None, # type: Optional[DurabilityType] + cas=None, # type: Optional[int] + span=None, # type: Optional[Any] + per_key_options=None, # type: Optional[Dict[str, PrependOptions]] + return_exceptions=None # type: Optional[bool] + ): + pass + + def __init__(self, **kwargs): + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + @classmethod + def get_valid_keys(cls): + return ['timeout', 'durability', 'cas', + 'span', 'per_key_options', 'return_exceptions'] + + +NoValueMultiOptions = Union[GetMultiOptions, ExistsMultiOptions, + RemoveMultiOptions, TouchMultiOptions, LockMultiOptions, UnlockMultiOptions] +MutationMultiOptions = Union[InsertMultiOptions, UpsertMultiOptions, ReplaceMultiOptions] +MutationOptions = Union[InsertOptions, + UpsertOptions, + ReplaceOptions, + RemoveOptions, + MutateInOptions, + AppendOptions, + PrependOptions, + IncrementOptions, + DecrementOptions] + + +class QueryOptions(QueryOptionsBase): + """Available options to for a N1QL (SQL++) query. + + .. warning:: + Importing options from ``couchbase.cluster`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + query operation timeout. + read_only (bool, optional): Specifies that this query should be executed in read-only mode, + disabling the ability for the query to make any changes to the data. Defaults to False. + scan_consistency (:class:`~couchbase.n1ql.QueryScanConsistency`, optional): Specifies the consistency + requirements when executing the query. + adhoc (bool, optional): Specifies whether this is an ad-hoc query, or if it should be prepared for + faster execution in the future. Defaults to True. + client_context_id (str, optional): The returned client context id for this query. Defaults to None. + max_parallelism (int, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + positional_parameters (Iterable[JSONType], optional): Positional values to be used for the placeholders + within the query. Defaults to None. + named_parameters (Iterable[Dict[str, JSONType]], optional): Named values to be used for the placeholders + within the query. Defaults to None. + pipeline_batch (int, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + pipeline_cap (int, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + profile (:class:`~couchbase.n1ql.QueryProfile`, optional): Specifies the level of profiling that should + be used for the query. Defaults to `Off`. + query_context (str, optional): Specifies the context within which this query should be executed. This can + be scoped to a scope or a collection within the dataset. Defaults to None. + scan_cap (int, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + scan_wait (timedelta, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + metrics (bool, optional): Specifies whether metrics should be captured as part of the execution of the query. + Defaults to False. + flex_index (bool, optional): Specifies whether flex-indexes should be enabled. Allowing the use of full-text + search from the query service. Defaults to False. + preserve_expiry (bool, optional): Specifies that any existing expiry on the document should be preserved. + Defaults to False. + use_replica (bool, optional): Specifies that the query engine should use replica nodes for KV fetches if the + active node is down. Defaults to None. + consistent_with (:class:`~couchbase.mutation_state.MutationState`, optional): Specifies a + :class:`~couchbase.mutation_state.MutationState` which the query should be consistent with. Defaults to + None. + serializer (:class:`~couchbase.serializer.Serializer`, optional): Specifies an explicit serializer + to use for this specific N1QL operation. Defaults to + :class:`~couchbase.serializer.DefaultJsonSerializer`. + raw (Dict[str, Any], optional): Specifies any additional parameters which should be passed to the query engine + when executing the query. Defaults to None. + """ + + +class AnalyticsOptions(AnalyticsOptionsBase): + """Available options to for an analytics query. + + .. warning:: + Importing options from ``couchbase.analytics`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + query operation timeout. + read_only (bool, optional): Specifies that this analytics query should be executed in read-only mode, + disabling the ability for the query to make any changes to the data. Defaults to False. + scan_consistency (:class:`~couchbase.analytics.AnalyticsScanConsistency`, optional): Specifies the consistency + requirements when executing the analytics query. + client_context_id (str, optional): The returned client context id for this analytics query. Defaults to None. + positional_parameters (Iterable[JSONType], optional): Positional values to be used for the placeholders + within the analytics query. Defaults to None. + named_parameters (Iterable[Dict[str, JSONType]], optional): Named values to be used for the placeholders + within the analytics query. Defaults to None. + priority (bool, optional): Indicates whether this analytics query should be executed with a specific priority + level. Defaults to False. + query_context (str, optional): Specifies the context within which this analytics query should be executed. + This can be scoped to a scope or a collection within the dataset. Defaults to None. + serializer (:class:`~couchbase.serializer.Serializer`, optional): Specifies an explicit serializer + to use for this specific analytics query. Defaults to + :class:`~couchbase.serializer.DefaultJsonSerializer`. + raw (Dict[str, Any], optional): Specifies any additional parameters which should be passed to the analytics + query engine when executing the analytics query. Defaults to None. + """ + + +class SearchOptions(SearchOptionsBase): + """Available options to for a search (FTS) query. + + .. warning:: + Importing options from ``couchbase.search`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + search query operation timeout. + limit (int, optional): Specifies the limit to the number of results that should be returned. + Defaults to None. + skip (int, optional): Specifies the number of results to skip from the index before returning + results. Defaults to None. + explain (bool, optional): Configures whether the result should contain the execution plan for the + search query. Defaults to False. + fields (List[str], optional): Specifies the list of fields which should be searched. Defaults to None. + highlight_style (:class:`~couchbase.search.HighlightStyle`, optional): Specifies the mode used for + highlighting. Defaults to None. + highlight_fields (List[str], optional): Specifies the list of fields that should be highlighted. + Defaults to None. + scan_consistency (:class:`~couchbase.search.SearchScanConsistency`, optional): Specifies the consistency + requirements when executing the search query. + facets (Dict[str, :class:`~couchbase.search.Facet`], optional): Specifies any facets that should be included + in the search query. Defaults to None. + client_context_id (str, optional): The returned client context id for this query. Defaults to None. + disable_scoring (bool, optional): Specifies that scoring should be disabled. This improves performance but + makes it impossible to sort based on how well a particular result scored. Defaults to False. + include_locations (bool optional): If set to True, will include the locations in the search result. + Defaults to False. + sort (Union[List[str],List[:class:`~couchbase.search.Sort`]], optional): Specifies a list of fields or search + :class:`~couchbase.search.Sort`'s to use when sorting the result sets. Defaults to None. + scope_name (string, optional): Specifies the scope which should be searched as part of the search + query. Defaults to None. + collections (List[str], optional): Specifies the collections which should be searched as part of the search + query. Defaults to None. + consistent_with (:class:`~couchbase.mutation_state.MutationState`, optional): Specifies a + :class:`~couchbase.mutation_state.MutationState` which the search query should be consistent with. + Defaults to None. + serializer (:class:`~couchbase.serializer.Serializer`, optional): Specifies an explicit serializer + to use for this specific search query. Defaults to + :class:`~couchbase.serializer.DefaultJsonSerializer`. + raw (Dict[str, Any], optional): Specifies any additional parameters which should be passed to the search query + engine when executing the search query. Defaults to None. + show_request (bool, optional): Specifies if the search response should contain the request for the search query. Defaults to False. + log_request (bool, optional): **UNCOMMITTED** Specifies if search request body should appear the log. Defaults to False. + log_response (bool, optional): **UNCOMMITTED** Specifies if search response should appear in the log. Defaults to False. + """ # noqa: E501 + + +class VectorSearchOptions(VectorSearchOptionsBase): + """Available options to for a FTS vector search. + + Args: + vector_query_combination (:class:`~couchbase.vector_search.VectorQueryCombination`, optional): Specifies logical operation + to use with multiple vector queries. + """ # noqa: E501 + + +class ViewOptions(ViewOptionsBase): + """Available options to for a view query. + + .. warning:: + Importing options from ``couchbase.bucket`` is deprecated. + All options should be imported from ``couchbase.options``. + + Args: + timeout (timedelta, optional): The timeout for this operation. Defaults to global + view query operation timeout. + scan_consistency (:class:`~couchbase.views.ViewScanConsistency`, optional): Specifies the consistency + requirements when executing the view query. Defaults to None. + limit (int, optional): Specifies the limit to the number of results that should be returned. + Defaults to None. + skip (int, optional): Specifies the number of results to skip from the index before returning + results. Defaults to None. + startkey (JSONType, optional): Specifies the first key that should be included in the results. + Defaults to None. + endkey (JSONType, optional): Specifies the last key that should be included in the results. + Defaults to None. + startkey_docid (str, optional): Specifies the first document ID that should be included in the results. + Defaults to None. + endkey_docid (str, optional): Specifies the last document ID that should be included in the results. + Defaults to None. + inclusive_end (bool, optional): Specifies whether the end key should be considered inclusive or exclusive. + Defaults to None. + group (bool, optional): Specifies whether the results should be grouped together. Defaults to None. + group_level (int, optional): Specifies the level to which results should be group. Defaults to None. + key (JSONType, optional): Specifies a specific key which should be fetched from the index. Defaults to None. + keys (List[JSONType], optional): Specifies a list of keys which should be fetched from the index. + Defaults to None. + order (:class:`~couchbase.views.ViewOrdering`, optional): Specifies the ordering that should be used when + returning results. Defaults to None. + reduce (bool, optional): Specifies whether reduction should be performed as part of the view query. + Defaults to None. + on_error (:class:`~couchbase.views.ViewErrorMode`, optional): Specifies the error-handling behaviour + that should be used when an error occurs. Defaults to None. + namespace(:class:`~couchbase.management.views.DesignDocumentNamespace`, optional): Specifies the namespace + for the design document. Defaults to ``Development``. + client_context_id (str, optional): The returned client context id for this view query. Defaults to None. + raw (Dict[str, str], optional): Specifies any additional parameters which should be passed to the view engine + when executing the view query. Defaults to None. + full_set (bool, optional): Specifies whether the query should force the entire set of document in the index + to be included in the result. Defaults to None. + """ + + +""" + +Couchbase Python SDK constrained integer classes + +""" + + +class ConstrainedInt(ConstrainedIntBase): + pass + + +class SignedInt64(SignedInt64Base): + pass + + +class UnsignedInt32(UnsignedInt32Base): + pass + + +class UnsignedInt64(UnsignedInt64Base): + pass + + +class DeltaValue(DeltaValueBase): + pass + + +class TransactionConfig: + _TXN_ALLOWED_KEYS = {"durability", "cleanup_window", "timeout", + "expiration_time", "cleanup_lost_attempts", "cleanup_client_attempts", + "metadata_collection", "scan_consistency"} + + @overload + def __init__(self, + durability=None, # type: Optional[ServerDurability] + cleanup_window=None, # type: Optional[timedelta] + kv_timeout=None, # type: Optional[timedelta] + expiration_time=None, # type: Optional[timedelta] + cleanup_lost_attempts=None, # type: Optional[bool] + cleanup_client_attempts=None, # type: Optional[bool] + metadata_collection=None, # type: Optional[TransactionKeyspace] + scan_consistency=None # type: Optional[QueryScanConsistency] + ): + """ + Configuration for Transactions. + + Args: + durability (:class:`ServerDurability`, optional): Desired durability level for all transaction operations. + cleanup_window (timedelta, optional): The query metadata is cleaned up over a the cleanup_window. + Longer windows mean less background activity, shorter intervals will clean things faster. + kv_timeout: (timedelta, optional): **DEPRECATED** Currently a no-op. KV operation timeout. + expiration_time: (timedelta, optional): **DEPRECATED** Use timeout instead. Maximum amount of time a transaction can take before rolling back. + cleanup_lost_attempts: (bool, optional): If False, then we don't do any background cleanup. + cleanup_client_attempts: (bool, optional): if False, we don't do any cleanup as a transaction finishes. + metadata_collection: (:class:`couchbase.transactions.TransactionKeyspace, optional): All transaction + metadata uses the specified bucket/scope/collection. + scan_consistency: (:class:`QueryScanConsistency`, optional): Scan consistency to use for all transactional + queries. + timeout: (timedelta, optional): Maximum amount of time a transaction can take before rolling back. + """ # noqa: E501 + + def __init__(self, # noqa: C901 + **kwargs # type: dict[str, Any] + ): # noqa: C901 + # CXXCBC-391: Adds support for ExtSDKIntegration which removes kv_timeout, the cluster kv_durable + # timeout is used internally + if 'kv_timeout' in kwargs: + kwargs.pop('kv_timeout') + Supportability.option_deprecated('kv_timeout') + kwargs = {k: v for k, v in kwargs.items() if k in TransactionConfig._TXN_ALLOWED_KEYS} + # convert everything here... + durability = kwargs.pop("durability", None) + if durability: + kwargs["durability_level"] = durability.level.value + if kwargs.get('cleanup_window', None): + kwargs['cleanup_window'] = int(kwargs['cleanup_window'].total_seconds() * 1000000) + coll = kwargs.pop("metadata_collection", None) + # CXXCBC-391: Adds support for ExtSDKIntegration which changes expiration_time -> timeout + if 'expiration_time' in kwargs or 'timeout' in kwargs: + Supportability.option_deprecated('expiration_time', 'timeout') + timeout = kwargs.pop('expiration_time', None) + # if timeout is also in the options, override expiration_time + if 'timeout' in kwargs: + timeout = kwargs.get('timeout', None) + if timeout: + kwargs['timeout'] = int(timeout.total_seconds() * 1000000) + if coll: + kwargs["metadata_bucket"] = coll.bucket + kwargs["metadata_scope"] = coll.scope + kwargs["metadata_collection"] = coll.collection + # don't pass None + if kwargs.get('scan_consistency', None): + kwargs['scan_consistency'] = kwargs['scan_consistency'].value + if kwargs["scan_consistency"] == "at_plus": + raise InvalidArgumentException("QueryScanConsistency.AT_PLUS not valid for transactions") + for key in [k for k, v in kwargs.items() if v is None]: + del (kwargs[key]) + self._base = transaction_config(**kwargs) + + +class TransactionOptions: + _TXN_ALLOWED_KEYS = {"durability", "timeout", "expiration_time", "scan_consistency", "metadata_collection"} + + @overload + def __init__(self, + durability=None, # type: Optional[ServerDurability] + kv_timeout=None, # type: Optional[timedelta] + expiration_time=None, # type: Optional[timedelta] + scan_consistency=None, # type: Optional[QueryScanConsistency] + metadata_collection=None, # type: Optional[Collection] + timeout=None, # type: Optional[timedelta] + ): + """ + Overrides a subset of the ``TransactionConfig`` parameters for a single query. + Args: + durability (:class:`ServerDurability`, optional): Desired durability level for all operations + in this transaction. + kv_timeout: (timedelta, optional): **DEPRECATED** Currently a no-op. KV timeout to use for this transaction. + expiration_time: (timedelta, optional): **DEPRECATED** Use timeout instead. Expiry for this transaction. + scan_consistency: (:class:`QueryScanConsistency`, optional): Scan consistency for queries in + this transaction. + metadata_collection: (:class: `couchbase.collection.Collection, optional): This transaction will + put all metadata in the specified bucket/scope/collection. + timeout: (timedelta, optional): Expiry for this transaction. + """ # noqa: E501 + + def __init__(self, + **kwargs # type: Dict[str, Any] + ): + # CXXCBC-391: Adds support for ExtSDKIntegration which removes kv_timeout, the cluster kv_durable + # timeout is used internally + if 'kv_timeout' in kwargs: + kwargs.pop('kv_timeout') + Supportability.option_deprecated('kv_timeout') + kwargs = {k: v for k, v in kwargs.items() if k in TransactionOptions._TXN_ALLOWED_KEYS} + # convert everything here... + durability = kwargs.pop("durability", None) + if durability: + kwargs["durability_level"] = durability.level.value + # CXXCBC-391: Adds support for ExtSDKIntegration which changes expiration_time -> timeout + if 'expiration_time' in kwargs or 'timeout' in kwargs: + Supportability.option_deprecated('expiration_time', 'timeout') + timeout = kwargs.pop('expiration_time', None) + # if timeout is also in the options, override expiration_time + if 'timeout' in kwargs: + timeout = kwargs.get('timeout', None) + if timeout: + kwargs['timeout'] = int(timeout.total_seconds() * 1000000) + if kwargs.get('scan_consistency', None): + kwargs['scan_consistency'] = kwargs['scan_consistency'].value + if kwargs["scan_consistency"] == "at_plus": + raise InvalidArgumentException("QueryScanConsistency.AT_PLUS not valid for transactions") + coll = kwargs.pop('metadata_collection', None) + if coll: + kwargs['metadata_bucket'] = coll.bucket + kwargs['metadata_scope'] = coll.scope + kwargs['metadata_collection'] = coll.collection + # don't pass None + for key in [k for k, v in kwargs.items() if v is None]: + del (kwargs[key]) + self._base = transaction_options(**kwargs) + + def __str__(self): + return f'TransactionOptions(base_:{self._base}' + + +# @TODO: lets replace this.... + +OptionsBase = dict +T = TypeVar("T", bound=OptionsBase) + + +class Forwarder(metaclass=ABCMeta): + def forward_args( + self, + arg_vars, # type: Optional[Dict[str,Any]] + *options # type: OptionsBase + ): + # type: (...) -> OptionsBase[str,Any] + arg_vars = copy.copy(arg_vars) if arg_vars else {} + temp_options = ( + copy.copy( + options[0]) if ( + options and options[0]) else OptionsBase()) + kwargs = arg_vars.pop("kwargs", {}) + temp_options.update(kwargs) + temp_options.update(arg_vars) + + end_options = {} + for k, v in temp_options.items(): + map_item = self.arg_mapping().get(k, None) + if not (map_item is None): + for out_k, out_f in map_item.items(): + converted = out_f(v) + if converted is not None: + end_options[out_k] = converted + else: + end_options[k] = v + return end_options + + @abstractmethod + def arg_mapping(self): + pass + + +class DefaultForwarder(Forwarder): + def arg_mapping(self): + return { + "spec": {"specs": lambda x: x}, + "id": {}, + "timeout": {"timeout": timedelta_as_microseconds}, + "expiry": {"expiry": timedelta_as_timestamp}, + "lock_time": {"lock_time": lambda x: int(x.total_seconds())}, + "self": {}, + "options": {}, + "durability": { + "durability": DurabilityParser.parse_durability}, + "disable_scoring": { + "disable_scoring": lambda dis_score: True if dis_score else None + }, + "preserve_expiry": {"preserve_expiry": lambda x: x}, + "report_id": {"report_id": lambda x: str(x)}, + "batch_byte_limit": {"batch_byte_limit": validate_int}, + "batch_item_limit": {"batch_item_limit": validate_int}, + "concurrency": {"concurrency": validate_int}, + "read_preference": {"read_preference": lambda r: r.value} + } + + +forward_args = DefaultForwarder().forward_args + + +class TransactionQueryOptions: + ALLOWED_KEYS = {"raw", "adhoc", "scan_consistency", "profile", "client_context_id", + "scan_wait", "read_only", "scan_cap", "pipeline_batch", "pipeline_cap", + "scope", "metrics", "max_parallelism", "positional_parameters", "named_parameters"} + + @overload + def __init__(self, + raw=None, # type: Optional[Dict[str, JSONType]] + adhoc=None, # type: Optional[bool] + scan_consistency=None, # type: Optional[QueryScanConsistency] + profile=None, # type: Optional[Any] + client_context_id=None, # type: Optional[str] + scan_wait=None, # type: Optional[timedelta] + read_only=None, # type: Optional[bool] + scan_cap=None, # type: Optional[int] + pipeline_batch=None, # type: Optional[int] + pipeline_cap=None, # type: Optional[int] + positional_parameters=None, # type: Optional[Iterable[JSONType]] + named_parameters=None, # type: Optional[Dict[str, JSONType]] + scope=None, # type: Optional[Any] + metrics=None, # type: Optional[bool] + max_parallelism=None # type: Optional[int] + ): + """ + QueryOptions for transactions. + + Args: + raw (Dict[str, Any], optional): Specifies any additional parameters which should be passed to the query + engine when executing the query. Defaults to None. + adhoc (bool, optional): Specifies whether this is an ad-hoc query, or if it should be prepared for + faster execution in the future. Defaults to True. + scan_consistency (:class:`~couchbase.analytics.AnalyticsScanConsistency`, optional): Specifies + the consistency requirements when executing the transactional query. + profile (:class:`~couchbase.n1ql.QueryProfile`, optional): Specifies the level of profiling that should + be used for the transactional query. Defaults to `Off`. + client_context_id (str, optional): Specifies an client id for this query. This is returned with the + response, and can be helpful when debugging. + scan_cap (int, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + scan_wait (timedelta, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + metrics (bool, optional): Specifies whether metrics should be captured as part of the execution of the + query. Defaults to False. + read_only: (bool, optional): Specifies that the query should be considered read-only, and not allowed to + mutate documents on the server-side. See query service reference for more details. + pipeline_batch (int, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + pipeline_cap (int, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + positional_parameters (Iterable[JSONType], optional): Positional values to be used for the placeholders + within the query. Defaults to None. + named_parameters (Dict[str, JSONType], optional): Named values to be used for the placeholders + within the query. Defaults to None. + scope (Union[:class:`~acouchbase.scope.Scope`,:class:`~couchbase.scope.Scope`], optional): Specify the + scope of the query. Defaults to None. + max_parallelism (int, optional): This is an advanced option, see the query service reference for more + information on the proper use and tuning of this option. Defaults to None. + """ + pass + + def __init__(self, # noqa: C901 + **kwargs # type: Dict[str, JSONType] + ): + kwargs = {k: v for k, v in kwargs.items() if k in TransactionQueryOptions.ALLOWED_KEYS} + # TODO: mapping similar to the options elsewhere. + scope = kwargs.pop("scope", None) + if scope: + kwargs["bucket_name"] = scope.bucket_name + kwargs["scope_name"] = scope.name + if kwargs.get("scan_wait", None): + kwargs["scan_wait"] = kwargs["scan_wait"].total_seconds/1000 + if kwargs.get("scan_consistency", None): + kwargs["scan_consistency"] = kwargs["scan_consistency"].value + if kwargs["scan_consistency"] == "at_plus": + raise InvalidArgumentException("QueryScanConsistency.AT_PLUS not valid for transactions") + raw = kwargs.pop('raw', None) + if raw: + kwargs['raw'] = dict() + for k, v in raw.items(): + kwargs['raw'][k] = DefaultJsonSerializer().serialize(v) + adhoc = kwargs.pop("adhoc", None) + if adhoc is not None: + kwargs["adhoc"] = adhoc + readonly = kwargs.pop("read_only", None) + if readonly is not None: + kwargs["readonly"] = readonly + profile = kwargs.pop("profile", None) + if profile: + kwargs["profile_mode"] = profile.value + positional = kwargs.pop("positional_parameters", None) + if positional: + kwargs["positional_parameters"] = list( + map(lambda param: DefaultJsonSerializer().serialize(param), positional)) + named = kwargs.pop("named_parameters", None) + if named: + kwargs["named_parameters"] = {key: DefaultJsonSerializer().serialize(val) for key, val in named.items()} + + self._base = transaction_query_options(query_args=kwargs) + + def split_scope_qualifier(self, + remove_backticks=True # type: Optional[bool] + ) -> Optional[Tuple[str, str]]: + scope_qualifier = self._base.to_dict().get('scope_qualifier', None) + if not scope_qualifier: + return None + + # expected format namespace:`bucket`:`scope` + namespace_tokens = scope_qualifier.split(':') + if len(namespace_tokens) != 2: + return None + + bucket_tokens = namespace_tokens[1].split('.') + if len(bucket_tokens) == 2: + if remove_backticks: + return bucket_tokens[0].replace('`', ''), bucket_tokens[1].replace('`', '') + return bucket_tokens[0], bucket_tokens[1] + + return None + + +class TransactionGetOptions(dict): + """Available options to for transaction get operation. + + Args: + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ # noqa: E501 + + @overload + def __init__(self, + transcoder=None # type: Optional[Transcoder] + ): + ... + + def __init__(self, **kwargs): + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class TransactionGetReplicaFromPreferredServerGroupOptions(dict): + """Available options to for transaction get_replica_from_preferred_server_group operation. + + Args: + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ + + @overload + def __init__(self, + transcoder=None # type: Optional[Transcoder] + ): + ... + + def __init__(self, **kwargs): + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class TransactionInsertOptions(dict): + """Available options to for transaction insert operation. + + Args: + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ # noqa: E501 + + @overload + def __init__(self, + transcoder=None # type: Optional[Transcoder] + ): + ... + + def __init__(self, **kwargs): + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) + + +class TransactionReplaceOptions(dict): + """Available options to for transaction replace operation. + + Args: + transcoder (:class:`~.transcoder.Transcoder`, optional): Specifies an explicit transcoder + to use for this specific operation. Defaults to :class:`~.transcoder.JsonTranscoder`. + """ # noqa: E501 + + @overload + def __init__(self, + transcoder=None # type: Optional[Transcoder] + ): + ... + + def __init__(self, **kwargs): + + kwargs = {k: v for k, v in kwargs.items() if v is not None} + super().__init__(**kwargs) diff --git a/couchbase/otel_tracer.py b/couchbase/otel_tracer.py new file mode 100644 index 000000000..ca85aa3c7 --- /dev/null +++ b/couchbase/otel_tracer.py @@ -0,0 +1,69 @@ +# Copyright 2021, Couchbase, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +from typing import Any + +from opentelemetry.trace import (Span, + Tracer, + set_span_in_context) + +from couchbase.tracing import CouchbaseSpan, CouchbaseTracer + + +class CouchbaseOtelSpan(CouchbaseSpan): + def __init__(self, + otel_span # type: Span + ): + super().__init__(otel_span) + + def set_attribute(self, + key, # type: str + value # type: Any + ): + # type: (...) -> None + self.span.set_attribute(key=key, value=value) + + def finish(self): + # type: (...) -> None + self.span.end() + + +class CouchbaseOtelTracer(CouchbaseTracer): + def __init__(self, + otel_tracer # type: Tracer + ): + # type: (...) -> CouchbaseOtelTracer + super().__init__(otel_tracer) + + def start_span(self, + name, # type: str + parent=None # type: CouchbaseOtelSpan + ): + # type: (...) -> CouchbaseOtelSpan + kwargs = {} + if parent: + kwargs['context'] = set_span_in_context(parent.span) + return CouchbaseOtelSpan( + self._external_tracer.start_span(name, **kwargs)) + + def __deepcopy__(self, memo): + """ + This prevents deepcopies, as the underlying opentelemetry tracer doesn't support a deepcopy. + :param memo: The object that we are copying into. + :return: The copy - None in this case + """ + memo[id(self)] = None + return None diff --git a/couchbase/replica_reads.py b/couchbase/replica_reads.py new file mode 100644 index 000000000..46f85a8c2 --- /dev/null +++ b/couchbase/replica_reads.py @@ -0,0 +1,6 @@ +from enum import IntEnum + + +class ReadPreference(IntEnum): + NO_PREFERENCE = 0 + SELECTED_SERVER_GROUP = 1 diff --git a/couchbase/result.py b/couchbase/result.py index 8f337ce59..89b2bd402 100644 --- a/couchbase/result.py +++ b/couchbase/result.py @@ -1,27 +1,1262 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. # -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from platform import python_implementation - -import couchbase._bootstrap -from couchbase._libcouchbase import ( - Result, - ValueResult, - OperationResult, - HttpResult, - MultiResult, - ObserveInfo, - AsyncResult) +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +from datetime import datetime +from typing import (Any, + Dict, + Optional, + Tuple, + Union) + +from acouchbase.analytics import AsyncAnalyticsRequest +from acouchbase.n1ql import AsyncN1QLRequest +from acouchbase.search import AsyncFullTextSearchRequest +from acouchbase.views import AsyncViewRequest +from couchbase.diagnostics import (ClusterState, + EndpointDiagnosticsReport, + EndpointPingReport, + EndpointState, + ServiceType) +from couchbase.exceptions import ErrorMapper, InvalidArgumentException +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.pycbc_core import result +from couchbase.subdocument import parse_subdocument_content_as, parse_subdocument_exists + + +class Result: + def __init__( + self, + orig, # type: result + ): + + self._orig = orig + + @property + def value(self) -> Optional[Any]: + """ + Optional[Any]: The content of the document, if it exists. + """ + return self._orig.raw_result.get("value", None) + + @property + def cas(self) -> Optional[int]: + """ + Optional[int]: The CAS of the document, if it exists + """ + return self._orig.raw_result.get("cas", 0) + + @property + def flags(self) -> Optional[int]: + """ + Optional[int]: Flags associated with the document. Used for transcoding. + """ + return self._orig.raw_result.get("flags", 0) + + @property + def key(self) -> Optional[str]: + """ + Optional[str]: Key for the operation, if it exists. + """ + return self._orig.raw_result.get("key", None) + + @property + def success(self) -> bool: + """ + bool: Indicates if the operation was successful or not. + """ + return self.cas != 0 + + +class ContentProxy: + """ + Used to provide access to Result content via Result.content_as[type] + """ + + def __init__(self, content): + self._content = content + + def __getitem__(self, + type_ # type: Any + ) -> Any: + """ + + :param type_: the type to attempt to cast the result to + :return: the content cast to the given type, if possible + """ + return type_(self._content) + + +class ContentSubdocProxy: + """ + Used to provide access to LookUpResult content via Result.content_as[type](index) + """ + + def __init__(self, content, key): + self._content = content + self._key = key + + def _parse_content_at_index(self, index, type_): + item = parse_subdocument_content_as(self._content, index, self._key) + return type_(item) + + def __getitem__(self, + type_ # type: Any + ) -> Any: + """ + + :param type_: the type to attempt to cast the result to + :return: the content cast to the given type, if possible + """ + return lambda index: self._parse_content_at_index(index, type_) + + +class DiagnosticsResult(Result): + + def __init__( + self, + orig, # type: result + ): + super().__init__(orig) + svc_endpoints = self._orig.raw_result.get("endpoints", None) + self._endpoints = {} + if svc_endpoints: + for service, endpoints in svc_endpoints.items(): + service_type = ServiceType(service) + self._endpoints[service_type] = [] + for endpoint in endpoints: + self._endpoints[service_type].append( + EndpointDiagnosticsReport(service_type, endpoint)) + + @property + def id(self) -> str: + """ + str: The unique identifier for this report. + """ + return self._orig.raw_result.get("id", None) + + @property + def version(self) -> int: + """ + int: The version number of this report. + """ + return self._orig.raw_result.get("version", None) + + @property + def sdk(self) -> str: + """ + str: The name of the SDK which generated this report. + """ + return self._orig.raw_result.get("sdk", None) + + @property + def endpoints(self) -> Dict[str, Any]: + """ + Dict[str, Any]: A map of service endpoints and their diagnostic status. + """ + return self._endpoints + + @property + def state(self) -> ClusterState: + """ + :class:`~couchbase.diagnostics.ClusterState`: The cluster state. + """ + num_found = 0 + num_connected = 0 + for endpoints in self._endpoints.values(): + for endpoint in endpoints: + num_found += 1 + if endpoint.state == EndpointState.Connected: + num_connected += 1 + + if num_found == num_connected: + return ClusterState.Online + if num_connected > 0: + return ClusterState.Degraded + return ClusterState.Offline + + def as_json(self) -> str: + """Returns a JSON formatted diagnostics report. + + Returns: + str: JSON formatted diagnostics report. + """ + return_val = { + 'version': self.version, + 'id': self.id, + 'sdk': self.sdk, + 'services': {k.value: list(map(lambda epr: epr.as_dict(), v)) for k, v in self.endpoints.items()} + } + + return json.dumps(return_val) + + def __repr__(self): + return "DiagnosticsResult:{}".format(self._orig) + + +class PingResult(Result): + + def __init__( + self, + orig, # type: result + ): + super().__init__(orig) + svc_endpoints = self._orig.raw_result.get("endpoints", None) + self._endpoints = {} + if svc_endpoints: + for service, endpoints in svc_endpoints.items(): + service_type = ServiceType(service) + self._endpoints[service_type] = [] + for endpoint in endpoints: + self._endpoints[service_type].append( + EndpointPingReport(service_type, endpoint)) + + @property + def id(self) -> str: + """ + str: The unique identifier for this report. + """ + return self._orig.raw_result.get("id", None) + + @property + def version(self) -> int: + """ + int: The version number of this report. + """ + return self._orig.raw_result.get("version", None) + + @property + def sdk(self) -> str: + """ + str: The name of the SDK which generated this report. + """ + return self._orig.raw_result.get("sdk", None) + + @property + def endpoints(self) -> Dict[str, Any]: + """ + Dict[str, Any]: A map of service endpoints and their ping status. + """ + return self._endpoints + + def as_json(self) -> str: + """Returns a JSON formatted diagnostics report. + + Returns: + str: JSON formatted diagnostics report. + """ + return_val = { + 'version': self.version, + 'id': self.id, + 'sdk': self.sdk, + 'services': {k.value: list(map(lambda epr: epr.as_dict(), v)) for k, v in self.endpoints.items()} + } + + return json.dumps(return_val) + + def __repr__(self): + return "PingResult:{}".format(self._orig) + + +class GetReplicaResult(Result): + + @property + def is_active(self) -> bool: + """ + ** DEPRECATED ** use is_replica + + bool: True if the result is the active document, False otherwise. + """ + return not self._orig.raw_result.get('is_replica') + + @property + def is_replica(self) -> bool: + """ + bool: True if the result is a replica, False otherwise. + """ + return self._orig.raw_result.get('is_replica') + + @property + def content_as(self) -> Any: + """ + Any: The contents of the document. + + Get the value as a dict:: + + res = collection.get_replica(key) + value = res.content_as[dict] + + """ + return ContentProxy(self.value) + + def __repr__(self): + return "GetReplicaResult:{}".format(self._orig) + + +class GetResult(Result): + + @property + def expiry_time(self) -> Optional[datetime]: + """ + Optional[datetime]: The expiry of the document, if it was requested. + """ + time_ms = self._orig.raw_result.get("expiry", None) + if time_ms: + return datetime.fromtimestamp(time_ms) + return None + + @property + def expiryTime(self) -> Optional[datetime]: + """ + ** DEPRECATED ** use expiry_time + + Optional[datetime]: The expiry of the document, if it was requested. + """ + # make this a datetime! + time_ms = self._orig.raw_result.get("expiry", None) + if time_ms: + return datetime.fromtimestamp(time_ms) + return None + + @property + def content_as(self) -> Any: + """ + Any: The contents of the document. + + Get the value as a dict:: + + res = collection.get(key) + value = res.content_as[dict] + + """ + return ContentProxy(self.value) + + def __repr__(self): + return "GetResult:{}".format(self._orig) + + +class MultiResult: + def __init__(self, + orig, # type: result + result_type, # type: Union[GetReplicaResult, GetResult] + return_exceptions # type: bool + ): + self._orig = orig + self._all_ok = self._orig.raw_result.pop('all_okay', False) + self._results = {} + self._result_type = result_type + for k, v in self._orig.raw_result.items(): + if isinstance(v, CouchbaseBaseException): + if not return_exceptions: + raise ErrorMapper.build_exception(v) + else: + self._results[k] = ErrorMapper.build_exception(v) + else: + if isinstance(v, list): + self._results[k] = v + else: + self._results[k] = result_type(v) + + @property + def all_ok(self) -> bool: + """ + bool: True if all operations succeeded, false otherwise. + """ + return self._all_ok + + @property + def exceptions(self) -> Dict[str, CouchbaseBaseException]: + """ + Dict[str, Exception]: Map of keys to their respective exceptions, if the + operation had an exception. + """ + exc = {} + for k, v in self._results.items(): + if not isinstance(v, self._result_type) and not isinstance(v, list): + exc[k] = v + return exc + + +class MultiGetReplicaResult(MultiResult): + def __init__(self, + orig, # type: result + return_exceptions # type: bool + ): + super().__init__(orig, GetReplicaResult, return_exceptions) + + @property + def results(self) -> Dict[str, GetReplicaResult]: + """ + Dict[str, :class:`.GetReplicaResult`]: Map of keys to their respective :class:`.GetReplicaResult`, if the + operation has a result. + """ + res = {} + for k, v in self._results.items(): + okay = (isinstance(v, GetReplicaResult) or + isinstance(v, list) and all(map(lambda r: isinstance(r, GetReplicaResult), v))) + if okay: + res[k] = v + return res + + def __repr__(self): + output_results = [] + for k, v in self._results.items(): + output_results.append(f'{k}:{v}') + + return f'MultiGetReplicaResult( {", ".join(output_results)} )' + + +class MultiGetResult(MultiResult): + def __init__(self, + orig, # type: result + return_exceptions # type: bool + ): + super().__init__(orig, GetResult, return_exceptions) + + @property + def results(self) -> Dict[str, GetResult]: + """ + Dict[str, :class:`.GetResult`]: Map of keys to their respective :class:`.GetResult`, if the + operation has a result. + """ + res = {} + for k, v in self._results.items(): + if isinstance(v, GetResult): + res[k] = v + return res + + def __repr__(self): + output_results = [] + for k, v in self._results.items(): + output_results.append(f'{k}:{v}') + + return f'MultiGetResult( {", ".join(output_results)} )' + + +class ExistsResult(Result): + + @property + def exists(self) -> bool: + """ + bool: True if the document exists, false otherwise. + """ + return self._orig.raw_result.get("exists", False) + + def __repr__(self): + return "ExistsResult:{}".format(self._orig) + + +class MultiExistsResult: + def __init__(self, + orig, # type: result + return_exceptions # type: bool + ): + + self._orig = orig + self._all_ok = self._orig.raw_result.pop('all_okay', False) + self._results = {} + for k, v in self._orig.raw_result.items(): + if isinstance(v, CouchbaseBaseException): + if not return_exceptions: + raise ErrorMapper.build_exception(v) + else: + self._results[k] = ErrorMapper.build_exception(v) + else: + self._results[k] = ExistsResult(v) + + @property + def all_ok(self) -> bool: + """ + bool: True if all operations succeeded, false otherwise. + """ + return self._all_ok + + @property + def exceptions(self) -> Dict[str, CouchbaseBaseException]: + """ + Dict[str, Exception]: Map of keys to their respective exceptions, if the + operation had an exception. + """ + exc = {} + for k, v in self._results.items(): + if not isinstance(v, ExistsResult): + exc[k] = v + return exc + + @property + def results(self) -> Dict[str, ExistsResult]: + """ + Dict[str, :class:`.MutationResult`]: Map of keys to their respective :class:`.MutationResult`, if the + operation has a result. + """ + res = {} + for k, v in self._results.items(): + if isinstance(v, ExistsResult): + res[k] = v + return res + + def __repr__(self): + output_results = [] + for k, v in self._results.items(): + output_results.append(f'{k}:{v}') + + return f'MultiExistsResult( {", ".join(output_results)} )' + + +class MutationResult(Result): + def __init__(self, + orig, # type: result + ): + super().__init__(orig) + self._raw_mutation_token = self._orig.raw_result.get('mutation_token', None) + self._mutation_token = None + + def mutation_token(self) -> Optional[MutationToken]: + """Get the operation's mutation token, if it exists. + + Returns: + Optional[:class:`.MutationToken`]: The operation's mutation token. + """ + if self._raw_mutation_token is not None and self._mutation_token is None: + self._mutation_token = MutationToken(self._raw_mutation_token.get()) + return self._mutation_token + + def __repr__(self): + return "MutationResult:{}".format(self._orig) + + +class MultiMutationResult: + def __init__(self, + orig, # type: result + return_exceptions # type: bool + ): + + self._orig = orig + self._all_ok = self._orig.raw_result.pop('all_okay', False) + self._results = {} + for k, v in self._orig.raw_result.items(): + if isinstance(v, CouchbaseBaseException): + if not return_exceptions: + raise ErrorMapper.build_exception(v) + else: + self._results[k] = ErrorMapper.build_exception(v) + else: + self._results[k] = MutationResult(v) + + @property + def all_ok(self) -> bool: + """ + bool: True if all operations succeeded, false otherwise. + """ + return self._all_ok + + @property + def exceptions(self) -> Dict[str, CouchbaseBaseException]: + """ + Dict[str, Exception]: Map of keys to their respective exceptions, if the + operation had an exception. + """ + exc = {} + for k, v in self._results.items(): + if not isinstance(v, MutationResult): + exc[k] = v + return exc + + @property + def results(self) -> Dict[str, MutationResult]: + """ + Dict[str, :class:`.MutationResult`]: Map of keys to their respective :class:`.MutationResult`, if the + operation has a result. + """ + res = {} + for k, v in self._results.items(): + if isinstance(v, MutationResult): + res[k] = v + return res + + def __repr__(self): + output_results = [] + for k, v in self._results.items(): + output_results.append(f'{k}:{v}') + + return f'MultiMutationResult( {", ".join(output_results)} )' + + +MultiResultType = Union[MultiGetResult, MultiMutationResult] + + +class MutationToken: + def __init__(self, token # type: Dict[str, Union[str, int]] + ): + self._token = token + + @property + def partition_id(self) -> int: + """ + int: The token's partition id. + """ + return self._token['partition_id'] + + @property + def partition_uuid(self) -> int: + """ + int: The token's partition uuid. + """ + return self._token['partition_uuid'] + + @property + def sequence_number(self) -> int: + """ + int: The token's sequence number. + """ + return self._token['sequence_number'] + + @property + def bucket_name(self) -> str: + """ + str: The token's bucket name. + """ + return self._token['bucket_name'] + + def as_tuple(self) -> Tuple[int, int, int, str]: + return (self.partition_id, self.partition_uuid, + self.sequence_number, self.bucket_name) + + def as_dict(self) -> Dict[str, Union[str, int]]: + return self._token + + def __repr__(self): + return "MutationToken:{}".format(self._token) + + def __hash__(self): + return hash(self.as_tuple()) + + def __eq__(self, other): + if not isinstance(other, MutationToken): + return False + return (self.partition_id == other.partition_id + and self.partition_uuid == other.partition_uuid + and self.sequence_number == other.sequence_number + and self.bucket_name == other.bucket_name) + + +class LookupInResult(Result): + def exists(self, # type: LookupInResult + index # type: int + ) -> bool: + """Check if the subdocument path exists. + + Raises: + :class:`~couchbase.exceptions.InvalidIndexException`: If the provided index is out of range. + + Returns: + bool: True if the path exists. False if the path does not exist. + """ + return parse_subdocument_exists(self.value, index, self.key) + + @property + def content_as(self) -> ContentSubdocProxy: + """ + :class:`.ContentSubdocProxy`: A proxy to return the value at the specified index. + + Get first value as a dict:: + + res = collection.lookup_in(key, (SD.get("geo"), SD.exists("city"))) + value = res.content_as[dict](0) + """ + return ContentSubdocProxy(self.value, self.key) + + def __repr__(self): + return "LookupInResult:{}".format(self._orig) + + +class LookupInReplicaResult(Result): + def exists(self, # type: LookupInReplicaResult + index # type: int + ) -> bool: + """Check if the subdocument path exists. + + Raises: + :class:`~couchbase.exceptions.InvalidIndexException`: If the provided index is out of range. + + Returns: + bool: True if the path exists. False if the path does not exist. + """ + return parse_subdocument_exists(self.value, index, self.key) + + @property + def content_as(self) -> ContentSubdocProxy: + """ + :class:`.ContentSubdocProxy`: A proxy to return the value at the specified index. + + Get first value as a dict:: + + res = collection.lookup_in(key, (SD.get("geo"), SD.exists("city"))) + value = res.content_as[dict](0) + """ + return ContentSubdocProxy(self.value, self.key) + + @property + def is_replica(self) -> bool: + """ + bool: True if the result is a replica, False otherwise. + """ + return self._orig.raw_result.get('is_replica') + + def __repr__(self): + return "LookupInReplicaResult:{}".format(self._orig) + + +class MutateInResult(MutationResult): + + @property + def content_as(self) -> ContentSubdocProxy: + """ + :class:`.ContentSubdocProxy`: A proxy to return the value at the specified index. + + Get first value as a str:: + + res = collection.mutate_in(key, (SD.upsert("city", "New City"), + SD.replace("faa", "CTY"))) + value = res.content_as[str](0) + """ + return ContentSubdocProxy(self.value, self.key) + + def __repr__(self): + return "MutateInResult:{}".format(self._orig) + + +class CounterResult(MutationResult): + + # Uncomment and delete previous property when ready to remove cas CounterResult. + # cas = RemoveProperty('cas') + + @property + def cas(self) -> int: + """ + .. warning:: + This property is deprecated and will be removed in a future version. + + int: **DEPRECATED** The CAS of the document. + """ + return self._orig.raw_result.get("cas", 0) + + @property + def content(self) -> Optional[int]: + """ + Optional[int]: The value of the document after the operation completed. + """ + return self._orig.raw_result.get("content", None) + + def __repr__(self): + # Uncomment and delete previous return when ready to remove cas from CounterResult. Or, ideally, + # remove cas from the cxx client's response. + # return "CounterResult:{}".format({k:v for k,v in self._orig.raw_result.items() if k != 'cas'}) + return "CounterResult:{}".format(self._orig.raw_result) + + +class MultiCounterResult: + def __init__(self, + orig, # type: result + return_exceptions # type: bool + ): + + self._orig = orig + self._all_ok = self._orig.raw_result.pop('all_okay', False) + self._results = {} + for k, v in self._orig.raw_result.items(): + if isinstance(v, CouchbaseBaseException): + if not return_exceptions: + raise ErrorMapper.build_exception(v) + else: + self._results[k] = ErrorMapper.build_exception(v) + else: + self._results[k] = CounterResult(v) + + @property + def all_ok(self) -> bool: + """ + bool: True if all operations succeeded, false otherwise. + """ + return self._all_ok + + @property + def exceptions(self) -> Dict[str, CouchbaseBaseException]: + """ + Dict[str, Exception]: Map of keys to their respective exceptions, if the + operation had an exception. + """ + exc = {} + for k, v in self._results.items(): + if not isinstance(v, CounterResult): + exc[k] = v + return exc + + @property + def results(self) -> Dict[str, CounterResult]: + """ + Dict[str, :class:`.MutationResult`]: Map of keys to their respective :class:`.MutationResult`, if the + operation has a result. + """ + res = {} + for k, v in self._results.items(): + if isinstance(v, CounterResult): + res[k] = v + return res + + def __repr__(self): + output_results = [] + for k, v in self._results.items(): + output_results.append(f'{k}:{v}') + + return f'MultiCounterResult( {", ".join(output_results)} )' + + +class ClusterInfoResult: + def __init__( + self, + orig # type: result + ): + self._orig = orig + # version string should be X.Y.Z-XXXX-YYYY + self._server_version_raw = None + self._server_version = None + self._server_version_short = None + self._server_build = None + self._is_enterprise = None + + @property + def nodes(self): + return self._orig.raw_result.get("nodes", None) + + @property + def server_version(self) -> Optional[str]: + if not self._server_version: + self._set_server_version() + + if self._server_version_raw: + self._server_version = self._server_version_raw[:10] + + return self._server_version + + @property + def server_version_short(self) -> Optional[float]: + """ + Optional[float]: The version of the connected Couchbase Server in Major.Minor form. + """ + if not self._server_version_short: + self._set_server_version() + + if self._server_version_raw: + self._server_version_short = float(self._server_version_raw[:3]) + + return self._server_version_short + + @property + def server_version_full(self) -> Optional[str]: + """ + Optional[str]: The full version details of the connected Couchbase Server. + """ + if not self._server_version_raw: + self._set_server_version() + + return self._server_version_raw + + @property + def server_version_build(self) -> Optional[int]: + """ + Optional[int]: The build version of the connected Couchbase Server. + """ + if not self._server_build: + self._set_server_version() + + if self._server_version_raw: + tokens = self._server_version_raw.split("-") + if len(tokens) == 3: + self._server_build = int(tokens[1]) + + return self._server_build + + @property + def is_enterprise(self) -> Optional[bool]: + """ + bool: True if connected Couchbase Server is Enterprise edition, false otherwise. + """ + if not self._is_enterprise: + self._set_server_version() + + if self._server_version_raw: + tokens = self._server_version_raw.split("-") + if len(tokens) == 3: + self._is_enterprise = tokens[2].upper() == "ENTERPRISE" + + return self._is_enterprise + + @property + def is_community(self) -> Optional[bool]: + """ + bool: True if connected Couchbase Server is Community edition, false otherwise. + """ + if not self._is_community: + self._set_server_version() + + if self._server_version_raw: + tokens = self._server_version_raw.split("-") + if len(tokens) == 3: + self._is_community = tokens[2].upper() == "COMMUNITY" + + return self._is_community + + def _set_server_version(self): + version = None + for n in self.nodes: + v = n["version"] + if version is None: + version = v + elif v != version: + # mixed versions -- not supported + version = None + break + + self._server_version_raw = version + + def __repr__(self): + return "ClusterInfoResult:{}".format(self._orig) + + +class HttpResult: + def __init__( + self, + orig # type: result + ): + self._orig = orig + + +class ScanResult(Result): + + def __init__(self, orig, ids_only): + super().__init__(orig) + self._ids_only = ids_only + + @property + def id(self) -> Optional[str]: + """ + Optional[str]: Id for the operation, if it exists. + """ + return self._orig.raw_result.get("key", None) + + @property + def ids_only(self) -> bool: + """ + bool: True is KV range scan request options set ids_only to True. False otherwise. + """ + return self._ids_only + + @property + def cas(self) -> int: + """ + Optional[int]: The CAS of the document, if it exists + """ + if self.ids_only: + raise InvalidArgumentException(("No cas available when scan is requested with " + "`ScanOptions` ids_only set to True.")) + return self._orig.raw_result.get("cas", 0) + + @property + def expiry_time(self) -> Optional[datetime]: + """ + Optional[datetime]: The expiry of the document, if it was requested. + """ + if self.ids_only: + raise InvalidArgumentException(("No expiry_time available when scan is requested with " + "`ScanOptions` ids_only set to True.")) + time_ms = self._orig.raw_result.get("expiry", None) + if time_ms: + return datetime.fromtimestamp(time_ms) + return None + + @property + def content_as(self) -> Any: + """ + Any: The contents of the document. + + Get the value as a dict:: + + res = collection.get(key) + value = res.content_as[dict] + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If called when KV range scan options set + without_content to True. + + """ + if self.ids_only: + raise InvalidArgumentException(("No content available when scan is requested with " + "`ScanOptions` ids_only set to True.")) + return ContentProxy(self.value) + + def __repr__(self): + return "ScanResult:{}".format(self._orig) + + +class ScanResultIterable: + def __init__( + self, + scan_request + ): + self._request = scan_request + + def rows(self): + """The rows which have been returned by the query. + + .. note:: + If using the *acouchbase* API be sure to use ``async for`` when looping over rows. + + Returns: + Iterable: Either an iterable or async iterable. + """ + # avoid circular import + from acouchbase.kv_range_scan import AsyncRangeScanRequest # noqa: F811 + if isinstance(self._request, AsyncRangeScanRequest): + return self.__aiter__() + return self.__iter__() + + def cancel_scan(self): + self._request.cancel_scan() + + def __iter__(self): + return self._request.__iter__() + + def __aiter__(self): + return self._request.__aiter__() + + def __repr__(self): + return "ScanResultIterable:{}".format(self._request) + + +class QueryResult: + def __init__( + self, + n1ql_request + ): + self._request = n1ql_request + + def rows(self): + """The rows which have been returned by the query. + + .. note:: + If using the *acouchbase* API be sure to use ``async for`` when looping over rows. + + Returns: + Iterable: Either an iterable or async iterable. + """ + if isinstance(self._request, AsyncN1QLRequest): + return self.__aiter__() + return self.__iter__() + + def execute(self): + """Convenience method to execute the query. + + Returns: + List[Any]: A list of query results. + + Example: + q_rows = cluster.query('SELECT * FROM `travel-sample` WHERE country LIKE 'United%' LIMIT 2;').execute() + + """ + return self._request.execute() + + def metadata(self): + """The meta-data which has been returned by the query. + + Returns: + :class:`~couchbase.n1ql.QueryMetaData`: An instance of :class:`~couchbase.n1ql.QueryMetaData`. + """ + return self._request.metadata() + + def __iter__(self): + return self._request.__iter__() + + def __aiter__(self): + return self._request.__aiter__() + + def __repr__(self): + return "QueryResult:{}".format(self._request) + + +class AnalyticsResult: + def __init__( + self, + analytics_request + ): + self._request = analytics_request + + def __repr__(self): + return "AnalyticsResult:{}".format(self._request) + + def rows(self): + """The rows which have been returned by the analytics query. + + .. note:: + If using the *acouchbase* API be sure to use ``async for`` when looping over rows. + + Returns: + Iterable: Either an iterable or async iterable. + """ + if isinstance(self._request, AsyncAnalyticsRequest): + return self.__aiter__() + return self.__iter__() + + def metadata(self): + """The meta-data which has been returned by the analytics query. + + Returns: + :class:`~couchbase.analytics.AnalyticsMetaData`: An instance of + :class:`~couchbase.analytics.AnalyticsMetaData`. + """ + return self._request.metadata() + + def __iter__(self): + return self._request.__iter__() + + def __aiter__(self): + return self._request.__aiter__() + + +class SearchResult: + def __init__( + self, + search_request + ): + self._request = search_request + + def __repr__(self): + return "SearchResult:{}".format(self._request) + + def rows(self): + """The rows which have been returned by the search query. + + .. note:: + If using the *acouchbase* API be sure to use ``async for`` when looping over rows. + + Returns: + Iterable: Either an iterable or async iterable. + """ + if isinstance(self._request, AsyncFullTextSearchRequest): + return self.__aiter__() + return self.__iter__() + + def metadata(self): + """The meta-data which has been returned by the search query. + + Returns: + :class:`~couchbase.search.SearchMetaData`: An instance of + :class:`~couchbase.search.SearchMetaData`. + """ + return self._request.metadata() + + def result_rows(self): + return self._request.result_rows() + + def facets(self): + return self._request.result_facets() + + def __iter__(self): + return self._request.__iter__() + + def __aiter__(self): + return self._request.__aiter__() + + +class ViewResult: + def __init__( + self, + search_request + ): + self._request = search_request + + def __repr__(self): + return "ViewResult:{}".format(self._request) + + def rows(self): + """The rows which have been returned by the view query. + + .. note:: + If using the *acouchbase* API be sure to use ``async for`` when looping over rows. + + Returns: + Iterable: Either an iterable or async iterable. + """ + if isinstance(self._request, AsyncViewRequest): + return self.__aiter__() + return self.__iter__() + + def metadata(self): + """The meta-data which has been returned by the view query. + + Returns: + :class:`~couchbase.views.ViewMetaData`: An instance of + :class:`~couchbase.views.ViewMetaData`. + """ + return self._request.metadata() + + def __iter__(self): + return self._request.__iter__() + + def __aiter__(self): + return self._request.__aiter__() + + +class OperationResult: + """ **DEPRECATED** """ + + def __init__(self, cas, mut_token): + self._mutation_token = mut_token + self._cas = cas + + @property + def cas(self) -> Optional[int]: + return self._cas + + def mutation_token(self) -> Optional[MutationToken]: + return self._mutation_token + + def __repr__(self): + return f'OperationResult(mutation_token={self._mutation_token}, cas={self._cas})' + + +class ValueResult: + """ **DEPRECATED** """ + + def __init__(self, cas, mut_token, value): + self._mutation_token = mut_token + self._cas = cas + self._value = value + + @property + def cas(self) -> Optional[int]: + return self._cas + + @property + def value(self) -> Optional[Any]: + return self._value + + def mutation_token(self) -> Optional[MutationToken]: + return self._mutation_token + + def __repr__(self): + return f'OperationResult(mutation_token={self._mutation_token}, cas={self._cas})' diff --git a/couchbase/scope.py b/couchbase/scope.py new file mode 100644 index 000000000..a6019194e --- /dev/null +++ b/couchbase/scope.py @@ -0,0 +1,38 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from couchbase.logic.scope import ScopeLogic + +""" +** DEPRECATION NOTICE ** + +Once the deprecated Scope import from couchbase.collection is removed, this class +can be replaced w/ the ScopeLogic class and the ScopeLogic class removed. The +hierarchy was created to help w/ 3.x imports. + +""" + + +class Scope(ScopeLogic): + """Create a Couchbase Scope instance. + + Exposes the operations which are available to be performed against a scope. Namely the ability to access + to Collections for performing operations. + + Args: + bucket (:class:`~couchbase.bucket.Bucket`): A :class:`~couchbase.bucket.Bucket` instance. + scope_name (str): Name of the scope. + + """ diff --git a/couchbase/search.py b/couchbase/search.py new file mode 100644 index 000000000..45712801c --- /dev/null +++ b/couchbase/search.py @@ -0,0 +1,157 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from couchbase.exceptions import NoChildrenException # noqa: F401 +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.search import DateFacet # noqa: F401 +from couchbase.logic.search import Facet # noqa: F401 +from couchbase.logic.search import HighlightStyle # noqa: F401 +from couchbase.logic.search import NumericFacet # noqa: F401 +from couchbase.logic.search import SearchDateRangeFacet # noqa: F401 +from couchbase.logic.search import SearchFacetResult # noqa: F401 +from couchbase.logic.search import SearchMetaData # noqa: F401 +from couchbase.logic.search import SearchMetrics # noqa: F401 +from couchbase.logic.search import SearchNumericRangeFacet # noqa: F401 +from couchbase.logic.search import SearchQueryBuilder # noqa: F401 +from couchbase.logic.search import SearchRow # noqa: F401 +from couchbase.logic.search import SearchRowFields # noqa: F401 +from couchbase.logic.search import SearchRowLocation # noqa: F401 +from couchbase.logic.search import SearchRowLocations # noqa: F401 +from couchbase.logic.search import SearchScanConsistency # noqa: F401 +from couchbase.logic.search import SearchTermFacet # noqa: F401 +from couchbase.logic.search import Sort # noqa: F401 +from couchbase.logic.search import SortField # noqa: F401 +from couchbase.logic.search import SortGeoDistance # noqa: F401 +from couchbase.logic.search import SortID # noqa: F401 +from couchbase.logic.search import SortScore # noqa: F401 +from couchbase.logic.search import TermFacet # noqa: F401 +from couchbase.logic.search import FullTextSearchRequestLogic +from couchbase.logic.search_queries import BooleanFieldQuery # noqa: F401 +from couchbase.logic.search_queries import BooleanQuery # noqa: F401 +from couchbase.logic.search_queries import ConjunctionQuery # noqa: F401 +from couchbase.logic.search_queries import DateRangeQuery # noqa: F401 +from couchbase.logic.search_queries import DisjunctionQuery # noqa: F401 +from couchbase.logic.search_queries import DocIdQuery # noqa: F401 +from couchbase.logic.search_queries import GeoBoundingBoxQuery # noqa: F401 +from couchbase.logic.search_queries import GeoDistanceQuery # noqa: F401 +from couchbase.logic.search_queries import GeoPolygonQuery # noqa: F401 +from couchbase.logic.search_queries import MatchAllQuery # noqa: F401 +from couchbase.logic.search_queries import MatchNoneQuery # noqa: F401 +from couchbase.logic.search_queries import MatchOperator # noqa: F401 +from couchbase.logic.search_queries import MatchPhraseQuery # noqa: F401 +from couchbase.logic.search_queries import MatchQuery # noqa: F401 +from couchbase.logic.search_queries import NumericRangeQuery # noqa: F401 +from couchbase.logic.search_queries import PhraseQuery # noqa: F401 +from couchbase.logic.search_queries import PrefixQuery # noqa: F401 +from couchbase.logic.search_queries import QueryStringQuery # noqa: F401 +from couchbase.logic.search_queries import RawQuery # noqa: F401 +from couchbase.logic.search_queries import RegexQuery # noqa: F401 +from couchbase.logic.search_queries import SearchQuery # noqa: F401 +from couchbase.logic.search_queries import TermQuery # noqa: F401 +from couchbase.logic.search_queries import TermRangeQuery # noqa: F401 +from couchbase.logic.search_queries import WildcardQuery # noqa: F401 +from couchbase.logic.search_request import SearchRequest # noqa: F401 +from couchbase.logic.supportability import Supportability + + +class FullTextSearchRequest(FullTextSearchRequestLogic): + def __init__(self, + connection, + encoded_query, + **kwargs + ): + super().__init__(connection, encoded_query, **kwargs) + + @classmethod + def generate_search_request(cls, connection, encoded_query, **kwargs): + return cls(connection, encoded_query, **kwargs) + + def execute(self): + return [r for r in list(self)] + + def _get_metadata(self): + try: + # @TODO: PYCBC-1524 + search_response = next(self._streaming_result) + self._set_metadata(search_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def __iter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_query() + + return self + + def _get_next_row(self): + if self.done_streaming is True: + return + + try: + row = next(self._streaming_result) + except StopIteration: + # @TODO: PYCBC-1524 + row = next(self._streaming_result) + + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopIteration + + return self._deserialize_row(row) + + def __next__(self): + try: + return self._get_next_row() + except StopIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + +""" +** DEPRECATION NOTICE ** + +The classes below are deprecated for 3.x compatibility. They should not be used. +Instead use: + * All options should be imported from `couchbase.options`. + +""" + +from couchbase.logic.options import SearchOptionsBase # nopep8 # isort:skip # noqa: E402 + + +@Supportability.import_deprecated('couchbase.search', 'couchbase.options') +class SearchOptions(SearchOptionsBase): # noqa: F811 + pass diff --git a/couchbase/serializer.py b/couchbase/serializer.py new file mode 100644 index 000000000..b76cf3739 --- /dev/null +++ b/couchbase/serializer.py @@ -0,0 +1,56 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from abc import ABC, abstractmethod +from typing import Any + + +class Serializer(ABC): + """Interface a Custom Serializer must implement + """ + + @abstractmethod + def serialize(self, + value # type: Any + ) -> bytes: + raise NotImplementedError() + + @abstractmethod + def deserialize(self, + value # type: bytes + ) -> Any: + raise NotImplementedError() + + @classmethod + def __subclasshook__(cls, subclass): + return (hasattr(subclass, 'serialize') and + callable(subclass.serialize) and + hasattr(subclass, 'deserialize') and + callable(subclass.deserialize)) + + +class DefaultJsonSerializer(Serializer): + def serialize(self, + value, # type: Any + ) -> bytes: + + return json.dumps(value, ensure_ascii=False).encode('utf-8') + + def deserialize(self, + value # type: bytes + ) -> Any: + + return json.loads(value.decode('utf-8')) diff --git a/couchbase/subdocument.py b/couchbase/subdocument.py new file mode 100644 index 000000000..a18f25a7a --- /dev/null +++ b/couchbase/subdocument.py @@ -0,0 +1,693 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from enum import IntEnum +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + List, + Optional, + Union) + +from couchbase.exceptions import (CouchbaseException, + DeltaInvalidException, + DocumentNotFoundException, + DocumentNotJsonException, + InvalidArgumentException, + InvalidIndexException, + NumberTooBigException, + PathExistsException, + PathInvalidException, + PathMismatchException, + PathNotFoundException, + PathTooBigException, + PathTooDeepException, + SubdocCantInsertValueException, + ValueTooDeepException) +from couchbase.logic.supportability import Supportability + +if TYPE_CHECKING: + from couchbase._utils import JSONType + +""" +couchbase++ couchbase/protocol/client_opcode.hxx +enum class subdoc_opcode : uint8_t { + get_doc = 0x00, + set_doc = 0x01, + remove_doc = 0x04, + get = 0xc5, + exists = 0xc6, + dict_add = 0xc7, + dict_upsert = 0xc8, + remove = 0xc9, + replace = 0xca, + array_push_last = 0xcb, + array_push_first = 0xcc, + array_insert = 0xcd, + array_add_unique = 0xce, + counter = 0xcf, + get_count = 0xd2, + replace_body_with_xattr = 0xd3, +}; + +""" + + +class SubDocOp(IntEnum): + GET_DOC = 0 + SET_DOC = 1 + REMOVE_DOC = 4 + GET = 197 + EXISTS = 198 + DICT_ADD = 199 + DICT_UPSERT = 200 + REMOVE = 201 + REPLACE = 202 + ARRAY_PUSH_LAST = 203 + ARRAY_PUSH_FIRST = 204 + ARRAY_INSERT = 205 + ARRAY_ADD_UNIQUE = 206 + COUNTER = 207 + GET_COUNT = 210 + REPLACE_BODY_WITH_XATTR = 211 + + +""" +couchbase++ couchbase/protocol/client_opcode.hxx +enum class status : std::uint16_t { + ... + subdoc_path_not_found = 0xc0, + subdoc_path_mismatch = 0xc1, + subdoc_path_invalid = 0xc2, + subdoc_path_too_big = 0xc3, + subdoc_doc_too_deep = 0xc4, + subdoc_value_cannot_insert = 0xc5, + subdoc_doc_not_json = 0xc6, + subdoc_num_range_error = 0xc7, + subdoc_delta_invalid = 0xc8, + subdoc_path_exists = 0xc9, + subdoc_value_too_deep = 0xca, + subdoc_invalid_combo = 0xcb, + subdoc_multi_path_failure = 0xcc, + subdoc_success_deleted = 0xcd, + subdoc_xattr_invalid_flag_combo = 0xce, + subdoc_xattr_invalid_key_combo = 0xcf, + subdoc_xattr_unknown_macro = 0xd0, + subdoc_xattr_unknown_vattr = 0xd1, + subdoc_xattr_cannot_modify_vattr = 0xd2, + subdoc_multi_path_failure_deleted = 0xd3, + subdoc_invalid_xattr_order = 0xd4, + subdoc_xattr_unknown_vattr_macro = 0xd5, + subdoc_can_only_revive_deleted_documents = 0xd6, + subdoc_deleted_document_cannot_have_value = 0xd7, +}; +""" + + +class SubDocStatus(IntEnum): + PathNotFound = 192 + PathMismatch = 193 + PathInvalid = 194 + PathTooBig = 195 + TooDeep = 196 + ValueCannotInsert = 197 + DocNotJson = 198 + NumRangeError = 199 + DeltaInvalid = 200 + PathExists = 201 + ValueTooDeep = 202 + + +class StoreSemantics(IntEnum): + """Define subdocument mutation operation store semantics. + """ + REPLACE = 0 + UPSERT = 1 + INSERT = 2 + + +class LookupInMacro(): + + @staticmethod + def document() -> str: + return '$document' + + @staticmethod + def expiry_time() -> str: + return '$document.exptime' + + @staticmethod + def cas() -> str: + return '$document.CAS' + + @staticmethod + def seq_no() -> str: + return '$document.seqno' + + @staticmethod + def last_modified() -> str: + return '$document.last_modified' + + @staticmethod + def is_deleted() -> str: + return '$document.deleted' + + @staticmethod + def value_size_bytes() -> str: + return '$document.value_bytes' + + @staticmethod + def rev_id() -> str: + return '$document.revid' + + +class MutationMacro(): + def __init__(self, value: str): + self._value = value + + @property + def value(self) -> str: + return self._value + + @classmethod + def cas(cls) -> MutationMacro: + return cls("${Mutation.CAS}") + + @classmethod + def seq_no(cls) -> MutationMacro: + return cls("${Mutation.seqno}") + + @classmethod + def value_crc32c(cls) -> MutationMacro: + return cls("${Mutation.value_crc32c}") + + +class Spec(tuple): + """Represents a sub-operation to perform.""" + + def __new__(cls, *args, **kwargs): + return super().__new__(cls, tuple(args)) + + def __repr__(self): + details = [] + # details.append(_SPECMAP.get(self[0])) + details.extend([repr(x) for x in self[1:]]) + return '{0}<{1}>'.format(self.__class__.__name__, + ', '.join(details)) + + +class ArrayValues(tuple): + def __new__(cls, *args, **kwargs): + return super(ArrayValues, cls).__new__(cls, tuple(args)) + + def __repr__(self): + return 'ArrayValues({0})'.format(tuple.__repr__(self)) + + +def parse_subdocument_content_as(content, # type: List[Dict[str, Any]] + index, # type: int + key, # type: str + ) -> Any: + if index > len(content) - 1 or index < 0: + raise InvalidIndexException(f"Provided index is invalid. Index={index}.") + + status = content[index].get('status', None) + if status is None: + raise DocumentNotFoundException(f"Could not find document. Key={key}.") + + op_code = content[index].get('opcode', None) + if op_code == SubDocOp.EXISTS: + return parse_subdocument_exists(content, index, key) + if status == 0: + return content[index].get('value', None) + + path = content[index].get('path', None) + parse_subdocument_status(status, path, key) + + +def parse_subdocument_exists(content, # type: List[Dict[str, Any]] + index, # type: int + key, # type: str + ) -> bool: + if index > len(content) - 1 or index < 0: + raise InvalidIndexException(f"Provided index is invalid. Index={index}.") + + status = content[index].get('status', None) + if status is None: + raise DocumentNotFoundException(f"Could not find document. Key={key}.") + + path = content[index].get('path', None) + if status == 0: + return True + elif status == SubDocStatus.PathNotFound: + return False + + parse_subdocument_status(status, path, key) + + +def parse_subdocument_status(status, path, key): # noqa: C901 + if status == SubDocStatus.PathNotFound: + raise PathNotFoundException(f"Path could not be found. Path={path}, key={key}.") + if status == SubDocStatus.PathMismatch: + raise PathMismatchException(f"Path mismatch. Path={path}, key={key}.") + if status == SubDocStatus.PathInvalid: + raise PathInvalidException(f"Path is invalid. Path={path}, key={key}.") + if status == SubDocStatus.PathTooBig: + msg = f"Path is too long, or contains too many independent components. Path={path}, key={key}." + raise PathTooBigException(msg) + if status == SubDocStatus.TooDeep: + raise PathTooDeepException(f"Path contains too many levels to parse. Path={path}, key={key}.") + if status == SubDocStatus.ValueCannotInsert: + raise SubdocCantInsertValueException(f"Cannot insert value. Path={path}, key={key}.") + if status == SubDocStatus.DocNotJson: + raise DocumentNotJsonException(f"Cannot operate on non-JSON document. Path={path}, key={key}.") + if status == SubDocStatus.NumRangeError: + msg = f"Value is outside the valid range for arithmetic operations. Path={path}, key={key}." + raise NumberTooBigException(msg) + if status == SubDocStatus.DeltaInvalid: + raise DeltaInvalidException(f"Delta value specified for operation is too large. Path={path}, key={key}.") + if status == SubDocStatus.PathExists: + raise PathExistsException(f"Path already exists. Path={path}, key={key}.") + if status == SubDocStatus.ValueTooDeep: + raise ValueTooDeepException(f"Value too deep for document. Path={path}, key={key}.") + + raise CouchbaseException(f"Unknown status. Status={status}, path={path}, key={key}") + + +def exists(path, # type: str + xattr=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` that returns whether a specific field exists in the document. + + Args: + path (str): The path to the field. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + return Spec(SubDocOp.EXISTS, path, xattr) + + +def get(path, # type: str + xattr=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for retrieving an element's value given a path. + + Args: + path (str): The path to the field. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + return Spec(SubDocOp.GET, path, xattr) + + +def count(path, # type: str + xattr=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` that returns the number of elements in the array referenced by the path. + + Args: + path (str): The path to the field. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + return Spec(SubDocOp.GET_COUNT, path, xattr) + + +def insert(path, # type: str + value, # type: Union[JSONType, MutationMacro] + create_parents=False, # type: Optional[bool] + xattr=False, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ) -> Spec: + """Creates a :class:`.Spec` for inserting a field into the document. Failing if the field already + exists at the specified path. + + Args: + path (str): The path to the field. + value (Union[JSONType, MutationMacro]): The value to insert. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + expand_macros = kwargs.get('expand_macros', False) + if isinstance(value, MutationMacro): + value = value.value + xattr = True + expand_macros = True + return Spec(SubDocOp.DICT_ADD, path, create_parents, xattr, expand_macros, value) + + +def upsert(path, # type: str + value, # type: Union[JSONType, MutationMacro] + create_parents=False, # type: Optional[bool] + xattr=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for upserting a field into the document. This updates the value of the specified field, + or creates the field if it does not exits. + + Args: + path (str): The path to the field. + value (Union[JSONType, MutationMacro]): The value to upsert. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + expand_macros = False + if isinstance(value, MutationMacro): + value = value.value + xattr = True + expand_macros = True + return Spec(SubDocOp.DICT_UPSERT, path, create_parents, xattr, expand_macros, value) + + +def replace(path, # type: str + value, # type: Union[JSONType, MutationMacro] + xattr=False, # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for replacing a field into the document. Failing if the field already + exists at the specified path. + + Args: + path (str): The path to the field. + value (Union[JSONType, MutationMacro]): The value to write. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + expand_macros = False + if isinstance(value, MutationMacro): + value = value.value + xattr = True + expand_macros = True + if not path: + return Spec(SubDocOp.SET_DOC, '', False, xattr, expand_macros, value) + return Spec(SubDocOp.REPLACE, path, False, xattr, expand_macros, value) + + +def remove(path, # type: str + xattr=False, # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for removing a field from a document. + + Args: + path (str): The path to the field. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + if not path: + return Spec(SubDocOp.REMOVE_DOC, '', False, xattr, False) + return Spec(SubDocOp.REMOVE, path, False, xattr, False) + + +def array_append(path, # type: str + *values, # type: Iterable[Any] + create_parents=False, # type: Optional[bool] + xattr=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for adding a value to the end of an array in a document. + + Args: + path (str): The path to an element of an array. + *values (Iterable[Any]): The values to add. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + expand_macros = False + if any(map(lambda m: isinstance(m, MutationMacro), values)): + values = [v.value if isinstance(v, MutationMacro) else v for v in values] + xattr = True + expand_macros = True + return Spec(SubDocOp.ARRAY_PUSH_LAST, path, create_parents, xattr, expand_macros, ArrayValues(*values)) + + +def array_prepend(path, # type: str + *values, # type: Iterable[Any] + create_parents=False, # type: Optional[bool] + xattr=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for adding a value to the beginning of an array in a document. + + Args: + path (str): The path to an element of an array. + *values (Iterable[Any]): The values to add. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + expand_macros = False + if any(map(lambda m: isinstance(m, MutationMacro), values)): + values = [v.value if isinstance(v, MutationMacro) else v for v in values] + xattr = True + expand_macros = True + return Spec(SubDocOp.ARRAY_PUSH_FIRST, path, create_parents, xattr, expand_macros, ArrayValues(*values)) + + +def array_insert(path, # type: str + *values, # type: Iterable[Any] + create_parents=False, # type: Optional[bool] + xattr=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for adding a value to a specified location in an array in a document. + The path should specify a specific index in the array and the new values are inserted at this location. + + Args: + path (str): The path to an element of an array. + *values (Iterable[Any]): The values to add. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + expand_macros = False + if any(map(lambda m: isinstance(m, MutationMacro), values)): + values = [v.value if isinstance(v, MutationMacro) else v for v in values] + xattr = True + expand_macros = True + return Spec(SubDocOp.ARRAY_INSERT, path, create_parents, xattr, expand_macros, ArrayValues(*values)) + + +def array_addunique(path, # type: str + value, # type: Union[str, int, float, bool, None] + create_parents=False, # type: Optional[bool] + xattr=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for adding unique values to an array in a document. This operation will only + add values if they do not already exist elsewhere in the array. + + Args: + path (str): The path to an element of an array. + value (Union[str, int, float, bool, None]): The value to add into the array. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + """ + expand_macros = False + if isinstance(value, MutationMacro): + value = value.value + xattr = True + expand_macros = True + return Spec(SubDocOp.ARRAY_ADD_UNIQUE, path, create_parents, xattr, expand_macros, value) + + +def counter(path, # type: str + delta, # type: int + xattr=False, # type: Optional[bool] + create_parents=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for incrementing or decrementing the value of a field in a document. If + the provided delta is >= 0 :meth:`~couchbase.subdocument.increment` is called, otherwise + :meth:`~couchbase.subdocument.decrement` is called. + + .. warning:: + This method is **deprecated** use :meth:`~.subdocument.increment` or :meth:`~couchbase.subdocument.decrement` + + Args: + path (str): The path to the field. + delta (int): The value to increment or decrement from the document. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + """ + if delta >= 0: + return increment(path, delta, xattr=xattr, create_parents=create_parents) + else: + return decrement(path, abs(delta), xattr=xattr, create_parents=create_parents) + + +def increment(path, # type: str + delta, # type: int + xattr=False, # type: Optional[bool] + create_parents=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for incrementing the value of a field in a document. + + Args: + path (str): The path to the field. + delta (int): The value to increment from the document. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the delta arugment is not >= 0 or not + of type int. + """ + if not isinstance(delta, int): + raise InvalidArgumentException("Delta must be integer") + if delta <= 0: + raise InvalidArgumentException( + "Delta must be integer greater than or equal to 0") + + return Spec(SubDocOp.COUNTER, path, create_parents, xattr, False, delta) + + +def decrement(path, # type: str + delta, # type: int + xattr=False, # type: Optional[bool] + create_parents=False # type: Optional[bool] + ) -> Spec: + """Creates a :class:`.Spec` for decrementing the value of a field in a document. + + Args: + path (str): The path to the field. + delta (int): The value to decrement from the document. + xattr (bool, optional): Whether this operation should reference the document body or the + extended attributes data for the document. + create_parents (bool, optional): Whether or not the path to the field should be created + if it does not already exist. + + Returns: + :class:`.Spec`: An instance of :class:`.Spec`. + + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentException`: If the delta arugment is not >= 0 or not + of type int. + """ + if not isinstance(delta, int): + raise InvalidArgumentException("Delta must be integer") + if delta <= 0: + raise InvalidArgumentException( + "Delta must be integer greater than or equal to 0") + + return Spec(SubDocOp.COUNTER, path, create_parents, xattr, False, -1 * delta) + + +def get_full() -> Spec: + """ + Fetches the entire document. + + :return: Spec + """ + return Spec(SubDocOp.GET_DOC, '', False) + + +def with_expiry() -> Spec: + """ + Fetches the expiry from the xattrs of the doc + + :return: Spec + """ + return Spec(SubDocOp.GET, LookupInMacro.expiry_time(), True) + + +def convert_macro_cas_to_cas(cas # type: str + ) -> int: + """ + Utility method to help encode CAS coming from MutationMacro.cas() stored in the xattr. + Due to a server bug, CAS is encoded backwards, but b/c of legacy users we cannot make a change. + """ + reversed_bytes = bytearray.fromhex(cas[2:] if cas.startswith('0x') else cas) + reversed_bytes.reverse() + return int(reversed_bytes.hex(), base=16) + + +""" +** DEPRECATION NOTICE ** + +The classes below are deprecated for 3.x compatibility. They should not be used. +Instead use: + * All options should be imported from `couchbase.options`. + * Scope object should be imported from `couchbase.scope`. + +""" + +from couchbase.logic.options import MutateInOptionsBase # nopep8 # isort:skip # noqa: E402 + + +@Supportability.import_deprecated('couchbase.subdocument', 'couchbase.options') +class MutateInOptions(MutateInOptionsBase): + pass diff --git a/couchbase/tests/admin/__init__.py b/couchbase/tests/admin/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/couchbase/tests/analytics_params_t.py b/couchbase/tests/analytics_params_t.py new file mode 100644 index 000000000..cba52ea1c --- /dev/null +++ b/couchbase/tests/analytics_params_t.py @@ -0,0 +1,161 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +from couchbase.analytics import (AnalyticsQuery, + AnalyticsScanConsistency, + AnalyticsStatus) +from couchbase.options import AnalyticsOptions +from tests.environments import CollectionType + + +class AnalyticsParamTestSuite: + TEST_MANIFEST = [ + 'test_encoded_consistency', + 'test_params_base', + 'test_params_client_context_id', + 'test_params_priority', + 'test_params_query_context', + 'test_params_read_only', + 'test_params_serializer', + 'test_params_timeout', + 'test_status' + ] + + @pytest.fixture(scope='class') + def base_opts(self): + return {'statement': 'SELECT * FROM default', + 'metrics': True} + + def test_encoded_consistency(self): + q_str = 'SELECT * FROM default' + q_opts = AnalyticsOptions(scan_consistency=AnalyticsScanConsistency.REQUEST_PLUS) + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + assert query.params.get('scan_consistency', None) == AnalyticsScanConsistency.REQUEST_PLUS.value + assert query.consistency == AnalyticsScanConsistency.REQUEST_PLUS.value + + q_opts = AnalyticsOptions(scan_consistency=AnalyticsScanConsistency.NOT_BOUNDED) + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + assert query.params.get('scan_consistency', None) == AnalyticsScanConsistency.NOT_BOUNDED.value + assert query.consistency == AnalyticsScanConsistency.NOT_BOUNDED.value + + def test_params_base(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = AnalyticsOptions() + query = AnalyticsQuery.create_query_object(q_str, q_opts) + assert query.params == base_opts + + def test_params_client_context_id(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = AnalyticsOptions(client_context_id='test-string-id') + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['client_context_id'] = 'test-string-id' + assert query.params == exp_opts + + def test_params_priority(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = AnalyticsOptions(priority=True) + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['priority'] = True + assert query.params == exp_opts + + def test_params_query_context(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = AnalyticsOptions(query_context='bucket.scope') + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['scope_qualifier'] = 'bucket.scope' + assert query.params == exp_opts + + def test_params_read_only(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = AnalyticsOptions(read_only=True) + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['readonly'] = True + assert query.params == exp_opts + + def test_params_serializer(self, base_opts): + from couchbase.serializer import DefaultJsonSerializer + + # serializer + serializer = DefaultJsonSerializer() + q_str = 'SELECT * FROM default' + q_opts = AnalyticsOptions(serializer=serializer) + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['serializer'] = serializer + assert query.params == exp_opts + + def test_params_timeout(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = AnalyticsOptions(timeout=timedelta(seconds=120)) + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 120000000 + assert query.params == exp_opts + + q_opts = AnalyticsOptions(timeout=20) + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 20000000 + assert query.params == exp_opts + + q_opts = AnalyticsOptions(timeout=25.5) + query = AnalyticsQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 25500000 + assert query.params == exp_opts + + @pytest.mark.parametrize('value, expected', [(k, v) for k, v in AnalyticsStatus.__members__.items()]) + def test_status(self, value, expected): + a_status = AnalyticsStatus[value] + assert isinstance(a_status, AnalyticsStatus) + assert a_status == expected + + +class ClassicAnalyticsParamTests(AnalyticsParamTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicAnalyticsParamTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicAnalyticsParamTests) if valid_test_method(meth)] + compare = set(AnalyticsParamTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.setup(request.param) + yield cb_base_env + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/analytics_t.py b/couchbase/tests/analytics_t.py new file mode 100644 index 000000000..7a9a60ac5 --- /dev/null +++ b/couchbase/tests/analytics_t.py @@ -0,0 +1,384 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading +from datetime import datetime, timedelta + +import pytest + +from couchbase.analytics import (AnalyticsMetaData, + AnalyticsMetrics, + AnalyticsStatus, + AnalyticsWarning) +from couchbase.exceptions import (AmbiguousTimeoutException, + DatasetNotFoundException, + DataverseNotFoundException) +from couchbase.options import AnalyticsOptions, UnsignedInt64 +from tests.environments import CollectionType +from tests.environments.analytics_environment import AnalyticsTestEnvironment + + +class AnalyticsCollectionTestSuite: + TEST_MANIFEST = [ + 'test_analytics_metadata', + 'test_analytics_query_in_thread', + 'test_analytics_with_metrics', + 'test_bad_query_context', + 'test_bad_scope_query', + 'test_cluster_query_context', + 'test_query_fully_qualified', + 'test_scope_query', + 'test_scope_query_fqdn', + 'test_scope_query_with_named_params_in_options', + 'test_scope_query_with_positional_params_in_options', + ] + + def test_analytics_metadata(self, cb_env): + result = cb_env.scope.analytics_query(f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2') + cb_env.assert_rows(result, 2) + metadata = result.metadata() # type: AnalyticsMetaData + assert isinstance(metadata, AnalyticsMetaData) + for id_meth in (metadata.client_context_id, metadata.request_id): + id_res = id_meth() + fail_msg = "{} failed".format(id_meth) + assert isinstance(id_res, str), fail_msg + assert metadata.status() == AnalyticsStatus.SUCCESS + assert isinstance(metadata.signature(), (str, dict)) + assert isinstance(metadata.warnings(), (list)) + for warning in metadata.warnings(): + assert isinstance(warning, AnalyticsWarning) + assert isinstance(warning.message(), str) + assert isinstance(warning.code(), int) + + def test_analytics_query_in_thread(self, cb_env): + results = [None] + + def run_test(scope, collection_name, assert_fn, results): + try: + result = scope.analytics_query(f"SELECT * FROM `{collection_name}` LIMIT 2") + assert_fn(result, 2) + assert result.metadata() is not None + except AssertionError: + results[0] = False + except Exception as ex: + results[0] = ex + else: + results[0] = True + + t = threading.Thread(target=run_test, + args=(cb_env.scope, cb_env.collection.name, cb_env.assert_rows, results)) + t.start() + t.join() + + assert len(results) == 1 + assert results[0] is True + + def test_analytics_with_metrics(self, cb_env): + initial = datetime.now() + result = cb_env.scope.analytics_query(f'SELECT * FROM `{cb_env.collection.name}` LIMIT 1') + cb_env.assert_rows(result, 1) + taken = datetime.now() - initial + metadata = result.metadata() # type: AnalyticsMetaData + metrics = metadata.metrics() + assert isinstance(metrics, AnalyticsMetrics) + assert isinstance(metrics.elapsed_time(), timedelta) + assert metrics.elapsed_time() < taken + assert metrics.elapsed_time() > timedelta(milliseconds=0) + assert isinstance(metrics.execution_time(), timedelta) + assert metrics.execution_time() < taken + assert metrics.execution_time() > timedelta(milliseconds=0) + + expected_counts = {metrics.error_count: 0, + metrics.result_count: 1, + metrics.warning_count: 0} + for method, expected in expected_counts.items(): + count_result = method() + fail_msg = "{} failed".format(method) + assert isinstance(count_result, UnsignedInt64), fail_msg + assert count_result == UnsignedInt64(expected), fail_msg + assert metrics.result_size() > UnsignedInt64(0) + assert isinstance(metrics.processed_objects(), UnsignedInt64) + assert metrics.error_count() == UnsignedInt64(0) + + def test_bad_query_context(self, cb_env): + # test w/ no context + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.collection.name} `LIMIT 2') + # @TODO: DatasetNotFoundException + with pytest.raises(DatasetNotFoundException): + cb_env.assert_rows(result, 2) + + # test w/ bad scope + q_context = f'default:`{cb_env.bucket.name}`.`fake-scope`' + result = cb_env.cluster.analytics_query( + f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2', AnalyticsOptions(query_context=q_context)) + # @TODO: DataverseNotFoundException + with pytest.raises(DataverseNotFoundException): + cb_env.assert_rows(result, 2) + + def test_bad_scope_query(self, cb_env): + q_context = f'default:`{cb_env.bucket.name}`.`fake-scope`' + result = cb_env.scope.analytics_query(f'SELECT * FROM {cb_env.fqdn} LIMIT 2', + AnalyticsOptions(query_context=q_context)) + with pytest.raises(DataverseNotFoundException): + cb_env.assert_rows(result, 2) + + q_context = f'default:`fake-bucket`.`{cb_env.scope.name}`' + result = cb_env.scope.analytics_query(f'SELECT * FROM {cb_env.fqdn} LIMIT 2', + query_context=q_context) + with pytest.raises(DataverseNotFoundException): + cb_env.assert_rows(result, 2) + + def test_cluster_query_context(self, cb_env): + q_context = f'default:`{cb_env.bucket.name}`.`{cb_env.scope.name}`' + # test with QueryOptions + a_opts = AnalyticsOptions(query_context=q_context) + result = cb_env.cluster.analytics_query( + f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2', a_opts) + cb_env.assert_rows(result, 2) + + # test with kwargs + result = cb_env.cluster.analytics_query( + f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2', query_context=q_context) + cb_env.assert_rows(result, 2) + + def test_query_fully_qualified(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM {cb_env.fqdn} LIMIT 2') + cb_env.assert_rows(result, 2) + + def test_scope_query(self, cb_env): + result = cb_env.scope.analytics_query(f'SELECT * FROM `{cb_env.collection.name}` LIMIT 2') + cb_env.assert_rows(result, 2) + + def test_scope_query_fqdn(self, cb_env): + # @TODO: look into this... + if cb_env.server_version_short >= 7.1: + pytest.skip("Analytics scope query format not allowed on server versions >= 7.1") + + result = cb_env.scope.analytics_query(f'SELECT * FROM {cb_env.fqdn} LIMIT 2', query_context='') + cb_env.assert_rows(result, 2) + + def test_scope_query_with_named_params_in_options(self, cb_env): + q_str = f'SELECT * FROM `{cb_env.collection.name}` WHERE batch LIKE $batch LIMIT 1' + result = cb_env.scope.analytics_query(q_str, + AnalyticsOptions(named_parameters={'batch': f'{cb_env.get_batch_id()}%'})) + cb_env.assert_rows(result, 1) + + def test_scope_query_with_positional_params_in_options(self, cb_env): + result = cb_env.scope.analytics_query(f'SELECT * FROM `{cb_env.collection.name}` WHERE batch LIKE $1 LIMIT 1', + AnalyticsOptions(positional_parameters=[f'{cb_env.get_batch_id()}%'])) + cb_env.assert_rows(result, 1) + + +class AnalyticsTestSuite: + TEST_MANIFEST = [ + 'test_analytics_metadata', + 'test_analytics_query_in_thread', + 'test_analytics_with_metrics', + 'test_query_named_parameters', + 'test_query_named_parameters_no_options', + 'test_query_named_parameters_override', + 'test_query_positional_params', + 'test_query_positional_params_no_option', + 'test_query_positional_params_override', + 'test_query_raw_options', + 'test_query_timeout', + 'test_simple_query', + ] + + def test_analytics_metadata(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` LIMIT 2') + cb_env.assert_rows(result, 2) + metadata = result.metadata() # type: AnalyticsMetaData + isinstance(metadata, AnalyticsMetaData) + for id_meth in (metadata.client_context_id, metadata.request_id): + id_res = id_meth() + fail_msg = "{} failed".format(id_meth) + assert isinstance(id_res, str), fail_msg + assert metadata.status() == AnalyticsStatus.SUCCESS + assert isinstance(metadata.signature(), (str, dict)) + assert isinstance(metadata.warnings(), (list)) + for warning in metadata.warnings(): + assert isinstance(warning, AnalyticsWarning) + assert isinstance(warning.message(), str) + assert isinstance(warning.code(), int) + + def test_analytics_query_in_thread(self, cb_env): + results = [None] + + def run_test(cluster, dataset_name, assert_fn, results): + try: + result = cluster.analytics_query(f"SELECT * FROM `{dataset_name}` LIMIT 2") + assert_fn(result, 2) + assert result.metadata() is not None + except AssertionError: + results[0] = False + except Exception as ex: + results[0] = ex + else: + results[0] = True + + t = threading.Thread(target=run_test, + args=(cb_env.cluster, cb_env.DATASET_NAME, cb_env.assert_rows, results)) + t.start() + t.join() + + assert len(results) == 1 + assert results[0] is True + + def test_analytics_with_metrics(self, cb_env): + initial = datetime.now() + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` LIMIT 1') + cb_env.assert_rows(result, 1) + taken = datetime.now() - initial + metadata = result.metadata() # type: AnalyticsMetaData + metrics = metadata.metrics() + assert isinstance(metrics, AnalyticsMetrics) + assert isinstance(metrics.elapsed_time(), timedelta) + assert metrics.elapsed_time() < taken + assert metrics.elapsed_time() > timedelta(milliseconds=0) + assert isinstance(metrics.execution_time(), timedelta) + assert metrics.execution_time() < taken + assert metrics.execution_time() > timedelta(milliseconds=0) + + expected_counts = {metrics.error_count: 0, + metrics.result_count: 1, + metrics.warning_count: 0} + for method, expected in expected_counts.items(): + count_result = method() + fail_msg = "{} failed".format(method) + assert isinstance(count_result, UnsignedInt64), fail_msg + assert count_result == UnsignedInt64(expected), fail_msg + assert metrics.result_size() > UnsignedInt64(0) + assert isinstance(metrics.processed_objects(), UnsignedInt64) + assert metrics.error_count() == UnsignedInt64(0) + + def test_query_named_parameters(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $atype LIMIT 1', + AnalyticsOptions(named_parameters={'atype': 'vehicle'})) + cb_env.assert_rows(result, 1) + + def test_query_named_parameters_no_options(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $atype LIMIT 1', + atype='vehicle') + cb_env.assert_rows(result, 1) + + def test_query_named_parameters_override(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $atype LIMIT 1', + AnalyticsOptions(named_parameters={'atype': 'abcdefg'}), + atype='vehicle') + cb_env.assert_rows(result, 1) + + def test_query_positional_params(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $1 LIMIT 1', + AnalyticsOptions(positional_parameters=["vehicle"])) + cb_env.assert_rows(result, 1) + + def test_query_positional_params_no_option(self, cb_env): + result = cb_env.cluster.analytics_query( + f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $1 LIMIT 1', 'vehicle') + cb_env.assert_rows(result, 1) + + def test_query_positional_params_override(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $1 LIMIT 1', + AnalyticsOptions(positional_parameters=['abcdefg']), 'vehicle') + cb_env.assert_rows(result, 1) + + def test_query_raw_options(self, cb_env): + # via raw, we should be able to pass any option + # if using named params, need to match full name param in query + # which is different for when we pass in name_parameters via their specific + # query option (i.e. include the $ when using raw) + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $atype LIMIT $1', + AnalyticsOptions(raw={'$atype': 'vehicle', 'args': [1]})) + cb_env.assert_rows(result, 1) + + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` WHERE `type` = $1 LIMIT 1', + AnalyticsOptions(raw={'args': ['vehicle']})) + cb_env.assert_rows(result, 1) + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_query_timeout(self, cb_env): + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + # Prior to PYCBC-1521, this test would fail as each request would override the cluster level analytics_timeout. + # If a timeout was not provided in the request, the default 75s timeout would be used. PYCBC-1521 corrects + # this behavior so this test will pass as we are essentially forcing an AmbiguousTimeoutException because + # we are setting the cluster level analytics_timeout such a small value. + timeout_opts = ClusterTimeoutOptions(analytics_timeout=timedelta(milliseconds=1)) + cluster = Cluster.connect(f'{conn_string}', ClusterOptions(auth, timeout_options=timeout_opts)) + # don't need to do this except for older server versions + _ = cluster.bucket(f'{cb_env.bucket.name}') + q_str = f'SELECT * FROM `{cb_env.DATASET_NAME}` LIMIT 1;' + with pytest.raises(AmbiguousTimeoutException): + res = cluster.analytics_query(q_str) + [r for r in res.rows()] + + # if we override the timeout w/in the request the query should succeed. + res = cluster.analytics_query(q_str, timeout=timedelta(seconds=10)) + rows = [r for r in res.rows()] + assert len(rows) > 0 + + def test_simple_query(self, cb_env): + result = cb_env.cluster.analytics_query(f'SELECT * FROM `{cb_env.DATASET_NAME}` LIMIT 1') + cb_env.assert_rows(result, 1) + + +class ClassicAnalyticsCollectionTests(AnalyticsCollectionTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicAnalyticsCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicAnalyticsCollectionTests) if valid_test_method(meth)] + compare = set(AnalyticsCollectionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = AnalyticsTestEnvironment.from_environment(cb_base_env) + cb_env.enable_analytics_mgmt() + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) + + +class ClassicAnalyticsTests(AnalyticsTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicAnalyticsTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicAnalyticsTests) if valid_test_method(meth)] + compare = set(AnalyticsTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = AnalyticsTestEnvironment.from_environment(cb_base_env) + cb_env.enable_analytics_mgmt() + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) diff --git a/couchbase/tests/analyticsmgmt_t.py b/couchbase/tests/analyticsmgmt_t.py new file mode 100644 index 000000000..cd10702ec --- /dev/null +++ b/couchbase/tests/analyticsmgmt_t.py @@ -0,0 +1,806 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.exceptions import (AnalyticsLinkExistsException, + AnalyticsLinkNotFoundException, + CouchbaseException, + DatasetAlreadyExistsException, + DatasetNotFoundException, + DataverseAlreadyExistsException, + DataverseNotFoundException, + InternalServerFailureException, + InvalidArgumentException) +from couchbase.management.analytics import (AnalyticsDataType, + AnalyticsEncryptionLevel, + AnalyticsLinkType, + AzureBlobExternalAnalyticsLink, + CouchbaseAnalyticsEncryptionSettings, + CouchbaseRemoteAnalyticsLink, + S3ExternalAnalyticsLink) +from couchbase.management.options import (ConnectLinkOptions, + CreateAnalyticsIndexOptions, + CreateDatasetOptions, + CreateDataverseOptions, + DisconnectLinkOptions, + DropAnalyticsIndexOptions, + DropDatasetOptions, + DropDataverseOptions, + GetLinksAnalyticsOptions) +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class AnalyticsManagementLinksTestSuite: + TEST_MANIFEST = [ + 'test_azure_link_fail_dataverse_not_found', + 'test_azure_link_fail_link_not_found', + 'test_couchbase_link_fail_dataverse_not_found', + 'test_couchbase_link_fail_link_not_found', + 'test_couchbase_remote_link_encode', + 'test_create_azure_block_link_fail_invalid_argument', + 'test_create_couchbase_link_fail_invalid_argument', + 'test_create_link_fail_link_exists', + 'test_create_s3_external_link', + 'test_create_s3_link_fail_invalid_argument', + 'test_drop_s3_external_link', + 'test_replace_s3_external_link', + 'test_s3_link_fail_dataverse_not_found', + 'test_s3_link_fail_link_not_found', + ] + + @pytest.fixture() + def bad_azure_blob_external_links(self, empty_dataverse_name): + links = [] + links.append(AzureBlobExternalAnalyticsLink('', + 'azurebloblink', + account_name='myaccount', + account_key='myaccountkey')) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + '', + account_name='myaccount', + account_key='myaccountkey')) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + 'azurebloblink')) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + 'azurebloblink', + account_name='myaccount')) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + 'azurebloblink', + account_key='myaccountkey')) + + links.append(AzureBlobExternalAnalyticsLink(empty_dataverse_name, + 'azurebloblink', + shared_access_signature='sharedaccesssignature')) + return links + + @pytest.fixture() + def bad_couchbase_remote_links(self, empty_dataverse_name): + links = [] + links.append(CouchbaseRemoteAnalyticsLink('', + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username='Administrator', + password='password')) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + '', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username='Administrator', + password='password')) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + '', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username='Administrator', + password='password')) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + password='password')) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username='Administrator')) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.HALF), + password='password')) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.HALF), + username='Administrator')) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL) + )) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL, + certificate=bytes('certificate', 'utf-8')) + )) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL, + certificate=bytes( + 'certificate', 'utf-8'), + client_certificate=bytes('clientcert', 'utf-8')) + )) + + links.append(CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL, + certificate=bytes( + 'certificate', 'utf-8'), + client_key=bytes('clientkey', 'utf-8')) + )) + + return links + + @pytest.fixture() + def bad_s3_external_links(self, empty_dataverse_name): + links = [] + links.append(S3ExternalAnalyticsLink('', + 's3link', + 'accesskey', + 'us-west-2', + secret_access_key='mysupersecretkey', + )) + + links.append(S3ExternalAnalyticsLink(empty_dataverse_name, + '', + 'accesskey', + 'us-west-2', + secret_access_key='mysupersecretkey', + )) + + links.append(S3ExternalAnalyticsLink(empty_dataverse_name, + 's3link', + '', + 'us-west-2', + secret_access_key='mysupersecretkey', + )) + + links.append(S3ExternalAnalyticsLink(empty_dataverse_name, + 's3link', + 'accesskey', + '', + secret_access_key='mysupersecretkey', + )) + + links.append(S3ExternalAnalyticsLink('', + 's3link', + 'accesskey', + 'us-west-2', + )) + return links + + @pytest.fixture() + def create_drop_empty_dataverse(self, cb_env, empty_dataverse_name): + cb_env.aixm.create_dataverse(empty_dataverse_name, ignore_if_exists=True) + yield + cb_env.aixm.drop_dataverse(empty_dataverse_name, ignore_if_not_exists=True) + + @pytest.fixture(scope='class') + def empty_dataverse_name(self, cb_env): + if cb_env.server_version_short >= 7.0: + name = 'empty/dataverse' + else: + name = 'empty_dataverse' + return name + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_azure_link_fail_dataverse_not_found(self, cb_env): + + link = AzureBlobExternalAnalyticsLink('notadataverse', + 'azurebloblink', + account_name='myaccount', + account_key='myaccountkey') + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.create_link(link) + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.replace_link(link) + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_azure_link_fail_link_not_found(self, cb_env, empty_dataverse_name): + + link = AzureBlobExternalAnalyticsLink(empty_dataverse_name, + 'azurebloblink', + account_name='myaccount', + account_key='myaccountkey') + + with pytest.raises(AnalyticsLinkNotFoundException): + cb_env.aixm.replace_link(link) + + with pytest.raises(AnalyticsLinkNotFoundException): + cb_env.aixm.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_couchbase_link_fail_dataverse_not_found(self, cb_env): + + link = CouchbaseRemoteAnalyticsLink("notadataverse", + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username='Administrator', + password='password') + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.create_link(link) + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.replace_link(link) + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_couchbase_link_fail_link_not_found(self, cb_env, empty_dataverse_name): + + link = CouchbaseRemoteAnalyticsLink(empty_dataverse_name, + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username='Administrator', + password='password') + + with pytest.raises(AnalyticsLinkNotFoundException): + cb_env.aixm.replace_link(link) + + with pytest.raises(AnalyticsLinkNotFoundException): + cb_env.aixm.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures('cb_env') + def test_couchbase_remote_link_encode(self): + link = CouchbaseRemoteAnalyticsLink('test_dataverse', + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.NONE), + username='Administrator', + password='password') + + encoded = link.as_dict() + assert isinstance(encoded, dict) + assert encoded.get('hostname') == 'localhost' + assert encoded.get('link_type') == AnalyticsLinkType.CouchbaseRemote.value + link_encryption = encoded.get('encryption', None) + assert isinstance(link_encryption, dict) + assert link_encryption.get('encryption_level') == AnalyticsEncryptionLevel.NONE.value + assert encoded.get('username') == 'Administrator' + assert encoded.get('password') == 'password' + + link = CouchbaseRemoteAnalyticsLink('test_dataverse', + 'cbremote', + 'localhost', + CouchbaseAnalyticsEncryptionSettings( + AnalyticsEncryptionLevel.FULL, + certificate=bytes( + 'certificate', 'utf-8'), + client_certificate=bytes( + 'clientcertificate', 'utf-8'), + client_key=bytes('clientkey', 'utf-8')), + ) + + encoded = link.as_dict() + assert isinstance(encoded, dict) + assert encoded.get('hostname') == 'localhost' + assert encoded.get('link_type') == AnalyticsLinkType.CouchbaseRemote.value + link_encryption = encoded.get('encryption', None) + assert isinstance(link_encryption, dict) + assert link_encryption.get('encryption_level') == AnalyticsEncryptionLevel.FULL.value + assert link_encryption.get('certificate') == 'certificate' + assert link_encryption.get('client_certificate') == 'clientcertificate' + assert link_encryption.get('client_key') == 'clientkey' + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_create_azure_block_link_fail_invalid_argument(self, cb_env, bad_azure_blob_external_links): + for link in bad_azure_blob_external_links: + with pytest.raises(InvalidArgumentException): + cb_env.aixm.create_link(link) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_create_couchbase_link_fail_invalid_argument(self, cb_env, bad_couchbase_remote_links): + for link in bad_couchbase_remote_links: + with pytest.raises(InvalidArgumentException): + cb_env.aixm.create_link(link) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_create_link_fail_link_exists(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + 's3link', + 'accesskey', + 'us-west-2', + secret_access_key='mysupersecretkey', + ) + + cb_env.aixm.create_link(link) + + with pytest.raises(AnalyticsLinkExistsException): + cb_env.aixm.create_link(link) + + cb_env.aixm.drop_link(link.name(), empty_dataverse_name) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_create_s3_external_link(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + 's3link', + 'accesskey', + 'us-west-2', + secret_access_key='mysupersecretkey', + ) + + cb_env.aixm.create_link(link) + + links = cb_env.aixm.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 1 + assert links[0].dataverse_name() == link.dataverse_name() + assert links[0].name() == link.name() + assert links[0].link_type() == AnalyticsLinkType.S3External + assert links[0]._region == link._region + assert links[0]._access_key_id == link._access_key_id + + cb_env.aixm.drop_link(link.name(), empty_dataverse_name) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_create_s3_link_fail_invalid_argument(self, cb_env, bad_s3_external_links): + for link in bad_s3_external_links: + with pytest.raises(InvalidArgumentException): + cb_env.aixm.create_link(link) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_drop_s3_external_link(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + 's3link', + 'accesskey', + 'us-west-2', + secret_access_key='mysupersecretkey', + ) + + cb_env.aixm.create_link(link) + + links = cb_env.aixm.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 1 + assert links[0].dataverse_name() == link.dataverse_name() + assert links[0].name() == link.name() + assert links[0].link_type() == AnalyticsLinkType.S3External + assert links[0]._region == link._region + assert links[0]._access_key_id == link._access_key_id + + cb_env.aixm.drop_link(link.name(), empty_dataverse_name) + + links = cb_env.aixm.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 0 + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_replace_s3_external_link(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + 's3link', + 'accesskey', + 'us-west-2', + secret_access_key='mysupersecretkey', + ) + + cb_env.aixm.create_link(link) + + links = cb_env.aixm.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 1 + assert links[0].dataverse_name() == link.dataverse_name() + assert links[0].name() == link.name() + assert links[0].link_type() == AnalyticsLinkType.S3External + assert links[0]._region == link._region + assert links[0]._access_key_id == link._access_key_id + + new_link = S3ExternalAnalyticsLink(empty_dataverse_name, + 's3link', + 'accesskey', + 'eu-west-2', + secret_access_key='mysupersecretkey1', + ) + + cb_env.aixm.replace_link(new_link) + + links = cb_env.aixm.get_links(GetLinksAnalyticsOptions( + dataverse_name=empty_dataverse_name, name=link.name())) + + assert len(links) == 1 + assert links[0].dataverse_name() == new_link.dataverse_name() + assert links[0].name() == new_link.name() + assert links[0].link_type() == AnalyticsLinkType.S3External + assert links[0]._region == new_link._region + assert links[0]._access_key_id == new_link._access_key_id + + cb_env.aixm.drop_link(link.name(), empty_dataverse_name) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_s3_link_fail_dataverse_not_found(self, cb_env): + + link = S3ExternalAnalyticsLink("notadataverse", + 's3link', + 'accesskey', + 'us-west-2', + secret_access_key='mysupersecretkey', + ) + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.create_link(link) + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.replace_link(link) + + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.drop_link(link.name(), link.dataverse_name()) + + @pytest.mark.usefixtures('create_drop_empty_dataverse') + def test_s3_link_fail_link_not_found(self, cb_env, empty_dataverse_name): + + link = S3ExternalAnalyticsLink(empty_dataverse_name, + 'notalink', + 'accesskey', + 'us-west-2', + secret_access_key='mysupersecretkey', + ) + + with pytest.raises(AnalyticsLinkNotFoundException): + cb_env.aixm.replace_link(link) + + with pytest.raises(AnalyticsLinkNotFoundException): + cb_env.aixm.drop_link(link.name(), link.dataverse_name()) + + +class AnalyticsManagementTestSuite: + DATASET_NAME = 'test-dataset' + + TEST_MANIFEST = [ + 'test_connect_disconnect_link', + 'test_create_dataset', + 'test_create_dataset_ignore_exists', + 'test_create_dataverse', + 'test_create_dataverse_ignore_exists', + 'test_create_index', + 'test_drop_dataset', + 'test_drop_dataset_ignore_not_exists', + 'test_drop_dataverse', + 'test_drop_dataverse_ignore_not_exists', + 'test_drop_index', + 'test_get_all_datasets', + 'test_get_pending_mutations', + 'test_v6_dataverse_name_parsing', + 'test_v7_dataverse_name_parsing', + ] + + @pytest.fixture() + def clean_drop(self, cb_env, empty_dataverse_name): + yield + cb_env.aixm.drop_dataset(self.DATASET_NAME, ignore_if_not_exists=True) + cb_env.aixm.drop_dataset(self.DATASET_NAME, + DropDatasetOptions(ignore_if_not_exists=True, + dataverse_name=empty_dataverse_name)) + cb_env.aixm.drop_dataverse(empty_dataverse_name, ignore_if_not_exists=True) + + @pytest.fixture() + def create_drop_empty_dataverse(self, cb_env, empty_dataverse_name): + cb_env.aixm.create_dataverse(empty_dataverse_name, ignore_if_exists=True) + yield + cb_env.aixm.drop_dataverse(empty_dataverse_name, ignore_if_not_exists=True) + + @pytest.fixture() + def create_empty_dataset(self, cb_env): + cb_env.aixm.create_dataset(self.DATASET_NAME, cb_env.bucket.name, ignore_if_exists=True) + + @pytest.fixture() + def create_empty_dataverse(self, cb_env, empty_dataverse_name): + cb_env.aixm.create_dataverse(empty_dataverse_name, ignore_if_exists=True) + + @pytest.fixture() + def drop_empty_dataset(self, cb_env): + yield + cb_env.aixm.drop_dataset(self.DATASET_NAME, ignore_if_not_exists=True) + + @pytest.fixture() + def drop_empty_dataverse(self, cb_env, empty_dataverse_name): + yield + cb_env.aixm.drop_dataverse(empty_dataverse_name, ignore_if_not_exists=True) + + @pytest.fixture(scope='class') + def empty_dataverse_name(self, cb_env): + if cb_env.server_version_short >= 7.0: + name = 'empty/dataverse' + else: + name = 'empty_dataverse' + return name + + @pytest.mark.usefixtures('create_empty_dataverse') + @pytest.mark.usefixtures('clean_drop') + def test_connect_disconnect_link(self, cb_env, empty_dataverse_name): + cb_env.aixm.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, + ignore_if_exists=True)) + cb_env.aixm.connect_link(ConnectLinkOptions(dataverse_name=empty_dataverse_name)) + + # # connect link should result in documents in the dataset, so... + # dataverse_name = self.mgr._scrub_dataverse_name(self.dataverse_name) + # self.assertRows( + # 'USE {}; SELECT * FROM `{}` LIMIT 1'.format(dataverse_name, self.dataset_name)) + # # manually stop it for now + # self.cluster.analytics_query( + # 'USE {}; DISCONNECT LINK Local'.format(dataverse_name, self.dataset_name)).metadata() + cb_env.aixm.disconnect_link(DisconnectLinkOptions(dataverse_name=empty_dataverse_name)) + + @pytest.mark.usefixtures("drop_empty_dataset") + def test_create_dataset(self, cb_env): + cb_env.aixm.create_dataset(self.DATASET_NAME, cb_env.bucket.name) + + @pytest.mark.usefixtures("drop_empty_dataset") + def test_create_dataset_ignore_exists(self, cb_env): + cb_env.aixm.create_dataset(self.DATASET_NAME, cb_env.bucket.name) + with pytest.raises(DatasetAlreadyExistsException): + cb_env.aixm.create_dataset(self.DATASET_NAME, cb_env.bucket.name) + + cb_env.aixm.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(ignore_if_exists=True)) + + @pytest.mark.usefixtures('drop_empty_dataverse') + def test_create_dataverse(self, cb_env, empty_dataverse_name): + cb_env.aixm.create_dataverse(empty_dataverse_name) + + @pytest.mark.usefixtures('drop_empty_dataverse') + def test_create_dataverse_ignore_exists(self, cb_env, empty_dataverse_name): + cb_env.aixm.create_dataverse( + empty_dataverse_name, CreateDataverseOptions(ignore_if_exists=True)) + + with pytest.raises(DataverseAlreadyExistsException): + cb_env.aixm.create_dataverse(empty_dataverse_name) + + @pytest.mark.usefixtures('create_empty_dataverse') + @pytest.mark.usefixtures('clean_drop') + def test_create_index(self, cb_env, empty_dataverse_name): + cb_env.aixm.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, ignore_if_exists=True)) + cb_env.aixm.create_index("test_idx", self.DATASET_NAME, + {'name': AnalyticsDataType.STRING, + 'description': AnalyticsDataType.STRING}, + CreateAnalyticsIndexOptions(dataverse_name=empty_dataverse_name)) + + def check_for_idx(idx): + indexes = cb_env.aixm.get_all_indexes() + for index in indexes: + if index.name == idx: + return + raise Exception( + "unable to find 'test_idx' in list of all indexes") + + TestEnvironment.try_n_times(10, 3, check_for_idx, 'test_idx') + + @pytest.mark.usefixtures('create_empty_dataset') + @pytest.mark.usefixtures("drop_empty_dataset") + def test_drop_dataset(self, cb_env): + cb_env.aixm.drop_dataset(self.DATASET_NAME) + + @pytest.mark.usefixtures('create_empty_dataset') + @pytest.mark.usefixtures("drop_empty_dataset") + def test_drop_dataset_ignore_not_exists(self, cb_env): + cb_env.aixm.drop_dataset(self.DATASET_NAME) + with pytest.raises(DatasetNotFoundException): + cb_env.aixm.drop_dataset(self.DATASET_NAME) + cb_env.aixm.drop_dataset(self.DATASET_NAME, DropDatasetOptions(ignore_if_not_exists=True)) + + @pytest.mark.usefixtures('create_empty_dataverse') + @pytest.mark.usefixtures('drop_empty_dataverse') + def test_drop_dataverse(self, cb_env, empty_dataverse_name): + cb_env.aixm.drop_dataverse(empty_dataverse_name) + + @pytest.mark.usefixtures('create_empty_dataverse') + @pytest.mark.usefixtures('drop_empty_dataverse') + def test_drop_dataverse_ignore_not_exists(self, cb_env, empty_dataverse_name): + cb_env.aixm.drop_dataverse(empty_dataverse_name) + with pytest.raises(DataverseNotFoundException): + cb_env.aixm.drop_dataverse(empty_dataverse_name) + cb_env.aixm.drop_dataverse(empty_dataverse_name, DropDataverseOptions(ignore_if_not_exists=True)) + + @pytest.mark.usefixtures('create_empty_dataverse') + @pytest.mark.usefixtures('clean_drop') + def test_drop_index(self, cb_env, empty_dataverse_name): + # create one first, if not already there + cb_env.aixm.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, ignore_if_exists=True)) + cb_env.aixm.create_index('test_idx', self.DATASET_NAME, + {'name': AnalyticsDataType.STRING, + 'description': AnalyticsDataType.STRING}, + CreateAnalyticsIndexOptions(dataverse_name=empty_dataverse_name)) + + def check_for_idx(idx): + indexes = cb_env.aixm.get_all_indexes() + for index in indexes: + if index.name == idx: + return + raise Exception( + "unable to find 'test_idx' in list of all indexes") + + TestEnvironment.try_n_times(10, 3, check_for_idx, 'test_idx') + cb_env.aixm.drop_index("test_idx", + self.DATASET_NAME, + DropAnalyticsIndexOptions(dataverse_name=empty_dataverse_name)) + TestEnvironment.try_n_times_till_exception(10, 3, check_for_idx, 'test_idx') + + @pytest.mark.usefixtures('create_empty_dataverse') + @pytest.mark.usefixtures('clean_drop') + def test_get_all_datasets(self, cb_env, empty_dataverse_name): + cb_env.aixm.create_dataset(self.DATASET_NAME, cb_env.bucket.name, ignore_if_exists=True) + cb_env.aixm.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, + ignore_if_exists=True)) + + datasets = cb_env.aixm.get_all_datasets() + local_ds = [ds for ds in datasets if ds.dataset_name == self.DATASET_NAME] + assert len(local_ds) == 2 + assert any(map(lambda ds: ds.dataverse_name == 'Default', local_ds)) is True + assert any(map(lambda ds: ds.dataverse_name == empty_dataverse_name, local_ds)) is True + + @pytest.mark.usefixtures('create_empty_dataverse') + @pytest.mark.usefixtures('clean_drop') + def test_get_pending_mutations(self, cb_env, empty_dataverse_name): + EnvironmentFeatures.check_if_feature_supported('analytics_pending_mutations', + cb_env.server_version_short, + cb_env.mock_server_type) + dv_name = empty_dataverse_name.replace('/', '.') + key = f'{dv_name}.{self.DATASET_NAME}' + result = cb_env.aixm.get_pending_mutations() + assert key not in result.keys() + cb_env.aixm.create_dataset(self.DATASET_NAME, + cb_env.bucket.name, + CreateDatasetOptions(dataverse_name=empty_dataverse_name, + ignore_if_exists=True)) + cb_env.aixm.connect_link(ConnectLinkOptions(dataverse_name=empty_dataverse_name)) + TestEnvironment.sleep(1) + result = cb_env.aixm.get_pending_mutations() + assert key in result.keys() + cb_env.aixm.disconnect_link(DisconnectLinkOptions(dataverse_name=empty_dataverse_name)) + + def test_v6_dataverse_name_parsing(self, cb_env): + if cb_env.server_version_short >= 7.0: + pytest.skip('Test only for 6.x versions') + + # test.test_dataverse, valid format which is valid >= 6.0, but not on 6.6...weird + if cb_env.server_version_short >= 6.6: + with pytest.raises(CouchbaseException): + cb_env.aixm.create_dataverse( + 'test.test_dataverse', CreateDataverseOptions(ignore_if_exists=True)) + else: + cb_env.aixm.create_dataverse( + 'test.test_dataverse', CreateDataverseOptions(ignore_if_exists=True)) + + cb_env.aixm.drop_dataverse('test.test_dataverse', ignore_if_not_exists=True) + + # test/test_dataverse, invalid format < 7.0 + with pytest.raises((InternalServerFailureException, CouchbaseException)): + cb_env.aixm.create_dataverse( + 'test/test_dataverse', CreateDataverseOptions(ignore_if_exists=True)) + + def test_v7_dataverse_name_parsing(self, cb_env): + if cb_env.server_version_short < 7.0: + pytest.skip('Test only for 7.x versions') + + # test.test_dataverse, valid format which is valid >= 6.6 + cb_env.aixm.create_dataverse( + 'test.test_dataverse', CreateDataverseOptions(ignore_if_exists=True)) + cb_env.aixm.drop_dataverse('test.test_dataverse') + + # test/test_dataverse, valideformat which is valid >= 7.0 + cb_env.aixm.create_dataverse( + 'test/test_dataverse', CreateDataverseOptions(ignore_if_exists=True)) + cb_env.aixm.drop_dataverse('test/test_dataverse') + + +class ClassicAnalyticsManagementLinksTests(AnalyticsManagementLinksTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicAnalyticsManagementLinksTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicAnalyticsManagementLinksTests) if valid_test_method(meth)] + compare = set(AnalyticsManagementLinksTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env, test_manifest_validated): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + EnvironmentFeatures.check_if_feature_supported('analytics_link_mgmt', + cb_base_env.server_version_short, + cb_base_env.mock_server_type) + + cb_base_env.enable_analytics_mgmt() + yield cb_base_env + cb_base_env.disable_analytics_mgmt() + + +class ClassicAnalyticsManagementTests(AnalyticsManagementTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicAnalyticsManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicAnalyticsManagementTests) if valid_test_method(meth)] + compare = set(AnalyticsManagementTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env, test_manifest_validated): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.enable_analytics_mgmt() + yield cb_base_env + cb_base_env.disable_analytics_mgmt() diff --git a/couchbase/tests/base.py b/couchbase/tests/base.py deleted file mode 100644 index 2100b4e85..000000000 --- a/couchbase/tests/base.py +++ /dev/null @@ -1,398 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from __future__ import absolute_import - -import os -import sys -import types -import platform - -try: - from unittest.case import SkipTest -except ImportError: - from nose.exc import SkipTest - -try: - from configparser import ConfigParser -except ImportError: - # Python <3.0 fallback - from ConfigParser import SafeConfigParser as ConfigParser - -from testresources import ResourcedTestCase, TestResourceManager - -from couchbase.exceptions import CouchbaseError -from couchbase.admin import Admin -from couchbase.mockserver import CouchbaseMock, BucketSpec, MockControlClient -from couchbase.result import ( - MultiResult, ValueResult, OperationResult, ObserveInfo, Result) -from couchbase._pyport import basestring -from couchbase.connstr import ConnectionString - -CONFIG_FILE = 'tests.ini' # in cwd - -class ClusterInformation(object): - def __init__(self): - self.host = "localhost" - self.port = 8091 - self.admin_username = "Administrator" - self.admin_password = "password" - self.bucket_name = "default" - self.bucket_password = "" - - def make_connargs(self, **overrides): - bucket = self.bucket_name - if 'bucket' in overrides: - bucket = overrides.pop('bucket') - connstr = 'http://{0}:{1}/{2}?'.format(self.host, self.port, bucket) - - if 'config_cache' in overrides: - connstr += 'config_cache=' - connstr += str(overrides.pop('config_cache')) - - ret = { - 'password': self.bucket_password, - 'connection_string': connstr - } - ret.update(overrides) - return ret - - def make_connection(self, conncls, **kwargs): - return conncls(**self.make_connargs(**kwargs)) - - def make_admin_connection(self): - return Admin(self.admin_username, self.admin_password, - self.host, self.port) - - -class ConnectionConfiguration(object): - def __init__(self, filename=CONFIG_FILE): - self._fname = filename - self.load() - - def load(self): - config = ConfigParser() - config.read(self._fname) - - info = ClusterInformation() - info.host = config.get('realserver', 'host') - info.port = config.getint('realserver', 'port') - info.admin_username = config.get('realserver', 'admin_username') - info.admin_password = config.get('realserver', 'admin_password') - info.bucket_name = config.get('realserver', 'bucket_name') - info.bucket_password = config.get('realserver', 'bucket_password') - - if config.getboolean('realserver', 'enabled'): - self.realserver_info = info - else: - self.realserver_info = None - - if (config.has_option("mock", "enabled") and - config.getboolean('mock', 'enabled')): - - self.mock_enabled = True - self.mockpath = config.get("mock", "path") - if config.has_option("mock", "url"): - self.mockurl = config.get("mock", "url") - else: - self.mockurl = None - else: - self.mock_enabled = False - - -class MockResourceManager(TestResourceManager): - def __init__(self, config): - super(MockResourceManager, self).__init__() - self._config = config - self._info = None - self._failed = False - - def _reset(self, *args, **kw): - pass - - def make(self, *args, **kw): - if not self._config.mock_enabled: - return None - - if self._info: - return self._info - - if self._failed: - raise Exception('Not invoking failed mock!') - - bspec_dfl = BucketSpec('default', 'couchbase') - mock = CouchbaseMock([bspec_dfl], - self._config.mockpath, - self._config.mockurl, - replicas=2, - nodes=4) - - try: - mock.start() - except: - self._failed = True - raise - - info = ClusterInformation() - info.bucket_name = "default" - info.port = mock.rest_port - info.host = "127.0.0.1" - info.admin_username = "Administrator" - info.admin_password = "password" - info.mock = mock - self._info = info - return info - - def isDirty(self): - return False - - -class RealServerResourceManager(TestResourceManager): - def __init__(self, config): - super(RealServerResourceManager, self).__init__() - self._config = config - - def make(self, *args, **kw): - return self._config.realserver_info - - def isDirty(self): - return False - - -class ApiImplementationMixin(object): - """ - This represents the interface which should be installed by an implementation - of the API during load-time - """ - @property - def factory(self): - """ - Return the main Connection class used for this implementation - """ - raise NotImplementedError() - - @property - def viewfactory(self): - """ - Return the view subclass used for this implementation - """ - raise NotImplementedError() - - @property - def should_check_refcount(self): - """ - Return whether the instance's reference cound should be checked at - destruction time - """ - raise NotImplementedError() - - cls_MultiResult = MultiResult - cls_ValueResult = ValueResult - cls_OperationResult = OperationResult - cls_ObserveInfo = ObserveInfo - cls_Result = Result - -GLOBAL_CONFIG = ConnectionConfiguration() - - -class CouchbaseTestCase(ResourcedTestCase): - resources = [ - ('_mock_info', MockResourceManager(GLOBAL_CONFIG)), - ('_realserver_info', RealServerResourceManager(GLOBAL_CONFIG)) - ] - - config = GLOBAL_CONFIG - - @property - def cluster_info(self): - for v in [self._realserver_info, self._mock_info]: - if v: - return v - raise Exception("Neither mock nor realserver available") - - @property - def is_realserver(self): - return self.cluster_info is self._realserver_info - - @property - def is_mock(self): - return self.cluster_info is self._mock_info - - @property - def realserver_info(self): - if not self._realserver_info: - raise SkipTest("Real server required") - return self._realserver_info - - @property - def mock(self): - try: - return self._mock_info.mock - except AttributeError: - return None - - @property - def mock_info(self): - if not self._mock_info: - raise SkipTest("Mock server required") - return self._mock_info - - - def setUp(self): - super(CouchbaseTestCase, self).setUp() - - if not hasattr(self, 'assertIsInstance'): - def tmp(self, a, *bases): - self.assertTrue(isinstance(a, bases)) - self.assertIsInstance = types.MethodType(tmp, self) - if not hasattr(self, 'assertIsNone'): - def tmp(self, a): - self.assertTrue(a is None) - self.assertIsNone = types.MethodType(tmp, self) - - self._key_counter = 0 - - def skipLcbMin(self, vstr): - """ - Test requires a libcouchbase version of at least vstr. - This may be a hex number (e.g. 0x020007) or a string (e.g. "2.0.7") - """ - - if isinstance(vstr, basestring): - components = vstr.split('.') - hexstr = "0x" - for comp in components: - if len(comp) > 2: - raise ValueError("Version component cannot be larger than 99") - hexstr += "{0:02}".format(int(comp)) - - vernum = int(hexstr, 16) - else: - vernum = vstr - components = [] - # Get the display - for x in range(0, 3): - comp = (vernum & 0xff << (x*8)) >> x*8 - comp = "{0:x}".format(comp) - components = [comp] + components - vstr = ".".join(components) - - rtstr, rtnum = self.factory.lcb_version() - if rtnum < vernum: - raise SkipTest(("Test requires {0} to run (have {1})") - .format(vstr, rtstr)) - - def skipIfMock(self): - pass - - def skipUnlessMock(self): - pass - - def make_connargs(self, **overrides): - return self.cluster_info.make_connargs(**overrides) - - def make_connection(self, **kwargs): - return self.cluster_info.make_connection(self.factory, **kwargs) - - def make_admin_connection(self): - return self.cluster_info.make_admin_connection() - - def gen_key(self, prefix=None): - if not prefix: - prefix = "python-couchbase-key_" - - ret = "{0}{1}".format(prefix, self._key_counter) - self._key_counter += 1 - return ret - - def gen_key_list(self, amount=5, prefix=None): - ret = [ self.gen_key(prefix) for x in range(amount) ] - return ret - - def gen_kv_dict(self, amount=5, prefix=None): - ret = {} - keys = self.gen_key_list(amount=amount, prefix=prefix) - for k in keys: - ret[k] = "Value_For_" + k - return ret - - -class ConnectionTestCase(CouchbaseTestCase): - def checkCbRefcount(self): - if not self.should_check_refcount: - return - - import gc - if platform.python_implementation() == 'PyPy': - return - - gc.collect() - for x in range(10): - oldrc = sys.getrefcount(self.cb) - if oldrc > 2: - gc.collect() - else: - break - - self.assertEqual(oldrc, 2) - - def setUp(self): - super(ConnectionTestCase, self).setUp() - self.cb = self.make_connection() - - def tearDown(self): - super(ConnectionTestCase, self).tearDown() - if hasattr(self, '_implDtorHook'): - self._implDtorHook() - else: - try: - self.checkCbRefcount() - finally: - del self.cb - - -class RealServerTestCase(ConnectionTestCase): - def setUp(self): - super(RealServerTestCase, self).setUp() - - if not self._realserver_info: - raise SkipTest("Need real server") - - @property - def cluster_info(self): - return self.realserver_info - - -# Class which sets up all the necessary Mock stuff -class MockTestCase(ConnectionTestCase): - def setUp(self): - super(MockTestCase, self).setUp() - self.skipUnlessMock() - self.mockclient = MockControlClient(self.mock.rest_port) - - def make_connection(self, **kwargs): - return self.mock_info.make_connection(self.factory, **kwargs) - - @property - def cluster_info(self): - return self.mock_info - - -class DDocTestCase(ConnectionTestCase): - pass - - -class ViewTestCase(ConnectionTestCase): - pass diff --git a/couchbase/tests/binary_collection_multi_t.py b/couchbase/tests/binary_collection_multi_t.py new file mode 100644 index 000000000..722f0ed29 --- /dev/null +++ b/couchbase/tests/binary_collection_multi_t.py @@ -0,0 +1,187 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.options import (DecrementMultiOptions, + DecrementOptions, + IncrementMultiOptions, + IncrementOptions, + SignedInt64) +from couchbase.result import (CounterResult, + MultiCounterResult, + MultiMutationResult, + MutationResult) +from tests.environments import CollectionType +from tests.environments.binary_environment import BinaryTestEnvironment + + +class BinaryCollectionMultiTestSuite: + + TEST_MANIFEST = [ + 'test_append_multi_bytes', + 'test_append_multi_string', + 'test_counter_multi_decrement', + 'test_counter_multi_decrement_non_default', + 'test_counter_multi_decrement_non_default_per_key', + 'test_counter_multi_increment', + 'test_counter_multi_increment_non_default', + 'test_counter_multi_increment_non_default_per_key', + 'test_prepend_multi_bytes', + 'test_prepend_multi_string', + ] + + def test_append_multi_bytes(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('bytes_empty', 4) + values = [b'foo', b'bar', b'baz', b'qux'] + keys_and_docs = dict(zip(keys, values)) + res = cb_env.collection.binary().append_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + def test_append_multi_string(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('utf8_empty', 4) + values = ['foo', 'bar', 'baz', 'qux'] + keys_and_docs = dict(zip(keys, values)) + res = cb_env.collection.binary().append_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + def test_counter_multi_decrement(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('counter_empty', 4) + res = cb_env.collection.binary().decrement_multi(keys) + assert isinstance(res, MultiCounterResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, CounterResult), res.results.values())) is True + + def test_counter_multi_decrement_non_default(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('counter_empty', 4) + res = cb_env.collection.binary().decrement_multi(keys, DecrementMultiOptions(initial=SignedInt64(3))) + assert isinstance(res, MultiCounterResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, CounterResult), res.results.values())) is True + for r in res.results.values(): + assert r.content == 3 + + def test_counter_multi_decrement_non_default_per_key(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('counter_empty', 4) + key1 = keys[0] + opts = DecrementMultiOptions(initial=SignedInt64(3), per_key_options={ + key1: DecrementOptions(initial=SignedInt64(100))}) + res = cb_env.collection.binary().decrement_multi(keys, opts) + assert isinstance(res, MultiCounterResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, CounterResult), res.results.values())) is True + for k, v in res.results.items(): + if k == key1: + assert v.content == 100 + else: + assert v.content == 3 + + def test_counter_multi_increment(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('counter_empty', 4) + res = cb_env.collection.binary().increment_multi(keys) + assert isinstance(res, MultiCounterResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, CounterResult), res.results.values())) is True + + def test_counter_multi_increment_non_default(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('counter_empty', 4) + res = cb_env.collection.binary().increment_multi(keys, IncrementMultiOptions(initial=SignedInt64(3))) + assert isinstance(res, MultiCounterResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, CounterResult), res.results.values())) is True + for r in res.results.values(): + assert r.content == 3 + + def test_counter_multi_increment_non_default_per_key(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('counter_empty', 4) + key1 = keys[0] + opts = IncrementMultiOptions(initial=SignedInt64(3), per_key_options={ + key1: IncrementOptions(initial=SignedInt64(100))}) + res = cb_env.collection.binary().increment_multi(keys, opts) + assert isinstance(res, MultiCounterResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, CounterResult), res.results.values())) is True + for k, v in res.results.items(): + if k == key1: + assert v.content == 100 + else: + assert v.content == 3 + + def test_prepend_multi_bytes(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('bytes_empty', 4) + values = [b'foo', b'bar', b'baz', b'qux'] + keys_and_docs = dict(zip(keys, values)) + res = cb_env.collection.binary().prepend_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + def test_prepend_multi_string(self, cb_env): + keys = cb_env.get_multiple_existing_docs_by_type('utf8_empty', 4) + values = ['foo', 'bar', 'baz', 'qux'] + keys_and_docs = dict(zip(keys, values)) + res = cb_env.collection.binary().prepend_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + +class ClassicBinaryCollectionMultiTests(BinaryCollectionMultiTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicBinaryCollectionMultiTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicBinaryCollectionMultiTests) if valid_test_method(meth)] + compare = set(BinaryCollectionMultiTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = BinaryTestEnvironment.from_environment(cb_base_env) + cb_env.enable_bucket_mgmt() + cb_env.setup(request.param, __name__) + + yield cb_env + + cb_env.teardown(request.param) diff --git a/couchbase/tests/binary_collection_t.py b/couchbase/tests/binary_collection_t.py new file mode 100644 index 000000000..f4cc50985 --- /dev/null +++ b/couchbase/tests/binary_collection_t.py @@ -0,0 +1,248 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.exceptions import DocumentNotFoundException, InvalidArgumentException +from couchbase.options import (DecrementOptions, + DeltaValue, + IncrementOptions, + SignedInt64) +from couchbase.result import CounterResult, MutationResult +from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder +from tests.environments import CollectionType +from tests.environments.binary_environment import BinaryTestEnvironment +from tests.environments.test_environment import TestEnvironment + + +class BinaryCollectionTestSuite: + + TEST_MANIFEST = [ + 'test_append_bytes', + 'test_append_bytes_not_empty', + 'test_append_string', + 'test_append_string_nokey', + 'test_append_string_not_empty', + 'test_counter_bad_delta_value', + 'test_counter_bad_initial_value', + 'test_counter_decrement', + 'test_counter_decrement_initial_value', + 'test_counter_decrement_non_default', + 'test_counter_decrement_no_initial_value', + 'test_counter_increment', + 'test_counter_increment_initial_value', + 'test_counter_increment_non_default', + 'test_counter_increment_no_initial_value', + 'test_prepend_bytes', + 'test_prepend_bytes_not_empty', + 'test_prepend_string', + 'test_prepend_string_nokey', + 'test_prepend_string_not_empty', + 'test_signed_int_64', + 'test_unsigned_int', + ] + + def test_append_bytes(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes_empty', key_only=True) + result = cb_env.collection.binary().append(key, b'XXX') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = cb_env.collection.get(key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'XXX' + + def test_append_bytes_not_empty(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + + result = cb_env.collection.binary().append(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = cb_env.collection.get(key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == value + b'foo' + + def test_append_string(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + result = cb_env.collection.binary().append(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = cb_env.collection.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + def test_append_string_not_empty(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('utf8') + result = cb_env.collection.binary().append(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = cb_env.collection.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == value + 'foo' + + def test_append_string_nokey(self, cb_env): + # @TODO(jc): 3.2.x SDK tests for NotStoredException + with pytest.raises(DocumentNotFoundException): + cb_env.collection.binary().append(TestEnvironment.NOT_A_KEY, 'foo') + + def test_counter_bad_delta_value(self, cb_env): + key = cb_env.get_existing_doc_by_type('counter_empty') + + with pytest.raises(InvalidArgumentException): + cb_env.collection.binary().increment(key, delta=5) + + with pytest.raises(InvalidArgumentException): + cb_env.collection.binary().decrement(key, delta=5) + + def test_counter_bad_initial_value(self, cb_env): + key = cb_env.get_existing_doc_by_type('counter_empty') + + with pytest.raises(InvalidArgumentException): + cb_env.collection.binary().increment(key, initial=100) + + with pytest.raises(InvalidArgumentException): + cb_env.collection.binary().decrement(key, initial=100) + + def test_counter_decrement(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('counter') + result = cb_env.collection.binary().decrement(key) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value - 1 + + def test_counter_decrement_initial_value(self, cb_env): + key = cb_env.get_existing_doc_by_type('counter_empty') + result = cb_env.collection.binary().decrement(key, DecrementOptions(initial=SignedInt64(100))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == 100 + + def test_counter_decrement_non_default(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('counter') + result = cb_env.collection.binary().decrement(key, DecrementOptions(delta=DeltaValue(3))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value - 3 + + def test_counter_decrement_no_initial_value(self, cb_env): + with pytest.raises(DocumentNotFoundException): + cb_env.collection.binary().decrement('non-existent-doc', DecrementOptions(initial=SignedInt64(-1))) + + def test_counter_increment(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('counter') + result = cb_env.collection.binary().increment(key) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value + 1 + + def test_counter_increment_initial_value(self, cb_env): + key = cb_env.get_existing_doc_by_type('counter_empty') + result = cb_env.collection.binary().increment(key, IncrementOptions(initial=SignedInt64(100))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == 100 + + def test_counter_increment_non_default(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('counter') + result = cb_env.collection.binary().increment(key, IncrementOptions(delta=DeltaValue(3))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value + 3 + + def test_counter_increment_no_initial_value(self, cb_env): + with pytest.raises(DocumentNotFoundException): + cb_env.collection.binary().increment('non-existent-doc', IncrementOptions(initial=SignedInt64(-1))) + + def test_prepend_bytes(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes_empty', key_only=True) + result = cb_env.collection.binary().prepend(key, b'XXX') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = cb_env.collection.get(key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'XXX' + + def test_prepend_bytes_not_empty(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + result = cb_env.collection.binary().prepend(key, b'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = cb_env.collection.get(key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'foo' + value + + def test_prepend_string(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + result = cb_env.collection.binary().prepend(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = cb_env.collection.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + def test_prepend_string_nokey(self, cb_env): + # @TODO(jc): 3.2.x SDK tests for NotStoredException + with pytest.raises(DocumentNotFoundException): + cb_env.collection.binary().prepend(TestEnvironment.NOT_A_KEY, 'foo') + + def test_prepend_string_not_empty(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('utf8') + result = cb_env.collection.binary().prepend(key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = cb_env.collection.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + value + + def test_unsigned_int(self): + with pytest.raises(InvalidArgumentException): + x = DeltaValue(-1) + with pytest.raises(InvalidArgumentException): + x = DeltaValue(0x7FFFFFFFFFFFFFFF + 1) + + x = DeltaValue(5) + assert 5 == x.value + + def test_signed_int_64(self): + with pytest.raises(InvalidArgumentException): + x = SignedInt64(-0x7FFFFFFFFFFFFFFF - 2) + + with pytest.raises(InvalidArgumentException): + x = SignedInt64(0x7FFFFFFFFFFFFFFF + 1) + + x = SignedInt64(0x7FFFFFFFFFFFFFFF) + assert 0x7FFFFFFFFFFFFFFF == x.value + x = SignedInt64(-0x7FFFFFFFFFFFFFFF - 1) + assert -0x7FFFFFFFFFFFFFFF - 1 == x.value + + +class ClassicBinaryCollectionTests(BinaryCollectionTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicBinaryCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicBinaryCollectionTests) if valid_test_method(meth)] + compare = set(BinaryCollectionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = BinaryTestEnvironment.from_environment(cb_base_env) + cb_env.enable_bucket_mgmt() + cb_env.setup(request.param, __name__) + + yield cb_env + + cb_env.teardown(request.param) diff --git a/couchbase/tests/binary_durability_t.py b/couchbase/tests/binary_durability_t.py new file mode 100644 index 000000000..623b4a231 --- /dev/null +++ b/couchbase/tests/binary_durability_t.py @@ -0,0 +1,299 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.durability import (ClientDurability, + DurabilityLevel, + PersistTo, + PersistToExtended, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import DurabilityImpossibleException +from couchbase.options import (AppendOptions, + DecrementOptions, + IncrementOptions, + PrependOptions) +from couchbase.transcoder import RawStringTranscoder +from tests.environments import CollectionType +from tests.environments.binary_environment import BinaryTestEnvironment +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class BinaryDurabilityTestSuite: + + TEST_MANIFEST = [ + 'test_client_durable_append', + 'test_client_durable_append_fail', + 'test_client_durable_append_single_node', + 'test_client_durable_decrement', + 'test_client_durable_decrement_fail', + 'test_client_durable_decrement_single_node', + 'test_client_durable_increment', + 'test_client_durable_increment_fail', + 'test_client_durable_increment_single_node', + 'test_client_durable_prepend', + 'test_client_durable_prepend_fail', + 'test_client_durable_prepend_single_node', + 'test_server_durable_append', + 'test_server_durable_append_single_node', + 'test_server_durable_decrement', + 'test_server_durable_decrement_single_node', + 'test_server_durable_increment', + 'test_server_durable_increment_single_node', + 'test_server_durable_prepend', + 'test_server_durable_prepend_single_node', + ] + + @pytest.fixture(scope='class') + def check_has_replicas(self, num_replicas): + if num_replicas == 0: + pytest.skip('No replicas to test durability.') + + @pytest.fixture(scope='class') + def check_multi_node(self, num_nodes): + if num_nodes == 1: + pytest.skip('Test only for clusters with more than a single node.') + + @pytest.fixture(scope='class') + def check_single_node(self, num_nodes): + if num_nodes != 1: + pytest.skip('Test only for clusters with a single node.') + + @pytest.fixture(scope='class') + def check_sync_durability_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('sync_durability', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def num_replicas(self, cb_env): + bucket_settings = TestEnvironment.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get('num_replicas') + return num_replicas + + @pytest.fixture(scope='class') + def num_nodes(self, cb_env): + return len(cb_env.cluster._cluster_info.nodes) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_append(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + cb_env.collection.binary().append(key, 'foo', AppendOptions(durability=durability)) + result = cb_env.collection.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_append_fail(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().append(key, 'foo', AppendOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_append_single_node(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().append(key, 'foo', AppendOptions(durability=durability)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_decrement(self, cb_env, num_replicas): + key, value = cb_env.get_existing_doc_by_type('counter') + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + result = cb_env.collection.binary().decrement(key, DecrementOptions(durability=durability)) + assert result.content == value - 1 + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_decrement_fail(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('counter', key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().decrement(key, DecrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_decrement_single_node(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('counter', key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().decrement(key, DecrementOptions(durability=durability)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_increment(self, cb_env, num_replicas): + key, value = cb_env.get_existing_doc_by_type('counter') + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + result = cb_env.collection.binary().increment(key, IncrementOptions(durability=durability)) + assert result.content == value + 1 + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_increment_fail(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('counter', key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().increment(key, IncrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_increment_single_node(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('counter', key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().increment(key, IncrementOptions(durability=durability)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_prepend(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + cb_env.collection.binary().prepend(key, 'foo', PrependOptions(durability=durability)) + result = cb_env.collection.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_prepend_fail(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().prepend(key, 'foo', PrependOptions(durability=durability)) + + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_prepend_single_node(self, cb_env, num_replicas): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().prepend(key, 'foo', PrependOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_append(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + cb_env.collection.binary().append(key, 'foo', AppendOptions(durability=durability)) + result = cb_env.collection.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_append_single_node(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().append(key, 'foo', AppendOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_decrement(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('counter') + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + result = cb_env.collection.binary().decrement(key, DecrementOptions(durability=durability)) + assert result.content == value - 1 + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_decrement_single_node(self, cb_env): + key = cb_env.get_existing_doc_by_type('counter', key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().decrement(key, DecrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_increment(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('counter') + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + result = cb_env.collection.binary().increment(key, IncrementOptions(durability=durability)) + assert result.content == value + 1 + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_increment_single_node(self, cb_env): + key = cb_env.get_existing_doc_by_type('counter', key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().increment(key, IncrementOptions(durability=durability)) + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_prepend(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + cb_env.collection.binary().append(key, 'foo', AppendOptions(durability=durability)) + result = cb_env.collection.get(key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + @pytest.mark.usefixtures("check_sync_durability_supported") + @pytest.mark.usefixtures("check_single_node") + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_prepend_single_node(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8_empty', key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.binary().prepend(key, 'foo', PrependOptions(durability=durability)) + + +class ClassicBinaryDurabilityTests(BinaryDurabilityTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicBinaryDurabilityTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicBinaryDurabilityTests) if valid_test_method(meth)] + compare = set(BinaryDurabilityTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = BinaryTestEnvironment.from_environment(cb_base_env) + cb_env.enable_bucket_mgmt() + cb_env.setup(request.param, __name__) + + yield cb_env + + cb_env.teardown(request.param) diff --git a/couchbase/tests/bucket_t.py b/couchbase/tests/bucket_t.py new file mode 100644 index 000000000..a422faf02 --- /dev/null +++ b/couchbase/tests/bucket_t.py @@ -0,0 +1,143 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from uuid import uuid4 + +import pytest + +from couchbase.diagnostics import (EndpointPingReport, + PingState, + ServiceType) +from couchbase.exceptions import InvalidArgumentException +from couchbase.options import PingOptions +from couchbase.result import PingResult +from tests.environments import CollectionType +from tests.test_features import EnvironmentFeatures + + +class BucketDiagnosticsTestSuite: + TEST_MANIFEST = [ + 'test_ping', + 'test_ping_as_json', + 'test_ping_invalid_services', + 'test_ping_mixed_services', + 'test_ping_report_id', + 'test_ping_restrict_services', + 'test_ping_str_services', + ] + + @pytest.fixture(scope="class") + def check_diagnostics_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('diagnostics', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping(self, cb_env): + bucket = cb_env.bucket + result = bucket.ping() + assert isinstance(result, PingResult) + + assert result.sdk is not None + assert result.id is not None + assert result.version is not None + assert result.endpoints is not None + for ping_reports in result.endpoints.values(): + for report in ping_reports: + assert isinstance(report, EndpointPingReport) + if report.state == PingState.OK: + assert report.id is not None + assert report.latency is not None + assert report.remote is not None + assert report.local is not None + assert report.service_type is not None + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_as_json(self, cb_env): + bucket = cb_env.bucket + result = bucket.ping() + assert isinstance(result, PingResult) + result_str = result.as_json() + assert isinstance(result_str, str) + result_json = json.loads(result_str) + assert result_json['version'] is not None + assert result_json['id'] is not None + assert result_json['sdk'] is not None + assert result_json['services'] is not None + for _, data in result_json['services'].items(): + if len(data): + assert data[0]['id'] is not None + assert data[0]['latency_us'] is not None + assert data[0]['remote'] is not None + assert data[0]['local'] is not None + assert data[0]['state'] is not None + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_invalid_services(self, cb_env): + bucket = cb_env.bucket + with pytest.raises(InvalidArgumentException): + bucket.ping(PingOptions(service_types=ServiceType.KeyValue)) + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_mixed_services(self, cb_env): + bucket = cb_env.bucket + services = [ServiceType.KeyValue, ServiceType.Query.value] + result = bucket.ping(PingOptions(service_types=services)) + assert len(result.endpoints) >= 1 + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_report_id(self, cb_env): + bucket = cb_env.bucket + report_id = uuid4() + result = bucket.ping(PingOptions(report_id=report_id)) + assert str(report_id) == result.id + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_restrict_services(self, cb_env): + bucket = cb_env.bucket + services = [ServiceType.KeyValue] + result = bucket.ping(PingOptions(service_types=services)) + keys = list(result.endpoints.keys()) + assert len(keys) == 1 + assert keys[0] == ServiceType.KeyValue + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_str_services(self, cb_env): + bucket = cb_env.bucket + services = [ServiceType.KeyValue.value, ServiceType.Query.value] + result = bucket.ping(PingOptions(service_types=services)) + assert len(result.endpoints) >= 1 + + +class ClassicBucketDiagnosticsTests(BucketDiagnosticsTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicBucketDiagnosticsTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicBucketDiagnosticsTests) if valid_test_method(meth)] + compare = set(BucketDiagnosticsTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.setup(request.param) + yield cb_base_env + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/bucketmgmt_t.py b/couchbase/tests/bucketmgmt_t.py new file mode 100644 index 000000000..fdceaaf79 --- /dev/null +++ b/couchbase/tests/bucketmgmt_t.py @@ -0,0 +1,486 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +from couchbase.bucket import Bucket +from couchbase.durability import DurabilityLevel +from couchbase.exceptions import (BucketAlreadyExistsException, + BucketDoesNotExistException, + BucketNotFlushableException, + FeatureUnavailableException, + InvalidArgumentException) +from couchbase.management.buckets import (BucketSettings, + BucketType, + ConflictResolutionType, + CreateBucketSettings, + StorageBackend) +from tests.environments import CollectionType +from tests.environments.bucket_mgmt_environment import BucketManagementTestEnvironment +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class BucketManagementTestSuite: + TEST_MANIFEST = [ + 'test_bucket_backend_default', + 'test_bucket_backend_ephemeral', + 'test_bucket_backend_magma', + 'test_bucket_backend_couchstore', + 'test_bucket_create', + 'test_bucket_create_durability', + 'test_bucket_create_fail', + 'test_bucket_create_replica_index_false', + 'test_bucket_create_replica_index_true', + 'test_bucket_custom_conflict_resolution', + 'test_bucket_create_history_retention', + 'test_bucket_create_history_retention_unsupported', + 'test_bucket_update_history_retention', + 'test_bucket_update_history_retention_unsupported', + 'test_bucket_drop_fail', + 'test_bucket_flush', + 'test_bucket_flush_fail', + 'test_bucket_list', + 'test_change_expiry', + 'test_cluster_sees_bucket', + ] + + @pytest.fixture(scope='class') + def check_bucket_mgmt_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('bucket_mgmt', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_bucket_min_durability_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('bucket_min_durability', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_bucket_storage_backend_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('bucket_storage_backend', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_custom_conflict_resolution_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('custom_conflict_resolution', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_non_deduped_history_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('non_deduped_history', + cb_env.server_version_short, + cb_env.mock_server_type) + + def supports_magma_128_buckets(self, cb_env): + return EnvironmentFeatures.supports_feature('magma_128_buckets', + cb_env.server_version_short, + cb_env.mock_server_type) is None + + @pytest.fixture() + def drop_bucket(self, cb_env): + yield + cb_env.drop_bucket() + + @pytest.mark.usefixtures('check_bucket_storage_backend_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_backend_default(self, cb_env): + bucket_name = cb_env.get_bucket_name() + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100, + flush_enabled=False)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + if self.supports_magma_128_buckets(cb_env): + assert bucket.storage_backend == StorageBackend.MAGMA + else: + assert bucket.storage_backend == StorageBackend.COUCHSTORE + + @pytest.mark.usefixtures('check_bucket_storage_backend_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_backend_ephemeral(self, cb_env): + bucket_name = cb_env.get_bucket_name() + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100, + bucket_type=BucketType.EPHEMERAL, + flush_enabled=False)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.storage_backend == StorageBackend.UNDEFINED + + @pytest.mark.usefixtures('check_bucket_storage_backend_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_backend_magma(self, cb_env): + bucket_name = cb_env.get_bucket_name() + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=1024, + flush_enabled=False, + storage_backend=StorageBackend.MAGMA)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.storage_backend == StorageBackend.MAGMA + + @pytest.mark.usefixtures('check_bucket_storage_backend_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_backend_couchstore(self, cb_env): + bucket_name = cb_env.get_bucket_name() + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100, + flush_enabled=False, + storage_backend=StorageBackend.COUCHSTORE)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.storage_backend == StorageBackend.COUCHSTORE + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_create(self, cb_env): + bucket_name = cb_env.get_bucket_name() + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + if cb_env.server_version_short >= 6.6: + assert bucket['minimum_durability_level'] == DurabilityLevel.NONE + + @pytest.mark.usefixtures('check_bucket_min_durability_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_create_durability(self, cb_env): + bucket_name = cb_env.get_bucket_name() + min_durability = DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE + cb_env.bm.create_bucket(CreateBucketSettings(name=bucket_name, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + minimum_durability_level=min_durability)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket["minimum_durability_level"] == min_durability + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_create_fail(self, cb_env): + bucket_name = cb_env.get_bucket_name() + settings = CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100) + cb_env.bm.create_bucket(settings) + cb_env.consistency.wait_until_bucket_present(bucket_name) + with pytest.raises(BucketAlreadyExistsException): + cb_env.bm.create_bucket(settings) + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_create_replica_index_false(self, cb_env): + bucket_name = cb_env.get_bucket_name() + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + replica_index=False)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.replica_index is False + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_create_replica_index_true(self, cb_env): + bucket_name = cb_env.get_bucket_name() + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + replica_index=True)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.replica_index is True + + @pytest.mark.usefixtures('check_custom_conflict_resolution_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_custom_conflict_resolution(self, cb_env): + bucket_name = cb_env.get_bucket_name() + if cb_env.is_developer_preview: + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100, + conflict_resolution_type=ConflictResolutionType.CUSTOM, + flush_enabled=False)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.conflict_resolution_type == ConflictResolutionType.CUSTOM + else: + with pytest.raises(FeatureUnavailableException): + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100, + conflict_resolution_type=ConflictResolutionType.CUSTOM, + flush_enabled=False)) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_create_history_retention(self, cb_env): + bucket_name = cb_env.get_bucket_name() + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.MAGMA, + ram_quota_mb=1024, + history_retention_collection_default=True, + history_retention_bytes=2**31, + history_retention_duration=timedelta(days=1) + ) + ) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.history_retention_collection_default + assert bucket.history_retention_bytes == 2**31 + assert bucket.history_retention_duration == timedelta(days=1) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_create_history_retention_unsupported(self, cb_env): + bucket_name = cb_env.get_bucket_name() + + # History retention not supported for CouchStore + with pytest.raises(InvalidArgumentException): + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.COUCHSTORE, + ram_quota_mb=1024, + history_retention_collection_default=True, + history_retention_bytes=2**31, + history_retention_duration=timedelta(days=1) + ) + ) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_update_history_retention(self, cb_env): + bucket_name = cb_env.get_bucket_name() + + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.MAGMA, + ram_quota_mb=1024, + ) + ) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket is not None + assert bucket.history_retention_collection_default + assert bucket.history_retention_bytes == 0 + assert bucket.history_retention_duration == timedelta(0) + + cb_env.bm.update_bucket( + BucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.MAGMA, + ram_quota_mb=1024, + history_retention_collection_default=True, + history_retention_bytes=2**31, + history_retention_duration=timedelta(minutes=10) + ) + ) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.history_retention_collection_default + assert bucket.history_retention_bytes == 2**31 + assert bucket.history_retention_duration == timedelta(minutes=10) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_update_history_retention_unsupported(self, cb_env): + bucket_name = cb_env.get_bucket_name() + + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.COUCHSTORE, + ram_quota_mb=256, + ) + ) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket is not None + assert bucket.history_retention_collection_default is None + assert bucket.history_retention_bytes is None + assert bucket.history_retention_duration is None + + with pytest.raises(InvalidArgumentException): + cb_env.bm.update_bucket( + BucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + storage_backend=StorageBackend.COUCHSTORE, + ram_quota_mb=256, + history_retention_collection_default=True, + history_retention_bytes=2**31, + history_retention_duration=timedelta(minutes=10) + ) + ) + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + def test_bucket_drop_fail(self, cb_env): + bucket_name = cb_env.get_bucket_name() + settings = CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100) + cb_env.bm.create_bucket(settings) + cb_env.consistency.wait_until_bucket_present(bucket_name) + cb_env.bm.drop_bucket(bucket_name) + cb_env.consistency.wait_until_bucket_dropped(bucket_name) + with pytest.raises(BucketDoesNotExistException): + cb_env.bm.drop_bucket(bucket_name) + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_flush(self, cb_env): + bucket_name = cb_env.get_bucket_name() + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100, + flush_enabled=True)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.flush_enabled is True + # flush the bucket + TestEnvironment.try_n_times(10, 3, cb_env.bm.flush_bucket, bucket.name) + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_flush_fail(self, cb_env): + bucket_name = cb_env.get_bucket_name() + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100, + flush_enabled=False)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + bucket = cb_env.bm.get_bucket(bucket_name) + assert bucket.flush_enabled is False + + with pytest.raises(BucketNotFlushableException): + cb_env.bm.flush_bucket(bucket_name) + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_bucket_list(self, cb_env): + bucket_names = cb_env.get_bucket_names() + for bucket_name in bucket_names: + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + + buckets = cb_env.bm.get_all_buckets() + assert set() == set(bucket_names).difference(set(map(lambda b: b.name, buckets))) + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_change_expiry(self, cb_env): + bucket_name = cb_env.get_bucket_name() + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + + # change bucket TTL + TestEnvironment.try_n_times(10, 3, cb_env.bm.update_bucket, BucketSettings( + name=bucket_name, max_expiry=timedelta(seconds=500))) + + def get_bucket_expiry_equal(name, expiry): + b = cb_env.bm.get_bucket(name) + + if not expiry == b.max_expiry: + raise Exception("not equal") + + TestEnvironment.try_n_times(10, 3, get_bucket_expiry_equal, bucket_name, timedelta(seconds=500)) + + @pytest.mark.usefixtures('check_bucket_mgmt_supported') + @pytest.mark.usefixtures('drop_bucket') + def test_cluster_sees_bucket(self, cb_env): + pytest.skip('Skip until seg fault root cause determined.') + bucket_name = cb_env.get_bucket_name() + # Create the bucket + cb_env.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + ram_quota_mb=100)) + cb_env.consistency.wait_until_bucket_present(bucket_name) + # cluster should be able to return it (though, not right away) + b = TestEnvironment.try_n_times(10, 2, cb_env.cluster.bucket, bucket_name) + assert b is not None + assert isinstance(b, Bucket) + + +@pytest.mark.flaky(reruns=3, reruns_delay=1) +class ClassicBucketManagementTests(BucketManagementTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicBucketManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicBucketManagementTests) if valid_test_method(meth)] + compare = set(BucketManagementTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = BucketManagementTestEnvironment.from_environment(cb_base_env) + cb_env.enable_bucket_mgmt() + cb_env.setup() + yield cb_env + cb_env.teardown() diff --git a/couchbase/tests/cases/__init__.py b/couchbase/tests/cases/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/couchbase/tests/cases/admin_t.py b/couchbase/tests/cases/admin_t.py deleted file mode 100644 index 94a2a595f..000000000 --- a/couchbase/tests/cases/admin_t.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import sys -import os - -from couchbase.admin import Admin -from couchbase.result import HttpResult -from couchbase.connstr import ConnectionString -from couchbase.exceptions import ( - ArgumentError, AuthError, CouchbaseError, - CouchbaseNetworkError, HTTPError) -from couchbase.tests.base import CouchbaseTestCase, SkipTest - -class AdminSimpleTest(CouchbaseTestCase): - def setUp(self): - super(AdminSimpleTest, self).setUp() - self.admin = self.make_admin_connection() - - def tearDown(self): - super(AdminSimpleTest, self).tearDown() - if self.should_check_refcount: - rc = sys.getrefcount(self.admin) - self.assertEqual(rc, 2) - - del self.admin - - def test_http_request(self): - htres = self.admin.http_request('pools/') - self.assertIsInstance(htres, HttpResult) - self.assertIsInstance(htres.value, dict) - self.assertEqual(htres.http_status, 200) - self.assertEqual(htres.url, 'pools/') - self.assertTrue(htres.success) - - def test_bad_request(self): - self.assertRaises(HTTPError, self.admin.http_request, '/badpath') - - excraised = 0 - try: - self.admin.http_request("/badpath") - except HTTPError as e: - excraised = 1 - self.assertIsInstance(e.objextra, HttpResult) - - self.assertTrue(excraised) - - def test_bad_args(self): - self.assertRaises(ArgumentError, - self.admin.http_request, - None) - - self.assertRaises(ArgumentError, - self.admin.http_request, - '/', - method='blahblah') - - def test_bad_auth(self): - self.assertRaises(AuthError, Admin, - 'baduser', 'badpass', - host=self.cluster_info.host, - port=self.cluster_info.port) - - def test_bad_host(self): - self.assertRaises(CouchbaseNetworkError, Admin, - 'user', 'pass', host='127.0.0.1', port=1) - - def test_bad_handle(self): - self.assertRaises(CouchbaseError, self.admin.upsert, "foo", "bar") - self.assertRaises(CouchbaseError, self.admin.get, "foo") - self.assertRaises(CouchbaseError, self.admin.append, "foo", "bar") - self.assertRaises(CouchbaseError, self.admin.remove, "foo") - self.assertRaises(CouchbaseError, self.admin.unlock, "foo", 1) - str(None) - - def test_actions(self): - if not self.is_realserver: - raise SkipTest('Real server must be used for admin tests') - - if not os.environ.get('PYCBC_TEST_ADMIN'): - raise SkipTest('PYCBC_TEST_ADMIN must be set in the environment') - - try: - # Remove the bucket, if it exists - self.admin.bucket_remove('dummy') - except CouchbaseError: - pass - - # Need to explicitly enable admin tests.. - # Create the bucket - self.admin.bucket_create(name='dummy', - ram_quota=100, bucket_password='letmein') - self.admin.wait_ready('dummy', timeout=15.0) - - # All should be OK, ensure we can connect: - connstr = ConnectionString.parse( - self.make_connargs()['connection_string']) - - connstr.bucket = 'dummy' - connstr = connstr.encode() - self.factory(connstr, password='letmein') - # OK, it exists - self.assertRaises(CouchbaseError, self.factory, connstr) - - # Change the password - self.admin.bucket_update('dummy', - self.admin.bucket_info('dummy'), - bucket_password='') - self.factory(connstr) # No password - - # Remove the bucket - self.admin.bucket_remove('dummy') - self.assertRaises(CouchbaseError, self.factory, connstr) diff --git a/couchbase/tests/cases/append_t.py b/couchbase/tests/cases/append_t.py deleted file mode 100644 index eff934a40..000000000 --- a/couchbase/tests/cases/append_t.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase import FMT_JSON, FMT_PICKLE, FMT_BYTES, FMT_UTF8 - -from couchbase.exceptions import (KeyExistsError, ValueFormatError, - ArgumentError, NotFoundError, - NotStoredError, CouchbaseError) - -from couchbase.tests.base import ConnectionTestCase - - -class AppendTest(ConnectionTestCase): - - def test_append_prepend(self): - key = self.gen_key("appendprepend") - vbase = "middle" - self.cb.upsert(key, vbase, format=FMT_UTF8) - self.cb.prepend(key, "begin ") - self.cb.append(key, " end") - self.assertEquals(self.cb.get(key).value, - "begin middle end") - - - def test_append_binary(self): - kname = self.gen_key("binary_append") - initial = b'\x10' - self.cb.upsert(kname, initial, format=FMT_BYTES) - self.cb.append(kname, b'\x20', format=FMT_BYTES) - self.cb.prepend(kname, b'\x00', format=FMT_BYTES) - - res = self.cb.get(kname) - self.assertEqual(res.value, b'\x00\x10\x20') - - def test_append_nostr(self): - key = self.gen_key("append_nostr") - self.cb.upsert(key, "value") - rv = self.cb.append(key, "a_string") - self.assertTrue(rv.cas) - - self.assertRaises(ValueFormatError, - self.cb.append, "key", { "some" : "object" }) - - def test_append_enoent(self): - key = self.gen_key("append_enoent") - self.cb.remove(key, quiet=True) - try: - self.cb.append(key, "value") - self.assertTrue(False, "Exception not thrown") - except CouchbaseError as e: - self.assertTrue(isinstance(e, NotStoredError) - or isinstance(e, NotFoundError)) - - def test_append_multi(self): - kv = self.gen_kv_dict(amount=4, prefix="append_multi") - - self.cb.upsert_multi(kv, format=FMT_UTF8) - self.cb.append_multi(kv) - self.cb.prepend_multi(kv) - - rvs = self.cb.get_multi(list(kv.keys())) - self.assertTrue(rvs.all_ok) - self.assertEqual(len(rvs), 4) - - for k, v in rvs.items(): - basekey = kv[k] - self.assertEqual(v.value, basekey * 3) diff --git a/couchbase/tests/cases/arithmetic_t.py b/couchbase/tests/cases/arithmetic_t.py deleted file mode 100644 index dd2488ad3..000000000 --- a/couchbase/tests/cases/arithmetic_t.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.exceptions import (NotFoundError, DeltaBadvalError) -from couchbase.tests.base import ConnectionTestCase - - -class ArithmeticTest(ConnectionTestCase): - - def test_trivial_incrdecr(self): - key = self.gen_key("trivial_incrdecr") - self.cb.remove(key, quiet=True) - rv_arith = self.cb.counter(key, initial=1, delta=1) - rv_get = self.cb.get(key) - - self.assertEqual(rv_arith.value, 1) - self.assertEqual(int(rv_get.value), 1) - - rv = self.cb.counter(key) - self.assertEquals(rv.value, 2) - - rv = self.cb.counter(key, delta=-1) - self.assertEquals(rv.value, 1) - self.assertEquals(int(self.cb.get(key).value), 1) - - rv = self.cb.counter(key, delta=-1) - self.assertEquals(rv.value, 0) - self.assertEquals(int(self.cb.get(key).value), 0) - - def test_incr_notfound(self): - key = self.gen_key("incr_notfound") - self.cb.remove(key, quiet=True) - self.assertRaises(NotFoundError, self.cb.counter, key) - - def test_incr_badval(self): - key = self.gen_key("incr_badval") - self.cb.upsert(key, "THIS IS SPARTA") - self.assertRaises(DeltaBadvalError, self.cb.counter, key) - - def test_incr_multi(self): - keys = self.gen_key_list(amount=5, prefix="incr_multi") - - def _multi_lim_assert(expected): - for k, v in self.cb.get_multi(keys).items(): - self.assertTrue(k in keys) - self.assertEqual(v.value, expected) - - self.cb.remove_multi(keys, quiet=True) - self.cb.counter_multi(keys, initial=5) - _multi_lim_assert(5) - - self.cb.counter_multi(keys) - _multi_lim_assert(6) - - self.cb.counter_multi(keys, delta=-1) - _multi_lim_assert(5) - - self.cb.counter_multi(keys, delta=10) - _multi_lim_assert(15) - - self.cb.counter_multi(keys, delta=-6) - _multi_lim_assert(9) - - self.cb.remove(keys[0]) - - self.assertRaises(NotFoundError, self.cb.counter_multi, keys) - - def test_incr_extended(self): - key = self.gen_key("incr_extended") - self.cb.remove(key, quiet=True) - rv = self.cb.counter(key, initial=10) - self.assertEquals(rv.value, 10) - srv = self.cb.upsert(key, "42", cas=rv.cas) - self.assertTrue(srv.success) - - # test with multiple values? - klist = self.gen_key_list(amount=5, prefix="incr_extended_list") - self.cb.remove_multi(klist, quiet=True) - rvs = self.cb.counter_multi(klist, initial=40) - [ self.assertEquals(x.value, 40) for x in rvs.values() ] - self.assertEquals(sorted(list(rvs.keys())), sorted(klist)) - - -if __name__ == '__main__': - unittest.main() diff --git a/couchbase/tests/cases/badargs_t.py b/couchbase/tests/cases/badargs_t.py deleted file mode 100644 index 9bd75a051..000000000 --- a/couchbase/tests/cases/badargs_t.py +++ /dev/null @@ -1,156 +0,0 @@ -from time import sleep - -from couchbase import FMT_JSON, FMT_PICKLE, FMT_UTF8, FMT_BYTES -from couchbase.exceptions import (KeyExistsError, ValueFormatError, - ArgumentError, NotFoundError, - NotStoredError) -from couchbase.tests.base import ConnectionTestCase - -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -class BadArgsTest(ConnectionTestCase): - - def test_bad_single(self): - - for k in ( - (), - ("key",), - {"key":"value"}, - [], - set(), - {}.keys(), - {}.values(), - ["key"], - None, - True, - False, - 0, - object()): - - print("Testing with key (%r)" % (k,)) - - self.assertRaises(ValueFormatError, self.cb.get, k) - self.assertRaises(ValueFormatError, self.cb.counter, k) - self.assertRaises(ValueFormatError, self.cb.delete, k) - self.assertRaises(ValueFormatError, self.cb.set, k, "value") - self.assertRaises(ValueFormatError, self.cb.set, "key", k, - format=FMT_UTF8) - self.assertRaises(ValueFormatError, self.cb.set, "key", k, - format=FMT_BYTES) - self.assertRaises(ValueFormatError, self.cb.append, "key", k) - - def test_bad_multi(self): - for k in ( - "key", - None, - [], - {}, - set(), - {}.keys(), - {}.values(), - 0, - object()): - print("Testing with keys (%r)" % (k,)) - - self.assertRaises(ArgumentError, self.cb.get_multi, k) - self.assertRaises(ArgumentError, self.cb.set_multi, k) - self.assertRaises(ArgumentError, self.cb.counter_multi, k) - self.assertRaises(ArgumentError, self.cb.delete_multi, k) - - def test_bad_timeout(self): - def _set_timeout(x): - self.cb.timeout = x - - self.assertRaises(Exception, _set_timeout, 0) - self.assertRaises(Exception, _set_timeout, -1) - self.assertRaises(Exception, _set_timeout, None) - self.assertRaises(Exception, _set_timeout, "a string") - - self.cb.timeout = 0.1 - self.cb.timeout = 1 - self.cb.timeout = 2.5 - - def test_bad_quiet(self): - def _set_quiet(x): - self.cb.quiet = x - - self.assertRaises(Exception, _set_quiet, "asfasf") - self.assertRaises(Exception, _set_quiet, None) - _set_quiet(True) - _set_quiet(False) - - def test_badargs_get(self): - self.assertRaises(ArgumentError, self.cb.get_multi, - {"key" : "string"}) - self.assertRaises(ArgumentError, self.cb.get_multi, - { "key" : object()} ) - self.assertRaises(ArgumentError, self.cb.get, "string", ttl="string") - self.assertRaises(ArgumentError, self.cb.lock, "string", ttl="string") - self.assertRaises(ArgumentError, self.cb.get, "string", ttl=object()) - - def test_bad_default_format(self): - def _set_fmt(x): - self.cb.default_format = x - self.assertEqual(self.cb.default_format, x) - - _set_fmt(FMT_JSON) - _set_fmt(FMT_BYTES) - _set_fmt(FMT_UTF8) - _set_fmt(FMT_PICKLE) - - self.assertRaises(ArgumentError, _set_fmt, "a format") - self.assertRaises(ArgumentError, _set_fmt, None) - self.assertRaises(ArgumentError, _set_fmt, False) - self.assertRaises(ArgumentError, _set_fmt, True) - self.assertRaises(ArgumentError, _set_fmt, object()) - - # TODO: Stricter format handling - - #self.assertRaises(ArgumentError, self.cb.set, - # "foo", "bar", format=-1) - - def test_negative_ttl(self): - for bad_ttl in (-1, - "ttl", - object(), - [1], - {'foo':'bar'}, - 2**100): - - print(bad_ttl) - self.assertRaises(ArgumentError, self.cb.get, "key", ttl=bad_ttl) - self.assertRaises(ArgumentError, self.cb.set, "key", "value", - ttl=bad_ttl) - self.assertRaises(ArgumentError, self.cb.touch, "key", ttl=bad_ttl) - self.assertRaises(ArgumentError, self.cb.counter, "key", ttl=bad_ttl) - self.assertRaises(ArgumentError, self.cb.lock, "key", ttl=bad_ttl) - - self.assertRaises(ArgumentError, self.cb.get_multi, - ["key"], ttl=bad_ttl) - self.assertRaises(ArgumentError, self.cb.get_multi, - { "key" : { 'ttl' : bad_ttl } }) - self.assertRaises(ArgumentError, self.cb.get_multi, - { "key" : bad_ttl } ) - self.assertRaises(ArgumentError, self.cb.counter_multi, - "key", ttl=bad_ttl) - self.assertRaises(ArgumentError, self.cb.lock_multi, - "key", ttl=bad_ttl) - - -if __name__ == '__main__': - unittest.main() diff --git a/couchbase/tests/cases/connection_t.py b/couchbase/tests/cases/connection_t.py deleted file mode 100644 index 000b1d342..000000000 --- a/couchbase/tests/cases/connection_t.py +++ /dev/null @@ -1,117 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import tempfile -import os - -from nose.plugins.attrib import attr - -from couchbase.exceptions import (AuthError, ArgumentError, - BucketNotFoundError, ConnectError, - CouchbaseNetworkError, - NotFoundError, InvalidError, - TimeoutError) -from couchbase.tests.base import CouchbaseTestCase, SkipTest -from couchbase.connstr import ConnectionString - - -class ConnectionTest(CouchbaseTestCase): - @attr('slow') - def test_server_not_found(self): - connargs = self.make_connargs() - cs = ConnectionString.parse(connargs['connection_string']) - cs.hosts = [ 'example.com' ] - connargs['connection_string'] = cs.encode() - self.assertRaises((CouchbaseNetworkError, TimeoutError), - self.factory, **connargs) - - cs.hosts = [ self.cluster_info.host + ':' + str(34567)] - self.assertRaises(CouchbaseNetworkError, self.factory, **connargs) - - def test_bucket(self): - cb = self.factory(**self.make_connargs()) - self.assertIsInstance(cb, self.factory) - - def test_bucket_not_found(self): - connargs = self.make_connargs(bucket='this_bucket_does_not_exist') - self.assertRaises(BucketNotFoundError, self.factory, **connargs) - - def test_quiet(self): - connparams = self.make_connargs() - cb = self.factory(**connparams) - self.assertRaises(NotFoundError, cb.get, 'missing_key') - - cb = self.factory(quiet=True, **connparams) - cb.remove('missing_key', quiet=True) - val1 = cb.get('missing_key') - self.assertFalse(val1.success) - - cb = self.factory(quiet=False, **connparams) - self.assertRaises(NotFoundError, cb.get, 'missing_key') - - - def test_configcache(self): - cachefile = None - # On Windows, the NamedTemporaryFile is deleted right when it's - # created. So we need to ensure it's not deleted, and delete it - # ourselves when it's closed - try: - cachefile = tempfile.NamedTemporaryFile(delete=False) - cb = self.factory(**self.make_connargs(config_cache=cachefile.name)) - self.assertTrue(cb.upsert("foo", "bar").success) - - cb2 = self.factory(**self.make_connargs(config_cache=cachefile.name)) - - self.assertTrue(cb2.upsert("foo", "bar").success) - self.assertEquals("bar", cb.get("foo").value) - - sb = os.stat(cachefile.name) - - # For some reason this fails on Windows? - self.assertTrue(sb.st_size > 0) - finally: - # On windows, we can't delete if the file is still being used - cachefile.close() - os.unlink(cachefile.name) - - # TODO, see what happens when bad path is used - # apparently libcouchbase does not report this failure. - - def test_invalid_hostname(self): - self.assertRaises(InvalidError, self.factory, - str('couchbase://12345:qwer###/default')) - - def test_multi_hosts(self): - passwd = self.cluster_info.bucket_password - cs = ConnectionString(bucket=self.cluster_info.bucket_name, - hosts=[self.cluster_info.host]) - - if not self.mock: - cb = self.factory(str(cs), password=passwd) - self.assertTrue(cb.upsert("foo", "bar").success) - - cs.hosts = [ self.cluster_info.host + ':' + str(self.cluster_info.port) ] - cs.scheme = 'http' - cb = self.factory(str(cs)) - self.assertTrue(cb.upsert("foo", "bar").success) - - cs.hosts.insert(0, 'localhost:1') - cb = self.factory(str(cs)) - self.assertTrue(cb.upsert("foo", "bar").success) - -if __name__ == '__main__': - unittest.main() diff --git a/couchbase/tests/cases/delete_t.py b/couchbase/tests/cases/delete_t.py deleted file mode 100644 index a69b83a4a..000000000 --- a/couchbase/tests/cases/delete_t.py +++ /dev/null @@ -1,123 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.exceptions import (KeyExistsError, NotFoundError) -from couchbase.tests.base import ConnectionTestCase - -class DeleteTest(ConnectionTestCase): - - def test_trivial_delete(self): - # Try to delete a key that exists. Ensure that the operation - # succeeds - - key = self.gen_key("trivial_delete") - rv = self.cb.upsert(key, 'value') - self.assertTrue(rv.success) - self.assertTrue(rv.cas > 0) - rv = self.cb.remove(key) - self.assertTrue(rv.success) - - def test_delete_notfound(self): - # Delete a key that does not exist. - # With 'quiet' ensure that it returns false. Without 'quiet', ensure that - # it raises a NotFoundError - - self.cb.remove("foo", quiet = True) - rv = self.cb.remove("foo", quiet = True) - self.assertFalse(rv.success) - self.assertRaises(NotFoundError, self.cb.remove, 'foo') - - def test_delete_cas(self): - # Delete with a CAS value. Ensure that it returns OK - - key = self.gen_key("delete_cas") - rv1 = self.cb.upsert(key, 'bar') - self.assertTrue(rv1.cas > 0) - rv2 = self.cb.remove(key, cas = rv1.cas) - self.assertTrue(rv2.success) - - def test_delete_badcas(self): - # Simple delete with a bad CAS - - key = self.gen_key("delete_badcas") - self.cb.upsert(key, 'bar') - self.assertRaises(KeyExistsError, - self.cb.remove, key, cas = 0xdeadbeef) - - def test_delete_multi(self): - # Delete passing a list of keys - - kvlist = self.gen_kv_dict(amount=5, prefix='delete_multi') - - rvs = self.cb.upsert_multi(kvlist) - self.assertTrue(len(rvs) == len(kvlist)) - rm_rvs = self.cb.remove_multi(list(rvs.keys())) - self.assertTrue(len(rm_rvs) == len(kvlist)) - self.assertTrue(rm_rvs.all_ok) - - for k, v in rm_rvs.items(): - self.assertTrue(k in kvlist) - self.assertTrue(v.success) - - def test_delete_dict(self): - # Delete passing a dict of key:cas pairs - - kvlist = self.gen_kv_dict(amount=5, prefix='delete_dict') - - rvs = self.cb.upsert_multi(kvlist) - self.assertTrue(rvs.all_ok) - - # We should just be able to pass it to 'delete' - rm_rvs = self.cb.remove_multi(rvs) - self.assertTrue(rm_rvs.all_ok) - for k, v in rm_rvs.items(): - self.assertTrue(v.success) - - def test_delete_mixed(self): - # Delete with mixed success-error keys. - # Test with mixed found/not-found - # Test with mixed cas-valid/cas-invalid - - self.cb.remove("foo", quiet = True) - - self.cb.upsert("bar", "a_value") - # foo does not exit, - - rvs = self.cb.remove_multi(('foo', 'bar'), quiet = True) - self.assertFalse(rvs.all_ok) - self.assertTrue(rvs['bar'].success) - self.assertFalse(rvs['foo'].success) - - # Now see what happens if we delete those with a bad CAS - kvs = self.gen_kv_dict(amount=3, prefix="delete_mixed_badcas") - keys = list(kvs.keys()) - cas_rvs = self.cb.upsert_multi(kvs) - - # Ensure set had no errors - set_errors = [] - for k, v in cas_rvs.items(): - if not v.success: - set_errors.append([k, v]) - self.assertTrue(len(set_errors) == 0) - - # Set one to have a bad CAS - cas_rvs[keys[0]] = 0xdeadbeef - self.assertRaises(KeyExistsError, - self.cb.remove_multi, cas_rvs) - -if __name__ == '__main__': - unittest.main() diff --git a/couchbase/tests/cases/design_t.py b/couchbase/tests/cases/design_t.py deleted file mode 100644 index e97a92412..000000000 --- a/couchbase/tests/cases/design_t.py +++ /dev/null @@ -1,95 +0,0 @@ -0# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import time - -from nose.plugins.attrib import attr - -from couchbase.tests.base import DDocTestCase -from couchbase.exceptions import HTTPError, CouchbaseError - -DNAME = "tmp" -VNAME = "a_view" - -DESIGN_JSON = { - 'language' : 'javascript', - 'views' : { - VNAME : { - 'map' : "function(doc) { emit(null,null); }" - } - } -} - -@attr('slow') -class DesignDocManagementTest(DDocTestCase): - def setUp(self): - super(DesignDocManagementTest, self).setUp() - self.skipIfMock() - self.mgr = self.cb.bucket_manager() - - try: - self.mgr.design_delete(DNAME, use_devmode=False, syncwait=5) - except HTTPError: - pass - - try: - self.mgr.design_delete(DNAME, use_devmode=True, syncwait=5) - except HTTPError: - pass - - def tearDown(self): - del self.mgr - super(DesignDocManagementTest, self).tearDown() - - - def test_design_management(self): - rv = self.mgr.design_create( - DNAME, DESIGN_JSON, use_devmode=True, syncwait=5) - self.assertTrue(rv.success) - - rv = self.cb._view(DNAME, VNAME, use_devmode=True, - params = { 'limit':10 }) - print(rv) - self.assertTrue(rv.success) - - rv = self.mgr.design_publish(DNAME, syncwait=5) - self.assertTrue(rv.success) - - rv = self.cb._view(DNAME, VNAME, use_devmode=False, - params = { 'limit':10 }) - self.assertTrue(rv.success) - - self.assertRaises(HTTPError, - self.cb._view, - DNAME, VNAME, - use_devmode=True) - - rv = self.mgr.design_delete(DNAME, use_devmode=False, syncwait=5) - self.assertTrue(rv.success) - - self.assertRaises(HTTPError, - self.cb._view, - DNAME, VNAME, - use_devmode=False) - - def test_design_headers(self): - rv = self.mgr.design_create(DNAME, DESIGN_JSON, use_devmode=True, - syncwait=5) - - rv = self.mgr.design_get(DNAME, use_devmode=True) - self.assertTrue(rv.headers) - print(rv.headers) - self.assertTrue('X-Couchbase-Meta' in rv.headers) diff --git a/couchbase/tests/cases/dupkeys_t.py b/couchbase/tests/cases/dupkeys_t.py deleted file mode 100644 index 212f99a43..000000000 --- a/couchbase/tests/cases/dupkeys_t.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import warnings - -from couchbase.tests.base import ConnectionTestCase -from couchbase.exceptions import NotFoundError, TemporaryFailError -import couchbase._libcouchbase as LCB - -class DupKeyTest(ConnectionTestCase): - def setUp(self): - super(DupKeyTest, self).setUp() - - def _assertWarned(self, wlog): - mcount = 0 - while wlog: - w = wlog.pop() - self.assertEqual(w.category, RuntimeWarning) - print(w.message) - mcount += 1 - - self.assertTrue(mcount) - warnings.resetwarnings() - - def test_duplicate_keys(self): - with warnings.catch_warnings(record=True) as wlog: - self.cb._privflags |= LCB.PYCBC_CONN_F_WARNEXPLICIT - warnings.resetwarnings() - - meths = (self.cb.get_multi, - self.cb.remove_multi, - self.cb.counter_multi) - - for m in meths: - print(m.__name__) - try: - m(("foo", "foo")) - except NotFoundError: - pass - self._assertWarned(wlog) - - - try: - self.cb.lock_multi(("foo", "foo"), ttl=5) - except NotFoundError: - pass - self._assertWarned(wlog) - - ktmp = self.gen_key("duplicate_keys") - rv = self.cb.upsert(ktmp, "value") - - try: - self.cb.unlock_multi((rv, rv)) - except (NotFoundError, TemporaryFailError): - pass - self._assertWarned(wlog) diff --git a/couchbase/tests/cases/empty_key_t.py b/couchbase/tests/cases/empty_key_t.py deleted file mode 100644 index 27be96a5a..000000000 --- a/couchbase/tests/cases/empty_key_t.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.exceptions import ArgumentError -from couchbase.tests.base import ConnectionTestCase - -class EmptyKeyTest(ConnectionTestCase): - - def test_empty_key(self): - fnargs = ( - (self.cb.set, ["", "value"]), - (self.cb.get, [""]), - (self.cb.lock, ["", {'ttl': 5}]), - (self.cb.counter, [""]), - (self.cb.unlock, ["", 1234]), - (self.cb.delete, [""]), - (self.cb.observe, [""]), - (self.cb.set_multi, [{"": "value"}]), - (self.cb.counter_multi, [("", "")]), - (self.cb.delete_multi, [("", "")]), - (self.cb.unlock_multi, [{"": 1234}]), - (self.cb.observe_multi, [("")]) - ) - - for fn, args in fnargs: - self.assertRaises(ArgumentError, fn, *args) diff --git a/couchbase/tests/cases/encodings_t.py b/couchbase/tests/cases/encodings_t.py deleted file mode 100644 index f550c8486..000000000 --- a/couchbase/tests/cases/encodings_t.py +++ /dev/null @@ -1,142 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase import ( - FMT_BYTES, FMT_JSON, FMT_PICKLE, FMT_UTF8, - FMT_LEGACY_MASK, FMT_COMMON_MASK) -from couchbase.exceptions import ValueFormatError, CouchbaseError -from couchbase.tests.base import ConnectionTestCase, SkipTest -from couchbase.transcoder import TranscoderPP, LegacyTranscoderPP - -BLOB_ORIG = b'\xff\xfe\xe9\x05\xdc\x05\xd5\x05\xdd\x05' - -class EncodingTest(ConnectionTestCase): - - def test_default_format(self): - self.assertEqual(self.cb.default_format, FMT_JSON) - - def test_unicode(self): - txt = BLOB_ORIG.decode('utf-16') - for f in (FMT_BYTES, FMT_PICKLE): - cas = self.cb.upsert(txt, txt.encode('utf-16'), format=f).cas - server_val = self.cb.get(txt).value - self.assertEquals(server_val, BLOB_ORIG) - - def test_json_unicode(self): - self.assertEqual(self.cb.default_format, FMT_JSON) - uc = BLOB_ORIG.decode('utf-16') - rv = self.cb.upsert(uc, uc) - self.assertTrue(rv.success) - rv = self.cb.get(uc) - self.assertEqual(rv.value, uc) - self.assertEqual(rv.key, uc) - - def test_json_compact(self): - # This ensures our JSON encoder doesn't store huge blobs of data in the - # server. This was added as a result of PYCBC-108 - self.assertEqual(self.cb.default_format, FMT_JSON) - uc = BLOB_ORIG.decode('utf-16') - key = self.gen_key('json_compact') - self.cb.upsert(key, uc, format=FMT_JSON) - self.cb.data_passthrough = 1 - rv = self.cb.get(key) - - expected = '"'.encode('utf-8') + uc.encode('utf-8') + '"'.encode('utf-8') - self.assertEqual(expected, rv.value) - - self.cb.data_passthrough = 0 - - def test_blob(self): - blob = b'\x00\x01\x00\xfe\xff\x01\x42' - for f in (FMT_BYTES, FMT_PICKLE): - cas = self.cb.upsert("key", blob, format=f).cas - self.assertTrue(cas) - rv = self.cb.get("key").value - self.assertEquals(rv, blob) - - def test_bytearray(self): - ba = bytearray(b"Hello World") - self.cb.upsert("key", ba, format=FMT_BYTES) - rv = self.cb.get("key") - self.assertEqual(ba, rv.value) - - def test_passthrough(self): - self.cb.data_passthrough = True - self.cb.upsert("malformed", "some json") - self.cb.append("malformed", "blobs") - rv = self.cb.get("malformed") - - self.assertTrue(rv.success) - self.assertEqual(rv.flags, FMT_JSON) - self.assertEqual(rv.value, b'"some json"blobs') - - self.cb.data_passthrough = False - self.assertRaises(ValueFormatError, self.cb.get, "malformed") - - def test_zerolength(self): - rv = self.cb.upsert("key", b"", format=FMT_BYTES) - self.assertTrue(rv.success) - rv = self.cb.get("key") - self.assertEqual(rv.value, b"") - - self.assertRaises(CouchbaseError, self.cb.upsert, "", "value") - - def test_blob_keys_py2(self): - if bytes == str: - rv = self.cb.upsert(b"\0", "value") - rv = self.cb.get(b"\0") - else: - self.assertRaises(ValueFormatError, self.cb.upsert, b"\0", "value") - - def test_compat_interop(self): - # Check that we can interact with older versions, and vice versa: - - # Some basic sanity checks: - self.assertEqual(0x00, FMT_JSON & FMT_LEGACY_MASK) - self.assertEqual(0x01, FMT_PICKLE & FMT_LEGACY_MASK) - self.assertEqual(0x02, FMT_BYTES & FMT_LEGACY_MASK) - self.assertEqual(0x04, FMT_UTF8 & FMT_LEGACY_MASK) - - self.cb.transcoder = TranscoderPP() - self.cb.upsert('foo', { 'foo': 'bar' }) # JSON - self.cb.transcoder = LegacyTranscoderPP() - rv = self.cb.get('foo') - self.assertIsInstance(rv.value, dict) - - # Set it back now - self.cb.upsert('foo', { 'foo': 'bar' }) - self.cb.transcoder = TranscoderPP() - rv = self.cb.get('foo') - self.assertIsInstance(rv.value, dict) - self.assertEqual(rv.flags, FMT_JSON & FMT_LEGACY_MASK) - - - ## Try with Bytes - self.cb.transcoder = TranscoderPP() - self.cb.upsert('bytesval', 'Hello World'.encode('utf-8'), format=FMT_BYTES) - self.cb.transcoder = LegacyTranscoderPP() - rv = self.cb.get('bytesval') - self.assertEqual(FMT_BYTES, rv.flags) - self.assertEqual('Hello World'.encode('utf-8'), rv.value) - - # Set it back - self.cb.transcoder = LegacyTranscoderPP() - self.cb.upsert('bytesval', 'Hello World'.encode('utf-8'), format=FMT_BYTES) - self.cb.transcoder = TranscoderPP() - rv = self.cb.get('bytesval') - self.assertEqual(FMT_BYTES & FMT_LEGACY_MASK, rv.flags) - self.assertEqual('Hello World'.encode('utf-8'), rv.value) diff --git a/couchbase/tests/cases/endure_t.py b/couchbase/tests/cases/endure_t.py deleted file mode 100644 index 68103713e..000000000 --- a/couchbase/tests/cases/endure_t.py +++ /dev/null @@ -1,116 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.exceptions import NotFoundError, ArgumentError, TimeoutError -from couchbase.tests.base import MockTestCase - -class EndureTest(MockTestCase): - #XXX: Require LCB 2.1.0 - - def test_excessive(self): - self.assertRaises(ArgumentError, - self.cb.set, - "foo", "bar", - persist_to=99, replicate_to=99) - - def test_embedded_endure_set(self): - key = self.gen_key("embedded_endure") - with self.cb.durability(persist_to=-1, replicate_to=-1, timeout=0.1): - def cb1(res): - self.mockclient.endure(key, - replica_count=self.mock.replicas, - value=90, - cas=res.cas) - - self.cb._dur_testhook = cb1 - rv = self.cb.upsert(key, "blah blah") - self.assertTrue(rv.success) - - - def cb2(res): - self.mockclient.unpersist(key, on_master=True, - replica_count=self.mock.replicas) - - self.cb._dur_testhook = cb2 - self.assertRaises(TimeoutError, self.cb.set, key, "value") - - def test_embedded_endure_delete(self): - key = self.gen_key("embedded_endure_delete") - cas = 12345 - - # Store it first - self.mockclient.endure(key, replica_count=self.mock.replicas, - on_master=True, - value=666666, cas=cas) - - with self.cb.durability(persist_to=-1, replicate_to=-1, timeout=0.1): - def cb1(res): - self.mockclient.purge(key, on_master=True, - replica_count=self.mock.replicas) - - res = self.cb.get(key) - - self.cb._dur_testhook = cb1 - rv_rm = self.cb.remove(key) - self.assertTrue(rv_rm.success) - - - - self.mockclient.endure(key, on_master=True, - replica_count=self.mock.replicas, - cas=cas, value="blah") - - self.cb._dur_testhook = None - self.assertRaises(TimeoutError, self.cb.delete, key) - - - def test_single_poll(self): - key = self.gen_key("endure_single_poll") - self.mockclient.endure(key, - on_master=True, - replica_count=self.mock.replicas, - value=90, - cas=1234) - - rv = self.cb.endure(key, - persist_to=-1, replicate_to=-1) - self.assertTrue(rv.success) - - # This will fail.. - self.mockclient.unpersist(key, - on_master=True, - replica_count=self.mock.replicas) - - obsres = self.cb.observe(key) - self.assertRaises(TimeoutError, - self.cb.endure, - key, persist_to=1, replicate_to=0, - timeout=0.1) - - self.mockclient.persist(key, on_master=True, replica_count=0) - rv = self.cb.endure(key, persist_to=1, replicate_to=0) - self.assertTrue(rv.success) - - self.assertRaises(TimeoutError, - self.cb.endure, - key, persist_to=2, - replicate_to=0, - timeout=0.1) - - rv = self.cb.endure(key, persist_to=0, - replicate_to=self.mock.replicas) - self.assertTrue(rv.success) diff --git a/couchbase/tests/cases/excextra_t.py b/couchbase/tests/cases/excextra_t.py deleted file mode 100644 index dfa3d45ba..000000000 --- a/couchbase/tests/cases/excextra_t.py +++ /dev/null @@ -1,85 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import couchbase.exceptions as E -from couchbase.tests.base import ConnectionTestCase - -# These tests try to see if the 'result' and 'all_results' appear properly -# also verify that other documented exception fields are present - -class ExceptionsTest(ConnectionTestCase): - - def test_simple_excextra(self): - exc = None - key = self.gen_key("simple_excextra") - self.cb.remove(key, quiet=True) - - try: - self.cb.get(key, quiet=False) - except E.CouchbaseError as e: - exc = e - - self.assertTrue(exc) - self.assertIsInstance(exc, E.CouchbaseError) - self.assertTrue(exc.message) - self.assertIsInstance(exc, E.NotFoundError) - self.assertEqual(exc.key, key) - self.assertIsInstance(exc.all_results, self.cls_MultiResult) - self.assertTrue(key in exc.all_results) - self.assertIsInstance(exc.all_results[key], self.cls_ValueResult) - self.assertEqual(exc.all_results[key].rc, exc.rc) - - str(exc) - repr(exc) - del exc - - def test_multi_exc(self): - kv_missing = self.gen_kv_dict(prefix="multi_exc_missing") - kv_existing = self.gen_kv_dict(prefix="multi_exc_existing") - self.cb.upsert_multi(kv_existing) - exc = None - try: - self.cb.get_multi(list(kv_missing.keys()) + list(kv_existing.keys()), - quiet=False) - except E.CouchbaseError as e: - exc = e - - self.assertTrue(exc) - self.assertIsInstance(exc, E.NotFoundError) - self.assertEqual(len(exc.all_results), - len(kv_missing) + len(kv_existing)) - - - res_ok, res_fail = exc.split_results() - all_results = exc.all_results - for k, v in kv_missing.items(): - self.assertTrue(k in all_results) - self.assertTrue(k in res_fail) - self.assertFalse(k in res_ok) - self.assertFalse(all_results[k].success) - - for k, v in kv_existing.items(): - self.assertTrue(k in all_results) - self.assertTrue(k in res_ok) - self.assertFalse(k in res_fail) - self.assertTrue(all_results[k].success) - self.assertTrue(all_results[k].value) - self.assertEqual(v, all_results[k].value) - - str(exc) - repr(exc) - del exc diff --git a/couchbase/tests/cases/flush_t.py b/couchbase/tests/cases/flush_t.py deleted file mode 100644 index 07ea728c5..000000000 --- a/couchbase/tests/cases/flush_t.py +++ /dev/null @@ -1,28 +0,0 @@ -# -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import MockTestCase - - -class FlushTest(MockTestCase): - def test_flush(self): - kv = self.gen_kv_dict(prefix='flush') - self.cb.upsert_multi(kv) - self.cb.flush() - rvs = self.cb.get_multi(kv.keys(), quiet=True) - for v in rvs.values(): - self.assertFalse(v.success) \ No newline at end of file diff --git a/couchbase/tests/cases/format_t.py b/couchbase/tests/cases/format_t.py deleted file mode 100644 index 02531282a..000000000 --- a/couchbase/tests/cases/format_t.py +++ /dev/null @@ -1,137 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import pickle - -from couchbase.tests.base import ConnectionTestCase, SkipTest -from couchbase.exceptions import ValueFormatError, ArgumentError -from couchbase import FMT_AUTO, FMT_JSON, FMT_BYTES, FMT_UTF8, FMT_PICKLE - -class FormatTest(ConnectionTestCase): - - def test_set_autoformat(self): - key = self.gen_key("set_autoformat") - jvals = (None, True, False, {}, [], tuple() ) - bvals = (b"\x01", bytearray([1,2,3])) - uvals = (b"\x42".decode('utf-8'), b'\xea\x80\x80'.decode("utf-8")) - pvals = (set([]), object()) - - for jv in jvals: - self.cb.upsert(key, jv, format=FMT_AUTO) - rv = self.cb.get(key, no_format=True) - self.assertEqual(rv.flags, FMT_JSON) - # We need 'decode' because Python3's byte type - self.assertEqual(rv.value.decode("utf-8"), json.dumps(jv)) - - for bv in bvals: - self.cb.upsert(key, bv, format=FMT_AUTO) - rv = self.cb.get(key, no_format=True) - self.assertEqual(rv.flags, FMT_BYTES) - self.assertEqual(rv.value, bv) - - for uv in uvals: - self.cb.upsert(key, uv, format=FMT_AUTO) - rv = self.cb.get(key, no_format=True) - self.assertEqual(rv.flags, FMT_UTF8) - self.assertEqual(rv.value, uv.encode("utf-8")) - - for pv in pvals: - self.cb.upsert(key, pv, format=FMT_AUTO) - rv = self.cb.get(key, no_format=True) - self.assertEqual(rv.flags, FMT_PICKLE) - self.assertEqual(rv.value, pickle.dumps(pv)) - - def test_set_format(self): - key = self.gen_key('set_format') - rv1 = self.cb.upsert(key, {'some': 'value1'}, format=FMT_JSON) - self.assertTrue(rv1.cas > 0) - - self.assertRaises(ValueFormatError, self.cb.upsert, - key, object(), format=FMT_JSON) - - rv3 = self.cb.upsert(key, {'some': 'value3'}, - format=FMT_PICKLE) - self.assertTrue(rv3.cas > 0) - rv4 = self.cb.upsert(key, object(), format=FMT_PICKLE) - self.assertTrue(rv4.cas > 0) - - self.assertRaises(ValueFormatError, self.cb.upsert, - key, {'some': 'value5'}, - format=FMT_BYTES) - self.assertRaises(ValueFormatError, self.cb.upsert, - key, { 'some' : 'value5.1'}, - format=FMT_UTF8) - - rv6 = self.cb.upsert(key, b'some value6', format=FMT_BYTES) - self.assertTrue(rv6.cas > 0) - - rv7 = self.cb.upsert(key, b"\x42".decode('utf-8'), - format=FMT_UTF8) - self.assertTrue(rv7.success) - - - def test_get_noformat(self): - k = self.gen_key("get_noformat") - self.cb.upsert(k, {"foo":"bar"}, format=FMT_JSON) - rv = self.cb.get(k, no_format=True) - self.assertEqual(rv.value, b'{"foo":"bar"}') - - kl = self.gen_key_list(prefix="get_noformat") - kv = {} - for k in kl: - kv[k] = {"foo" : "bar"} - - self.cb.upsert_multi(kv) - rvs = self.cb.get_multi(kv.keys(), no_format=True) - for k, v in rvs.items(): - self.assertEqual(v.value, b'{"foo":"bar"}') - - - def test_get_format(self): - - raise(SkipTest("get-with-format not implemented")) - - self.cb.upsert('key_format1', {'some': 'value1'}, format=FMT_JSON) - val1 = self.cb.get('key_format1') - self.assertEqual(val1, {'some': 'value1'}) - - self.cb.upsert('key_format2', {'some': 'value2'}, format=FMT_PICKLE) - val2 = self.cb.get('key_format2') - self.assertEqual(val2, {'some': 'value2'}) - - self.cb.upsert('key_format3', b'some value3', format=FMT_BYTES) - val3 = self.cb.get('key_format3') - self.assertEqual(val3, b'some value3') - - - self.cb.upsert('key_format4', {'some': 'value4'}, format=FMT_JSON) - val4 = self.cb.get('key_format4', format=FMT_BYTES) - self.assertEqual(val4, b'{"some": "value4"}') - - self.cb.upsert('key_format5', {'some': 'value5'}, format=FMT_PICKLE) - val5 = self.cb.get('key_format5', format=FMT_BYTES) - self.assertEqual(pickle.loads(val5), {'some': 'value5'}) - - - self.cb.upsert('key_format6', {'some': 'value6'}, format=FMT_JSON) - self.assertRaises(ValueFormatError, self.cb.get, 'key_format6', - format=FMT_PICKLE) - - self.cb.upsert('key_format7', {'some': 'value7'}, format=FMT_PICKLE) - self.assertRaises(ValueFormatError, self.cb.get, 'key_format7', - format=FMT_JSON) diff --git a/couchbase/tests/cases/get_t.py b/couchbase/tests/cases/get_t.py deleted file mode 100644 index b73df65e0..000000000 --- a/couchbase/tests/cases/get_t.py +++ /dev/null @@ -1,192 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import pickle -from time import sleep - -from nose.plugins.attrib import attr - -from couchbase import FMT_JSON, FMT_PICKLE, FMT_UTF8, FMT_BYTES - -from couchbase.exceptions import ( - CouchbaseError, ValueFormatError, NotFoundError) -from couchbase.result import MultiResult, Result -from couchbase.tests.base import ConnectionTestCase, SkipTest - -class GetTest(ConnectionTestCase): - - def test_trivial_get(self): - key = self.gen_key('trivial_get') - self.cb.upsert(key, 'value1') - rv = self.cb.get(key) - self.assertEqual(rv.value, 'value1') - - rvs = self.cb.get_multi([key]) - self.assertIsInstance(rvs, self.cls_MultiResult) - self.assertEqual(len(rvs), 1) - self.assertEqual(rvs[key].value, 'value1') - - - def test_get_missing_key(self): - rv = self.cb.get('key_missing_1', quiet=True) - self.assertIsNone(rv.value) - self.assertFalse(rv.success) - - # Get with quiet=False - self.assertRaises(NotFoundError, self.cb.get, 'key_missing_1', - quiet=False) - - def test_multi_get(self): - kv = self.gen_kv_dict(amount=3, prefix='get_multi') - rvs = self.cb.upsert_multi(kv) - self.assertTrue(rvs.all_ok) - - k_subset = list(kv.keys())[:2] - - rvs1 = self.cb.get_multi(k_subset) - self.assertEqual(len(rvs1), 2) - self.assertEqual(rvs1[k_subset[0]].value, kv[k_subset[0]]) - self.assertEqual(rvs1[k_subset[1]].value, kv[k_subset[1]]) - - rv2 = self.cb.get_multi(kv.keys()) - self.assertEqual(rv2.keys(), kv.keys()) - - - def test_multi_mixed(self): - kv_missing = self.gen_kv_dict(amount=3, prefix='multi_missing_mixed') - kv_existing = self.gen_kv_dict(amount=3, prefix='multi_existing_mixed') - - self.cb.remove_multi(list(kv_missing.keys()) + list(kv_existing.keys()), - quiet=True) - - self.cb.upsert_multi(kv_existing) - - rvs = self.cb.get_multi( - list(kv_existing.keys()) + list(kv_missing.keys()), - quiet=True) - - - self.assertFalse(rvs.all_ok) - - for k, v in kv_missing.items(): - self.assertTrue(k in rvs) - self.assertFalse(rvs[k].success) - self.assertTrue(rvs[k].value is None) - self.assertTrue(NotFoundError._can_derive(rvs[k].rc)) - - for k, v in kv_existing.items(): - self.assertTrue(k in rvs) - self.assertTrue(rvs[k].success) - self.assertEqual(rvs[k].value, kv_existing[k]) - self.assertEqual(rvs[k].rc, 0) - - # Try this again, but without quiet - cb_exc = None - try: - self.cb.get_multi(list(kv_existing.keys()) + list(kv_missing.keys())) - except NotFoundError as e: - cb_exc = e - - self.assertTrue(cb_exc) - all_res = cb_exc.all_results - self.assertTrue(all_res) - self.assertFalse(all_res.all_ok) - - for k, v in kv_existing.items(): - self.assertTrue(k in all_res) - self.assertTrue(all_res[k].success) - self.assertEqual(all_res[k].value, v) - self.assertEqual(all_res[k].rc, 0) - - for k, v in kv_missing.items(): - self.assertTrue(k in all_res) - self.assertFalse(all_res[k].success) - self.assertTrue(all_res[k].value is None) - - del cb_exc - - - def test_extended_get(self): - key = self.gen_key(prefix='key_extended') - orig_cas1 = self.cb.upsert(key, 'value1').cas - rv = self.cb.get(key) - val1, flags1, cas1 = rv.value, rv.flags, rv.cas - self.assertEqual(val1, 'value1') - self.assertEqual(flags1, FMT_JSON) - self.assertEqual(cas1, orig_cas1) - - # Test named tuples - result1 = self.cb.get(key) - self.assertEqual(result1.value, 'value1') - self.assertEqual(result1.flags, FMT_JSON) - self.assertEqual(result1.cas, orig_cas1) - - # Single get as array - result2 = self.cb.get_multi([key]) - self.assertIsInstance(result2, self.cls_MultiResult) - self.assertTrue(key in result2) - self.assertEqual(result2[key].value, 'value1') - self.assertEqual(result2[key].flags, FMT_JSON) - self.assertEqual(result2[key].cas, orig_cas1) - - key2 = self.gen_key('key_extended_2') - cas2 = self.cb.upsert(key2, 'value2').cas - - key3 = self.gen_key('key_extended_3') - cas3 = self.cb.upsert(key3, 'value3').cas - results = self.cb.get_multi([key2, key3]) - - self.assertEqual(results[key3].value, 'value3') - self.assertEqual(results[key3].flags, FMT_JSON) - self.assertEqual(results[key3].cas, cas3) - - rv = self.cb.get('missing_key', quiet=True) - val4, flags4, cas4 = rv.value, rv.flags, rv.cas - self.assertEqual(val4, None) - self.assertEqual(flags4, 0x00) - self.assertEqual(cas4, 0) - - @attr('slow') - def test_get_ttl(self): - key = self.gen_key('get_ttl') - self.cb.remove(key, quiet=True) - self.cb.upsert(key, "a_value") - rv = self.cb.get(key, ttl=1) - self.assertEqual(rv.value, "a_value") - sleep(2) - rv = self.cb.get(key, quiet=True) - self.assertFalse(rv.success) - self.assertTrue(NotFoundError._can_derive(rv.rc)) - - @attr('slow') - def test_get_multi_ttl(self): - kvs = self.gen_kv_dict(amount=2, prefix='get_multi_ttl') - - self.cb.upsert_multi(kvs) - rvs = self.cb.get_multi(list(kvs.keys()), ttl=1) - for k, v in rvs.items(): - self.assertEqual(v.value, kvs[k]) - - sleep(2) - rvs = self.cb.get_multi(list(kvs.keys()), quiet=True) - for k, v in rvs.items(): - self.assertFalse(v.success) - self.assertTrue(k in kvs) - self.assertTrue(NotFoundError._can_derive(v.rc)) - -if __name__ == '__main__': - unittest.main() diff --git a/couchbase/tests/cases/iops_t.py b/couchbase/tests/cases/iops_t.py deleted file mode 100644 index 2d5fa7608..000000000 --- a/couchbase/tests/cases/iops_t.py +++ /dev/null @@ -1,42 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import CouchbaseTestCase -from couchbase.iops.select import SelectIOPS - -# For now, this just checks that basic set/get doesn't explode -# We'll definitely want to add more here before we consider it stable - -class IopsTest(CouchbaseTestCase): - def setUp(self): - super(IopsTest, self).setUp() - - def _iops_connection(self, **kwargs): - return self.make_connection(_iops=SelectIOPS(), **kwargs) - - def test_creation(self): - self._iops_connection() - self.assertTrue(True) - - def test_simple_ops(self): - cb = self._iops_connection() - key = self.gen_key("iops-simple") - value = "some_value" - cb.upsert(key, value) - rv = cb.get(key) - self.assertTrue(rv.success) - self.assertEqual(rv.value, value) diff --git a/couchbase/tests/cases/itertypes_t.py b/couchbase/tests/cases/itertypes_t.py deleted file mode 100644 index 5237a61b4..000000000 --- a/couchbase/tests/cases/itertypes_t.py +++ /dev/null @@ -1,84 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import ConnectionTestCase -from couchbase.exceptions import ArgumentError, ValueFormatError -from couchbase.user_constants import FMT_UTF8 - -class ItertypeTest(ConnectionTestCase): - - def test_itertypes(self): - kvs = self.gen_kv_dict(amount=10, prefix='itertypes') - intlist = set(self.gen_key_list(amount=3, prefix='setobject')) - - self.cb.remove_multi(kvs.keys(), quiet=True) - self.cb.upsert_multi(kvs) - self.cb.get_multi(kvs.keys()) - self.cb.get_multi(kvs.values(), quiet=True) - - self.cb.counter_multi(intlist, initial=10) - self.cb.get_multi(intlist) - - def test_bad_elements(self): - badlist = ("key1", None, "key2") - for fn in (self.cb.counter_multi, - self.cb.delete_multi, - self.cb.get_multi): - self.assertRaises( - (ArgumentError, ValueFormatError), - fn, badlist) - - self.assertRaises( - (ArgumentError, ValueFormatError), - self.cb.set_multi, - { None: "value" }) - - self.assertRaises(ValueFormatError, - self.cb.set_multi, - { "Value" : None}, - format=FMT_UTF8) - - def test_iterclass(self): - class IterTemp(object): - def __init__(self, gen_ints = False, badlen=False): - self.current = 0 - self.max = 5 - self.gen_ints = gen_ints - self.badlen = badlen - - def __iter__(self): - while self.current < self.max: - ret = self.current - if not self.gen_ints: - ret = "Key_" + str(ret) - self.current += 1 - yield ret - - def __len__(self): - if self.badlen: - return 100 - return self.max - - self.cb.remove_multi(IterTemp(gen_ints=False), quiet=True) - self.cb.counter_multi(IterTemp(gen_ints = False), initial=10) - self.cb.get_multi(IterTemp(gen_ints=False)) - self.cb.remove_multi(IterTemp(gen_ints = False)) - - # Try with a mismatched len-iter - self.assertRaises(ArgumentError, - self.cb.get_multi, - IterTemp(gen_ints=False, badlen=True)) diff --git a/couchbase/tests/cases/itmops_t.py b/couchbase/tests/cases/itmops_t.py deleted file mode 100644 index 74759afad..000000000 --- a/couchbase/tests/cases/itmops_t.py +++ /dev/null @@ -1,195 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import ConnectionTestCase -from couchbase.items import Item, ItemSequence, ItemOptionDict -from couchbase.exceptions import ( - NotFoundError, ValueFormatError, ArgumentError, KeyExistsError) -from couchbase.user_constants import FMT_BYTES, FMT_UTF8 - -class ItemTest(ConnectionTestCase): - """ - This class tests the new 'Item' API - """ - - def setUp(self): - super(ItemTest, self).setUp() - - def test_construction(self): - # Test whether we can construct a simple Item - it = Item("some_key", "some_value") - it.cas = 123456 - it.flags = 1000 - - self.assertEqual(it.key, "some_key") - self.assertEqual(it.value, "some_value") - self.assertEqual(it.cas, 123456) - self.assertEqual(it.flags, 1000) - hash(it) - - def test_simple_get(self): - k = self.gen_key("itm_simple_get") - it = Item(k, "simple_value") - - rvs = self.cb.upsert_multi(ItemSequence([it])) - self.assertTrue(rvs.all_ok) - - it_out = rvs[it.key] - self.assertEqual(it_out, it) - - it = Item() - it.key = k - itcoll = ItemSequence([it]) - - rvs = self.cb.get_multi(ItemSequence([it])) - self.assertTrue(rvs.all_ok) - it_out = rvs[it.key] - self.assertEqual(it_out, it) - self.assertEqual(it_out.value, "simple_value") - - # Now, set it again - self.cb.replace_multi(itcoll) - - # Now, delete it - self.cb.remove_multi(itcoll) - - self.assertRaises(NotFoundError, - self.cb.get_multi, itcoll) - - def test_item_format(self): - # Tests whether things like 'CAS' and 'format' are honored - k = self.gen_key("itm_format_options") - it = Item(k, {}) - itcoll = ItemOptionDict() - itcoll.dict[it] = { "format" : FMT_BYTES } - self.assertRaises(ValueFormatError, self.cb.upsert_multi, itcoll) - - def test_items_append(self): - k = self.gen_key("itm_append") - it = Item(k, "MIDDLE") - itcoll = ItemOptionDict() - itcoll.add(it) - - self.cb.upsert_multi(itcoll, format=FMT_UTF8) - - itcoll.add(it, fragment="_END") - self.cb.append_items(itcoll, format=FMT_UTF8) - self.assertEqual(it.value, "MIDDLE_END") - - itcoll.add(it, fragment="BEGIN_") - self.cb.prepend_items(itcoll, format=FMT_UTF8) - self.assertEqual(it.value, "BEGIN_MIDDLE_END") - - rv = self.cb.get(it.key) - self.assertEqual(rv.value, "BEGIN_MIDDLE_END") - - # Try without a 'fragment' specifier - self.assertRaises(ArgumentError, - self.cb.append_items, ItemSequence([it])) - itcoll.add(it) - self.assertRaises(ArgumentError, - self.cb.append_items, itcoll) - - def test_items_ignorecas(self): - k = self.gen_key("itm_ignorecas") - it = Item(k, "a value") - itcoll = ItemOptionDict() - itcoll.add(it) - self.cb.upsert_multi(itcoll) - self.assertTrue(it.cas) - - # Set it again - rv = self.cb.upsert(it.key, it.value) - self.assertTrue(rv.cas) - self.assertFalse(rv.cas == it.cas) - - # Should raise an error without ignore_cas - self.assertRaises(KeyExistsError, self.cb.upsert_multi, itcoll) - self.assertTrue(it.cas) - - itcoll.add(it, ignore_cas=True) - self.cb.upsert_multi(itcoll) - rv = self.cb.get(it.key) - self.assertEqual(rv.cas, it.cas) - - def test_subclass_descriptors(self): - class MyItem(Item): - def __init__(self): - pass - @property - def value(self): - return "This should not be present!!!" - @value.setter - def value(self, other): - return - - k = self.gen_key("itm_desc") - it = MyItem() - it.key = k - it.value = "hi!" - self.assertRaises(ArgumentError, - self.cb.upsert_multi, - ItemSequence([it])) - - def test_apiwrap(self): - it = Item(self.gen_key("item_apiwrap")) - self.cb.upsert_multi(it.as_itcoll()) - self.assertTrue(it.cas) - - # Set with 'ignorecas' - it.cas = 1234 - self.cb.upsert_multi(it.as_itcoll(ignore_cas=True)) - - self.cb.upsert_multi(ItemSequence(it)) - - def test_invalid_item(self): - itcoll = ItemOptionDict() - itcoll.add(None) - self.assertRaises(ArgumentError, self.cb.upsert_multi, itcoll) - - self.assertRaises(ArgumentError, - self.cb.upsert_multi, ItemSequence([None])) - - def test_create_and_add(self): - itcoll = ItemOptionDict() - itcoll.create_and_add('foo', value='fooValue', cas=123, persist_to=-1) - itcoll.create_and_add('bar', value='barValue', cas=321, replicate_to=-1) - - dd = itcoll.dict - self.assertEqual(2, len(dd)) - - def _find_item(key): - for k, v in dd.items(): - if k.key == key: - return k, v - - foo_item, foo_options = _find_item('foo') - self.assertIsInstance(foo_item, Item) - self.assertEqual('foo', foo_item.key) - self.assertEqual('fooValue', foo_item.value) - self.assertEqual(123, foo_item.cas) - self.assertEqual(1, len(foo_options)) - self.assertEqual(-1, foo_options['persist_to']) - - bar_item, bar_options = _find_item('bar') - self.assertIsInstance(bar_item, Item) - self.assertEqual('bar', bar_item.key) - self.assertEqual('barValue', bar_item.value) - self.assertEqual(321, bar_item.cas) - self.assertEqual(1, len(bar_options)) - self.assertEqual(-1, bar_options['replicate_to']) - diff --git a/couchbase/tests/cases/lock_t.py b/couchbase/tests/cases/lock_t.py deleted file mode 100644 index 4af33334d..000000000 --- a/couchbase/tests/cases/lock_t.py +++ /dev/null @@ -1,121 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from time import sleep - -from nose.plugins.attrib import attr - -from couchbase.exceptions import ( - CouchbaseError, TemporaryFailError, KeyExistsError, ArgumentError) - -from couchbase.tests.base import ConnectionTestCase - - -class LockTest(ConnectionTestCase): - - def test_simple_lock(self): - k = self.gen_key('lock') - v = "locked_value" - self.cb.upsert(k, v) - rv = self.cb.lock(k, ttl=5) - - self.assertTrue(rv.success) - self.assertEqual(rv.value, v) - self.assertRaises(KeyExistsError, self.cb.set, k, v) - - self.assertRaises(TemporaryFailError, self.cb.lock, k, ttl=5) - - # Test set-while-locked - self.assertRaises(KeyExistsError, self.cb.set, k, v) - - self.assertRaises(TemporaryFailError, self.cb.unlock, k, cas=0xdeadbeef) - - rv = self.cb.unlock(k, rv.cas) - self.assertTrue(rv.success) - - # Unlocked with key already unlocked - self.assertRaises(TemporaryFailError, - self.cb.unlock, - k, - 1234) - - rv = self.cb.upsert(k, v) - self.assertTrue(rv.success) - - @attr('slow') - def test_timed_lock(self): - k = self.gen_key('lock') - v = "locked_value" - self.cb.upsert(k, v) - rv = self.cb.lock(k, ttl=1) - sleep(2) - self.cb.upsert(k, v) - - def test_multi_lock(self): - kvs = self.gen_kv_dict(prefix='lock_multi') - - self.cb.upsert_multi(kvs) - rvs = self.cb.lock_multi(kvs.keys(), ttl=5) - self.assertTrue(rvs.all_ok) - self.assertEqual(len(rvs), len(kvs)) - for k, v in rvs.items(): - self.assertEqual(v.value, kvs[k]) - - rvs = self.cb.unlock_multi(rvs) - - def test_unlock_multi(self): - key = self.gen_key(prefix='unlock_multi') - val = "lock_value" - self.cb.upsert(key, val) - - rv = self.cb.lock(key, ttl=5) - rvs = self.cb.unlock_multi({key:rv.cas}) - self.assertTrue(rvs.all_ok) - self.assertTrue(rvs[key].success) - - rv = self.cb.lock(key, ttl=5) - self.assertTrue(rv.success) - rvs = self.cb.unlock_multi({key:rv}) - self.assertTrue(rvs.all_ok) - self.assertTrue(rvs[key].success) - - def test_missing_expiry(self): - self.assertRaises(ArgumentError, - self.cb.lock, "foo") - self.assertRaises(ArgumentError, self.cb.lock_multi, - ("foo", "bar")) - - def test_missing_cas(self): - self.assertRaises(ArgumentError, - self.cb.unlock_multi, - ("foo", "bar")) - self.assertRaises(ArgumentError, - self.cb.unlock_multi, - {"foo":0, "bar":0}) - - def test_resobjs(self): - keys = self.gen_kv_dict(prefix="Lock_test_resobjs") - self.cb.upsert_multi(keys) - rvs = self.cb.lock_multi(keys.keys(), ttl=5) - self.cb.unlock_multi(rvs) - - kv_single = list(keys.keys())[0] - - rv = self.cb.lock(kv_single, ttl=5) - self.assertTrue(rv.cas) - - self.cb.unlock_multi([rv]) diff --git a/couchbase/tests/cases/lockmode_t.py b/couchbase/tests/cases/lockmode_t.py deleted file mode 100644 index 706ea0748..000000000 --- a/couchbase/tests/cases/lockmode_t.py +++ /dev/null @@ -1,89 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from threading import Thread, Lock -import time - - -from couchbase.exceptions import CouchbaseError, ObjectThreadError -from couchbase.tests.base import CouchbaseTestCase -from couchbase import LOCKMODE_WAIT, LOCKMODE_EXC, LOCKMODE_NONE - -class LockmodeTest(CouchbaseTestCase): - def test_lockmode_defaults(self): - # default is LOCKMODE_EXC - key = self.gen_key("lockmode_defaults") - cb = self.make_connection() - self.assertEqual(cb.lockmode, LOCKMODE_EXC) - cb._thr_lockop(0) - cb._thr_lockop(1) - cb.upsert(key, "value") - - cb = self.make_connection(lockmode=LOCKMODE_NONE) - self.assertEqual(cb.lockmode, LOCKMODE_NONE) - - self.assertRaises(ObjectThreadError, - cb._thr_lockop, 1) - self.assertRaises(ObjectThreadError, - cb._thr_lockop, 0) - cb.upsert(key, "value") - - cb = self.make_connection(lockmode=LOCKMODE_WAIT) - self.assertEqual(cb.lockmode, LOCKMODE_WAIT) - cb._thr_lockop(0) - cb._thr_lockop(1) - cb.upsert(key, "value") - - cb = self.make_connection(lockmode=LOCKMODE_WAIT, unlock_gil=False) - self.assertEqual(cb.lockmode, LOCKMODE_NONE) - cb.upsert(key, "value") - - def test_lockmode_exc(self): - key = self.gen_key("lockmode_exc") - - cb = self.make_connection() - cb._thr_lockop(0) - self.assertRaises(ObjectThreadError, - cb.upsert, - key, "bar") - cb._thr_lockop(1) - - # Ensure the old value is not buffered - cb.upsert(key, "baz") - self.assertEqual(cb.get(key).value, "baz") - - def test_lockmode_wait(self): - key = self.gen_key("lockmode_wait") - cb = self.make_connection(lockmode=LOCKMODE_WAIT, unlock_gil=True) - - d = { - 'ended' : 0 - } - - def runfunc(): - cb.upsert(key, "value") - d['ended'] = time.time() - - cb._thr_lockop(0) - t = Thread(target=runfunc) - t.start() - - time.sleep(0.5) - time_unlocked = time.time() - cb._thr_lockop(1) - - t.join() - self.assertTrue(d['ended'] >= time_unlocked) diff --git a/couchbase/tests/cases/misc_t.py b/couchbase/tests/cases/misc_t.py deleted file mode 100644 index 8baf5123d..000000000 --- a/couchbase/tests/cases/misc_t.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -try: - from urlparse import urlparse -except ImportError: - from urllib.parse import urlparse - -from couchbase.tests.base import ConnectionTestCase -from couchbase.user_constants import FMT_JSON, FMT_AUTO, FMT_JSON, FMT_PICKLE -from couchbase.exceptions import ClientTemporaryFailError -from couchbase.exceptions import CouchbaseError - -class MiscTest(ConnectionTestCase): - - def test_server_nodes(self): - nodes = self.cb.server_nodes - self.assertIsInstance(nodes, (list, tuple)) - self.assertTrue(len(nodes) > 0) - for n in nodes: - self.assertIsInstance(n, str) - - def _set_nodes(): - self.cb.server_nodes = 'sdf' - self.assertRaises((AttributeError, TypeError), _set_nodes) - - def test_lcb_version(self): - verstr, vernum = self.factory.lcb_version() - self.assertIsInstance(verstr, str) - self.assertIsInstance(vernum, int) - - def test_bucket(self): - bucket_str = self.cb.bucket - connstr = self.make_connargs()['connection_string'] - expected = urlparse(connstr).path - - self.assertEqual('/' + bucket_str, expected) - - def test_conn_repr(self): - repr(self.cb) - - - def test_connection_defaults(self): - # This will only work on the basic Connection class - from couchbase.bucket import Bucket - ctor_params = self.make_connargs() - # XXX: Change these if any of the defaults change - defaults = { - 'quiet' : False, - 'default_format' : FMT_JSON, - 'unlock_gil' : True, - 'transcoder' : None - } - - cb_ctor = Bucket(**ctor_params) - - for option, value in defaults.items(): - actual = getattr(cb_ctor, option) - self.assertEqual(actual, value) - - - def test_closed(self): - cb = self.cb - self.assertFalse(cb.closed) - cb._close() - self.assertTrue(cb.closed) - self.assertRaises(ClientTemporaryFailError, self.cb.get, "foo") - - - def test_fmt_args(self): - # Regression - cb = self.make_connection(default_format=123) - self.assertEqual(cb.default_format, 123) - - key = self.gen_key("fmt_auto_ctor") - cb = self.make_connection(default_format = FMT_AUTO) - cb.upsert("foo", set([])) - rv = cb.get("foo") - self.assertEqual(rv.flags, FMT_PICKLE) - - - def test_cntl(self): - cb = self.make_connection() - # Get the timeout - rv = cb._cntl(0x01) - self.assertEqual(75000000, rv) - - cb._cntl(0x01, rv) - # Doesn't crash? good enough - - # Try with something invalid - self.assertRaises(CouchbaseError, cb._cntl, 0xf000) - self.assertRaises(CouchbaseError, cb._cntl, 0x01, "string") - - # Try with something else now. Operation timeout - rv = cb._cntl(0x00, value_type="timeout") - self.assertEqual(2.5, rv) - - rv = cb._cntl(0x00, value_type="uint32_t") - self.assertEqual(2500000, rv) - - # Modification: - cb._cntl(0x00, 100000, value_type="uint32_t") - rv = cb._cntl(0x00, value_type="timeout") - self.assertEqual(0.1, rv) - - def test_newer_ctls(self): - cb = self.make_connection() - self.skipLcbMin("2.3.1") - rv = cb._cntl(0x1f, value_type="string") # LCB_CNTL_CHANGESET - "" + rv # String - - # CONFIG_CACHE_LOADED - rv = cb._cntl(0x15, value_type="int") # - self.assertEqual(0, rv) - - def test_cntl_string(self): - cb = self.make_connection() - cb._cntlstr("operation_timeout", "5.0") - self.assertEqual(5.0, cb.timeout) - - def test_vbmap(self): - # We don't know what the vbucket map is supposed to be, so just - # check it doesn't fail - cb = self.make_connection() - vb, ix = cb._vbmap("hello") - int(vb) - int(ix) - - def test_logging(self): - # Assume we don't have logging here.. - import couchbase - import couchbase._libcouchbase as lcb - - self.assertFalse(lcb.lcb_logging()) - - logfn = lambda x: x - lcb.lcb_logging(logfn) - self.assertEqual(logfn, lcb.lcb_logging()) - - couchbase.enable_logging() - self.assertTrue(lcb.lcb_logging()) - couchbase.disable_logging() - self.assertFalse(lcb.lcb_logging()) - - def test_compat_timeout(self): - cb = self.make_connection(timeout=7.5) - self.assertEqual(7.5, cb.timeout) \ No newline at end of file diff --git a/couchbase/tests/cases/n1qlstrings_t.py b/couchbase/tests/cases/n1qlstrings_t.py deleted file mode 100644 index e02ad16b8..000000000 --- a/couchbase/tests/cases/n1qlstrings_t.py +++ /dev/null @@ -1,91 +0,0 @@ -# -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from __future__ import print_function -import json - -from couchbase.tests.base import CouchbaseTestCase -from couchbase.n1ql import N1QLQuery, CONSISTENCY_REQUEST, CONSISTENCY_NONE - - -class N1QLStringTest(CouchbaseTestCase): - def setUp(self): - super(N1QLStringTest, self).setUp() - - def test_encode_namedargs(self): - qstr = 'SELECT * FROM default WHERE field1=$arg1 AND field2=$arg2' - q = N1QLQuery(qstr, arg1='foo', arg2='bar') - - self.assertEqual(qstr, q.statement) - - dval = json.loads(q.encoded) - self.assertEqual(qstr, dval['statement']) - self.assertEqual('foo', dval['$arg1']) - self.assertEqual('bar', dval['$arg2']) - - def test_encode_posargs(self): - qstr = 'SELECT * FROM default WHERE field1=$1 AND field2=$arg2' - q = N1QLQuery(qstr, 'foo', 'bar') - dval = json.loads(q.encoded) - self.assertEqual(qstr, dval['statement']) - self.assertEqual('foo', dval['args'][0]) - self.assertEqual('bar', dval['args'][1]) - - def test_encode_mixed_args(self): - qstr = 'SELECT * FROM default WHERE field1=$1 AND field2=$arg2' - q = N1QLQuery(qstr, 'foo', arg2='bar') - dval = json.loads(q.encoded) - self.assertEqual('bar', dval['$arg2']) - self.assertEqual('foo', dval['args'][0]) - self.assertEqual(1, len(dval['args'])) - - def test_encoded_consistency(self): - qstr = 'SELECT * FROM default' - q = N1QLQuery(qstr) - q.consistency = CONSISTENCY_REQUEST - dval = json.loads(q.encoded) - self.assertEqual('request_plus', dval['scan_consistency']) - - q.consistency = CONSISTENCY_NONE - dval = json.loads(q.encoded) - self.assertEqual('none', dval['scan_consistency']) - - def test_encode_scanvec(self): - # The value is a vbucket's sequence number, - # and guard is a vbucket's UUID. - - q = N1QLQuery('SELECT * FROM default') - - q._add_scanvec((42, 3004, 3)) - dval = json.loads(q.encoded) - sv_exp = { - '42': {'value': 3, 'guard': '3004'} - } - - self.assertEqual('at_plus', dval['scan_consistency']) - self.assertEqual(sv_exp, dval['scan_vector']) - - # Ensure the vb field gets updated. No duplicates! - q._add_scanvec((42, 3004, 4)) - sv_exp['42']['value'] = 4 - dval = json.loads(q.encoded) - self.assertEqual(sv_exp, dval['scan_vector']) - - q._add_scanvec((91, 7779, 23)) - dval = json.loads(q.encoded) - sv_exp['91'] = {'guard': '7779', 'value': 23} - self.assertEqual(sv_exp, dval['scan_vector']) diff --git a/couchbase/tests/cases/observe_t.py b/couchbase/tests/cases/observe_t.py deleted file mode 100644 index 707c00dc9..000000000 --- a/couchbase/tests/cases/observe_t.py +++ /dev/null @@ -1,90 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import ConnectionTestCase, MockTestCase -from couchbase.result import ObserveInfo -from couchbase.user_constants import OBS_MASK, OBS_FOUND, OBS_PERSISTED - -class ObserveTest(ConnectionTestCase): - - def test_single_observe(self): - key = self.gen_key("test_single_observe") - self.cb.upsert(key, "value") - rv = self.cb.observe(key) - grv = self.cb.get(key) - print(rv) - - self.assertTrue(rv.success) - self.assertIsInstance(rv.value, list) - self.assertTrue(rv.value) - - found_master = False - - - for oi in rv.value: - self.assertIsInstance(oi, self.cls_ObserveInfo) - oi.cas - oi.from_master - self.assertEqual(oi.flags, oi.flags & OBS_MASK) - - if oi.from_master: - found_master = True - self.assertTrue(oi.flags & (OBS_FOUND) == OBS_FOUND) - self.assertEqual(oi.cas, grv.cas) - - self.assertTrue(found_master) - repr(oi) - str(oi) - - def test_multi_observe(self): - kexist = self.gen_key("test_multi_observe-exist") - kmissing = self.gen_key("test_multi_observe-missing") - self.cb.upsert(kexist, "value") - self.cb.remove(kmissing, quiet=True) - grv = self.cb.get(kexist) - - mres = self.cb.observe_multi((kexist, kmissing)) - self.assertTrue(mres.all_ok) - self.assertEqual(len(mres), 2) - - v_exist = mres[kexist] - v_missing = mres[kmissing] - - for v in (v_exist.value, v_missing.value): - self.assertIsInstance(v, list) - self.assertTrue(len(v)) - found_master = False - - for oi in v: - self.assertIsInstance(oi, self.cls_ObserveInfo) - oi.flags - oi.cas - if oi.from_master: - found_master = True - - -class ConnectionObserveMasterTest(MockTestCase): - def test_master_observe(self): - self.skipLcbMin("2.3.0") - key = self.gen_key("test_master_observe") - rv = self.cb.upsert(key, "value") - obs_all = self.cb.observe(key) - self.assertTrue(len(obs_all.value) > 1) - obs_master = self.cb.observe(key, master_only=True) - self.assertEqual(len(obs_master.value), 1) - obs_val = obs_master.value[0] - self.assertTrue(obs_val.from_master) - self.assertEqual(obs_val.cas, rv.cas) diff --git a/couchbase/tests/cases/pipeline_t.py b/couchbase/tests/cases/pipeline_t.py deleted file mode 100644 index 170e0b8d0..000000000 --- a/couchbase/tests/cases/pipeline_t.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.exceptions import PipelineError, NotFoundError, ArgumentError -from couchbase.tests.base import ConnectionTestCase -from couchbase import FMT_UTF8 - -class PipelineTest(ConnectionTestCase): - - def test_simple_pipeline(self): - k = self.gen_key("pipeline_test") - with self.cb.pipeline(): - self.cb.remove(k, quiet=True) - self.cb.insert(k, "MIDDLE", format=FMT_UTF8) - self.cb.prepend(k, "BEGIN_") - self.cb.append(k, "_END") - - # No errors - rv = self.cb.get(k) - self.assertEqual(rv.value, "BEGIN_MIDDLE_END") - - def test_empty_pipeline(self): - k = self.gen_key("empty_pipeline") - - with self.cb.pipeline(): - pass - - self.cb.upsert(k, "a value") - rv = self.cb.get(k) - self.assertEqual(rv.value, "a value") - - def test_pipeline_results(self): - k = self.gen_key("pipeline_results") - pipeline = self.cb.pipeline() - with pipeline: - self.cb.remove(k, quiet=True) - self.cb.upsert(k, "blah") - self.cb.get(k) - self.cb.remove(k) - - results = pipeline.results - self.assertEqual(len(results), 4) - - self.assertTrue(results[1].success) - self.assertEqual(results[1].key, k) - - self.assertTrue(results[2].success) - self.assertEqual(results[2].key, k) - self.assertEqual(results[2].value, "blah") - - self.assertTrue(results[3].success) - - def test_pipeline_operrors(self): - k = self.gen_key("pipeline_errors") - v = "hahahaha" - self.cb.remove(k, quiet=True) - - def run_pipeline(): - with self.cb.pipeline(): - self.cb.get(k, quiet=False) - self.cb.upsert(k, v) - self.assertRaises(NotFoundError, run_pipeline) - - rv = self.cb.upsert("foo", "bar") - self.assertTrue(rv.success) - - def test_pipeline_state_errors(self): - def fun(): - with self.cb.pipeline(): - with self.cb.pipeline(): - pass - - self.assertRaises(PipelineError, fun) - - def fun(): - with self.cb.pipeline(): - list(self.cb.query("design", "view")) - - self.assertRaises(PipelineError, fun) - - def test_pipeline_argerrors(self): - k = self.gen_key("pipeline_argerrors") - self.cb.remove(k, quiet=True) - - pipeline = self.cb.pipeline() - - def fun(): - with pipeline: - self.cb.upsert(k, "foo") - self.cb.get("foo", "bar") - self.cb.get(k) - - self.assertRaises(ArgumentError, fun) - self.assertEqual(len(pipeline.results), 1) - self.assertEqual(self.cb.get(k).value, "foo") - - def test_multi_pipeline(self): - kvs = self.gen_kv_dict(prefix="multi_pipeline") - - pipeline = self.cb.pipeline() - with pipeline: - self.cb.upsert_multi(kvs) - self.cb.get_multi(kvs.keys()) - - self.assertEqual(len(pipeline.results), 2) - for mres in pipeline.results: - for k in kvs: - self.assertTrue(k in mres) - self.assertTrue(mres[k].success) - - - for k, v in pipeline.results[1].items(): - self.assertEqual(v.value, kvs[k]) diff --git a/couchbase/tests/cases/results_t.py b/couchbase/tests/cases/results_t.py deleted file mode 100644 index 44876996e..000000000 --- a/couchbase/tests/cases/results_t.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import ConnectionTestCase - -INT_TYPES = None -try: - INT_TYPES = (long, int) -except: - INT_TYPES = (int,) - -class ResultsTest(ConnectionTestCase): - - def __test_oprsesult(self, rv, check_exact=True, exprc=0): - # Ensure they can be stringified - self.assertIsInstance(rv, self.cls_OperationResult) - self.assertIsInstance(rv, self.cls_Result) - - if check_exact: - self.assertEqual(rv.__class__, self.cls_OperationResult) - - self.assertIsInstance(rv.cas, INT_TYPES) - self.assertIsInstance(rv.rc, INT_TYPES) - - self.assertEqual(rv.rc, exprc) - if exprc == 0: - self.assertTrue(rv.success) - - self.assertIsInstance(rv.errstr, str) - - self.assertIsInstance(repr(rv), str) - self.assertIsInstance(str(rv), str) - - def __test_valresult(self, rv, value): - self.assertEqual(rv.__class__, self.cls_ValueResult) - self.__test_oprsesult(rv, check_exact=False) - - self.assertEqual(rv.value, value) - self.assertIsInstance(rv.flags, INT_TYPES) - - def test_results(self): - # Test OperationResult/ValueResult fields - key = self.gen_key("opresult") - rv = self.cb.upsert(key, "value") - self.__test_oprsesult(rv) - - rv = self.cb.remove(key) - self.__test_oprsesult(rv) - - rv = self.cb.upsert(key, "value") - self.__test_oprsesult(rv) - - rv = self.cb.lock(key, ttl=10) - self.__test_valresult(rv, "value") - rv = self.cb.unlock(key, rv.cas) - self.__test_oprsesult(rv) - rv = self.cb.get(key) - self.__test_valresult(rv, "value") - rv = self.cb.remove(key) - self.__test_oprsesult(rv) - - rv = self.cb.counter(key, initial=10) - self.__test_valresult(rv, 10) - rv = self.cb.get(key) - self.__test_valresult(rv, 10) - - rv = self.cb.touch(key) - self.__test_oprsesult(rv) - - def test_multi_results(self): - kvs = self.gen_kv_dict(prefix="multi_results") - rvs = self.cb.upsert_multi(kvs) - self.assertIsInstance(rvs, self.cls_MultiResult) - [ self.__test_oprsesult(x) for x in rvs.values() ] - repr(rvs) - str(rvs) - - rvs = self.cb.get_multi(kvs.keys()) - self.assertIsInstance(rvs, self.cls_MultiResult) - self.assertTrue(rvs.all_ok) - - [ self.__test_valresult(v, kvs[k]) for k, v in rvs.items()] - - rvs = self.cb.remove_multi(kvs.keys()) - [ self.__test_oprsesult(x) for x in rvs.values() ] diff --git a/couchbase/tests/cases/rget_t.py b/couchbase/tests/cases/rget_t.py deleted file mode 100644 index c7a8cd4db..000000000 --- a/couchbase/tests/cases/rget_t.py +++ /dev/null @@ -1,97 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.exceptions import NotFoundError, ArgumentError - -from couchbase.tests.base import MockTestCase -from couchbase.mockserver import MockControlClient - -class ReplicaGetTest(MockTestCase): - def setUp(self): - super(ReplicaGetTest, self).setUp() - self.skipUnlessMock() - self.skipLcbMin("2.0.7") - self.mockclient = MockControlClient(self.mock.rest_port) - - def test_get_kw(self): - key = self.gen_key("get_kw") - # Set on all replicas - self.mockclient.cache(key, - on_master=False, - replica_count=self.mock.replicas, - value=99, - cas=1234) - - self.assertRaises(NotFoundError, - self.cb.get, key) - - rv = self.cb.get(key, replica=True) - self.assertTrue(rv.success) - self.assertEqual(rv.value, 99) - - def _check_single_replica(self, ix): - key = self.gen_key("get_kw_ix") - - # Ensure the key is removed... - self.mockclient.purge(key, - on_master=True, - replica_count=self.mock.replicas) - - # Getting it should raise an error - self.assertRaises(NotFoundError, self.cb.get, key) - - # So should getting it from any replica - self.assertRaises(NotFoundError, self.cb.rget, key) - - # And so should getting it from a specific index - for jx in range(self.mock.replicas): - self.assertRaises(NotFoundError, self.cb.rget, key, - replica_index=jx) - - # Store the key on the desired replica - self.mockclient.cache(key, - on_master=False, - replicas=[ix], - value=ix, - cas=12345) - - # Getting it from a replica should ultimately succeed - self.cb.get(key, replica=True) - rv = self.cb.rget(key) - self.assertTrue(rv.success) - self.assertEqual(rv.value, ix) - - # Getting it from our specified replica should succeed - rv = self.cb.rget(key, replica_index=ix) - self.assertTrue(rv.success) - self.assertEqual(rv.value, ix) - - # Getting it from any other replica should fail - for jx in range(self.mock.replicas): - if jx == ix: - continue - - self.assertRaises(NotFoundError, - self.cb.rget, - key, - replica_index=jx) - - - def test_get_ix(self): - key = self.gen_key("get_kw_ix") - for ix in range(self.mock.replicas): - self._check_single_replica(ix) diff --git a/couchbase/tests/cases/set_converters_t.py b/couchbase/tests/cases/set_converters_t.py deleted file mode 100644 index 06f0c5008..000000000 --- a/couchbase/tests/cases/set_converters_t.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import json -import pickle - -from couchbase.tests.base import ConnectionTestCase -import couchbase -import couchbase._libcouchbase as LCB - -class ConverertSetTest(ConnectionTestCase): - def _swap_converters(self, swapfunc, kbase, new_enc, new_dec): - kencode = kbase + "_encode" - kdecode = kbase + "_decode" - - old_enc = LCB._get_helper(kencode) - old_dec = LCB._get_helper(kdecode) - - old = swapfunc(new_enc, new_dec) - self.assertEqual(old[0], old_enc) - self.assertEqual(old[1], old_dec) - return old - - def test_json_conversions(self): - d = { - 'encode' : 0, - 'decode' : 0 - } - - def _encode(val): - d['encode'] += 1 - return json.dumps(val) - - def _decode(val): - d['decode'] += 1 - return json.loads(val) - - old = self._swap_converters(couchbase.set_json_converters, - "json", - _encode, - _decode) - - key = self.gen_key("test_json_conversion") - - self.cb.upsert(key, ["value"], format=couchbase.FMT_JSON) - rv = self.cb.get(key) - self.assertEqual(rv.value, ["value"]) - self.assertEqual(1, d['encode']) - self.assertEqual(1, d['decode']) - - self._swap_converters(couchbase.set_json_converters, - "json", - old[0], - old[1]) - - def test_pickle_conversions(self): - d = { - 'encode' : 0, - 'decode' : 0 - } - - def _encode(val): - d['encode'] += 1 - return pickle.dumps(val) - - def _decode(val): - d['decode'] += 1 - return pickle.loads(val) - - key = self.gen_key("test_pickle_conversions") - old = self._swap_converters(couchbase.set_pickle_converters, - "pickle", - _encode, - _decode) - fn = set([1,2,3]) - self.cb.upsert(key, fn, format=couchbase.FMT_PICKLE) - rv = self.cb.get(key) - self.assertEqual(rv.value, fn) - self.assertEqual(1, d['encode']) - self.assertEqual(1, d['decode']) - - self._swap_converters(couchbase.set_pickle_converters, - "pickle", - old[0], - old[1]) diff --git a/couchbase/tests/cases/set_t.py b/couchbase/tests/cases/set_t.py deleted file mode 100644 index 9a48ed7a1..000000000 --- a/couchbase/tests/cases/set_t.py +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from time import sleep - -from nose.plugins.attrib import attr - -from couchbase import FMT_JSON, FMT_PICKLE, FMT_BYTES, FMT_UTF8 -from couchbase.exceptions import (KeyExistsError, ValueFormatError, - ArgumentError, NotFoundError, - NotStoredError) -from couchbase.tests.base import ConnectionTestCase - - -class UpsertTest(ConnectionTestCase): - - def test_trivial_set(self): - rv = self.cb.upsert(self.gen_key(), 'value1') - self.assertTrue(rv) - self.assertTrue(rv.cas > 0) - rv = self.cb.upsert(self.gen_key(), 'value2') - self.assertTrue(rv.cas > 0) - - def test_set_with_cas(self): - key = self.gen_key('cas') - rv1 = self.cb.upsert(key, 'value1') - self.assertTrue(rv1.cas > 0) - - self.assertRaises(KeyExistsError, self.cb.upsert, - key, 'value2', cas=rv1.cas+1) - - rv2 = self.cb.upsert(key, 'value3', cas=rv1.cas) - self.assertTrue(rv2.cas > 0) - self.assertNotEqual(rv1.cas, rv2.cas) - - rv3 = self.cb.upsert(key, 'value4') - self.assertTrue(rv3.cas > 0) - self.assertNotEqual(rv3.cas, rv2.cas) - self.assertNotEqual(rv3.cas, rv1.cas) - - @attr('slow') - def test_set_with_ttl(self): - key = self.gen_key('ttl') - self.cb.upsert(key, 'value_ttl', ttl=2) - rv = self.cb.get(key) - self.assertEqual(rv.value, 'value_ttl') - # Make sure the key expires - sleep(3) - self.assertRaises(NotFoundError, self.cb.get, key) - - def test_set_objects(self): - key = self.gen_key('set_objects') - for v in (None, False, True): - for fmt in (FMT_JSON, FMT_PICKLE): - rv = self.cb.upsert(key, v, format=fmt) - self.assertTrue(rv.success) - rv = self.cb.get(key) - self.assertTrue(rv.success) - self.assertEqual(rv.value, v) - - def test_multi_set(self): - kv = self.gen_kv_dict(prefix='set_multi') - rvs = self.cb.upsert_multi(kv) - self.assertTrue(rvs.all_ok) - for k, v in rvs.items(): - self.assertTrue(v.success) - self.assertTrue(v.cas > 0) - - for k, v in rvs.items(): - self.assertTrue(k in rvs) - self.assertTrue(rvs[k].success) - - self.assertRaises((ArgumentError,TypeError), self.cb.upsert_multi, kv, - cas = 123) - - def test_add(self): - key = self.gen_key('add') - self.cb.remove(key, quiet=True) - rv = self.cb.insert(key, "value") - self.assertTrue(rv.cas) - - self.assertRaises(KeyExistsError, - self.cb.insert, key, "value") - - def test_replace(self): - key = self.gen_key('replace') - rv = self.cb.upsert(key, "value") - self.assertTrue(rv.success) - - rv = self.cb.replace(key, "value") - self.assertTrue(rv.cas) - - rv = self.cb.replace(key, "value", cas=rv.cas) - self.assertTrue(rv.cas) - - self.assertRaises(KeyExistsError, - self.cb.replace, key, "value", cas=0xdeadbeef) - - self.cb.remove(key, quiet=True) - self.assertRaises(NotFoundError, - self.cb.replace, key, "value") - - -if __name__ == '__main__': - unittest.main() diff --git a/couchbase/tests/cases/spatial_t.py b/couchbase/tests/cases/spatial_t.py deleted file mode 100644 index 2a1b05ea0..000000000 --- a/couchbase/tests/cases/spatial_t.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from couchbase.tests.base import RealServerTestCase -from couchbase.user_constants import FMT_JSON -from couchbase.views.params import SpatialQuery - -DESIGN_JSON = { - 'language': 'javascript', - 'spatial': { - 'simpleGeo': - ''' - function(doc) { - if (doc.loc) { - emit({"type":"Point", "coordinates": doc.loc}, null); - } - } - '''.replace("\n", '') - } -} - -DOCS_JSON = { - 'mountain-view_ca_usa': { - 'locname': ['Oakland', 'CA', 'USA'], - 'loc': [-122, 37] - }, - 'reno_nv_usa': { - 'locname': ['Reno', 'NV', 'USA'], - 'loc': [-119, 39] - }, - 'guayaquil_guayas_ec': { - 'locname': ['Guayaquil', 'Guayas', 'Ecuador'], - 'loc': [-79, -2] - }, - 'banos_tungurahua_ec': { - 'locname': ['Banos', 'Tungurahua', 'Ecuador'], - 'loc': [-78, -1] - } -} - - -class SpatialTest(RealServerTestCase): - def setUp(self): - super(SpatialTest, self).setUp() - mgr = self.cb.bucket_manager() - ret = mgr.design_create('geo', DESIGN_JSON, use_devmode=False) - self.assertTrue(ret.success) - self.assertTrue(self.cb.upsert_multi(DOCS_JSON, format=FMT_JSON).all_ok) - - def test_simple_spatial(self): - spq = SpatialQuery() - - # Get all locations within five degress of the equator - spq.start_range = [None, -5] - spq.end_range = [None, 5] - rows_found = [r for r in self.cb.query('geo', 'simpleGeo', query=spq)] - self.assertEqual(2, len(rows_found)) - - # Get everything on the US west - spq.start_range = [-130, None] - spq.end_range = [-110, None] - rows_found = [r for r in self.cb.query('geo', 'simpleGeo', query=spq)] - self.assertEqual(2, len(rows_found)) - - # Sanity check: Ensure we actually did filtering earlier on! - spq = SpatialQuery() - rows_found = [r for r in self.cb.query('geo', 'simpleGeo', query=spq)] - self.assertTrue(len(rows_found) > 2) diff --git a/couchbase/tests/cases/stats_t.py b/couchbase/tests/cases/stats_t.py deleted file mode 100644 index 8fc81c7ea..000000000 --- a/couchbase/tests/cases/stats_t.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import ConnectionTestCase, RealServerTestCase - - -# For Python 2/3 compatibility -try: - basestring -except NameError: - basestring = str - - -class StatsTest(ConnectionTestCase): - - def test_trivial_stats_without_argument(self): - stats = self.cb.stats() - self.assertIsInstance(stats, dict) - self.assertTrue('curr_connections' in stats) - val = list(stats['curr_connections'].values())[0] - self.assertIsInstance(val, (float,int)) - key, info = list(stats.items())[0] - self.assertIsInstance(key, basestring) - self.assertIsInstance(info, dict) - - def test_stats_with_argument(self): - stats = self.cb.stats('memory') - self.assertIsInstance(stats, dict) - self.assertTrue('mem_used' in stats) - self.assertFalse('ep_tap_count' in stats) - key, info = list(stats.items())[0] - self.assertIsInstance(key, basestring) - self.assertIsInstance(info, dict) - - def test_stats_with_argument_list(self): - stats = self.cb.stats(['memory', 'tap']) - self.assertIsInstance(stats, dict) - self.assertTrue('mem_used' in stats) - self.assertTrue('ep_tap_count' in stats) - key, info = list(stats.items())[0] - self.assertIsInstance(key, basestring) - self.assertIsInstance(info, dict) - - -if __name__ == '__main__': - unittest.main() diff --git a/couchbase/tests/cases/touch_t.py b/couchbase/tests/cases/touch_t.py deleted file mode 100644 index 19bd03619..000000000 --- a/couchbase/tests/cases/touch_t.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import time - -from nose.plugins.attrib import attr - -from couchbase.tests.base import ConnectionTestCase -import couchbase.exceptions as E - - -@attr('slow') -class TouchTest(ConnectionTestCase): - def setUp(self): - super(TouchTest, self).setUp() - self.cb = self.make_connection() - - def test_trivial_touch(self): - key = self.gen_key("trivial_touch") - self.cb.upsert(key, "value", ttl=1) - rv = self.cb.touch(key, ttl=0) - self.assertTrue(rv.success) - time.sleep(2) - rv = self.cb.get(key) - self.assertTrue(rv.success) - self.assertEqual(rv.value, "value") - - self.cb.touch(key, ttl=1) - time.sleep(2) - rv = self.cb.get(key, quiet=True) - self.assertFalse(rv.success) - self.assertTrue(E.NotFoundError._can_derive(rv.rc)) - - def test_trivial_multi_touch(self): - kv = self.gen_kv_dict(prefix="trivial_multi_touch") - self.cb.upsert_multi(kv, ttl=1) - time.sleep(2) - rvs = self.cb.get_multi(kv.keys(), quiet=True) - self.assertFalse(rvs.all_ok) - - self.cb.upsert_multi(kv, ttl=1) - self.cb.touch_multi(kv.keys(), ttl=0) - rvs = self.cb.get_multi(kv.keys()) - self.assertTrue(rvs.all_ok) - - self.cb.touch_multi(kv.keys(), ttl=1) - time.sleep(2) - rvs = self.cb.get_multi(kv.keys(), quiet=True) - self.assertFalse(rvs.all_ok) - - def test_dict_touch_multi(self): - k_missing = self.gen_key("dict_touch_multi_missing") - k_existing = self.gen_key("dict_touch_multi_existing") - - self.cb.upsert_multi( - {k_missing : "missing_val", k_existing : "existing_val"}) - - self.cb.touch_multi({k_missing : 1, k_existing : 3}) - time.sleep(2) - rvs = self.cb.get_multi([k_missing, k_existing], quiet=True) - self.assertTrue(rvs[k_existing].success) - self.assertFalse(rvs[k_missing].success) - time.sleep(2) - rv = self.cb.get(k_existing, quiet=True) - self.assertFalse(rv.success) diff --git a/couchbase/tests/cases/transcoder_t.py b/couchbase/tests/cases/transcoder_t.py deleted file mode 100644 index 9a9d3b8c6..000000000 --- a/couchbase/tests/cases/transcoder_t.py +++ /dev/null @@ -1,225 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json - -from couchbase.tests.base import ConnectionTestCase -from couchbase.transcoder import TranscoderPP, Transcoder -from couchbase.bucket import Bucket -from couchbase import FMT_UTF8 -import couchbase.exceptions as E - -# This won't test every single permutation of the transcoder, but will check -# mainly to see if error messages are appropriate and how the application handles -# a misbehaving transcoder. The Transcoder API is fairly simple.. so - -def gen_func(fname): - def fn(self, *args): - if fname in self._op_next: - val = self._op_next[fname] - if hasattr(val, '__call__'): - return val() - return self._op_next[fname] - - return getattr(self._tc, fname)(*args) - return fn - -class MangledTranscoder(object): - """ - This is a custom transcoder class where we can optionally set a 'next_value' - field for a specific operation. If this field is empty, then the default - method is used - """ - def __init__(self): - self._tc = TranscoderPP() - self._op_next = {} - - def set_all(self, val): - for n in ('encode_key', 'encode_value', 'decode_key', 'decode_value'): - self._op_next[n] = val - - def set_next(self, ftype, val): - self._op_next[ftype] = val - - decode_key = gen_func('decode_key') - encode_key = gen_func('encode_key') - decode_value = gen_func('decode_value') - encode_value = gen_func('encode_value') - -class TranscoderTest(ConnectionTestCase): - - def test_simple_transcoder(self): - tc = TranscoderPP() - self.cb.transcoder = tc - - key = self.gen_key("simple_transcoder") - obj_values = ({}, [], -1, None, False, True) - for curval in obj_values: - self.cb.upsert(key, curval) - ret = self.cb.get(key) - self.assertEqual(ret.value, curval) - - - # Try to test some bad transcoders: - - def test_empty_transcoder(self): - for v in (None, False, 0): - self.cb.transcoder = v - self.cb.upsert("foo", "bar") - - def test_bad_transcoder(self): - self.cb.transcoder = None - - key = self.gen_key("bad_transcoder") - self.cb.upsert(key, "value") - self.cb.transcoder = object() - self.assertRaises(E.ValueFormatError, self.cb.upsert, key, "bar") - self.assertRaises(E.ValueFormatError, self.cb.get, key) - - - mangled = MangledTranscoder() - # Ensure we actually work - self.cb.transcoder = mangled - self.cb.upsert(key, "value") - self.cb.get(key) - - - for badret in (None, (), [], ""): - mangled.set_all(badret) - self.assertRaises(E.ValueFormatError, self.cb.upsert, key, "value") - self.assertRaises(E.ValueFormatError, self.cb.get, key) - - mangled._op_next.clear() - # Try with only bad keys: - mangled._op_next['encode_key'] = None - self.assertRaises(E.ValueFormatError, self.cb.upsert, key, "value") - - - def test_transcoder_bad_encvals(self): - mangled = MangledTranscoder() - self.cb.transcoder = mangled - - key = self.gen_key("transcoder_bad_encvals") - - # Various tests for 'bad_value': - encrets = ( - - # None - None, - - # Valid string, but not inside tuple - b"string", - - # Tuple, but invalid contents - (None, None), - - # Tuple, valid string, but invalid size (no length) - (b"valid string"), - - # Tuple, valid flags but invalid string - (None, 0xf00), - - # Valid tuple, but flags are too big - (b"string", 2**40), - - # Tuple, but bad leading string - ([], 42) - ) - - for encret in encrets: - print(encret) - mangled._op_next['encode_value'] = encret - self.assertRaises(E.ValueFormatError, self.cb.upsert, key, "value") - - def test_transcoder_kdec_err(self): - key = self.gen_key("transcoder_kenc_err") - mangled = MangledTranscoder() - self.cb.transcoder = mangled - key = self.gen_key('kdec_err') - self.cb.upsert(key, 'blah', format=FMT_UTF8) - def exthrow(): - raise UnicodeDecodeError() - - mangled.set_next('decode_value', exthrow) - self.assertRaises(E.ValueFormatError, self.cb.get, key) - - - - def test_transcoder_anyobject(self): - # This tests the versatility of the transcoder object - key = self.gen_key("transcoder_anyobject") - mangled = MangledTranscoder() - self.cb.transcoder = mangled - - mangled._op_next['encode_key'] = key.encode("utf-8") - mangled._op_next['encode_value'] = (b"simple_value", 10) - - objs = (object(), None, MangledTranscoder(), True, False) - for o in objs: - mangled._op_next['decode_key'] = o - mangled._op_next['decode_value'] = o - self.cb.upsert(o, o, format=o) - rv = self.cb.get(o) - self.assertEqual(rv.value, o) - - - def test_transcoder_unhashable_keys(self): - key = self.gen_key("transcoder_unhashable_keys") - mangled = MangledTranscoder() - mangled._op_next['encode_key'] = key.encode("utf-8") - mangled._op_next['encode_value'] = (b"simple_value", 10) - self.cb.transcoder = mangled - - # As MultiResult objects must be able to store its keys in a dictionary - # we cannot allow unhashable types. These are such examples - unhashable = ({}, [], set()) - for o in unhashable: - mangled._op_next['decode_key'] = o - mangled._op_next['decode_value'] = o - self.assertRaises(E.ValueFormatError, self.cb.upsert, o, o) - self.assertRaises(E.ValueFormatError, self.cb.get, o, quiet=True) - - def test_transcoder_class(self): - # Test whether we can pass a class for a transcoder - key = self.gen_key("transcoder_class") - c = Bucket(**self.make_connargs(transcoder=TranscoderPP)) - c.upsert(key, "value") - - c = Bucket(**self.make_connargs(transcoder=TranscoderPP)) - c.upsert(key, "value") - - def test_mask_sanity(self): - from couchbase import FMT_COMMON_MASK, FMT_LEGACY_MASK - self.assertEqual(FMT_COMMON_MASK, 0xFF000000) - self.assertEqual(FMT_LEGACY_MASK, 0x00000007) - - def test_pycbc295(self): - # Test that we ignore the legacy flags and use the common flags - # instead - custom_tc = MangledTranscoder() - orig_tc = Transcoder() - - c = self.make_connection() - c.transcoder = custom_tc - custom_tc._op_next['encode_value'] = ( - json.dumps({'Hello': 'World'}), - 0x02000001 - ) - key = self.gen_key('pycbc295') - c.upsert(key, 'whatevs') - c.transcoder = orig_tc - rv = c.get(key) - self.assertIsInstance(rv.value, (dict,)) \ No newline at end of file diff --git a/couchbase/tests/cases/verinfo_t.py b/couchbase/tests/cases/verinfo_t.py deleted file mode 100644 index a9e752313..000000000 --- a/couchbase/tests/cases/verinfo_t.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest -from couchbase.tests.base import SkipTest - - -try: - from couchbase_version import VersionInfo -except ImportError: - VersionInfo = None - - -class VersionInfoTest(unittest.TestCase): - def setUp(self): - super(VersionInfoTest, self).setUp() - if VersionInfo is None: - raise SkipTest("Don't have couchbase_version") - - def test_version_parse(self): - info = VersionInfo('2.0.1-99-gblahblah') - self.assertEqual('2.0.1.dev99+gblahblah', info.package_version) - - info = VersionInfo('2.0.0-dp3-34-gff') - self.assertEqual('2.0.0a3.dev34+gff', info.package_version) - - info = VersionInfo('2.0.0-0-gdeadbeef') - self.assertEqual('2.0.0', info.package_version) - - info = VersionInfo('2.0.0-beta-9-gff') - self.assertEqual('2.0.0b0.dev9+gff', info.package_version) - - info = VersionInfo('2.0.0-beta3-0-gff') - self.assertEqual('2.0.0b3', info.package_version) - - -if __name__ == '__main__': - import unittest - unittest.main() \ No newline at end of file diff --git a/couchbase/tests/cases/view_iterator_t.py b/couchbase/tests/cases/view_iterator_t.py deleted file mode 100644 index 844da6976..000000000 --- a/couchbase/tests/cases/view_iterator_t.py +++ /dev/null @@ -1,381 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import ViewTestCase, SkipTest -from couchbase.views.iterator import ( - View, ViewRow, RowProcessor, AlreadyQueriedError, MAX_URI_LENGTH) - -from couchbase.views.params import Query, UNSPEC -from couchbase.exceptions import CouchbaseError -from couchbase.result import Result -from couchbase.exceptions import ArgumentError, CouchbaseError, HTTPError -from couchbase._pyport import xrange - - -# We'll be using the beer-sample database as it has a sufficiently large -# dataset with well-defined return values - -class Brewery(object): - __ALL_BREWERIES = {} - __BY_ID = {} - - def __init__(self, id, doc): - self._id = id - self.name = doc['name'] - self.city = doc['city'] - self.code = doc['code'] - self.country = doc['country'] - self.phone = doc['phone'] - self.website = doc['website'] - self.type = doc['type'] - self.updated = doc['updated'] - self.description = doc['description'] - self.address = doc['address'] - - self.__ALL_BREWERIES[self.name] = self - self.__BY_ID[self._id] = self - - @classmethod - def get_brewery(cls, name): - return cls.__ALL_BREWERIES.get(name) - - @classmethod - def by_id(cls, name): - return cls.__BY_ID.get(name) - -class Beer(object): - __ALL_BEERS = {} - __BY_ID = {} - - @classmethod - def get_beer(cls, name): - return cls.__ALL_BEERS.get(name) - - @classmethod - def by_id(cls, name): - return cls.__BY_ID.get(name) - - def __init__(self, id, doc): - self._id = id - self.name = doc['name'] - self.brewery = Brewery.by_id(doc['brewery_id']) - self.abv = doc['abv'] - self.updated = doc['updated'] - self.description = doc['description'] - self.category = doc['category'] - self.style = doc['style'] - - self.__ALL_BEERS[self.name] = self - self.__BY_ID[self._id] = self - - -class BreweryBeerRowProcessor(object): - """ - This specialized processor will attempt to fetch the name of the beers - and breweries which it gets, trying to ensure maximum efficiency. - - This only returns beers, skipping over any breweries. - """ - def __init__(self): - # Iterates over names of beers. We get them via 'get_beer'. - self._riter = None - - def handle_rows(self, rows, connection, include_docs): - """ - This shows an example of an efficient 'include_docs' algorithm - which fetches the beers and relevant breweries in a single sweep, - skipping over those that are already cached locally. - """ - - breweries_to_fetch = set() - beers_to_fetch = set() - - # The order of the keys returned in the result set. - retkeys = [] - pre_included = {} - - for r in rows: - if len(r['key']) == 1: - # It's a brewery - continue - - brewery_id, beer_id = r['key'] - retkeys.append(beer_id) - - if not Brewery.by_id(brewery_id): - breweries_to_fetch.add(brewery_id) - - if not Beer.by_id(beer_id): - beers_to_fetch.add(beer_id) - - if r.get('__DOCRESULT__'): - pre_included[r['id']] = r['__DOCRESULT__'] - - self._riter = iter(retkeys) - - if beers_to_fetch or breweries_to_fetch: - if not include_docs and not pre_included: - raise ValueError( - "Don't have all documents, but include_docs was set to False") - - keys_to_fetch = list(breweries_to_fetch) + list(beers_to_fetch) - keys_to_fetch = [x for x in keys_to_fetch if x not in pre_included] - docs = {} - docs.update(pre_included) - - if keys_to_fetch: - docs.update(connection.get_multi(keys_to_fetch)) - - for brewery in breweries_to_fetch: - Brewery(brewery, docs[brewery].value) - - for beer in beers_to_fetch: - Beer(beer, docs[beer].value) - - return iter(self) - - def __iter__(self): - if not self._riter: - return - - for b in self._riter: - beer = Beer.by_id(b) - assert beer, "Eh?" - - yield beer - - self._riter = None - - -class ViewIteratorTest(ViewTestCase): - def setUp(self): - super(ViewIteratorTest, self).setUp() - self.skipIfMock() - - def make_connection(self): - try: - return super(ViewIteratorTest, - self).make_connection(bucket='beer-sample') - except CouchbaseError: - raise SkipTest("Need 'beer-sample' bucket for this") - - def test_simple_query(self): - ret = self.cb.query("beer", "brewery_beers", limit=3) - self.assertIsInstance(ret, View) - self.assertIsInstance(ret.row_processor, RowProcessor) - - count = 0 - rows = list(ret) - self.assertEqual(len(rows), 3) - for r in rows: - self.assertIsInstance(r, ViewRow) - - def test_include_docs(self): - ret = self.cb.query("beer", "brewery_beers", limit=10, - include_docs=True) - rows = list(ret) - self.assertEqual(len(rows), 10) - for r in rows: - self.assertIsInstance(r.doc, self.cls_Result) - doc = r.doc - mc_doc = self.cb.get(r.docid, quiet=True) - self.assertEqual(doc.cas, mc_doc.cas) - self.assertEqual(doc.value, mc_doc.value) - self.assertTrue(doc.success) - - # Try with reduce - self.assertRaises(ArgumentError, - self.cb.query, - "beer", "by_location", - reduce=True, - include_docs=True) - - def test_bad_view(self): - ret = self.cb.query("beer", "bad_view") - self.assertIsInstance(ret, View) - self.assertRaises(HTTPError, - tuple, ret) - - def test_streaming(self): - ret = self.cb.query("beer", "brewery_beers", streaming=True, limit=100) - rows = list(ret) - self.assertEqual(len(rows), 100) - - # Get all the views - ret = self.cb.query("beer", "brewery_beers", streaming=True) - rows = list(ret) - self.assertTrue(len(rows)) - self.assertEqual(len(rows), ret.indexed_rows) - - self.assertTrue(ret.raw.value) - self.assertIsInstance(ret.raw.value, dict) - self.assertTrue('total_rows' in ret.raw.value) - - def test_streaming_dtor(self): - # Ensure that the internal lcb_http_request_t is destroyed if the - # Python object is destroyed before the results are done. - - ret = self.cb.query("beer", "brewery_beers", streaming=True) - v = iter(ret) - try: - v.next() - except AttributeError: - v.__next__() - - del ret - - def test_mixed_query(self): - self.assertRaises(ArgumentError, - self.cb.query, - "d", "v", - query=Query(), - limit=10) - - self.cb.query("d","v", query=Query(limit=5).update(skip=15)) - - def test_range_query(self): - q = Query() - - q.mapkey_range = [ - ["abbaye_de_maredsous"], - ["abbaye_de_maredsous", Query.STRING_RANGE_END] - ] - - q.inclusive_end = True - - ret = self.cb.query("beer", "brewery_beers", query=q) - rows = list(ret) - self.assertEqual(len(rows), 4) - - q.mapkey_range = [ ["u"], ["v"] ] - ret = self.cb.query("beer", "brewery_beers", query=q) - self.assertEqual(len(list(ret)), 88) - - q.mapkey_range = [ ["u"], ["uppper"+Query.STRING_RANGE_END]] - ret = self.cb.query("beer", "brewery_beers", query=q) - rows = list(ret) - self.assertEqual(len(rows), 56) - - def test_key_query(self): - q = Query() - q.mapkey_single = ["abbaye_de_maredsous"] - ret = self.cb.query("beer", "brewery_beers", query=q) - rows = list(ret) - self.assertEqual(len(rows), 1) - - q.mapkey_single = UNSPEC - q.mapkey_multi = [["abbaye_de_maredsous"], - ["abbaye_de_maredsous", "abbaye_de_maredsous-8"]] - ret = self.cb.query("beer", "brewery_beers", query=q) - rows = list(ret) - self.assertEqual(len(rows), 2) - - def test_row_processor(self): - rp = BreweryBeerRowProcessor() - q = Query(limit=20) - - ret = self.cb.query("beer", "brewery_beers", - query=q, - row_processor=rp, - include_docs=True) - - beers = list(ret) - for b in beers: - self.assertIsInstance(b, Beer) - self.assertIsInstance(b.brewery, Brewery) - - ret = self.cb.query("beer", "brewery_beers", - query=q, - row_processor=rp, - include_docs=False) - - list(ret) - - ret = self.cb.query("beer", "brewery_beers", - row_processor=rp, - include_docs=False, - limit=40) - - self.assertRaises(ValueError, list, ret) - - def test_already_queried(self): - ret = self.cb.query("beer", "brewery_beers", limit=5) - list(ret) - self.assertRaises(AlreadyQueriedError, list, ret) - - def test_no_rows(self): - ret = self.cb.query("beer", "brewery_beers", limit=0) - for row in ret: - raise Exception("...") - - - def test_long_uri(self): - qobj = Query() - qobj.mapkey_multi = [ str(x) for x in xrange(MAX_URI_LENGTH) ] - ret = self.cb.query("beer", "brewery_beers", query=qobj) - # No assertions, just make sure it didn't break - for row in ret: - raise Exception("...") - - # Apparently only the "keys" parameter is supposed to be in POST. - # Let's fetch 100 items now - keys = [r.key for r in self.cb.query("beer", "brewery_beers", limit=100)] - self.assertEqual(100, len(keys)) - - kslice = keys[90:] - self.assertEqual(10, len(kslice)) - rows = [x for x in self.cb.query("beer", "brewery_beers", mapkey_multi=kslice, limit=5)] - self.assertEqual(5, len(rows)) - for row in rows: - self.assertTrue(row.key in kslice) - - - def _verify_data(self, ret): - list(ret) - data = ret.raw.value - self.assertTrue('rows' in data) - self.assertTrue('total_rows' in data) - self.assertTrue('debug_info' in data) - - - def test_http_data(self): - q = Query(limit=30, debug=True) - self._verify_data(self.cb.query("beer", "brewery_beers", streaming=False, - query=q)) - - def test_http_data_streaming(self): - q = Query(limit=30, debug=True) - self._verify_data(self.cb.query("beer", "brewery_beers", streaming=True, - query=q)) - - def test_pycbc_206(self): - # Set up the view.. - mgr = self.cb.bucket_manager() - design = mgr.design_get('beer', use_devmode=False).value - if not 'with_value' in design['views']: - - design['views']['with_value'] = { - 'map': 'function(doc,meta) { emit(meta.id,doc.name); }' - } - - ret = mgr.design_create('beer', design, use_devmode=0) - self.assertTrue(ret.success) - - # Streaming with values - view = self.cb.query("beer", "with_value", streaming=True) - rows = list(view) - self.assertTrue(len(rows)) diff --git a/couchbase/tests/cases/view_t.py b/couchbase/tests/cases/view_t.py deleted file mode 100644 index 5fc16c341..000000000 --- a/couchbase/tests/cases/view_t.py +++ /dev/null @@ -1,125 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import json - -from couchbase.tests.base import ViewTestCase -from couchbase.user_constants import FMT_JSON -from couchbase.exceptions import HTTPError - -DESIGN_JSON = { - 'language' : 'javascript', - 'views' : { - 'recent_posts' : { - 'map' : - """ - function(doc) { - if (doc.date && doc.title) { - emit(doc.date, doc.title); - } - } - """.replace("\n", '') - } - } -} - -DOCS_JSON = { - "bought-a-cat" : { - "title" : "Bought a Cat", - "body" : "I went to the pet store earlier and brought home a " - "little kitty", - "date" : "2009/01/30 18:04:11" - }, - "biking" : { - "title" : "Biking", - "body" : "My biggest hobby is mountainbiking. The other day..", - "date" : "2009/01/30 18:04:11" - }, - "hello-world" : { - "title" : "Hello World", - "body" : "Well hello and welcome to my new blog", - "date" : "2009/01/15 15:52:20" - } -} - -class ViewTest(ViewTestCase): - def setUp(self): - super(ViewTest, self).setUp() - self.skipIfMock() - mgr = self.cb.bucket_manager() - ret = mgr.design_create('blog', DESIGN_JSON, use_devmode=False) - self.assertTrue(ret.success) - self.assertTrue(self.cb.upsert_multi(DOCS_JSON, format=FMT_JSON).all_ok) - - def test_simple_view(self): - ret = self.cb._view("blog", "recent_posts", - params={ 'stale' : 'false' }) - self.assertTrue(ret.success) - rows = ret.value - self.assertIsInstance(rows, dict) - print(rows) - self.assertTrue(rows['total_rows'] >= 3) - self.assertTrue(len(rows['rows']) == rows['total_rows']) - - def test_with_params(self): - ret = self.cb._view("blog", "recent_posts", - params={'limit':1}) - self.assertTrue(ret.success) - rows = ret.value['rows'] - self.assertEqual(len(rows), 1) - - def test_with_strparam(self): - ret = self.cb._view("blog", "recent_posts", params='limit=2') - self.assertTrue(ret.success) - self.assertEqual(len(ret.value['rows']), 2) - - def test_with_jparams(self): - jkey_pure = '2009/01/15 15:52:20' - - ret = self.cb._view("blog", "recent_posts", - params={ - 'startkey' : jkey_pure, - 'endkey' : jkey_pure, - 'inclusive_end' : 'true' - }) - print(ret) - self.assertTrue(ret.success) - rows = ret.value['rows'] - self.assertTrue(len(rows) == 1) - single_row = rows[0] - self.assertEqual(single_row['id'], 'hello-world') - self.assertEqual(single_row['key'], jkey_pure) - - - jkey_pure = [] - for v in DOCS_JSON.values(): - curdate = v['date'] - jkey_pure.append(curdate) - - ret = self.cb._view("blog", "recent_posts", - params={ - 'keys' : jkey_pure - }) - self.assertTrue(ret.success) - self.assertTrue(len(ret.value['rows']), 3) - for row in ret.value['rows']: - self.assertTrue(row['id'] in DOCS_JSON) - self.assertTrue(row['key'] in jkey_pure) - - def test_missing_view(self): - self.assertRaises(HTTPError, - self.cb._view, - "nonexist", "designdoc") diff --git a/couchbase/tests/cases/viewstrings_t.py b/couchbase/tests/cases/viewstrings_t.py deleted file mode 100644 index 6935a7395..000000000 --- a/couchbase/tests/cases/viewstrings_t.py +++ /dev/null @@ -1,279 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# These tests are largely ported from php-ext-couchbase -import json - -from couchbase.views.params import (make_options_string, - Query, - ulp, - UNSPEC, - _HANDLER_MAP) - -from couchbase.exceptions import ArgumentError -from couchbase.tests.base import CouchbaseTestCase - -class ViewStringTest(CouchbaseTestCase): - def setUp(self): - super(ViewStringTest, self).setUp() - - def _assert_vopteq(self, expected, key, value): - s = make_options_string({key:value}) - self.assertEqual(s, expected) - - def _assert_vopteq_multi(self, d, key, value): - q = Query(**{key:value}) - enc = q.encoded - res = {} - for kvp in enc.split("&"): - k, v = kvp.split("=") - res[k] = v - - d = d.copy() - for k in d: - d[k] = ulp.quote(d[k]) - - self.assertEqual(res, d) - - def test_stale_params(self): - self._assert_vopteq('stale=ok', 'stale', True) - self._assert_vopteq('stale=false', 'stale', False) - self._assert_vopteq('stale=update_after', "stale", "update_after") - self._assert_vopteq('stale=ok', 'stale', 1) - self._assert_vopteq('stale=false', 'stale', 0) - self._assert_vopteq('stale=false', 'stale', "false") - - - def test_bad_stale(self): - self.assertRaises(ArgumentError, - self._assert_vopteq, - 'stale=blahblah', 'stale', 'blahblha') - self.assertRaises(ArgumentError, - self._assert_vopteq, - 'stale=None', 'stale', None) - - - def test_unrecognized_params(self): - self.assertRaises(ArgumentError, - self._assert_vopteq, - 'frobble=gobble', 'frobble', 'gobble') - - def test_misc_booleans(self): - bparams = ('descending', - 'reduce', - 'inclusive_end', - 'full_set', - 'group') - - for p in bparams: - # with string "false" - self._assert_vopteq(p+"=false", - p, - "false") - - # with string "true" - self._assert_vopteq(p+"=true", - p, - "true") - - self.assertRaises(ArgumentError, - self._assert_vopteq, - p+'=gobble', p, 'gobble') - - self.assertRaises(ArgumentError, - self._assert_vopteq, - p+'=None', p, None) - - def test_misc_numeric(self): - nparams = ( - 'connection_timeout', - 'group_level', - 'skip') - - for p in nparams: - self._assert_vopteq(p+'=42', - p, - 42) - - self._assert_vopteq(p+'=42', - p, - "42") - - self.assertRaises(ArgumentError, - self._assert_vopteq, - p+'=true', p, True) - - self.assertRaises(ArgumentError, - self._assert_vopteq, - p+'=blah', p, 'blah') - - self._assert_vopteq(p+'=0', p, 0) - self._assert_vopteq(p+'=0', p, "0") - self._assert_vopteq(p+'=-1', p, -1) - - def test_encode_string_to_json(self): - jparams = ( - 'endkey', - 'key', - 'startkey') - - values = ( - 'dummy', - 42, - None, - True, - False, - { "chicken" : "broth" }, - ["noodle", "soup"], - ["lone element"], - ("empty tuple",) - ) - - for p in jparams: - for v in values: - expected = p + '=' + ulp.quote(json.dumps(v)) - print("Expected", expected) - self._assert_vopteq(expected, p, v) - - self.assertRaises(ArgumentError, - self._assert_vopteq, - "blah", p, object()) - - - def test_encode_to_jarray(self): - jparams = ('keys',) #add more here - values = ( - ['foo', 'bar'], - ['foo']) - - badvalues = (True, - False, - {"foo":"bar"}, - 1, - "string") - - for p in jparams: - for v in values: - - print(v) - expected = p + '=' + ulp.quote(json.dumps(v)) - self._assert_vopteq(expected, p, v) - - for v in badvalues: - self.assertRaises(ArgumentError, - self._assert_vopteq, - "blah", p, v) - - - def test_passthrough(self): - values = ( - "blah", - -1, - "-invalid/uri&char") - - for p in _HANDLER_MAP.keys(): - for v in values: - expected = "{0}={1}".format(p, v) - got = make_options_string({p:v}, passthrough=True) - self.assertEqual(expected, got) - - - # Ensure we still can't use unrecognized params - self.assertRaises(ArgumentError, - make_options_string, - {'foo':'bar'}, - passthrough=True) - - - # ensure we still can't use "stupid" params - badvals = (object(), None, True, False) - for bv in badvals: - self.assertRaises(ArgumentError, - make_options_string, - {'stale':bv}, - passthrough=True) - - - def test_unrecognized(self): - keys = ("new_param", "another_param") - values = ("blah", -1, "-invalid-uri-char^&") - for p in keys: - for v in values: - got = make_options_string({p:v}, - unrecognized_ok=True) - expected = "{0}={1}".format(p, v) - self.assertEqual(expected, got) - - - badvals = (object(), True, False, None) - for bv in badvals: - self.assertRaises(ArgumentError, - make_options_string, - {'foo':bv}, - unrecognized_ok=True) - - def test_string_params(self): - # This test is mainly to see that 'stupid' things don't make - # their way through as strings, like booleans and None - sparams = ('endkey_docid', - 'startkey_docid') - - goodvals = ("string", -1, "OHAI!", '&&escape_me_nao&&') - badvals = (True, False, None, object(), []) - - for p in sparams: - for v in goodvals: - expected = "{0}={1}".format(p, ulp.quote(str(v))) - self._assert_vopteq(expected, p, v) - - for v in badvals: - self.assertRaises(ArgumentError, - make_options_string, - {p:v}) - - - def test_ranges(self): - expected = "startkey={0}".format(ulp.quote(json.dumps("foo"))) - self._assert_vopteq(expected, "mapkey_range", ["foo"]) - self._assert_vopteq_multi( - {'startkey' : json.dumps("foo"), - 'endkey' : json.dumps("bar") }, - "mapkey_range", - ["foo", "bar"]) - - - expected = "startkey_docid=bar" - self._assert_vopteq(expected, "dockey_range", ["bar"]) - self._assert_vopteq_multi( - {'startkey_docid' : "range_begin", - 'endkey_docid' : "range_end"}, - "dockey_range", - ["range_begin", "range_end"]) - - for p in ('mapkey_range', 'dockey_range'): - self._assert_vopteq('', p, []) - self._assert_vopteq('', p, UNSPEC) - self._assert_vopteq('', p, [UNSPEC,UNSPEC]) - self._assert_vopteq('', p, [UNSPEC]) - - self.assertRaises(ArgumentError, - self._assert_vopteq, - "blah", p, [object()]) - - self.assertRaises(ArgumentError, - self._assert_vopteq, - "blah", p, None) diff --git a/couchbase/tests/cluster_t.py b/couchbase/tests/cluster_t.py new file mode 100644 index 000000000..10f661464 --- /dev/null +++ b/couchbase/tests/cluster_t.py @@ -0,0 +1,249 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta +from uuid import uuid4 + +import pytest + +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.diagnostics import (ClusterState, + EndpointPingReport, + EndpointState, + PingState, + ServiceType) +from couchbase.exceptions import (InternalServerFailureException, + InvalidArgumentException, + ParsingFailedException, + QueryIndexNotFoundException) +from couchbase.options import (ClusterOptions, + DiagnosticsOptions, + PingOptions) +from couchbase.result import DiagnosticsResult, PingResult +from tests.environments import CollectionType +from tests.test_features import EnvironmentFeatures + + +class ClusterDiagnosticsTestSuite: + TEST_MANIFEST = [ + 'test_diagnostics', + 'test_diagnostics_after_query', + 'test_diagnostics_as_json', + 'test_multiple_close_cluster', + 'test_ping', + 'test_ping_as_json', + 'test_ping_invalid_services', + 'test_ping_mixed_services', + 'test_ping_report_id', + 'test_ping_restrict_services', + 'test_ping_str_services', + ] + + @pytest.fixture(scope="class") + def check_diagnostics_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('diagnostics', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.mark.usefixtures('check_diagnostics_supported') + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_diagnostics(self, cb_env): + cluster = cb_env.cluster + report_id = str(uuid4()) + result = cluster.diagnostics( + DiagnosticsOptions(report_id=report_id)) + assert isinstance(result, DiagnosticsResult) + assert result.id == report_id + assert result.sdk is not None + assert result.version is not None + assert result.state != ClusterState.Offline + + kv_endpoints = result.endpoints[ServiceType.KeyValue] + assert len(kv_endpoints) > 0 + assert kv_endpoints[0].id is not None + assert kv_endpoints[0].local is not None + assert kv_endpoints[0].remote is not None + assert kv_endpoints[0].last_activity_us is not None + assert kv_endpoints[0].state == EndpointState.Connected + assert kv_endpoints[0].service_type == ServiceType.KeyValue + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_diagnostics_after_query(self, cb_env): + cluster = cb_env.cluster + # lets make sure there is at least 1 row + key, value = cb_env.get_new_doc() + cb_env.collection.upsert(key, value) + bucket_name = cb_env.bucket.name + report_id = str(uuid4()) + # query should fail, but diagnostics should + # still return a query service type + try: + rows = cluster.query(f'SELECT * FROM `{bucket_name}` LIMIT 1').execute() + assert len(rows) > 0 + except (InternalServerFailureException, ParsingFailedException, QueryIndexNotFoundException): + pass + except Exception as e: + print(f'exception: {e.__class__.__name__}, {e}') + raise e + + result = cluster.diagnostics( + DiagnosticsOptions(report_id=report_id)) + assert result.id == report_id + + query_diag_result = result.endpoints[ServiceType.Query] + assert len(query_diag_result) >= 1 + for q in query_diag_result: + assert q.id is not None + assert q.local is not None + assert q.remote is not None + assert isinstance(q.last_activity_us, timedelta) + assert q.state == EndpointState.Connected + assert q.service_type == ServiceType.Query + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_diagnostics_as_json(self, cb_env): + cluster = cb_env.cluster + report_id = str(uuid4()) + result = cluster.diagnostics( + DiagnosticsOptions(report_id=report_id)) + + assert isinstance(result, DiagnosticsResult) + result_str = result.as_json() + assert isinstance(result_str, str) + result_json = json.loads(result_str) + assert result_json['version'] is not None + assert result_json['id'] is not None + assert result_json['sdk'] is not None + assert result_json['services'] is not None + for _, data in result_json['services'].items(): + if len(data): + assert data[0]['id'] is not None + assert data[0]['last_activity_us'] is not None + assert data[0]['remote'] is not None + assert data[0]['local'] is not None + assert data[0]['state'] is not None + + # TODO: really we could use a separate test class to test things like opening/closing buckets, cluster, etc... + # i.e. things that are not query, diagnostics, etc... For now, lets just have this here + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_multiple_close_cluster(self, cb_env): + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + opts = ClusterOptions(auth) + cluster = Cluster.connect(conn_string, opts) + for _ in range(10): + cluster.close() + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping(self, cb_env): + cluster = cb_env.cluster + result = cluster.ping() + assert isinstance(result, PingResult) + + assert result.sdk is not None + assert result.id is not None + assert result.version is not None + assert result.endpoints is not None + for ping_reports in result.endpoints.values(): + for report in ping_reports: + assert isinstance(report, EndpointPingReport) + if report.state == PingState.OK: + assert report.id is not None + assert report.latency is not None + assert report.remote is not None + assert report.local is not None + assert report.service_type is not None + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_as_json(self, cb_env): + cluster = cb_env.cluster + result = cluster.ping() + assert isinstance(result, PingResult) + result_str = result.as_json() + assert isinstance(result_str, str) + result_json = json.loads(result_str) + assert result_json['version'] is not None + assert result_json['id'] is not None + assert result_json['sdk'] is not None + assert result_json['services'] is not None + for _, data in result_json['services'].items(): + if len(data): + assert data[0]['id'] is not None + assert data[0]['latency_us'] is not None + assert data[0]['remote'] is not None + assert data[0]['local'] is not None + assert data[0]['state'] is not None + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_invalid_services(self, cb_env): + cluster = cb_env.cluster + with pytest.raises(InvalidArgumentException): + cluster.ping(PingOptions(service_types=ServiceType.KeyValue)) + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_mixed_services(self, cb_env): + cluster = cb_env.cluster + services = [ServiceType.KeyValue, ServiceType.Query.value] + result = cluster.ping(PingOptions(service_types=services)) + assert len(result.endpoints) >= 1 + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_report_id(self, cb_env): + cluster = cb_env.cluster + report_id = uuid4() + result = cluster.ping(PingOptions(report_id=report_id)) + assert str(report_id) == result.id + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_restrict_services(self, cb_env): + cluster = cb_env.cluster + services = [ServiceType.KeyValue] + result = cluster.ping(PingOptions(service_types=services)) + keys = list(result.endpoints.keys()) + assert len(keys) == 1 + assert keys[0] == ServiceType.KeyValue + + @pytest.mark.usefixtures('check_diagnostics_supported') + def test_ping_str_services(self, cb_env): + cluster = cb_env.cluster + services = [ServiceType.KeyValue.value, ServiceType.Query.value] + result = cluster.ping(PingOptions(service_types=services)) + assert len(result.endpoints) >= 1 + + +class ClassicClusterDiagnosticsTests(ClusterDiagnosticsTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicClusterDiagnosticsTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicClusterDiagnosticsTests) if valid_test_method(meth)] + compare = set(ClusterDiagnosticsTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.setup(request.param) + yield cb_base_env + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/collection_multi_t.py b/couchbase/tests/collection_multi_t.py new file mode 100644 index 000000000..4e7f950c3 --- /dev/null +++ b/couchbase/tests/collection_multi_t.py @@ -0,0 +1,606 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +from couchbase.diagnostics import ServiceType +from couchbase.exceptions import (CouchbaseException, + DocumentExistsException, + DocumentNotFoundException, + DocumentUnretrievableException, + InvalidArgumentException) +from couchbase.options import (GetAllReplicasMultiOptions, + GetAnyReplicaMultiOptions, + GetMultiOptions, + InsertMultiOptions, + InsertOptions, + ReplaceMultiOptions, + TouchMultiOptions, + UpsertMultiOptions, + UpsertOptions) +from couchbase.replica_reads import ReadPreference +from couchbase.result import (ExistsResult, + GetReplicaResult, + GetResult, + MultiExistsResult, + MultiGetReplicaResult, + MultiGetResult, + MultiMutationResult, + MutationResult) +from tests.environments import CollectionType +from tests.environments.collection_multi_environment import CollectionMultiTestEnvironment +from tests.environments.test_environment import TestEnvironment +from tests.mock_server import MockServerType +from tests.test_features import EnvironmentFeatures + + +class CollectionMultiTestSuite: + + TEST_MANIFEST = [ + 'test_multi_exists_invalid_input', + 'test_multi_exists_not_exist', + 'test_multi_exists_simple', + 'test_multi_get_all_replicas_fail', + 'test_multi_get_all_replicas_invalid_input', + 'test_multi_get_all_replicas_simple', + 'test_multi_get_all_replicas_read_preference', + 'test_multi_get_any_replica_fail', + 'test_multi_get_any_replica_invalid_input', + 'test_multi_get_any_replica_simple', + 'test_multi_get_any_replica_read_preference', + 'test_multi_get_fail', + 'test_multi_get_invalid_input', + 'test_multi_get_simple', + 'test_multi_insert_fail', + 'test_multi_insert_global_opts', + 'test_multi_insert_invalid_input', + 'test_multi_insert_key_opts', + 'test_multi_insert_simple', + 'test_multi_lock_and_unlock_simple', + 'test_multi_lock_invalid_input', + 'test_multi_remove_fail', + 'test_multi_remove_invalid_input', + 'test_multi_remove_simple', + 'test_multi_replace_fail', + 'test_multi_replace_global_opts', + 'test_multi_replace_invalid_input', + 'test_multi_replace_key_opts', + 'test_multi_replace_simple', + 'test_multi_touch_fail', + 'test_multi_touch_invalid_input', + 'test_multi_touch_simple', + 'test_multi_unlock_invalid_input', + 'test_multi_upsert_global_opts', + 'test_multi_upsert_invalid_input', + 'test_multi_upsert_key_opts', + 'test_multi_upsert_simple', + ] + + @pytest.fixture(scope='class') + def check_multi_node(self, num_nodes): + if num_nodes == 1: + pytest.skip('Test only for clusters with more than a single node.') + + @pytest.fixture(scope='class') + def check_replicas(self, cb_env): + if cb_env.is_mock_server and cb_env.mock_server_type == MockServerType.GoCAVES: + pytest.skip('GoCaves inconstent w/ replicas') + bucket_settings = TestEnvironment.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get('num_replicas') + ping_res = cb_env.bucket.ping() + kv_endpoints = ping_res.endpoints.get(ServiceType.KeyValue, None) + if kv_endpoints is None or len(kv_endpoints) < (num_replicas + 1): + pytest.skip('Not all replicas are online') + + @pytest.fixture(scope='class') + def check_server_groups_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('server_groups', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.fixture(scope='class') + def num_nodes(self, cb_env): + return len(cb_env.cluster._cluster_info.nodes) + + def test_multi_exists_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.exists_multi(keys) + assert isinstance(res, MultiExistsResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, ExistsResult), res.results.values())) is True + for r in res.results.values(): + assert r.exists is True + + def test_multi_exists_not_exist(self, cb_env): + keys_and_docs = cb_env.FAKE_DOCS + keys = list(keys_and_docs.keys()) + res = cb_env.collection.exists_multi(keys) + assert isinstance(res, MultiExistsResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, ExistsResult), res.results.values())) is True + for r in res.results.values(): + assert r.exists is False + + def test_multi_exists_invalid_input(self, cb_env): + keys_and_docs = { + 'test-key1': {'what': 'a test doc!', 'id': 'test-key1'}, + 'test-key2': {'what': 'a test doc!', 'id': 'test-key2'}, + 'test-key3': {'what': 'a test doc!', 'id': 'test-key3'}, + 'test-key4': {'what': 'a test doc!', 'id': 'test-key4'} + } + with pytest.raises(InvalidArgumentException): + cb_env.collection.exists_multi(keys_and_docs) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + def test_multi_get_all_replicas_fail(self, cb_env): + keys_and_docs = cb_env.FAKE_DOCS + keys = list(keys_and_docs.keys()) + res = cb_env.collection.get_all_replicas_multi(keys) + assert isinstance(res, MultiGetReplicaResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), CouchbaseException), res.exceptions.values())) is True + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get_all_replicas_multi(keys, return_exceptions=False) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get_all_replicas_multi(keys, GetAnyReplicaMultiOptions(return_exceptions=False)) + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + def test_multi_get_all_replicas_invalid_input(self, cb_env): + keys_and_docs = { + 'test-key1': {'what': 'a test doc!', 'id': 'test-key1'}, + 'test-key2': {'what': 'a test doc!', 'id': 'test-key2'}, + 'test-key3': {'what': 'a test doc!', 'id': 'test-key3'}, + 'test-key4': {'what': 'a test doc!', 'id': 'test-key4'} + } + with pytest.raises(InvalidArgumentException): + cb_env.collection.get_all_replicas_multi(keys_and_docs) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + def test_multi_get_all_replicas_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.get_all_replicas_multi(keys) + assert isinstance(res, MultiGetReplicaResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, list), res.results.values())) is True + for k, v in res.results.items(): + for replica in v: + assert isinstance(replica, GetReplicaResult) + assert isinstance(replica.is_replica, bool) + assert replica.content_as[dict] == keys_and_docs[k] + + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.usefixtures("check_server_groups_supported") + def test_multi_get_all_replicas_read_preference(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.get_all_replicas_multi( + keys, GetAllReplicasMultiOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP, + return_exceptions=True)) + assert isinstance(res, MultiGetReplicaResult) + assert res.all_ok is False + assert isinstance(res.results, dict) + assert all(map(lambda r: isinstance(r, DocumentUnretrievableException), res.exceptions.values())) is True + assert len(res.results) == 0 + + @pytest.mark.usefixtures("check_replicas") + def test_multi_get_any_replica_fail(self, cb_env): + keys_and_docs = cb_env.FAKE_DOCS + keys = list(keys_and_docs.keys()) + res = cb_env.collection.get_any_replica_multi(keys) + assert isinstance(res, MultiGetReplicaResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), CouchbaseException), res.exceptions.values())) is True + + with pytest.raises(DocumentUnretrievableException): + cb_env.collection.get_any_replica_multi(keys, return_exceptions=False) + + with pytest.raises(DocumentUnretrievableException): + cb_env.collection.get_any_replica_multi(keys, GetAnyReplicaMultiOptions(return_exceptions=False)) + + @pytest.mark.usefixtures("check_replicas") + def test_multi_get_any_replica_invalid_input(self, cb_env): + keys_and_docs = { + 'test-key1': {'what': 'a test doc!', 'id': 'test-key1'}, + 'test-key2': {'what': 'a test doc!', 'id': 'test-key2'}, + 'test-key3': {'what': 'a test doc!', 'id': 'test-key3'}, + 'test-key4': {'what': 'a test doc!', 'id': 'test-key4'} + } + with pytest.raises(InvalidArgumentException): + cb_env.collection.get_any_replica_multi(keys_and_docs) + + @pytest.mark.usefixtures("check_replicas") + def test_multi_get_any_replica_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.get_any_replica_multi(keys) + assert isinstance(res, MultiGetReplicaResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, GetReplicaResult), res.results.values())) is True + for k, v in res.results.items(): + assert isinstance(v.is_replica, bool) + assert v.content_as[dict] == keys_and_docs[k] + + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.usefixtures("check_server_groups_supported") + def test_multi_get_any_replica_read_preference(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.get_any_replica_multi( + keys, GetAnyReplicaMultiOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP, + return_exceptions=True)) + assert isinstance(res, MultiGetReplicaResult) + assert res.all_ok is False + assert isinstance(res.results, dict) + assert all(map(lambda r: isinstance(r, DocumentUnretrievableException), res.exceptions.values())) is True + assert len(res.results) == 0 + + def test_multi_get_fail(self, cb_env): + keys_and_docs = cb_env.FAKE_DOCS + keys = list(keys_and_docs.keys()) + res = cb_env.collection.get_multi(keys) + assert isinstance(res, MultiGetResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), CouchbaseException), res.exceptions.values())) is True + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get_multi(keys, return_exceptions=False) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get_multi(keys, GetMultiOptions(return_exceptions=False)) + + def test_multi_get_invalid_input(self, cb_env): + keys_and_docs = { + 'test-key1': {'what': 'a test doc!', 'id': 'test-key1'}, + 'test-key2': {'what': 'a test doc!', 'id': 'test-key2'}, + 'test-key3': {'what': 'a test doc!', 'id': 'test-key3'}, + 'test-key4': {'what': 'a test doc!', 'id': 'test-key4'} + } + with pytest.raises(InvalidArgumentException): + cb_env.collection.get_multi(keys_and_docs) + + def test_multi_get_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.get_multi(keys) + assert isinstance(res, MultiGetResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, GetResult), res.results.values())) is True + for k, v in res.results.items(): + assert v.content_as[dict] == keys_and_docs[k] + + def test_multi_insert_fail(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + res = cb_env.collection.insert_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), CouchbaseException), res.exceptions.values())) is True + + with pytest.raises(DocumentExistsException): + cb_env.collection.insert_multi(keys_and_docs, return_exceptions=False) + + with pytest.raises(DocumentExistsException): + cb_env.collection.insert_multi(keys_and_docs, InsertMultiOptions(return_exceptions=False)) + + def test_multi_insert_global_opts(self, cb_env): + keys_and_docs = cb_env.get_new_docs(4) + opts = InsertMultiOptions(expiry=timedelta(seconds=2)) + res = cb_env.collection.insert_multi(keys_and_docs, opts) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify they all expired... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys())) + + def test_multi_insert_invalid_input(self, cb_env): + keys = ['test-key1', 'test-key2', 'test-key3', 'test-key4'] + with pytest.raises(InvalidArgumentException): + cb_env.collection.insert_multi(keys) + + def test_multi_insert_key_opts(self, cb_env): + keys_and_docs = cb_env.get_new_docs(4) + key1 = list(keys_and_docs.keys())[0] + opts = InsertMultiOptions(expiry=timedelta(seconds=2), per_key_options={ + key1: InsertOptions(expiry=timedelta(seconds=0))}) + res = cb_env.collection.insert_multi(keys_and_docs, opts) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify they all expired... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys()), okay_key=key1) + + def test_multi_insert_simple(self, cb_env): + keys_and_docs = cb_env.get_new_docs(4) + res = cb_env.collection.insert_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + def test_multi_lock_and_unlock_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.lock_multi(keys, timedelta(seconds=5)) + assert isinstance(res, MultiGetResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, GetResult), res.results.values())) is True + + res = cb_env.collection.unlock_multi(res) + assert isinstance(res, dict) + + def test_multi_lock_invalid_input(self, cb_env): + keys_and_docs = { + 'test-key1': {'what': 'a test doc!', 'id': 'test-key1'}, + 'test-key2': {'what': 'a test doc!', 'id': 'test-key2'}, + 'test-key3': {'what': 'a test doc!', 'id': 'test-key3'}, + 'test-key4': {'what': 'a test doc!', 'id': 'test-key4'} + } + with pytest.raises(InvalidArgumentException): + cb_env.collection.lock_multi(keys_and_docs, timedelta(seconds=5)) + + def test_multi_remove_fail(self, cb_env): + keys_and_docs = cb_env.FAKE_DOCS + keys = list(keys_and_docs.keys()) + res = cb_env.collection.remove_multi(keys) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), CouchbaseException), res.exceptions.values())) is True + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.remove_multi(keys, return_exceptions=False) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.remove_multi(keys, ReplaceMultiOptions(return_exceptions=False)) + + def test_multi_remove_invalid_input(self, cb_env): + keys_and_docs = { + 'test-key1': {'what': 'a test doc!', 'id': 'test-key1'}, + 'test-key2': {'what': 'a test doc!', 'id': 'test-key2'}, + 'test-key3': {'what': 'a test doc!', 'id': 'test-key3'}, + 'test-key4': {'what': 'a test doc!', 'id': 'test-key4'} + } + with pytest.raises(InvalidArgumentException): + cb_env.collection.remove_multi(keys_and_docs) + + def test_multi_remove_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.remove_multi(keys) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify they all expired... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys())) + + def test_multi_replace_fail(self, cb_env): + keys_and_docs = cb_env.FAKE_DOCS + res = cb_env.collection.replace_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), CouchbaseException), res.exceptions.values())) is True + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.replace_multi(keys_and_docs, return_exceptions=False) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.replace_multi(keys_and_docs, ReplaceMultiOptions(return_exceptions=False)) + + def test_multi_replace_global_opts(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + opts = ReplaceMultiOptions(expiry=timedelta(seconds=2)) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + res = cb_env.collection.replace_multi(keys_and_docs, opts) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify they all expired... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys())) + + def test_multi_replace_invalid_input(self, cb_env): + keys = ['test-key1', 'test-key2', 'test-key3', 'test-key4'] + with pytest.raises(InvalidArgumentException): + cb_env.collection.replace_multi(keys) + + def test_multi_replace_key_opts(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + key1 = list(keys_and_docs.keys())[0] + opts = ReplaceMultiOptions(expiry=timedelta(seconds=2), per_key_options={ + key1: UpsertOptions(expiry=timedelta(seconds=0))}) + res = cb_env.collection.replace_multi(keys_and_docs, opts) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify they all expired... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys()), okay_key=key1) + + def test_multi_replace_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + res = cb_env.collection.replace_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + for k, v in keys_and_docs.items(): + r = cb_env.collection.get(k) + assert r.content_as[dict] == v + + def test_multi_touch_fail(self, cb_env): + keys_and_docs = cb_env.FAKE_DOCS + keys = list(keys_and_docs.keys()) + res = cb_env.collection.touch_multi(keys, timedelta(seconds=2)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), CouchbaseException), res.exceptions.values())) is True + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.touch_multi(keys, timedelta(seconds=2), return_exceptions=False) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.touch_multi(keys, timedelta(seconds=2), TouchMultiOptions(return_exceptions=False)) + + def test_multi_touch_invalid_input(self, cb_env): + keys_and_docs = { + 'test-key1': {'what': 'a test doc!', 'id': 'test-key1'}, + 'test-key2': {'what': 'a test doc!', 'id': 'test-key2'}, + 'test-key3': {'what': 'a test doc!', 'id': 'test-key3'}, + 'test-key4': {'what': 'a test doc!', 'id': 'test-key4'} + } + with pytest.raises(InvalidArgumentException): + cb_env.collection.touch_multi(keys_and_docs, timedelta(seconds=2)) + + def test_multi_touch_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + res = cb_env.collection.touch_multi(keys, timedelta(seconds=2)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify they all expired... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys())) + + def test_multi_unlock_invalid_input(self, cb_env): + keys_and_docs = { + 'test-key1': {'what': 'a test doc!', 'id': 'test-key1'}, + 'test-key2': {'what': 'a test doc!', 'id': 'test-key2'}, + 'test-key3': {'what': 'a test doc!', 'id': 'test-key3'}, + 'test-key4': {'what': 'a test doc!', 'id': 'test-key4'} + } + with pytest.raises(InvalidArgumentException): + cb_env.collection.unlock_multi(keys_and_docs) + + with pytest.raises(InvalidArgumentException): + cb_env.collection.unlock_multi(list(keys_and_docs.keys())) + + def test_multi_upsert_global_opts(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + opts = UpsertMultiOptions(expiry=timedelta(seconds=2)) + res = cb_env.collection.upsert_multi(keys_and_docs, opts) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify they all expired... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys())) + + def test_multi_upsert_invalid_input(self, cb_env): + keys = ['test-key1', 'test-key2', 'test-key3', 'test-key4'] + with pytest.raises(InvalidArgumentException): + cb_env.collection.upsert_multi(keys) + + def test_multi_upsert_key_opts(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + key1 = list(keys_and_docs.keys())[0] + opts = UpsertMultiOptions(expiry=timedelta(seconds=2), per_key_options={ + key1: UpsertOptions(expiry=timedelta(seconds=0))}) + res = cb_env.collection.upsert_multi(keys_and_docs, opts) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + # lets verify they all expired... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys()), okay_key=key1) + + def test_multi_upsert_simple(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + res = cb_env.collection.upsert_multi(keys_and_docs) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + +class ClassicCollectionMultiTests(CollectionMultiTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicCollectionMultiTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicCollectionMultiTests) if valid_test_method(meth)] + compare = set(CollectionMultiTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = CollectionMultiTestEnvironment.from_environment(cb_base_env) + cb_env.enable_bucket_mgmt() + cb_env.setup(request.param) + + yield cb_env + + cb_env.teardown(request.param) diff --git a/couchbase/tests/collection_t.py b/couchbase/tests/collection_t.py new file mode 100644 index 000000000..1c3b76d77 --- /dev/null +++ b/couchbase/tests/collection_t.py @@ -0,0 +1,700 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta +from time import time + +import pytest + +import couchbase.subdocument as SD +from couchbase.diagnostics import ServiceType +from couchbase.exceptions import (AmbiguousTimeoutException, + CasMismatchException, + DocumentExistsException, + DocumentLockedException, + DocumentNotFoundException, + DocumentNotLockedException, + DocumentUnretrievableException, + InvalidArgumentException, + TemporaryFailException) +from couchbase.options import (GetAllReplicasOptions, + GetAnyReplicaOptions, + GetOptions, + InsertOptions, + ReplaceOptions, + UpsertOptions) +from couchbase.replica_reads import ReadPreference +from couchbase.result import (ExistsResult, + GetReplicaResult, + GetResult, + MutationResult) +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment +from tests.mock_server import MockServerType +from tests.test_features import EnvironmentFeatures + + +class CollectionTestSuite: + FIFTY_YEARS = 50 * 365 * 24 * 60 * 60 + THIRTY_DAYS = 30 * 24 * 60 * 60 + + TEST_MANIFEST = [ + 'test_document_expiry_values', + 'test_does_not_exists', + 'test_exists', + 'test_expiry_really_expires', + 'test_get', + 'test_get_after_lock', + 'test_get_all_replicas', + 'test_get_all_replicas_fail', + 'test_get_all_replicas_results', + 'test_get_all_replicas_read_preference', + 'test_get_and_lock', + 'test_get_and_lock_replace_with_cas', + 'test_get_and_touch', + 'test_get_and_touch_no_expire', + 'test_get_any_replica', + 'test_get_any_replica_fail', + 'test_get_any_replica_read_preference', + 'test_get_fails', + 'test_get_options', + 'test_get_with_expiry', + 'test_insert', + 'test_insert_document_exists', + 'test_project', + 'test_project_bad_path', + 'test_project_project_not_list', + 'test_project_too_many_projections', + 'test_remove', + 'test_remove_fail', + 'test_replace', + 'test_replace_fail', + 'test_replace_preserve_expiry', + 'test_replace_preserve_expiry_fail', + 'test_replace_preserve_expiry_not_used', + 'test_replace_with_cas', + 'test_touch', + 'test_touch_no_expire', + 'test_unlock', + 'test_unlock_wrong_cas', + 'test_unlock_not_locked', + 'test_upsert', + 'test_upsert_preserve_expiry', + 'test_upsert_preserve_expiry_not_used', + ] + + @pytest.fixture(scope='class') + def check_preserve_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('preserve_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_xattr_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('xattr', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_not_locked_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('kv_not_locked', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def num_replicas(self, cb_env): + bucket_settings = TestEnvironment.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + return num_replicas + + @pytest.fixture(scope='class') + def check_replicas(self, cb_env, num_replicas): + if cb_env.is_mock_server and cb_env.mock_server_type == MockServerType.GoCAVES: + pytest.skip('GoCaves inconstent w/ replicas') + ping_res = cb_env.bucket.ping() + kv_endpoints = ping_res.endpoints.get(ServiceType.KeyValue, None) + if kv_endpoints is None or len(kv_endpoints) < (num_replicas + 1): + pytest.skip("Not all replicas are online") + + @pytest.fixture(scope='class') + def num_nodes(self, cb_env): + return len(cb_env.cluster._cluster_info.nodes) + + @pytest.fixture(scope='class') + def check_multi_node(self, num_nodes): + if num_nodes == 1: + pytest.skip("Test only for clusters with more than a single node.") + + @pytest.fixture(scope='class') + def check_server_groups_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('server_groups', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.fixture(scope="class") + def skip_if_go_caves(self, cb_env): + if cb_env.is_mock_server and cb_env.mock_server_type == MockServerType.GoCAVES: + pytest.skip("GoCAVES does not like this operation.") + + @pytest.mark.usefixtures('check_xattr_supported') + @pytest.mark.parametrize("expiry", [FIFTY_YEARS + 1, + FIFTY_YEARS, + THIRTY_DAYS - 1, + THIRTY_DAYS, + int(time() - 1.0), + 60, + -1]) + def test_document_expiry_values(self, cb_env, expiry): + key, value = cb_env.get_new_doc() + # expiry should be >= 0 (0 being don't expire) + if expiry == -1: + with pytest.raises(InvalidArgumentException): + cb_env.collection.upsert(key, value, expiry=timedelta(seconds=expiry)) + else: + before = int(time() - 1.0) + result = cb_env.collection.upsert(key, value, expiry=timedelta(seconds=expiry)) + assert result.cas is not None + + expiry_path = '$document.exptime' + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + res_expiry = res.content_as[int](0) + + after = int(time() + 1.0) + before + expiry <= res_expiry <= after + expiry + + def test_does_not_exists(self, cb_env): + result = cb_env.collection.exists(TestEnvironment.NOT_A_KEY) + assert isinstance(result, ExistsResult) + assert result.exists is False + + def test_exists(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = cb_env.collection.exists(key) + assert isinstance(result, ExistsResult) + assert result.exists is True + + def test_expiry_really_expires(self, cb_env): + key, value = cb_env.get_new_doc() + result = cb_env.collection.upsert(key, value, UpsertOptions( + expiry=timedelta(seconds=2))) + assert result.cas != 0 + + TestEnvironment.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + def test_get(self, cb_env): + key, value = cb_env.get_existing_doc() + result = cb_env.collection.get(key) + assert isinstance(result, GetResult) + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + assert result.content_as[dict] == value + + def test_get_after_lock(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + orig = cb_env.collection.get_and_lock(key, timedelta(seconds=5)) + assert isinstance(orig, GetResult) + result = cb_env.collection.get(key) + assert orig.content_as[dict] == result.content_as[dict] + assert orig.cas != result.cas + + # @TODO(jc): cxx client raises ambiguous timeout w/ retry reason: kv_temporary_failure + TestEnvironment.try_n_times_till_exception(10, 1, + cb_env.collection.unlock, key, orig.cas, + expected_exceptions=(TemporaryFailException, + DocumentNotLockedException)) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + def test_get_all_replicas(self, cb_env): + key, value = cb_env.get_existing_doc() + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get_all_replicas, key) + # make sure we can iterate over results + while True: + try: + res = next(result) + assert isinstance(res, GetReplicaResult) + assert isinstance(res.is_replica, bool) + assert value == res.content_as[dict] + except StopIteration: + break + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + def test_get_all_replicas_fail(self, cb_env): + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get_all_replicas('not-a-key') + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + def test_get_all_replicas_results(self, cb_env, num_replicas): + key, value = cb_env.get_existing_doc() + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get_all_replicas, key) + active_cnt = 0 + replica_cnt = 0 + for res in result: + assert isinstance(res, GetReplicaResult) + assert isinstance(res.is_replica, bool) + assert value == res.content_as[dict] + if res.is_replica: + replica_cnt += 1 + else: + active_cnt += 1 + + assert active_cnt == 1 + if num_replicas > 0: + assert replica_cnt >= active_cnt + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.usefixtures("check_server_groups_supported") + def test_get_all_replicas_read_preference(self, cb_env): + key, value = cb_env.get_existing_doc() + # No preferred server group was specified in the cluster options so this should raise + # DocumentUnretrievableException + with pytest.raises(DocumentUnretrievableException): + cb_env.collection.get_all_replicas( + key, GetAllReplicasOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP)) + + def test_get_and_lock(self, cb_env): + key, value = cb_env.get_existing_doc() + result = cb_env.collection.get_and_lock(key, timedelta(seconds=3)) + assert isinstance(result, GetResult) + with pytest.raises(DocumentLockedException): + cb_env.collection.upsert(key, value) + + TestEnvironment.try_n_times(10, 1, cb_env.collection.upsert, key, value) + + def test_get_and_lock_replace_with_cas(self, cb_env): + key, value = cb_env.get_existing_doc() + result = cb_env.collection.get_and_lock(key, timedelta(seconds=5)) + assert isinstance(result, GetResult) + cas = result.cas + # TODO: handle retry reasons, looks to be where we can get the locked + # exception + with pytest.raises((AmbiguousTimeoutException, DocumentLockedException)): + cb_env.collection.upsert(key, value) + + cb_env.collection.replace(key, value, ReplaceOptions(cas=cas)) + TestEnvironment.try_n_times_till_exception(10, 1, + cb_env.collection.unlock, key, cas, + expected_exceptions=(TemporaryFailException, + DocumentNotLockedException)) + + def test_get_and_touch(self, cb_env): + key, value = cb_env.get_new_doc() + cb_env.collection.upsert(key, value) + TestEnvironment.try_n_times(10, 1, cb_env.collection.get, key) + result = cb_env.collection.get_and_touch(key, timedelta(seconds=2)) + assert isinstance(result, GetResult) + TestEnvironment.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + @pytest.mark.flaky(reruns=3, reruns_delay=1) + def test_get_and_touch_no_expire(self, cb_env): + key, value = cb_env.get_existing_doc() + # @TODO(jc): GoCAVES does not seem to like nested doc structures in this scenario, fails + # on the last get consistently + if cb_env.is_mock_server and 'manufacturer' in value: + value.pop('manufacturer') + cb_env.collection.upsert(key, value) + TestEnvironment.try_n_times(10, 1, cb_env.collection.get, key) + cb_env.collection.get_and_touch(key, timedelta(seconds=15)) + g_result = cb_env.collection.get(key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is not None + cb_env.collection.get_and_touch(key, timedelta(seconds=0)) + # this get seems to be a problem + g_result = cb_env.collection.get(key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is None + + @pytest.mark.usefixtures("check_replicas") + def test_get_any_replica(self, cb_env): + key, value = cb_env.get_existing_doc() + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get_any_replica, key) + assert isinstance(result, GetReplicaResult) + assert isinstance(result.is_replica, bool) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_replicas") + def test_get_any_replica_fail(self, cb_env): + with pytest.raises(DocumentUnretrievableException): + cb_env.collection.get_any_replica('not-a-key') + + @pytest.mark.usefixtures("check_multi_node") + @pytest.mark.usefixtures("check_replicas") + @pytest.mark.usefixtures("check_server_groups_supported") + def test_get_any_replica_read_preference(self, cb_env): + key, value = cb_env.get_existing_doc() + # No preferred server group was specified in the cluster options so this should raise + # DocumentUnretrievableException + with pytest.raises(DocumentUnretrievableException): + cb_env.collection.get_any_replica( + key, GetAnyReplicaOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP)) + + def test_get_options(self, cb_env): + key, value = cb_env.get_existing_doc() + result = cb_env.collection.get(key, GetOptions( + timeout=timedelta(seconds=2), with_expiry=False)) + assert isinstance(result, GetResult) + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + assert result.content_as[dict] == value + + def test_get_fails(self, cb_env): + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(TestEnvironment.NOT_A_KEY) + + @pytest.mark.usefixtures('check_xattr_supported') + def test_get_with_expiry(self, cb_env): + key, value = cb_env.get_new_doc() + cb_env.collection.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=1000))) + expiry_path = '$document.exptime' + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry = res.content_as[int](0) + assert expiry is not None + assert expiry > 0 + expires_in = (datetime.fromtimestamp(expiry) - datetime.now()).total_seconds() + # when running local, this can be be up to 1050, so just make sure > 0 + assert expires_in > 0 + + def test_insert(self, cb_env): + key, value = cb_env.get_new_doc() + result = cb_env.collection.insert(key, value, InsertOptions( + timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + def test_insert_document_exists(self, cb_env): + key, value = cb_env.get_existing_doc() + with pytest.raises(DocumentExistsException): + cb_env.collection.insert(key, value) + + def test_project(self, cb_env): + # @TODO(jc): Why does caves not like the dealership type??? + key, value = cb_env.get_existing_doc() + result = cb_env.collection.upsert(key, value, UpsertOptions( + expiry=timedelta(seconds=2))) + + def cas_matches(cb, new_cas): + r = cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + TestEnvironment.try_n_times(10, 3, cas_matches, cb_env.collection, result.cas) + result = cb_env.collection.get(key, GetOptions(project=['batch'])) + assert 'batch' in result.content_as[dict] + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + + def test_project_bad_path(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + # CXXCBC-295 - b8bb98c31d377100934dd4b33998f0a118df41e8, bad path no longer raises PathNotFoundException + result = cb_env.collection.get(key, GetOptions(project=['qzx'])) + assert result.cas is not None + res_dict = result.content_as[dict] + assert res_dict == {} + assert 'qzx' not in res_dict + + def test_project_project_not_list(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + with pytest.raises(InvalidArgumentException): + cb_env.collection.get(key, GetOptions(project='thiswontwork')) + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_project_too_many_projections(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + value = {f'f{i}': i for i in range(1, 21)} + cb_env.collection.upsert(key, value) + + project = [f'f{i}' for i in range(1, 19)] + result = cb_env.collection.get(key, GetOptions(project=project)) + + assert result.cas is not None + res_dict = result.content_as[dict] + assert res_dict != {} + assert res_dict.get('f20') is None + for field in project: + assert res_dict.get(field) is not None + + def test_remove(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = cb_env.collection.remove(key) + assert isinstance(result, MutationResult) + + with pytest.raises(DocumentNotFoundException): + TestEnvironment.try_n_times_till_exception(3, + 1, + cb_env.collection.get, + key, + expected_exceptions=(DocumentNotFoundException,), + raise_exception=True) + + def test_remove_fail(self, cb_env): + with pytest.raises(DocumentNotFoundException): + cb_env.collection.remove(TestEnvironment.NOT_A_KEY) + + def test_replace(self, cb_env): + key, value = cb_env.get_existing_doc() + result = cb_env.collection.replace(key, value, ReplaceOptions( + timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + def test_replace_with_cas(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + _, value1 = cb_env.get_new_doc() + result = cb_env.collection.get(key) + old_cas = result.cas + result = cb_env.collection.replace(key, value1, ReplaceOptions(cas=old_cas)) + assert isinstance(result, MutationResult) + assert result.cas != old_cas + + # try same cas again, must fail. + with pytest.raises(CasMismatchException): + cb_env.collection.replace(key, value1, ReplaceOptions(cas=old_cas)) + + def test_replace_fail(self, cb_env): + with pytest.raises(DocumentNotFoundException): + cb_env.collection.replace(TestEnvironment.NOT_A_KEY, {"some": "content"}) + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_replace_preserve_expiry(self, cb_env): + key, value = cb_env.get_existing_doc() + _, value1 = cb_env.get_new_doc() + + cb_env.collection.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = '$document.exptime' + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + cb_env.collection.replace(key, value1, ReplaceOptions(preserve_expiry=True)) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + TestEnvironment.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_replace_preserve_expiry_fail(self, cb_env): + key, value = cb_env.get_existing_doc() + + opts = ReplaceOptions(expiry=timedelta(seconds=5), preserve_expiry=True) + with pytest.raises(InvalidArgumentException): + cb_env.collection.replace(key, value, opts) + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_replace_preserve_expiry_not_used(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_existing_doc() + _, value1 = cb_env.get_new_doc() + + cb.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = '$document.exptime' + res = TestEnvironment.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + cb.replace(key, value1) + res = TestEnvironment.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + TestEnvironment.sleep(3.0) + result = cb.get(key) + assert isinstance(result, GetResult) + assert result.content_as[dict] == value1 + + def test_touch(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = cb_env.collection.touch(key, timedelta(seconds=2)) + assert isinstance(result, MutationResult) + TestEnvironment.try_n_times_till_exception(10, + 1, + cb_env.collection.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + @pytest.mark.flaky(reruns=3, reruns_delay=1) + def test_touch_no_expire(self, cb_env): + key, value = cb_env.get_existing_doc() + # @TODO(jc): GoCAVES does not seem to like nested doc structures in this scenario, fails + # on the last get consistently + if cb_env.is_mock_server and 'manufacturer' in value: + value.pop('manufacturer') + cb_env.collection.touch(key, timedelta(seconds=0)) + # this get seems to be a problem + g_result = cb_env.collection.get(key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is None + + def test_unlock(self, cb_env): + key, value = cb_env.get_existing_doc() + result = cb_env.collection.get_and_lock(key, timedelta(seconds=5)) + assert isinstance(result, GetResult) + cb_env.collection.unlock(key, result.cas) + cb_env.collection.upsert(key, value) + + def test_unlock_wrong_cas(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = cb_env.collection.get_and_lock(key, timedelta(seconds=5)) + cas = result.cas + # @TODO(jc): MOCK - TemporaryFailException + with pytest.raises(CasMismatchException): + cb_env.collection.unlock(key, 100) + + TestEnvironment.try_n_times_till_exception(10, 1, + cb_env.collection.unlock, key, cas, + expected_exceptions=(TemporaryFailException, + DocumentNotLockedException)) + + @pytest.mark.usefixtures("check_not_locked_supported") + def test_unlock_not_locked(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = cb_env.collection.get_and_lock(key, timedelta(seconds=5)) + cas = result.cas + cb_env.collection.unlock(key, cas) + with pytest.raises(DocumentNotLockedException): + cb_env.collection.unlock(key, cas) + + def test_upsert(self, cb_env): + key, value = cb_env.get_existing_doc() + result = cb_env.collection.upsert(key, value, UpsertOptions( + timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_upsert_preserve_expiry(self, cb_env): + key, value = cb_env.get_existing_doc() + _, value1 = cb_env.get_new_doc() + + cb_env.collection.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = '$document.exptime' + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + cb_env.collection.upsert(key, value1, UpsertOptions(preserve_expiry=True)) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + TestEnvironment.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_upsert_preserve_expiry_not_used(self, cb_env): + key, value = cb_env.get_existing_doc() + _, value1 = cb_env.get_new_doc() + cb_env.collection.upsert(key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = '$document.exptime' + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + cb_env.collection.upsert(key, value1) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + TestEnvironment.sleep(3.0) + result = cb_env.collection.get(key) + assert isinstance(result, GetResult) + assert result.content_as[dict] == value1 + + +class ClassicCollectionTests(CollectionTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicCollectionTests) if valid_test_method(meth)] + compare = set(CollectionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.enable_bucket_mgmt() + cb_base_env.setup(request.param, num_docs=100) + yield cb_base_env + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/collectionmgmt_t.py b/couchbase/tests/collectionmgmt_t.py new file mode 100644 index 000000000..a97c823a7 --- /dev/null +++ b/couchbase/tests/collectionmgmt_t.py @@ -0,0 +1,585 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +from couchbase.exceptions import (BucketDoesNotExistException, + CollectionAlreadyExistsException, + CollectionNotFoundException, + DocumentNotFoundException, + FeatureUnavailableException, + InvalidArgumentException, + ScopeAlreadyExistsException, + ScopeNotFoundException) +from couchbase.management.buckets import StorageBackend +from couchbase.management.collections import (CollectionSpec, + CreateCollectionSettings, + UpdateCollectionSettings) +from tests.environments import CollectionType +from tests.environments.collection_mgmt_environment import CollectionManagementTestEnvironment +from tests.environments.test_environment import EnvironmentFeatures, TestEnvironment + + +class CollectionManagementTestSuite: + TEST_MANIFEST = [ + 'test_collection_goes_in_correct_bucket', + 'test_create_collection', + 'test_create_collection_already_exists', + 'test_create_collection_bad_scope', + 'test_create_collection_max_expiry', + 'test_create_collection_max_expiry_default', + 'test_create_collection_max_expiry_invalid', + 'test_create_collection_max_expiry_no_expiry', + 'test_create_scope', + 'test_create_scope_already_exists', + 'test_create_scope_and_collection', + 'test_drop_collection', + 'test_drop_collection_not_found', + 'test_drop_collection_scope_not_found', + 'test_drop_scope', + 'test_drop_scope_not_found', + 'test_get_all_scopes', + 'test_create_collection_history_retention', + 'test_create_collection_history_retention_unsupported', + 'test_update_collection_history_retention', + 'test_update_collection_history_retention_unsupported', + 'test_update_collection_max_expiry', + 'test_update_collection_max_expiry_bucket_default', + 'test_update_collection_max_expiry_invalid', + 'test_update_collection_max_expiry_no_expiry', + 'test_deprecated_create_collection', + 'test_deprecated_create_collection_already_exists', + 'test_deprecated_create_collection_bad_scope', + 'test_deprecated_create_collection_max_ttl', + 'test_deprecated_create_scope_and_collection', + 'test_deprecated_drop_collection', + 'test_deprecated_drop_collection_not_found', + 'test_deprecated_drop_collection_scope_not_found', + ] + + @pytest.fixture(scope="class") + def check_non_deduped_history_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('non_deduped_history', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_update_collection_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('update_collection', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_update_collection_max_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('update_collection_max_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_negative_collection_max_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('negative_collection_max_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + def test_collection_goes_in_correct_bucket(self, cb_env): + collection_name = cb_env.get_collection_name() + scope_name = '_default' + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + # make sure it actually is in the other-bucket + assert cb_env.get_collection(scope_name, collection_name) is not None + # also be sure this isn't in the default bucket + assert cb_env.get_collection(scope_name, + collection_name, + bucket_name=cb_env.bucket.name) is None + + def test_create_collection(self, cb_env): + # create a collection under default_ scope + collection_name = cb_env.get_collection_name() + scope_name = '_default' + cb_env.cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.bucket.name, scope_name, collection_name) + coll = cb_env.get_collection(scope_name, collection_name, bucket_name=cb_env.bucket.name) + assert coll.scope_name == scope_name + assert coll is not None + + def test_create_collection_already_exists(self, cb_env): + collection_name = cb_env.get_collection_name() + scope_name = '_default' + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + # verify the collection exists w/in other-bucket + assert cb_env.get_collection(scope_name, collection_name) is not None + # now, it will fail if we try to create it again... + with pytest.raises(CollectionAlreadyExistsException): + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + + def test_create_collection_bad_scope(self, cb_env): + collection_name = cb_env.get_collection_name() + scope_name = 'im-a-fake-scope' + with pytest.raises(ScopeNotFoundException): + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + + def test_create_collection_max_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = cb_env.get_collection_name() + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=2)) + + cb_env.test_bucket_cm.create_collection(scope_name, collection_name, settings) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=2) + + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(collection_name) + key = 'test-coll-key0' + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + TestEnvironment.try_n_times(10, 1, coll.upsert, key, {'some': 'thing'}) + TestEnvironment.try_n_times(10, 1, coll.get, key) + TestEnvironment.try_n_times_till_exception(4, + 1, + coll.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_create_collection_max_expiry_default(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = cb_env.get_collection_name() + scope_name = '_default' + + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_create_collection_max_expiry_invalid(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + collection_name = cb_env.get_collection_name() + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=-20)) + + with pytest.raises(InvalidArgumentException): + cb_env.test_bucket_cm.create_collection(scope_name, collection_name, settings) + + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_create_collection_max_expiry_no_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = cb_env.get_collection_name() + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=-1)) + cb_env.test_bucket_cm.create_collection(scope_name, collection_name, settings) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=-1) + + def test_create_scope(self, cb_env): + scope_name = cb_env.get_scope_name() + cb_env.test_bucket_cm.create_scope(scope_name) + cb_env.consistency.wait_until_scope_present(cb_env.test_bucket.name, scope_name) + + assert cb_env.get_scope(scope_name) is not None + + def test_create_scope_already_exists(self, cb_env): + scope_name = cb_env.get_scope_name() + cb_env.test_bucket_cm.create_scope(scope_name) + cb_env.consistency.wait_until_scope_present(cb_env.test_bucket.name, scope_name) + + assert cb_env.get_scope(scope_name) is not None + with pytest.raises(ScopeAlreadyExistsException): + cb_env.test_bucket_cm.create_scope(scope_name) + + def test_create_scope_and_collection(self, cb_env): + scope_name = cb_env.get_scope_name() + collection_name = cb_env.get_collection_name() + cb_env.test_bucket_cm.create_scope(scope_name) + cb_env.consistency.wait_until_scope_present(cb_env.test_bucket.name, scope_name) + + assert cb_env.get_scope(scope_name) is not None + + collection = CollectionSpec(collection_name, scope_name) + cb_env.test_bucket_cm.create_collection(collection) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + + def test_drop_collection(self, cb_env): + collection_name = cb_env.get_collection_name() + scope_name = '_default' + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + # verify the collection exists w/in other-bucket + assert cb_env.get_collection(scope_name, collection_name) is not None + # attempt to drop it again will raise CollectionNotFoundException + cb_env.test_bucket_cm.drop_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_dropped(cb_env.test_bucket.name, scope_name, collection_name) + with pytest.raises(CollectionNotFoundException): + cb_env.test_bucket_cm.drop_collection(scope_name, collection_name) + + def test_drop_collection_not_found(self, cb_env): + collection_name = 'fake-collection' + scope_name = '_default' + with pytest.raises(CollectionNotFoundException): + cb_env.test_bucket_cm.drop_collection(scope_name, collection_name) + + def test_drop_collection_scope_not_found(self, cb_env): + collection_name = 'fake-collection' + scope_name = 'fake-scope' + with pytest.raises(ScopeNotFoundException): + cb_env.test_bucket_cm.drop_collection(scope_name, collection_name) + + def test_drop_scope(self, cb_env): + scope_name = cb_env.get_scope_name() + cb_env.test_bucket_cm.create_scope(scope_name) + cb_env.consistency.wait_until_scope_present(cb_env.test_bucket.name, scope_name) + assert cb_env.get_scope(scope_name) is not None + cb_env.test_bucket_cm.drop_scope(scope_name) + cb_env.consistency.wait_until_scope_dropped(cb_env.test_bucket.name, scope_name) + cb_env.add_dropped_scope(scope_name) + with pytest.raises(ScopeNotFoundException): + cb_env.test_bucket_cm.drop_scope(scope_name) + + def test_drop_scope_not_found(self, cb_env): + with pytest.raises(ScopeNotFoundException): + cb_env.test_bucket_cm.drop_scope('some-random-scope') + + def test_get_all_scopes(self, cb_env): + scope_names = cb_env.get_scope_names() + scopes = cb_env.test_bucket_cm.get_all_scopes() + assert (sum(s.name[0] != '_' for s in scopes) == len(scope_names)) + # should have a _default scope + assert any(map(lambda s: s.name == '_default', scopes)) + for scope_name in scope_names: + assert any(map(lambda s: s.name == scope_name, scopes)) + for s in scopes: + for c in s.collections: + assert isinstance(c, CollectionSpec) + assert isinstance(c.name, str) + assert isinstance(c.scope_name, str) + assert isinstance(c.max_expiry, timedelta) + assert c.history is None or isinstance(c.history, bool) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + def test_create_collection_history_retention(self, cb_env): + bucket_name = 'test-magma-bucket' + scope_name = '_default' + collection_name = cb_env.get_collection_name() + + cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA) + bucket = TestEnvironment.try_n_times(3, 5, cb_env.cluster.bucket, bucket_name) + cm = bucket.collections() + + cm.create_collection(scope_name, collection_name, CreateCollectionSettings(history=True)) + cb_env.consistency.wait_until_collection_present(bucket.name, scope_name, collection_name) + collection_spec = cb_env.get_collection(scope_name, collection_name, bucket_name=bucket_name) + assert collection_spec is not None + assert collection_spec.history + + TestEnvironment.try_n_times_till_exception(10, + 3, + cb_env.bm.drop_bucket, + bucket_name, + expected_exceptions=(BucketDoesNotExistException,)) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + def test_create_collection_history_retention_unsupported(self, cb_env): + scope_name = '_default' + collection_name = cb_env.get_collection_name() + + # Couchstore does not support history retention + with pytest.raises(FeatureUnavailableException): + cb_env.test_bucket_cm.create_collection( + scope_name, collection_name, CreateCollectionSettings(history=True)) + + with pytest.raises(FeatureUnavailableException): + cb_env.test_bucket_cm.create_collection( + scope_name, collection_name, CreateCollectionSettings(history=False)) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('check_update_collection_supported') + def test_update_collection_history_retention(self, cb_env): + bucket_name = 'test-magma-bucket' + scope_name = '_default' + collection_name = cb_env.get_collection_name() + + cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA) + bucket = TestEnvironment.try_n_times(3, 5, cb_env.cluster.bucket, bucket_name) + cm = bucket.collections() + + cm.create_collection(scope_name, collection_name, CreateCollectionSettings(history=False)) + cb_env.consistency.wait_until_collection_present(bucket.name, scope_name, collection_name) + collection_spec = cb_env.get_collection(scope_name, collection_name, bucket_name=bucket_name) + assert collection_spec is not None + assert not collection_spec.history + + cm.update_collection(scope_name, collection_name, UpdateCollectionSettings(history=True)) + cb_env.consistency.wait_until_collection_has_settings( + bucket.name, scope_name, collection_name, {'history': True}) + collection_spec = cb_env.get_collection(scope_name, collection_name, bucket_name=bucket_name) + assert collection_spec is not None + assert collection_spec.history + + TestEnvironment.try_n_times_till_exception(10, + 3, + cb_env.bm.drop_bucket, + bucket_name, + expected_exceptions=(BucketDoesNotExistException,)) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('check_update_collection_supported') + def test_update_collection_history_retention_unsupported(self, cb_env): + scope_name = '_default' + collection_name = cb_env.get_collection_name() + + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + collection_spec = cb_env.get_collection(scope_name, collection_name) + assert collection_spec is not None + assert collection_spec.history is False + + # Couchstore does not support history retention + with pytest.raises(FeatureUnavailableException): + cb_env.test_bucket_cm.update_collection(scope_name, collection_name, UpdateCollectionSettings(history=True)) + + # Collection history retention setting remains unchanged + collection_spec = cb_env.get_collection(scope_name, collection_name) + assert collection_spec is not None + assert collection_spec.history is False + + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + def test_update_collection_max_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = cb_env.get_collection_name() + scope_name = '_default' + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=2)) + cb_env.test_bucket_cm.update_collection(scope_name, collection_name, settings) + cb_env.consistency.wait_until_collection_has_settings( + cb_env.test_bucket.name, scope_name, collection_name, {'maxTTL': 2}) + + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=2) + + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(collection_name) + key = 'test-coll-key0' + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + TestEnvironment.try_n_times(10, 1, coll.upsert, key, {'some': 'thing'}) + TestEnvironment.try_n_times(10, 1, coll.get, key) + TestEnvironment.try_n_times_till_exception(4, + 1, + coll.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_update_collection_max_expiry_bucket_default(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = cb_env.get_collection_name() + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=5)) + + cb_env.test_bucket_cm.create_collection(scope_name, collection_name, settings) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=5) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=0)) + cb_env.test_bucket_cm.update_collection(scope_name, collection_name, settings) + cb_env.consistency.wait_until_collection_has_settings( + cb_env.test_bucket.name, scope_name, collection_name, {'maxTTL': 0}) + + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_update_collection_max_expiry_invalid(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + collection_name = cb_env.get_collection_name() + scope_name = '_default' + + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=-20)) + with pytest.raises(InvalidArgumentException): + cb_env.test_bucket_cm.update_collection(scope_name, collection_name, settings) + + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_update_collection_max_expiry_no_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = cb_env.get_collection_name() + scope_name = '_default' + cb_env.test_bucket_cm.create_collection(scope_name, collection_name) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=-1)) + cb_env.test_bucket_cm.update_collection(scope_name, collection_name, settings) + cb_env.consistency.wait_until_collection_has_settings( + cb_env.test_bucket.name, scope_name, collection_name, {'maxTTL': -1}) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=-1) + + def test_deprecated_create_collection(self, cb_env): + # create a collection under default_ scope + collection_name = cb_env.get_collection_name() + collection = CollectionSpec(collection_name) + cb_env.cm.create_collection(collection) + cb_env.consistency.wait_until_collection_present(cb_env.bucket.name, '_default', collection_name) + coll = cb_env.get_collection(collection.scope_name, collection.name, bucket_name=cb_env.bucket.name) + assert collection.scope_name == '_default' + assert coll is not None + + def test_deprecated_create_collection_already_exists(self, cb_env): + collection_name = cb_env.get_collection_name() + collection = CollectionSpec(collection_name) + cb_env.test_bucket_cm.create_collection(collection) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, '_default', collection_name) + # verify the collection exists w/in other-bucket + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + # now, it will fail if we try to create it again... + with pytest.raises(CollectionAlreadyExistsException): + cb_env.test_bucket_cm.create_collection(collection) + + def test_deprecated_create_collection_bad_scope(self, cb_env): + collection_name = cb_env.get_collection_name() + collection = CollectionSpec(collection_name, 'im-a-fake-scope') + with pytest.raises(ScopeNotFoundException): + cb_env.test_bucket_cm.create_collection(collection) + + def test_deprecated_create_collection_max_ttl(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection TTL.") + + collection_name = cb_env.get_collection_name() + collection = CollectionSpec(collection_name, max_ttl=timedelta(seconds=2)) + + cb_env.test_bucket_cm.create_collection(collection) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, '_default', collection_name) + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(collection.name) + key = 'test-coll-key0' + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + TestEnvironment.try_n_times(10, 1, coll.upsert, key, {'some': 'thing'}) + TestEnvironment.try_n_times(10, 1, coll.get, key) + TestEnvironment.try_n_times_till_exception(4, + 1, + coll.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + def test_deprecated_create_scope_and_collection(self, cb_env): + scope_name = cb_env.get_scope_name() + collection_name = cb_env.get_collection_name() + cb_env.test_bucket_cm.create_scope(scope_name) + cb_env.consistency.wait_until_scope_present(cb_env.test_bucket.name, scope_name) + assert cb_env.get_scope(scope_name) is not None + collection = CollectionSpec(collection_name, scope_name) + cb_env.test_bucket_cm.create_collection(collection) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, scope_name, collection_name) + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + + def test_deprecated_drop_collection(self, cb_env): + collection_name = cb_env.get_collection_name() + collection = CollectionSpec(collection_name) + cb_env.test_bucket_cm.create_collection(collection) + cb_env.consistency.wait_until_collection_present(cb_env.test_bucket.name, '_default', collection_name) + # verify the collection exists w/in other-bucket + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + # attempt to drop it again will raise CollectionNotFoundException + cb_env.test_bucket_cm.drop_collection(collection) + cb_env.consistency.wait_until_collection_dropped(cb_env.test_bucket.name, '_default', collection_name) + with pytest.raises(CollectionNotFoundException): + cb_env.test_bucket_cm.drop_collection(collection) + + def test_deprecated_drop_collection_not_found(self, cb_env): + collection = CollectionSpec('fake-collection') + with pytest.raises(CollectionNotFoundException): + cb_env.test_bucket_cm.drop_collection(collection) + + def test_deprecated_drop_collection_scope_not_found(self, cb_env): + collection = CollectionSpec('fake-collection', 'fake-scope') + with pytest.raises(ScopeNotFoundException): + cb_env.test_bucket_cm.drop_collection(collection) + + +class ClassicCollectionManagementTests(CollectionManagementTestSuite): + + @pytest.fixture(scope='class', autouse=True) + def validate_test_manifest(self): + def valid_test_method(meth): + attr = getattr(ClassicCollectionManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicCollectionManagementTests) if valid_test_method(meth)] + test_list = set(CollectionManagementTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest invalid. Missing/extra tests: {test_list}.') + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env): + cb_env = CollectionManagementTestEnvironment.from_environment(cb_base_env) + cb_env.enable_bucket_mgmt().enable_collection_mgmt() + cb_env.setup() + yield cb_env + cb_env.teardown() diff --git a/couchbase/tests/connection_t.py b/couchbase/tests/connection_t.py new file mode 100644 index 000000000..b2cdd14b6 --- /dev/null +++ b/couchbase/tests/connection_t.py @@ -0,0 +1,1005 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import platform +import warnings +from copy import copy +from datetime import timedelta + +import pytest + +from couchbase.auth import CertificateAuthenticator, PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.exceptions import (CouchbaseException, + InvalidArgumentException, + UnAmbiguousTimeoutException) +from couchbase.logic.cluster import ClusterLogic +from couchbase.options import (CONFIG_PROFILES, + ClusterOptions, + ClusterTimeoutOptions, + ClusterTracingOptions, + ConfigProfile, + IpProtocol, + KnownConfigProfiles, + TLSVerifyMode) +from couchbase.serializer import DefaultJsonSerializer +from couchbase.transcoder import JSONTranscoder +from tests.environments import CollectionType + + +class ConnectionTestSuite: + TEST_MANIFEST = [ + 'test_cluster_auth_fail', + 'test_cluster_cert_auth', + 'test_cluster_cert_auth_fail', + 'test_cluster_cert_auth_ts_connstr', + 'test_cluster_cert_auth_ts_kwargs', + 'test_cluster_ldap_auth', + 'test_cluster_ldap_auth_real', + 'test_cluster_legacy_sasl_mech_force', + 'test_cluster_legacy_sasl_mech_force_real', + 'test_cluster_legacy_ssl_no_verify', + 'test_cluster_options', + 'test_cluster_pw_auth', + 'test_cluster_pw_auth_fail', + 'test_cluster_pw_auth_with_cert', + 'test_cluster_pw_auth_with_cert_connstr', + 'test_cluster_pw_auth_with_cert_kwargs', + 'test_cluster_sasl_mech_default', + 'test_cluster_sasl_mech_legacy_multiple', + 'test_cluster_sasl_mech_multiple', + 'test_cluster_timeout_options', + 'test_cluster_timeout_options_fail', + 'test_cluster_timeout_options_kwargs', + 'test_cluster_tracing_options', + 'test_cluster_tracing_options_fail', + 'test_cluster_tracing_options_kwargs', + 'test_config_profile_fail', + 'test_custom_config_profile', + 'test_custom_config_profile_fail', + 'test_invalid_connection_strings', + 'test_connection_string_options', + 'test_valid_connection_strings', + 'test_wan_config_profile', + 'test_wan_config_profile_with_auth', + ] + + def test_cluster_auth_fail(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + with pytest.raises(InvalidArgumentException): + ClusterLogic(conn_string) + + def test_cluster_cert_auth(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + + expected_auth = { + 'cert_path': 'path/to/a/cert', + 'key_path': 'path/to/a/key', + 'trust_store_path': 'path/to/truststore' + } + + auth = CertificateAuthenticator(**expected_auth) + cluster = ClusterLogic(conn_string, ClusterOptions(auth)) + auth_opts, _ = cluster._get_connection_opts() + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + + # check as kwargs + cluster = ClusterLogic(conn_string, authenticator=auth) + auth_opts, _ = cluster._get_connection_opts() + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + + def test_cluster_cert_auth_fail(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + ts_path = 'path/to/truststore' + expected_auth = { + 'cert_path': 'path/to/a/cert', + 'key_path': 'path/to/a/key', + 'trust_store_path': ts_path + } + + with pytest.raises(InvalidArgumentException): + CertificateAuthenticator(expected_auth) + + auth_copy = copy(expected_auth) + auth_copy['cert_path'] = None + with pytest.raises(InvalidArgumentException): + CertificateAuthenticator(**auth_copy) + + auth_copy = copy(expected_auth) + auth_copy['cert_path'] = {} + with pytest.raises(InvalidArgumentException): + CertificateAuthenticator(**auth_copy) + + auth_copy = copy(expected_auth) + auth_copy['key_path'] = None + with pytest.raises(InvalidArgumentException): + CertificateAuthenticator(**auth_copy) + + auth_copy = copy(expected_auth) + auth_copy['key_path'] = {} + with pytest.raises(InvalidArgumentException): + CertificateAuthenticator(**auth_copy) + + auth_copy = copy(expected_auth) + auth_copy['trust_store_path'] = {} + with pytest.raises(InvalidArgumentException): + CertificateAuthenticator(**auth_copy) + + auth = CertificateAuthenticator(**expected_auth) + with pytest.raises(InvalidArgumentException): + ClusterLogic(conn_string, ClusterOptions(auth), trust_store_path=ts_path) + + with pytest.raises(InvalidArgumentException): + ClusterLogic(conn_string, authenticator=auth, trust_store_path=ts_path) + + def test_cluster_cert_auth_ts_connstr(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + + expected_auth = { + 'cert_path': 'path/to/a/cert', + 'key_path': 'path/to/a/key' + } + + auth = CertificateAuthenticator(**expected_auth) + ts_path = 'path/to/truststore' + cluster = ClusterLogic(f'{conn_string}?truststorepath={ts_path}', ClusterOptions(auth)) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + assert cluster_opts is not None + assert isinstance(cluster_opts, dict) + assert 'trust_store_path' in cluster_opts + assert cluster_opts['trust_store_path'] == ts_path + + # check as kwargs + cluster = ClusterLogic(f'{conn_string}?truststorepath={ts_path}', + authenticator=auth, + trust_store_path=ts_path) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + assert cluster_opts is not None + assert isinstance(cluster_opts, dict) + assert 'trust_store_path' in cluster_opts + assert cluster_opts['trust_store_path'] == ts_path + + def test_cluster_cert_auth_ts_kwargs(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + + expected_auth = { + 'cert_path': 'path/to/a/cert', + 'key_path': 'path/to/a/key' + } + + auth = CertificateAuthenticator(**expected_auth) + ts_path = 'path/to/truststore' + cluster = ClusterLogic(conn_string, ClusterOptions(auth), trust_store_path=ts_path) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + assert cluster_opts is not None + assert isinstance(cluster_opts, dict) + assert 'trust_store_path' in cluster_opts + assert cluster_opts['trust_store_path'] == ts_path + + # check as kwargs + cluster = ClusterLogic(conn_string, authenticator=auth, trust_store_path=ts_path) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + assert cluster_opts is not None + assert isinstance(cluster_opts, dict) + assert 'trust_store_path' in cluster_opts + assert cluster_opts['trust_store_path'] == ts_path + + def test_cluster_ldap_auth(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator.ldap_compatible(username, pw) + assert isinstance(auth, PasswordAuthenticator) + expected_auth = auth.as_dict() + cluster = ClusterLogic(conn_string, ClusterOptions(auth)) + auth_opts = cluster._get_connection_opts(auth_only=True) + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_cluster_ldap_auth_real(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + auth = PasswordAuthenticator.ldap_compatible(username, pw) + cluster = Cluster.connect(conn_string, ClusterOptions(auth)) + + client_opts = cluster._get_client_connection_info() + assert client_opts['credentials'] is not None + assert client_opts['credentials']['allowed_sasl_mechanisms'] == ['PLAIN'] + + def test_cluster_legacy_sasl_mech_force(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + expected_auth = auth.as_dict() + expected_auth['allowed_sasl_mechanisms'] = ['PLAIN'] + cluster = ClusterLogic(f'{conn_string}?sasl_mech_force=PLAIN', ClusterOptions(auth)) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert cluster_opts is not None + assert auth_opts is not None + assert auth_opts == expected_auth + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_cluster_legacy_sasl_mech_force_real(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + cluster = Cluster.connect(f'{conn_string}?sasl_mech_force=PLAIN', ClusterOptions(auth)) + + client_opts = cluster._get_client_connection_info() + assert client_opts['credentials'] is not None + assert client_opts['credentials']['allowed_sasl_mechanisms'] == ['PLAIN'] + + def test_cluster_legacy_ssl_no_verify(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster = ClusterLogic(f'{conn_string}?ssl=no_verify', ClusterOptions(auth)) + cluster_opts = cluster._get_connection_opts(conn_only=True) + assert cluster_opts is not None + assert cluster_opts['tls_verify'] == 'none' + + def test_cluster_options(self, couchbase_config): + opts = { + 'enable_tls': True, + 'enable_mutation_tokens': True, + 'enable_tcp_keep_alive': True, + 'ip_protocol': IpProtocol.Any, + 'enable_dns_srv': True, + 'show_queries': True, + 'enable_unordered_execution': True, + 'enable_clustermap_notification': True, + 'enable_compression': True, + 'enable_tracing': True, + 'enable_metrics': True, + 'network': 'external', + 'tls_verify': TLSVerifyMode.NO_VERIFY, + 'disable_mozilla_ca_certificates': False, + 'serializer': DefaultJsonSerializer(), + 'transcoder': JSONTranscoder(), + 'tcp_keep_alive_interval': timedelta(seconds=30), + 'config_poll_interval': timedelta(seconds=30), + 'config_poll_floor': timedelta(seconds=30), + 'max_http_connections': 10, + 'logging_meter_emit_interval': timedelta(seconds=30), + 'num_io_threads': 1, + 'dump_configuration': True, + 'preferred_server_group': 'group1', + 'enable_app_telemetry': True, + 'app_telemetry_endpoint': 'ws://localhost:8093', + "app_telemetry_backoff": timedelta(seconds=10), + "app_telemetry_ping_interval": timedelta(seconds=60), + "app_telemetry_ping_timeout": timedelta(seconds=5), + } + + expected_opts = copy(opts) + # serializer is not passed to connection options + expected_opts.pop('serializer') + # transcoder is not passed to connection options + expected_opts.pop('transcoder') + # timedeltas are translated to microseconds + expected_opts['tcp_keep_alive_interval'] = 30000000 + expected_opts['config_poll_interval'] = 30000000 + expected_opts['config_poll_floor'] = 30000000 + expected_opts['app_telemetry_backoff'] = 10000000 + expected_opts['app_telemetry_ping_interval'] = 60000000 + expected_opts['app_telemetry_ping_timeout'] = 5000000 + # IpProtocol is translated to string and has another name + expected_opts.pop('ip_protocol') + expected_opts['use_ip_protocol'] = 'any' + # TLSVerifyMode is translated to string + expected_opts['tls_verify'] = 'none' + # change name of logging meter emit int. + expected_opts.pop('logging_meter_emit_interval') + expected_opts['emit_interval'] = 30000000 + + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + + # check via ClusterOptions + cluster_opts = ClusterOptions(auth, **opts) + cluster = ClusterLogic(conn_string, cluster_opts) + cluster_opts = cluster._get_connection_opts(conn_only=True) + user_agent = cluster_opts.pop('user_agent_extra', None) + assert cluster_opts is not None + assert isinstance(cluster_opts, dict) + assert cluster_opts == expected_opts + assert user_agent == f'python/{platform.python_version()}' + + # check via kwargs + cluster_opts = ClusterOptions(auth) + cluster = ClusterLogic(conn_string, cluster_opts, **opts) + cluster_opts = cluster._get_connection_opts(conn_only=True) + user_agent = cluster_opts.pop('user_agent_extra', None) + assert cluster_opts is not None + assert isinstance(cluster_opts, dict) + assert cluster_opts == expected_opts + assert user_agent == f'python/{platform.python_version()}' + + def test_cluster_pw_auth(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + expected_auth = auth.as_dict() + cluster = ClusterLogic(conn_string, ClusterOptions(auth)) + auth_opts = cluster._get_connection_opts(auth_only=True) + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + assert auth_opts['allowed_sasl_mechanisms'] is None + + # check as kwargs + cluster = ClusterLogic(conn_string, authenticator=auth) + auth_opts = cluster._get_connection_opts(auth_only=True) + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + assert auth_opts['allowed_sasl_mechanisms'] is None + + def test_cluster_pw_auth_fail(self): + expected_auth = { + 'username': 'Administrator', + 'password': 'password', + } + + # this is OK + auth = PasswordAuthenticator(**expected_auth) + assert isinstance(auth, PasswordAuthenticator) + + with pytest.raises(InvalidArgumentException): + PasswordAuthenticator(None, None) + + auth_copy = copy(expected_auth) + auth_copy['username'] = None + with pytest.raises(InvalidArgumentException): + PasswordAuthenticator(*auth_copy.values()) + + auth_copy = copy(expected_auth) + auth_copy['password'] = None + with pytest.raises(InvalidArgumentException): + PasswordAuthenticator(*auth_copy.values()) + + with pytest.raises(InvalidArgumentException): + PasswordAuthenticator(*expected_auth.values(), cert_path={}) + + with pytest.raises(InvalidArgumentException): + PasswordAuthenticator(*expected_auth.values(), allowed_sasl_mechanisms={}) + + with pytest.raises(InvalidArgumentException): + PasswordAuthenticator(*expected_auth.values(), allowed_sasl_mechanisms=['PLAIN', 1]) + + def test_cluster_pw_auth_with_cert(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw, cert_path='path/to/cert') + expected_auth = auth.as_dict() + cluster = ClusterLogic(conn_string, ClusterOptions(auth)) + auth_opts = cluster._get_connection_opts(auth_only=True) + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + + # check as kwargs + cluster = ClusterLogic(conn_string, authenticator=auth) + auth_opts = cluster._get_connection_opts(auth_only=True) + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + + def test_cluster_pw_auth_with_cert_connstr(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cert_path = 'path/to/cert' + expected_auth = auth.as_dict() + expected_auth['cert_path'] = cert_path + cluster = ClusterLogic(f'{conn_string}?certpath={cert_path}', ClusterOptions(auth)) + auth_opts = cluster._get_connection_opts(auth_only=True) + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + + def test_cluster_pw_auth_with_cert_kwargs(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cert_path = 'path/to/cert' + expected_auth = auth.as_dict() + expected_auth['cert_path'] = cert_path + cluster = ClusterLogic(conn_string, ClusterOptions(auth), cert_path=cert_path) + auth_opts = cluster._get_connection_opts(auth_only=True) + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + + # check as kwargs + cluster = ClusterLogic(conn_string, authenticator=auth, cert_path=cert_path) + auth_opts = cluster._get_connection_opts(auth_only=True) + assert auth_opts is not None + assert isinstance(auth_opts, dict) + assert auth_opts == expected_auth + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_cluster_sasl_mech_default(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + cluster = Cluster.connect(f'{conn_string}', ClusterOptions(auth)) + + client_opts = cluster._get_client_connection_info() + assert client_opts['credentials'] is not None + assert client_opts['credentials']['allowed_sasl_mechanisms'] == [] + + def test_cluster_sasl_mech_legacy_multiple(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + expected_auth = auth.as_dict() + expected_auth['allowed_sasl_mechanisms'] = ['SCRAM-SHA512', 'SCRAM-SHA256'] + conn_str = f'{conn_string}?sasl_mech_force=SCRAM-SHA512&sasl_mech_force=SCRAM-SHA256' + cluster = ClusterLogic(conn_str, ClusterOptions(auth)) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert cluster_opts is not None + assert auth_opts is not None + assert auth_opts == expected_auth + + cluster = ClusterLogic(f'{conn_string}?sasl_mech_force=SCRAM-SHA512,SCRAM-SHA256', ClusterOptions(auth)) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert cluster_opts is not None + assert auth_opts is not None + assert auth_opts == expected_auth + + def test_cluster_sasl_mech_multiple(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + expected_auth = auth.as_dict() + expected_auth['allowed_sasl_mechanisms'] = ['SCRAM-SHA512', 'SCRAM-SHA256'] + conn_str = f'{conn_string}?allowed_sasl_mechanisms=SCRAM-SHA512&allowed_sasl_mechanisms=SCRAM-SHA256' + cluster = ClusterLogic(conn_str, ClusterOptions(auth)) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert cluster_opts is not None + assert auth_opts is not None + assert auth_opts == expected_auth + + cluster = ClusterLogic(f'{conn_string}?allowed_sasl_mechanisms=SCRAM-SHA512,SCRAM-SHA256', ClusterOptions(auth)) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert cluster_opts is not None + assert auth_opts is not None + assert auth_opts == expected_auth + + auth = PasswordAuthenticator(username, pw, allowed_sasl_mechanisms='SCRAM-SHA512,SCRAM-SHA256') + cluster = ClusterLogic(f'{conn_string}', ClusterOptions(auth)) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert cluster_opts is not None + assert auth_opts is not None + assert auth_opts == expected_auth + + auth = PasswordAuthenticator(username, pw, allowed_sasl_mechanisms=['SCRAM-SHA512', 'SCRAM-SHA256']) + cluster = ClusterLogic(f'{conn_string}', ClusterOptions(auth)) + auth_opts, cluster_opts = cluster._get_connection_opts() + assert cluster_opts is not None + assert auth_opts is not None + assert auth_opts == expected_auth + + @pytest.mark.parametrize('opts, expected_opts', + [({'bootstrap_timeout': timedelta(seconds=30)}, + {'bootstrap_timeout': 30000000}), + ({'resolve_timeout': timedelta(seconds=30)}, + {'resolve_timeout': 30000000}), + ({'connect_timeout': timedelta(seconds=30)}, + {'connect_timeout': 30000000}), + ({'kv_timeout': timedelta(seconds=30)}, + {'key_value_timeout': 30000000}), + ({'kv_durable_timeout': timedelta(seconds=30)}, + {'key_value_durable_timeout': 30000000}), + ({'views_timeout': timedelta(seconds=30)}, + {'view_timeout': 30000000}), + ({'query_timeout': timedelta(seconds=30)}, + {'query_timeout': 30000000}), + ({'analytics_timeout': timedelta(seconds=30)}, + {'analytics_timeout': 30000000}), + ({'search_timeout': timedelta(seconds=30)}, + {'search_timeout': 30000000}), + ({'management_timeout': timedelta(seconds=30)}, + {'management_timeout': 30000000}), + ({'dns_srv_timeout': timedelta(seconds=30)}, + {'dns_srv_timeout': 30000000}), + ({'idle_http_connection_timeout': timedelta(seconds=30)}, + {'idle_http_connection_timeout': 30000000}), + ({'config_idle_redial_timeout': timedelta(seconds=30)}, + {'config_idle_redial_timeout': 30000000}), + ({'bootstrap_timeout': timedelta(seconds=60), + 'kv_timeout': timedelta(seconds=5), + 'query_timeout': timedelta(seconds=30), + 'search_timeout': timedelta(seconds=120), + 'management_timeout': timedelta(seconds=60)}, + {'bootstrap_timeout': 60000000, + 'key_value_timeout': 5000000, + 'query_timeout': 30000000, + 'search_timeout': 120000000, + 'management_timeout': 60000000}), + ]) + def test_cluster_timeout_options(self, couchbase_config, opts, expected_opts): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster_opts = ClusterOptions(auth, timeout_options=ClusterTimeoutOptions(**opts)) + cluster = ClusterLogic(conn_string, cluster_opts) + cluster_opts = cluster._get_connection_opts(conn_only=True) + timeout_opts = cluster_opts.get('timeout_options', None) + assert timeout_opts is not None + assert isinstance(timeout_opts, dict) + assert timeout_opts == expected_opts + + @pytest.mark.parametrize('opts', + [ + {'bootstrap_timeout': 30}, + {'resolve_timeout': 30}, + {'connect_timeout': 30}, + {'kv_timeout': 30}, + {'kv_durable_timeout': 30}, + {'views_timeout': 30}, + {'query_timeout': 30}, + {'analytics_timeout': 30}, + {'search_timeout': 30}, + {'management_timeout': 30}, + {'dns_srv_timeout': 30}, + {'idle_http_connection_timeout': 30}, + {'config_idle_redial_timeout': 30}, + {'bootstrap_timeout': 60000000, + 'kv_timeout': 5000000, + 'query_timeout': 30000000, + 'search_timeout': 120000000, + 'management_timeout': 60000000} + ]) + def test_cluster_timeout_options_fail(self, couchbase_config, opts): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster_opts = ClusterOptions(auth) + with pytest.raises(InvalidArgumentException): + ClusterLogic(conn_string, cluster_opts, **opts) + + cluster_opts = ClusterOptions(auth, timeout_options=opts) + with pytest.raises(InvalidArgumentException): + ClusterLogic(conn_string, cluster_opts) + + @pytest.mark.parametrize('opts, expected_opts', + [({'bootstrap_timeout': timedelta(seconds=30)}, + {'bootstrap_timeout': 30000000}), + ({'resolve_timeout': timedelta(seconds=30)}, + {'resolve_timeout': 30000000}), + ({'connect_timeout': timedelta(seconds=30)}, + {'connect_timeout': 30000000}), + ({'kv_timeout': timedelta(seconds=30)}, + {'key_value_timeout': 30000000}), + ({'kv_durable_timeout': timedelta(seconds=30)}, + {'key_value_durable_timeout': 30000000}), + ({'views_timeout': timedelta(seconds=30)}, + {'view_timeout': 30000000}), + ({'query_timeout': timedelta(seconds=30)}, + {'query_timeout': 30000000}), + ({'analytics_timeout': timedelta(seconds=30)}, + {'analytics_timeout': 30000000}), + ({'search_timeout': timedelta(seconds=30)}, + {'search_timeout': 30000000}), + ({'management_timeout': timedelta(seconds=30)}, + {'management_timeout': 30000000}), + ({'dns_srv_timeout': timedelta(seconds=30)}, + {'dns_srv_timeout': 30000000}), + ({'idle_http_connection_timeout': timedelta(seconds=30)}, + {'idle_http_connection_timeout': 30000000}), + ({'config_idle_redial_timeout': timedelta(seconds=30)}, + {'config_idle_redial_timeout': 30000000}), + ({'bootstrap_timeout': timedelta(seconds=60), + 'kv_timeout': timedelta(seconds=5), + 'query_timeout': timedelta(seconds=30), + 'search_timeout': timedelta(seconds=120), + 'management_timeout': timedelta(seconds=60)}, + {'bootstrap_timeout': 60000000, + 'key_value_timeout': 5000000, + 'query_timeout': 30000000, + 'search_timeout': 120000000, + 'management_timeout': 60000000}), + ]) + def test_cluster_timeout_options_kwargs(self, couchbase_config, opts, expected_opts): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster_opts = ClusterOptions(auth) + cluster = ClusterLogic(conn_string, cluster_opts, **opts) + cluster_opts = cluster._get_connection_opts(conn_only=True) + timeout_opts = cluster_opts.get('timeout_options', None) + assert timeout_opts is not None + assert isinstance(timeout_opts, dict) + assert timeout_opts == expected_opts + + @pytest.mark.parametrize('opts, expected_opts', + [({'tracing_threshold_kv': timedelta(milliseconds=30)}, + {'key_value_threshold': 30000}), + ({'tracing_threshold_view': timedelta(milliseconds=30)}, + {'view_threshold': 30000}), + ({'tracing_threshold_query': timedelta(milliseconds=30)}, + {'query_threshold': 30000}), + ({'tracing_threshold_search': timedelta(milliseconds=30)}, + {'search_threshold': 30000}), + ({'tracing_threshold_analytics': timedelta(milliseconds=30)}, + {'analytics_threshold': 30000}), + ({'tracing_threshold_eventing': timedelta(milliseconds=30)}, + {'eventing_threshold': 30000}), + ({'tracing_threshold_management': timedelta(milliseconds=30)}, + {'management_threshold': 30000}), + ({'tracing_threshold_queue_size': 20}, + {'threshold_sample_size': 20}), + ({'tracing_threshold_queue_flush_interval': timedelta( + seconds=30)}, {'threshold_emit_interval': 30000000}), + ({'tracing_orphaned_queue_size': 20}, + {'orphaned_sample_size': 20}), + ({'tracing_orphaned_queue_flush_interval': timedelta( + seconds=30)}, {'orphaned_emit_interval': 30000000}), + ({'tracing_threshold_kv': timedelta(milliseconds=60), + 'tracing_threshold_query': timedelta(milliseconds=5), + 'tracing_threshold_management': timedelta(milliseconds=30), + 'tracing_threshold_queue_size': 20}, + {'key_value_threshold': 60000, + 'query_threshold': 5000, + 'management_threshold': 30000, + 'threshold_sample_size': 20}), + ]) + def test_cluster_tracing_options(self, couchbase_config, opts, expected_opts): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster_opts = ClusterOptions(auth, tracing_options=ClusterTracingOptions(**opts)) + cluster = ClusterLogic(conn_string, cluster_opts) + cluster_opts = cluster._get_connection_opts(conn_only=True) + tracing_options = cluster_opts.get('tracing_options', None) + assert tracing_options is not None + assert isinstance(tracing_options, dict) + assert tracing_options == expected_opts + + @pytest.mark.parametrize('opts', + [ + {'tracing_threshold_kv': 30}, + {'tracing_threshold_view': 30}, + {'tracing_threshold_query': 30}, + {'tracing_threshold_search': 30}, + {'tracing_threshold_analytics': 30}, + {'tracing_threshold_eventing': 30}, + {'tracing_threshold_management': 30}, + {'tracing_threshold_queue_flush_interval': 30}, + {'tracing_orphaned_queue_flush_interval': 30}, + {'tracing_threshold_kv': 60000000, + 'tracing_threshold_query': 5000000, + 'tracing_threshold_search': 30000000, + 'tracing_threshold_management': 120000000, + 'tracing_threshold_queue_flush_interval': 60000000} + ]) + def test_cluster_tracing_options_fail(self, couchbase_config, opts): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster_opts = ClusterOptions(auth) + with pytest.raises(InvalidArgumentException): + ClusterLogic(conn_string, cluster_opts, **opts) + + cluster_opts = ClusterOptions(auth, tracing_options=opts) + with pytest.raises(InvalidArgumentException): + ClusterLogic(conn_string, cluster_opts) + + @pytest.mark.parametrize('opts, expected_opts', + [({'tracing_threshold_kv': timedelta(milliseconds=30)}, + {'key_value_threshold': 30000}), + ({'tracing_threshold_view': timedelta(milliseconds=30)}, + {'view_threshold': 30000}), + ({'tracing_threshold_query': timedelta(milliseconds=30)}, + {'query_threshold': 30000}), + ({'tracing_threshold_search': timedelta(milliseconds=30)}, + {'search_threshold': 30000}), + ({'tracing_threshold_analytics': timedelta(milliseconds=30)}, + {'analytics_threshold': 30000}), + ({'tracing_threshold_eventing': timedelta(milliseconds=30)}, + {'eventing_threshold': 30000}), + ({'tracing_threshold_management': timedelta(milliseconds=30)}, + {'management_threshold': 30000}), + ({'tracing_threshold_queue_size': 20}, + {'threshold_sample_size': 20}), + ({'tracing_threshold_queue_flush_interval': timedelta( + seconds=30)}, {'threshold_emit_interval': 30000000}), + ({'tracing_orphaned_queue_size': 20}, + {'orphaned_sample_size': 20}), + ({'tracing_orphaned_queue_flush_interval': timedelta( + seconds=30)}, {'orphaned_emit_interval': 30000000}), + ({'tracing_threshold_kv': timedelta(milliseconds=60), + 'tracing_threshold_query': timedelta(milliseconds=5), + 'tracing_threshold_management': timedelta(milliseconds=30), + 'tracing_threshold_queue_size': 20}, + {'key_value_threshold': 60000, + 'query_threshold': 5000, + 'management_threshold': 30000, + 'threshold_sample_size': 20}), + ]) + def test_cluster_tracing_options_kwargs(self, couchbase_config, opts, expected_opts): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster_opts = ClusterOptions(auth) + cluster = ClusterLogic(conn_string, cluster_opts, **opts) + cluster_opts = cluster._get_connection_opts(conn_only=True) + tracing_options = cluster_opts.get('tracing_options', None) + assert tracing_options is not None + assert isinstance(tracing_options, dict) + assert tracing_options == expected_opts + + def test_config_profile_fail(self, couchbase_config): + username, pw = couchbase_config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + opts = ClusterOptions(auth) + with pytest.raises(InvalidArgumentException): + opts.apply_profile('test_profile') + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_custom_config_profile(self, couchbase_config): + expected_opts = {'bootstrap_timeout': 10000, + 'resolve_timeout': 2000, + 'dns_srv_timeout': 5000, + 'connect_timeout': 5000, + 'key_value_timeout': 5000, + 'key_value_durable_timeout': 5000, + 'view_timeout': 60000, + 'query_timeout': 60000, + 'analytics_timeout': 60000, + 'search_timeout': 60000, + 'management_timeout': 60000} + + class TestProfile(ConfigProfile): + def __init__(self): + super().__init__() + + def apply(self, + options # type: ClusterOptions + ) -> None: + options['kv_timeout'] = timedelta(seconds=5) + options['kv_durable_timeout'] = timedelta(seconds=5) + options['dns_srv_timeout'] = timedelta(seconds=5) + options['connect_timeout'] = timedelta(seconds=5) + options['analytics_timeout'] = timedelta(seconds=60) + options['query_timeout'] = timedelta(seconds=60) + options['search_timeout'] = timedelta(seconds=60) + options['management_timeout'] = timedelta(seconds=60) + options['views_timeout'] = timedelta(seconds=60) + + CONFIG_PROFILES.register_profile('test_profile', TestProfile()) + + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + opts = ClusterOptions(auth) + opts.apply_profile('test_profile') + cluster = Cluster.connect(conn_string, opts) + client_opts = cluster._get_client_connection_info() + for k in expected_opts.keys(): + assert client_opts[k] == expected_opts[k] + + profile = CONFIG_PROFILES.unregister_profile('test_profile') + assert isinstance(profile, ConfigProfile) + + def test_custom_config_profile_fail(self): + + class TestProfile(): + def __init__(self): + super().__init__() + + def apply(self, + options # type: ClusterOptions + ) -> None: + options['kv_timeout'] = timedelta(seconds=5) + options['kv_durable_timeout'] = timedelta(seconds=5) + options['connect_timeout'] = timedelta(seconds=5) + options['analytics_timeout'] = timedelta(seconds=60) + options['query_timeout'] = timedelta(seconds=60) + options['search_timeout'] = timedelta(seconds=60) + options['management_timeout'] = timedelta(seconds=60) + options['views_timeout'] = timedelta(seconds=60) + + with pytest.raises(InvalidArgumentException): + CONFIG_PROFILES.register_profile('test_profile', TestProfile()) + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.parametrize("conn_str", ['http://host1,http://host2', + 'https://host2:8091,host3:8091', + 'http://::ffff:00ee:2122', + 'couchbase://12345:qwer###/default']) + def test_invalid_connection_strings(self, conn_str): + try: + Cluster(conn_str, authenticator=PasswordAuthenticator( + 'Administrator', 'password'), bootstrap_timeout=timedelta(seconds=1)) + except (InvalidArgumentException, UnAmbiguousTimeoutException): + pass + except Exception as ex: + pytest.fail(f'Unexpected exception occurred: {ex}') + + @pytest.mark.parametrize("conn_str", ['10.0.0.1:8091', + 'http://10.0.0.1', + 'couchbase://10.0.0.1', + 'couchbases://10.0.0.1:11222,10.0.0.2,10.0.0.3:11207', + 'couchbase://10.0.0.1;10.0.0.2:11210;10.0.0.3', + 'couchbase://[3ffe:2a00:100:7031::1]', + 'couchbases://[::ffff:192.168.0.1]:11207,[::ffff:192.168.0.2]:11207', + 'couchbase://test.local:11210?key=value', + 'http://fqdn', + 'http://fqdn?key=value', + 'couchbases://fqdn' + ]) + def test_valid_connection_strings(self, conn_str): + expected_opts = {'timeout_options': {'bootstrap_timeout': 1000000}} + try: + if conn_str == '10.0.0.1:8091': + with warnings.catch_warnings(record=True) as w: + # Cause all warnings to always be triggered. + warnings.simplefilter("always") + + cl = ClusterLogic(conn_str, authenticator=PasswordAuthenticator( + 'Administrator', 'password'), bootstrap_timeout=timedelta(seconds=1)) + assert len(w) == 1 + assert issubclass(w[-1].category, DeprecationWarning) + assert "deprecated" in str(w[-1].message) + else: + cl = ClusterLogic(conn_str, authenticator=PasswordAuthenticator( + 'Administrator', 'password'), bootstrap_timeout=timedelta(seconds=1)) + + user_agent = cl._cluster_opts.pop('user_agent_extra', None) + assert expected_opts == cl._cluster_opts + assert user_agent == f'python/{platform.python_version()}' + expected_conn_str = conn_str.split('?')[0] + assert expected_conn_str == cl._connstr + except CouchbaseException: + pass + except Exception as ex: + pytest.fail(f'Unexpected exception occurred: {ex}') + + @pytest.mark.parametrize('conn_str, expected_opts', + [('couchbase://10.0.0.1?num_io_threads=1&dump_configuration=true', + {'num_io_threads': 1, 'dump_configuration': True}), + ('couchbase://10.0.0.1?max_http_connections=4&disable_mozilla_ca_certificates=False', + {'max_http_connections': 4, 'disable_mozilla_ca_certificates': False}), + ('couchbase://localhost?enable_app_telemetry=true&app_telemetry_endpoint=ws://localhost:8091', # noqa: E501 + {'enable_app_telemetry': True, 'app_telemetry_endpoint': 'ws://localhost:8091'}), + ('couchbase://10.0.0.1?an_invalid_option=10', + {}), + ]) + def test_connection_string_options(self, conn_str, expected_opts): + try: + cl = ClusterLogic(conn_str, authenticator=PasswordAuthenticator('Administrator', 'password')) + + user_agent = cl._cluster_opts.pop('user_agent_extra', None) + assert expected_opts == cl._cluster_opts + assert user_agent == f'python/{platform.python_version()}' + expected_conn_str = conn_str.split('?')[0] + assert expected_conn_str == cl._connstr + except CouchbaseException: + pass + except Exception as ex: + pytest.fail(f'Unexpected exception occurred: {ex}') + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.parametrize('profile', + [KnownConfigProfiles.WanDevelopment, 'wan_development']) + def test_wan_config_profile(self, couchbase_config, profile): + expected_opts = {'bootstrap_timeout': 120000, + 'resolve_timeout': 20000, + 'connect_timeout': 20000, + 'dns_srv_timeout': 20000, + 'key_value_timeout': 20000, + 'key_value_durable_timeout': 20000, + 'view_timeout': 120000, + 'query_timeout': 120000, + 'analytics_timeout': 120000, + 'search_timeout': 120000, + 'management_timeout': 120000} + + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + opts = ClusterOptions(auth) + opts.apply_profile(profile) + cluster = Cluster.connect(conn_string, opts) + client_opts = cluster._get_client_connection_info() + for k in expected_opts.keys(): + assert client_opts[k] == expected_opts[k] + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.parametrize('profile', + [KnownConfigProfiles.WanDevelopment, 'wan_development']) + def test_wan_config_profile_with_auth(self, couchbase_config, profile): + expected_opts = {'bootstrap_timeout': 120000, + 'resolve_timeout': 20000, + 'connect_timeout': 20000, + 'dns_srv_timeout': 20000, + 'key_value_timeout': 20000, + 'key_value_durable_timeout': 20000, + 'view_timeout': 120000, + 'query_timeout': 120000, + 'analytics_timeout': 120000, + 'search_timeout': 120000, + 'management_timeout': 120000} + + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + cluster = Cluster.connect(conn_string, ClusterOptions.create_options_with_profile(auth, profile)) + client_opts = cluster._get_client_connection_info() + for k in expected_opts.keys(): + assert client_opts[k] == expected_opts[k] + + +class ClassicConnectionTests(ConnectionTestSuite): + + @pytest.fixture(scope='class', autouse=True) + def manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicConnectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicConnectionTests) if valid_test_method(meth)] + test_list = set(ConnectionTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest not validated. Missing/extra tests: {test_list}.') + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, request): + cb_base_env.setup(request.param) + yield cb_base_env + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/datastructures_t.py b/couchbase/tests/datastructures_t.py new file mode 100644 index 000000000..1a281f17f --- /dev/null +++ b/couchbase/tests/datastructures_t.py @@ -0,0 +1,324 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.datastructures import (CouchbaseList, + CouchbaseMap, + CouchbaseQueue, + CouchbaseSet) +from couchbase.exceptions import (DocumentNotFoundException, + InvalidArgumentException, + QueueEmpty) +from couchbase.result import OperationResult +from tests.environments import CollectionType + + +class DatastructuresTestSuite: + + TEST_MANIFEST = [ + 'test_list', + 'test_map', + 'test_queue', + 'test_sets', + ] + + def test_list(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + cb_list = cb_env.collection.couchbase_list(key) + assert isinstance(cb_list, CouchbaseList) + + cb_list.append('world') + rv = cb_list.get_at(0) + assert str(rv) == 'world' + + cb_list.prepend('hello') + rv = cb_list.get_at(0) + assert str(rv) == 'hello' + + rv = cb_list.get_at(1) + assert str(rv) == 'world' + assert 2 == cb_list.size() + + cb_list.remove_at(1) + res = cb_env.collection.get(key) + assert ['hello'] == res.content_as[list] + + cb_list.append('world') + res = cb_env.collection.get(key) + assert ['hello', 'world'] == res.content_as[list] + + cb_list.set_at(1, 'after') + res = cb_env.collection.get(key) + assert ['hello', 'after'] == res.content_as[list] + + res = cb_list.get_all() + assert ['hello', 'after'] == res + + res = cb_list.index_of('after') + assert res == 1 + + expected = ['hello', 'after'] + for idx, v in enumerate(cb_list): + assert expected[idx] == v + + cb_list.clear() + + assert 0 == cb_list.size() + + def test_map(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + cb_map = cb_env.collection.couchbase_map(key) + assert isinstance(cb_map, CouchbaseMap) + + cb_map.add('key1', 'val1') + + rv = cb_map.get('key1') + assert rv == 'val1' + + assert 1 == cb_map.size() + + with pytest.raises(InvalidArgumentException): + cb_map.remove('key2') + + cb_map.add('key2', 'val2') + + keys = cb_map.keys() + assert ['key1', 'key2'] == keys + + values = cb_map.values() + assert ['val1', 'val2'] == values + + assert cb_map.exists('key1') is True + assert cb_map.exists('no-key') is False + + expected_keys = ['key1', 'key2'] + expected_values = ['val1', 'val2'] + for k, v in cb_map.items(): + assert k in expected_keys + assert v in expected_values + + with pytest.raises(TypeError): + for k, v in cb_map: + assert k in expected_keys + assert v in expected_values + + cb_map.remove('key1') + assert 1 == cb_map.size() + + cb_map.clear() + assert 0 == cb_map.size() + + def test_sets(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + cb_set = cb_env.collection.couchbase_set(key) + assert isinstance(cb_set, CouchbaseSet) + + rv = cb_set.add(123) + rv = cb_set.add(123) + assert 1 == cb_set.size() + assert cb_set.contains(123) is True + + rv = cb_set.remove(123) + assert 0 == cb_set.size() + rv = cb_set.remove(123) + assert rv is None + assert cb_set.contains(123) is False + + cb_set.add(1) + cb_set.add(2) + cb_set.add(3) + cb_set.add(4) + + values = cb_set.values() + assert values == [1, 2, 3, 4] + cb_set.clear() + assert 0 == cb_set.size() + + def test_queue(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + cb_queue = cb_env.collection.couchbase_queue(key) + assert isinstance(cb_queue, CouchbaseQueue) + + cb_queue.push(1) + cb_queue.push(2) + cb_queue.push(3) + + # Pop the items now + assert cb_queue.pop() == 1 + assert cb_queue.pop() == 2 + assert cb_queue.pop() == 3 + with pytest.raises(QueueEmpty): + cb_queue.pop() + + cb_queue.push(1) + cb_queue.push(2) + cb_queue.push(3) + + assert cb_queue.size() == 3 + + expected = [3, 2, 1] + for idx, v in enumerate(cb_queue): + assert expected[idx] == v + + cb_queue.clear() + + assert 0 == cb_queue.size() + + +class LegacyDatastructuresTestSuite: + + TEST_MANIFEST = [ + 'test_list', + 'test_map', + 'test_queue', + 'test_sets', + ] + + def test_list(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.list_append(key, 'world') + + cb_env.collection.list_append(key, 'world', create=True) + rv = cb_env.collection.list_get(key, 0) + assert str(rv) == 'world' + + cb_env.collection.list_prepend(key, 'hello') + rv = cb_env.collection.list_get(key, 0) + assert str(rv) == 'hello' + rv = cb_env.collection.list_get(key, 1) + assert str(rv) == 'world' + assert 2 == cb_env.collection.list_size(key) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.list_get('not-a-key', 0) + + cb_env.collection.list_remove(key, 1) + res = cb_env.collection.get(key) + assert ['hello'] == res.content_as[list] + + cb_env.collection.list_append(key, 'world') + res = cb_env.collection.get(key) + assert ['hello', 'world'] == res.content_as[list] + + cb_env.collection.list_set(key, 1, 'after') + res = cb_env.collection.get(key) + assert ['hello', 'after'] == res.content_as[list] + + def test_map(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.map_add(key, 'key1', 'val1') + + cb_env.collection.map_add(key, 'key1', 'val1', create=True) + + rv = cb_env.collection.map_get(key, 'key1') + assert rv == 'val1' + + assert 1 == cb_env.collection.map_size(key) + + with pytest.raises(IndexError): + cb_env.collection.map_remove(key, 'key2') + + cb_env.collection.map_remove(key, 'key1') + assert 0 == cb_env.collection.map_size(key) + + def test_sets(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.set_add(key, 123) + + rv = cb_env.collection.set_add(key, 123, create=True) + assert isinstance(rv, OperationResult) + rv = cb_env.collection.set_add(key, 123) + assert rv is None + assert 1 == cb_env.collection.set_size(key) + assert cb_env.collection.set_contains(key, 123) is True + + rv = cb_env.collection.set_remove(key, 123) + assert 0 == cb_env.collection.set_size(key) + rv = cb_env.collection.set_remove(key, 123) + assert rv is None + assert cb_env.collection.set_contains(key, 123) is False + + def test_queue(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.queue_push(key, 1) + + rv = cb_env.collection.queue_push(key, 1, create=True) + assert isinstance(rv, OperationResult) + cb_env.collection.queue_push(key, 2) + cb_env.collection.queue_push(key, 3) + + # Pop the items now + assert cb_env.collection.queue_pop(key) == 1 + assert cb_env.collection.queue_pop(key) == 2 + assert cb_env.collection.queue_pop(key) == 3 + with pytest.raises(QueueEmpty): + cb_env.collection.queue_pop(key) + + +class ClassicDatastructuresTests(DatastructuresTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicDatastructuresTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicDatastructuresTests) if valid_test_method(meth)] + compare = set(DatastructuresTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.enable_bucket_mgmt() + cb_base_env.setup(request.param, __name__) + + yield cb_base_env + + cb_base_env.teardown(request.param) + + +class ClassicLegacyDatastructuresTests(LegacyDatastructuresTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicLegacyDatastructuresTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicLegacyDatastructuresTests) if valid_test_method(meth)] + compare = set(LegacyDatastructuresTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.enable_bucket_mgmt() + cb_base_env.setup(request.param, __name__) + + yield cb_base_env + + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/durability_multi_t.py b/couchbase/tests/durability_multi_t.py new file mode 100644 index 000000000..493d3e64e --- /dev/null +++ b/couchbase/tests/durability_multi_t.py @@ -0,0 +1,374 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.diagnostics import ServiceType +from couchbase.durability import (ClientDurability, + DurabilityLevel, + PersistTo, + PersistToExtended, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import DurabilityImpossibleException +from couchbase.options import (InsertMultiOptions, + RemoveMultiOptions, + ReplaceMultiOptions, + UpsertMultiOptions) +from couchbase.result import MultiMutationResult, MutationResult +from tests.environments import CollectionType +from tests.environments.collection_multi_environment import CollectionMultiTestEnvironment +from tests.environments.test_environment import TestEnvironment +from tests.mock_server import MockServerType +from tests.test_features import EnvironmentFeatures + + +class DurabilityTestSuite: + + TEST_MANIFEST = [ + 'test_client_durable_insert_multi', + 'test_client_durable_insert_multi_fail', + 'test_client_durable_remove_multi', + 'test_client_durable_remove_multi_fail', + 'test_client_durable_replace_multi', + 'test_client_durable_replace_multi_fail', + 'test_client_durable_upsert_multi', + 'test_client_durable_upsert_multi_fail', + 'test_server_durable_insert_multi', + 'test_server_durable_insert_multi_single_node', + 'test_server_durable_remove_multi', + 'test_server_durable_remove_multi_single_node', + 'test_server_durable_replace_multi', + 'test_server_durable_replace_multi_single_node', + 'test_server_durable_upsert_multi', + 'test_server_durable_upsert_multi_single_node', + ] + + @pytest.fixture(scope='class') + def check_replicas(self, cb_env): + if cb_env.is_mock_server and cb_env.mock_server_type == MockServerType.GoCAVES: + pytest.skip('GoCaves inconstent w/ replicas') + bucket_settings = TestEnvironment.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get('num_replicas') + ping_res = cb_env.bucket.ping() + kv_endpoints = ping_res.endpoints.get(ServiceType.KeyValue, None) + if kv_endpoints is None or len(kv_endpoints) < (num_replicas + 1): + pytest.skip('Not all replicas are online') + + @pytest.fixture(scope='class') + def check_multi_node(self, num_nodes): + if num_nodes == 1: + pytest.skip('Test only for clusters with more than a single node.') + + @pytest.fixture(scope='class') + def check_single_node(self, num_nodes): + if num_nodes != 1: + pytest.skip('Test only for clusters with a single node.') + + @pytest.fixture(scope='class') + def check_sync_durability_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('sync_durability', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def num_replicas(self, cb_env): + bucket_settings = TestEnvironment.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + return num_replicas + + @pytest.fixture(scope='class') + def num_nodes(self, cb_env): + return len(cb_env.cluster._cluster_info.nodes) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_client_durable_insert_multi(self, cb_env, num_replicas): + keys_and_docs = cb_env.get_new_docs(4) + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + res = cb_env.collection.insert_multi(keys_and_docs, InsertMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_client_durable_insert_multi_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip('Too many replicas enabled.') + keys_and_docs = cb_env.get_new_docs(4) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + res = cb_env.collection.insert_multi(keys_and_docs, InsertMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), DurabilityImpossibleException), res.exceptions.values())) is True + + keys_and_docs = cb_env.get_new_docs(4) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.insert_multi(keys_and_docs, durability=durability, return_exceptions=False) + + keys_and_docs = cb_env.get_new_docs(4) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.insert_multi(keys_and_docs, InsertMultiOptions(durability=durability, + return_exceptions=False)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_client_durable_remove_multi(self, cb_env, num_replicas): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + durability = ClientDurability(persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + res = cb_env.collection.remove_multi(keys, RemoveMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify all docs were removed... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys())) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_client_durable_remove_multi_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + res = cb_env.collection.remove_multi(keys, RemoveMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), DurabilityImpossibleException), res.exceptions.values())) is True + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_client_durable_replace_multi(self, cb_env, num_replicas): + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + res = cb_env.collection.replace_multi(keys_and_docs, ReplaceMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_client_durable_replace_multi_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + res = cb_env.collection.replace_multi(keys_and_docs, ReplaceMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), DurabilityImpossibleException), res.exceptions.values())) is True + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_client_durable_upsert_multi(self, cb_env, num_replicas): + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + res = cb_env.collection.upsert_multi(keys_and_docs, UpsertMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_client_durable_upsert_multi_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + res = cb_env.collection.upsert_multi(keys_and_docs, UpsertMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), DurabilityImpossibleException), res.exceptions.values())) is True + + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.upsert_multi(keys_and_docs, durability=durability, return_exceptions=False) + + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.upsert_multi(keys_and_docs, UpsertMultiOptions(durability=durability, + return_exceptions=False)) + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_server_durable_insert_multi(self, cb_env): + keys_and_docs = cb_env.get_new_docs(4) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = cb_env.collection.insert_multi(keys_and_docs, InsertMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_replicas') + def test_server_durable_insert_multi_single_node(self, cb_env): + keys_and_docs = cb_env.get_new_docs(4) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = cb_env.collection.insert_multi(keys_and_docs, InsertMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), DurabilityImpossibleException), res.exceptions.values())) is True + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_server_durable_remove_multi(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = cb_env.collection.remove_multi(keys, RemoveMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + # lets verify all docs were removed... + TestEnvironment.try_n_times(5, 3, cb_env.check_all_not_found, cb_env, list(keys_and_docs.keys())) + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_replicas') + def test_server_durable_remove_multi_single_node(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + keys = list(keys_and_docs.keys()) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = cb_env.collection.remove_multi(keys, RemoveMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), DurabilityImpossibleException), res.exceptions.values())) is True + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_server_durable_replace_multi(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = cb_env.collection.replace_multi(keys_and_docs, ReplaceMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_replicas') + def test_server_durable_replace_multi_single_node(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = cb_env.collection.replace_multi(keys_and_docs, ReplaceMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), DurabilityImpossibleException), res.exceptions.values())) is True + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_replicas') + def test_server_durable_upsert_multi(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = cb_env.collection.upsert_multi(keys_and_docs, UpsertMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is True + assert isinstance(res.results, dict) + assert res.exceptions == {} + assert all(map(lambda r: isinstance(r, MutationResult), res.results.values())) is True + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_replicas') + def test_server_durable_upsert_multi_single_node(self, cb_env): + keys_and_docs = cb_env.get_docs(4) + for _, v in keys_and_docs.items(): + v['what'] = 'An updated doc!' + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + res = cb_env.collection.upsert_multi(keys_and_docs, UpsertMultiOptions(durability=durability)) + assert isinstance(res, MultiMutationResult) + assert res.all_ok is False + assert res.results == {} + assert isinstance(res.exceptions, dict) + assert all(map(lambda e: issubclass(type(e), DurabilityImpossibleException), res.exceptions.values())) is True + + +class ClassicDurabilityTests(DurabilityTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicDurabilityTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicDurabilityTests) if valid_test_method(meth)] + compare = set(DurabilityTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = CollectionMultiTestEnvironment.from_environment(cb_base_env) + cb_env.enable_bucket_mgmt() + cb_env.setup(request.param) + + yield cb_env + + cb_env.teardown(request.param) diff --git a/couchbase/tests/durability_t.py b/couchbase/tests/durability_t.py new file mode 100644 index 000000000..61ed4c439 --- /dev/null +++ b/couchbase/tests/durability_t.py @@ -0,0 +1,382 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +import couchbase.subdocument as SD +from couchbase.durability import (ClientDurability, + DurabilityLevel, + PersistTo, + PersistToExtended, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import DocumentNotFoundException, DurabilityImpossibleException +from couchbase.options import (InsertOptions, + MutateInOptions, + RemoveOptions, + ReplaceOptions, + UpsertOptions) +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class DurabilityTestSuite: + + TEST_MANIFEST = [ + 'test_client_durable_insert', + 'test_client_durable_insert_fail', + 'test_client_durable_mutate_in', + 'test_client_durable_mutate_in_fail', + 'test_client_durable_remove', + 'test_client_durable_remove_fail', + 'test_client_durable_replace', + 'test_client_durable_replace_fail', + 'test_client_durable_upsert', + 'test_client_durable_upsert_fail', + 'test_client_durable_upsert_single_node', + 'test_client_persist_to_extended', + 'test_server_durable_insert', + 'test_server_durable_insert_single_node', + 'test_server_durable_mutate_in', + 'test_server_durable_mutate_in_single_node', + 'test_server_durable_remove', + 'test_server_durable_remove_single_node', + 'test_server_durable_replace', + 'test_server_durable_replace_single_node', + 'test_server_durable_upsert', + 'test_server_durable_upsert_single_node', + ] + + @pytest.fixture(scope='class') + def check_has_replicas(self, num_replicas): + if num_replicas == 0: + pytest.skip('No replicas to test durability.') + + @pytest.fixture(scope='class') + def check_multi_node(self, num_nodes): + if num_nodes == 1: + pytest.skip('Test only for clusters with more than a single node.') + + @pytest.fixture(scope='class') + def check_single_node(self, num_nodes): + if num_nodes != 1: + pytest.skip('Test only for clusters with a single node.') + + @pytest.fixture(scope='class') + def check_sync_durability_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('sync_durability', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def num_replicas(self, cb_env): + bucket_settings = TestEnvironment.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + return num_replicas + + @pytest.fixture(scope='class') + def num_nodes(self, cb_env): + return len(cb_env.cluster._cluster_info.nodes) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_insert(self, cb_env, num_replicas): + key, value = cb_env.get_new_doc() + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + cb_env.collection.insert(key, value, InsertOptions(durability=durability)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_insert_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip('Too many replicas enabled.') + key, value = cb_env.get_new_doc() + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.insert(key, value, InsertOptions(durability=durability)) + + # @TODO: why DurabilityImpossibleException not raised? + # @pytest.mark.usefixtures('check_multi_node') + # @pytest.mark.usefixtures('check_has_replicas') + # def test_client_durable_insert_single_node(self, cb_env, num_replicas): + # key, value = cb_env.get_new_doc() + + # durability = ClientDurability( + # persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + # with pytest.raises(DurabilityImpossibleException): + # cb_env.collection.insert(key, value, InsertOptions(durability=durability)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_mutate_in(self, cb_env, num_replicas): + key, value = cb_env.get_existing_doc() + value['make'] = 'New Make' + value['model'] = 'New Model' + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + cb_env.collection.mutate_in(key, + (SD.upsert('make', 'New Make'), SD.replace('model', 'New Model')), + MutateInOptions(durability=durability)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_mutate_in_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip('Too many replicas enabled.') + key = cb_env.get_existing_doc(key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.mutate_in(key, (SD.upsert('make', 'New Make'),), MutateInOptions(durability=durability)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_remove(self, cb_env, num_replicas): + key = cb_env.get_existing_doc(key_only=True) + durability = ClientDurability(persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + cb_env.collection.remove(key, RemoveOptions(durability=durability)) + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_remove_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + key = cb_env.get_existing_doc(key_only=True) + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.remove(key, RemoveOptions(durability=durability)) + + # @TODO: why DurabilityImpossibleException not raised? + # @pytest.mark.usefixtures('check_multi_node') + # @pytest.mark.usefixtures('check_has_replicas') + # def test_client_durable_remove_single_node(self, cb_env, num_replicas): + # key = cb_env.get_existing_doc(key_only=True) + + # durability = ClientDurability( + # persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + # with pytest.raises(DurabilityImpossibleException): + # cb_env.collection.remove(key, RemoveOptions(durability=durability)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_replace(self, cb_env, num_replicas): + key, value = cb_env.get_existing_doc() + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + cb_env.collection.replace(key, value, ReplaceOptions(durability=durability)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_replace_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + key, value = cb_env.get_existing_doc() + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.replace(key, value, ReplaceOptions(durability=durability)) + + # @TODO: why DurabilityImpossibleException not raised? + # @pytest.mark.usefixtures('check_multi_node') + # @pytest.mark.usefixtures('check_has_replicas') + # def test_client_durable_replace_single_node(self, cb_env, num_replicas): + # key, value = cb_env.get_existing_doc() + + # durability = ClientDurability( + # persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + + # with pytest.raises(DurabilityImpossibleException): + # cb_env.collection.replace(key, value, ReplaceOptions(durability=durability)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_upsert(self, cb_env, num_replicas): + key, value = cb_env.get_existing_doc() + durability = ClientDurability( + persist_to=PersistTo.ONE, replicate_to=ReplicateTo(num_replicas)) + cb_env.collection.upsert(key, value, + UpsertOptions(durability=durability), timeout=timedelta(seconds=3)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_upsert_fail(self, cb_env, num_replicas): + if num_replicas > 2: + pytest.skip("Too many replicas enabled.") + key, value = cb_env.get_existing_doc() + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.upsert(key, value, UpsertOptions(durability=durability)) + + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_client_durable_upsert_single_node(self, cb_env, num_replicas): + key, value = cb_env.get_existing_doc() + durability = ClientDurability( + persist_to=PersistToExtended.FOUR, replicate_to=ReplicateTo(num_replicas)) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.upsert(key, value, UpsertOptions(durability=durability)) + + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + @pytest.mark.parametrize('persist_to', [PersistToExtended.NONE, PersistToExtended.ACTIVE, PersistToExtended.ONE]) + def test_client_persist_to_extended(self, cb_env, persist_to): + key, value = cb_env.get_existing_doc() + durability = ClientDurability( + persist_to=persist_to, replicate_to=ReplicateTo.ONE) + cb_env.collection.upsert(key, value, UpsertOptions(durability=durability)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_insert(self, cb_env): + key, value = cb_env.get_new_doc() + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + cb_env.collection.insert(key, value, InsertOptions(durability=durability)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_insert_single_node(self, cb_env): + key, value = cb_env.get_new_doc() + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.insert(key, value, InsertOptions(durability=durability)) + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_mutate_in(self, cb_env): + key, value = cb_env.get_existing_doc() + value['make'] = 'New Make' + value['model'] = 'New Model' + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + cb_env.collection.mutate_in(key, + (SD.upsert('make', 'New Make'), SD.replace('model', 'New Model')), + MutateInOptions(durability=durability)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_mutate_in_single_node(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.mutate_in(key, (SD.upsert('make', 'New Make'),), MutateInOptions(durability=durability)) + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_remove(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + cb_env.collection.remove(key, RemoveOptions(durability=durability)) + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_remove_single_node(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.remove(key, RemoveOptions(durability=durability)) + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_replace(self, cb_env): + key, value = cb_env.get_existing_doc() + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + cb_env.collection.replace(key, value, ReplaceOptions(durability=durability)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_replace_single_node(self, cb_env): + key, value = cb_env.get_existing_doc() + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.replace(key, value, ReplaceOptions(durability=durability)) + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_multi_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_upsert(self, cb_env): + key, value = cb_env.get_existing_doc() + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + cb_env.collection.upsert(key, value, UpsertOptions(durability=durability)) + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('check_sync_durability_supported') + @pytest.mark.usefixtures('check_single_node') + @pytest.mark.usefixtures('check_has_replicas') + def test_server_durable_upsert_single_node(self, cb_env): + key, value = cb_env.get_existing_doc() + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + with pytest.raises(DurabilityImpossibleException): + cb_env.collection.upsert(key, value, UpsertOptions(durability=durability)) + + +class ClassicDurabilityTests(DurabilityTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicDurabilityTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicDurabilityTests) if valid_test_method(meth)] + compare = set(DurabilityTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.enable_bucket_mgmt() + cb_base_env.setup(request.param) + + yield cb_base_env + + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/eventingmgmt_t.py b/couchbase/tests/eventingmgmt_t.py new file mode 100644 index 000000000..2e0e796e6 --- /dev/null +++ b/couchbase/tests/eventingmgmt_t.py @@ -0,0 +1,594 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +from couchbase.exceptions import (CollectionAlreadyExistsException, + EventingFunctionAlreadyDeployedException, + EventingFunctionCollectionNotFoundException, + EventingFunctionNotBootstrappedException, + EventingFunctionNotDeployedException, + EventingFunctionNotFoundException, + EventingFunctionNotUnDeployedException) +from couchbase.management.collections import CollectionSpec +from couchbase.management.eventing import (EventingFunction, + EventingFunctionBucketAccess, + EventingFunctionBucketBinding, + EventingFunctionConstantBinding, + EventingFunctionDcpBoundary, + EventingFunctionDeploymentStatus, + EventingFunctionKeyspace, + EventingFunctionLanguageCompatibility, + EventingFunctionProcessingStatus, + EventingFunctionSettings, + EventingFunctionsStatus, + EventingFunctionState, + EventingFunctionStatus, + EventingFunctionUrlAuthBasic, + EventingFunctionUrlAuthBearer, + EventingFunctionUrlAuthDigest, + EventingFunctionUrlBinding, + EventingFunctionUrlNoAuth) +from couchbase.management.options import GetFunctionOptions, UpsertFunctionOptions +from tests.environments import CollectionType +from tests.environments.eventing_mgmt_environment import EventingManagementTestEnvironment +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class EventingManagementTestSuite: + TEST_MANIFEST = [ + 'test_constant_bindings', + 'test_deploy_function', + 'test_deploy_function_fail', + 'test_drop_function', + 'test_drop_function_fail', + 'test_functions_status', + 'test_get_all_functions', + 'test_get_function', + 'test_get_function_fail', + 'test_options_simple', + 'test_pause_function_fail', + 'test_pause_and_resume_function', + 'test_resume_function_fail', + 'test_undeploy_function', + 'test_undeploy_function_fail', + 'test_upsert_function', + 'test_upsert_function_fail', + 'test_url_bindings', + 'test_with_scope_and_collection', + ] + + @pytest.fixture() + def create_and_drop_eventing_function(self, cb_env): + cb_env.efm.upsert_function(cb_env.BASIC_FUNC) + yield + cb_env.efm.drop_function(cb_env.BASIC_FUNC.name) + TestEnvironment.try_n_times_till_exception(10, + 1, + cb_env.efm.get_function, + cb_env.BASIC_FUNC.name, + EventingFunctionNotFoundException) + + @pytest.fixture() + def create_eventing_function(self, cb_env): + cb_env.efm.upsert_function(cb_env.BASIC_FUNC) + + @pytest.fixture() + def drop_eventing_function(self, cb_env): + yield + cb_env.efm.drop_function(cb_env.TEST_EVT_NAME) + TestEnvironment.try_n_times_till_exception(10, + 1, + cb_env.efm.get_function, + cb_env.BASIC_FUNC.name, + EventingFunctionNotFoundException) + + @pytest.fixture() + def undeploy_and_drop_eventing_function(self, cb_env): + yield + cb_env.efm.undeploy_function(cb_env.TEST_EVT_NAME) + cb_env.wait_until_status(15, 2, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + cb_env.efm.drop_function(cb_env.TEST_EVT_NAME) + TestEnvironment.try_n_times_till_exception(10, + 1, + cb_env.efm.get_function, + cb_env.BASIC_FUNC.name, + EventingFunctionNotFoundException) + + def test_constant_bindings(self, cb_env): + # TODO: look into why timeout occurs when providing > 1 constant + # binding + local_func = EventingFunction( + 'test-evt-const-func', + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace(cb_env.EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc', + name=EventingFunctionKeyspace(cb_env.EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ], + constant_bindings=[ + EventingFunctionConstantBinding( + alias='testConstant', literal='1234'), + EventingFunctionConstantBinding( + alias='testConstant1', literal="\"another test value\"") + ] + ) + + cb_env.efm.upsert_function(local_func) + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, local_func.name) + func = TestEnvironment.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + def test_deploy_function(self, cb_env): + # deploy function -- but first verify in undeployed state + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + cb_env.efm.deploy_function(cb_env.BASIC_FUNC.name) + cb_env.wait_until_status(20, 3, EventingFunctionState.Deployed, cb_env.BASIC_FUNC.name) + func = TestEnvironment.try_n_times(5, 1, cb_env.efm.get_function, cb_env.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status has changed + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Deployed + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + def test_deploy_function_fail(self, cb_env): + with pytest.raises(EventingFunctionNotFoundException): + cb_env.efm.deploy_function("not-a-function") + + # deploy function -- but first verify in undeployed state + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + cb_env.efm.deploy_function(cb_env.BASIC_FUNC.name) + cb_env.wait_until_status(20, 3, EventingFunctionState.Deployed, cb_env.BASIC_FUNC.name) + with pytest.raises(EventingFunctionAlreadyDeployedException): + cb_env.efm.deploy_function(cb_env.BASIC_FUNC.name) + + @pytest.mark.usefixtures('create_eventing_function') + def test_drop_function(self, cb_env): + cb_env.efm.drop_function(cb_env.BASIC_FUNC.name) + TestEnvironment.try_n_times_till_exception(10, + 1, + cb_env.efm.get_function, + cb_env.BASIC_FUNC.name, + EventingFunctionNotFoundException) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + def test_drop_function_fail(self, cb_env): + with pytest.raises( + (EventingFunctionNotDeployedException, + EventingFunctionNotFoundException) + ): + cb_env.efm.drop_function("not-a-function") + + # deploy function -- but first verify in undeployed state + cb_env.wait_until_status(15, 2, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + cb_env.efm.deploy_function(cb_env.BASIC_FUNC.name) + # now, wait for it to be deployed + cb_env.wait_until_status(20, 3, EventingFunctionState.Deployed, cb_env.BASIC_FUNC.name) + + with pytest.raises(EventingFunctionNotUnDeployedException): + cb_env.efm.drop_function(cb_env.BASIC_FUNC.name) + + def test_functions_status(self, cb_env): + new_funcs = [ + EventingFunction( + 'test-evt-func-1', + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace('beer-sample'), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc1', + name=EventingFunctionKeyspace('beer-sample'), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ), + EventingFunction( + 'test-evt-func-2', + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace('beer-sample'), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_5_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc2', + name=EventingFunctionKeyspace('beer-sample'), + access=EventingFunctionBucketAccess.ReadOnly + ) + ] + ) + ] + for func in new_funcs: + cb_env.efm.upsert_function(func) + TestEnvironment.try_n_times(5, 3, cb_env.efm.get_function, func.name) + + funcs = cb_env.efm.functions_status() + assert isinstance(funcs, EventingFunctionsStatus) + for func in funcs.functions: + assert isinstance(func, EventingFunctionStatus) + + def test_get_all_functions(self, cb_env): + new_funcs = [ + EventingFunction( + 'test-evt-func-1', + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace('beer-sample'), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc1', + name=EventingFunctionKeyspace('beer-sample'), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ), + EventingFunction( + 'test-evt-func-2', + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace('beer-sample'), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_5_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc2', + name=EventingFunctionKeyspace('beer-sample'), + access=EventingFunctionBucketAccess.ReadOnly + ) + ] + ) + ] + for func in new_funcs: + cb_env.efm.upsert_function(func) + TestEnvironment.try_n_times(5, 3, cb_env.efm.get_function, func.name) + + funcs = cb_env.efm.get_all_functions() + for func in funcs: + cb_env.validate_eventing_function(func) + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + def test_get_function(self, cb_env): + func = TestEnvironment.try_n_times(5, 3, cb_env.efm.get_function, cb_env.BASIC_FUNC.name) + cb_env.validate_eventing_function(func) + + def test_get_function_fail(self, cb_env): + with pytest.raises(EventingFunctionNotFoundException): + cb_env.efm.get_function('not-a-function') + + def test_options_simple(self, cb_env): + local_func = EventingFunction( + 'test-evt-func-1', + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace('beer-sample'), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc1', + name=EventingFunctionKeyspace('beer-sample'), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + cb_env.efm.upsert_function(local_func, UpsertFunctionOptions(timeout=timedelta(seconds=20))) + func = TestEnvironment.try_n_times(5, + 3, + cb_env.efm.get_function, + local_func.name, + GetFunctionOptions(timeout=timedelta(seconds=15))) + cb_env.validate_eventing_function(func, shallow=True) + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + def test_pause_function_fail(self, cb_env): + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + + with pytest.raises(EventingFunctionNotFoundException): + cb_env.efm.pause_function('not-a-function') + + with pytest.raises(EventingFunctionNotBootstrappedException): + cb_env.efm.pause_function(cb_env.BASIC_FUNC.name) + + @pytest.mark.usefixtures('create_eventing_function') + @pytest.mark.usefixtures('undeploy_and_drop_eventing_function') + def test_pause_and_resume_function(self, cb_env): + # make sure function has been deployed + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + cb_env.efm.deploy_function(cb_env.BASIC_FUNC.name) + cb_env.wait_until_status(20, 3, EventingFunctionState.Deployed, cb_env.BASIC_FUNC.name) + # pause function - verify status is paused + cb_env.efm.pause_function(cb_env.BASIC_FUNC.name) + func = TestEnvironment.try_n_times(5, 1, cb_env.efm.get_function, cb_env.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function processing status + assert func.settings.processing_status == EventingFunctionProcessingStatus.Paused + cb_env.wait_until_status(15, 2, EventingFunctionState.Paused, cb_env.BASIC_FUNC.name) + # resume function + cb_env.efm.resume_function(cb_env.BASIC_FUNC.name) + func = TestEnvironment.try_n_times(5, 1, cb_env.efm.get_function, cb_env.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function processing status + assert func.settings.processing_status == EventingFunctionProcessingStatus.Running + # verify function state + cb_env.wait_until_status(20, 3, EventingFunctionState.Deployed, cb_env.BASIC_FUNC.name) + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + def test_resume_function_fail(self, cb_env): + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + + with pytest.raises(EventingFunctionNotFoundException): + cb_env.efm.pause_function('not-a-function') + + with pytest.raises(EventingFunctionNotBootstrappedException): + cb_env.efm.pause_function(cb_env.BASIC_FUNC.name) + + @pytest.mark.usefixtures('create_and_drop_eventing_function') + def test_undeploy_function(self, cb_env): + # deploy function -- but first verify in undeployed state + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + cb_env.efm.deploy_function(cb_env.BASIC_FUNC.name) + cb_env.wait_until_status(20, 3, EventingFunctionState.Deployed, cb_env.BASIC_FUNC.name) + func = TestEnvironment.try_n_times(5, 1, cb_env.efm.get_function, cb_env.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Deployed + # now, undeploy function + cb_env.efm.undeploy_function(cb_env.BASIC_FUNC.name) + cb_env.wait_until_status(15, 2, EventingFunctionState.Undeployed, cb_env.BASIC_FUNC.name) + func = TestEnvironment.try_n_times(5, 1, cb_env.efm.get_function, cb_env.BASIC_FUNC.name) + cb_env.validate_eventing_function(func, shallow=True) + # verify function deployement status has changed + assert func.settings.deployment_status == EventingFunctionDeploymentStatus.Undeployed + + def test_undeploy_function_fail(self, cb_env): + with pytest.raises( + (EventingFunctionNotDeployedException, + EventingFunctionNotFoundException) + ): + cb_env.efm.undeploy_function('not-a-function') + + def test_upsert_function(self, cb_env): + local_func = EventingFunction( + 'test-evt-func-1', + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace('beer-sample'), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc1', + name=EventingFunctionKeyspace('beer-sample'), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + cb_env.efm.upsert_function(local_func) + func = TestEnvironment.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func, shallow=True) + + def test_upsert_function_fail(self, cb_env): + # bad appcode + local_func = EventingFunction( + 'test-evt-func-1', + 'func OnUpdate(doc, meta) {\n log("Doc created/updated", meta.id);\n}\n\n', + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace('beer-sample'), + settings=EventingFunctionSettings.new_settings( + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_0_0 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc1', + name=EventingFunctionKeyspace('beer-sample'), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + + # @TODO: couchbase++ seg faults on this... + # with pytest.raises(EventingFunctionCompilationFailureException): + # cb_env.efm.upsert_function(local_func) + + local_func.code = cb_env.SIMPLE_EVT_CODE + local_func.source_keyspace = EventingFunctionKeyspace( + 'beer-sample', "test-scope", "test-collection" + ) + with pytest.raises(EventingFunctionCollectionNotFoundException): + cb_env.efm.upsert_function(local_func) + + def test_url_bindings(self, cb_env): + local_func = EventingFunction( + "test-evt-url-func", + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace('default'), + source_keyspace=EventingFunctionKeyspace(cb_env.EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc', + name=EventingFunctionKeyspace(cb_env.EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ], + url_bindings=[ + EventingFunctionUrlBinding( + hostname='http://localhost:5000', + alias='urlBinding1', + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlNoAuth() + ), + EventingFunctionUrlBinding( + hostname='http://localhost:5001', + alias='urlBinding2', + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthBasic('username', 'password') + ), + EventingFunctionUrlBinding( + hostname='http://localhost:5002', + alias='urlBinding3', + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthBearer( + 'IThinkTheBearerTokenIsSupposedToGoHere' + ) + ), + EventingFunctionUrlBinding( + hostname='http://localhost:5003', + alias='urlBinding4', + allow_cookies=True, + validate_ssl_certificate=False, + auth=EventingFunctionUrlAuthDigest('username', 'password') + ) + ] + ) + + cb_env.efm.upsert_function(local_func) + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, local_func.name) + func = TestEnvironment.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + def test_with_scope_and_collection(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('collections', + cb_env.server_version_short, + cb_env.mock_server_type) + + coll_name = 'test-collection-1' + collection_spec = CollectionSpec(coll_name, cb_env.TEST_SCOPE) + + # Make sure the collection is there... + TestEnvironment.try_n_times_till_exception(5, + 3, + cb_env.cm.create_collection, + collection_spec, + expected_exceptions=(CollectionAlreadyExistsException,)) + + local_func = EventingFunction( + 'test-evt-func-coll', + cb_env.SIMPLE_EVT_CODE, + cb_env.evt_version, + metadata_keyspace=EventingFunctionKeyspace( + cb_env.bucket.name, + cb_env.TEST_SCOPE, + cb_env.TEST_COLLECTION + ), + source_keyspace=EventingFunctionKeyspace( + cb_env.bucket.name, + cb_env.TEST_SCOPE, + coll_name + ), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc', + name=EventingFunctionKeyspace(cb_env.bucket.name), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + cb_env.efm.upsert_function(local_func) + cb_env.wait_until_status(10, 1, EventingFunctionState.Undeployed, local_func.name) + func = TestEnvironment.try_n_times(5, 3, cb_env.efm.get_function, local_func.name) + cb_env.validate_eventing_function(func) + + cb_env.efm.drop_function(local_func.name) + + +@pytest.mark.flaky(reruns=5, reruns_delay=1) +class ClassicEventingManagementTests(EventingManagementTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicEventingManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicEventingManagementTests) if valid_test_method(meth)] + compare = set(EventingManagementTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env, test_manifest_validated): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = EventingManagementTestEnvironment.from_environment(cb_base_env) + cb_env.setup() + yield cb_env + cb_env.teardown() + + +@pytest.mark.flaky(reruns=5, reruns_delay=1) +class ClassicScopeEventingManagementTests(EventingManagementTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicEventingManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicEventingManagementTests) if valid_test_method(meth)] + compare = set(EventingManagementTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env, test_manifest_validated): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = EventingManagementTestEnvironment.from_environment(cb_base_env).enable_scope_eventing_mgmt() + cb_env.setup(collection_type=CollectionType.NAMED) + yield cb_env + cb_env.disable_scope_eventing_mgmt().teardown(collection_type=CollectionType.NAMED) diff --git a/couchbase/tests/exceptions_t.py b/couchbase/tests/exceptions_t.py new file mode 100644 index 000000000..9d9e1c7ad --- /dev/null +++ b/couchbase/tests/exceptions_t.py @@ -0,0 +1,79 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +import pytest + +import couchbase.exceptions as E +from tests.environments import CollectionType + + +class ExceptionTestSuite: + TEST_MANIFEST = [ + 'test_exceptions_create_only_message', + ] + + @pytest.fixture(scope='class', name='cb_exceptions') + def get_couchbase_exceptions(self): + couchbase_exceptions = [] + skip_list = [ + 'CouchbaseException', + 'CryptoException', + 'EncryptionFailureException', + 'DecryptionFailureException', + 'CryptoKeyNotFoundException', + 'InvalidCryptoKeyException', + 'EncrypterNotFoundException', + 'DecrypterNotFoundException', + 'EncrypterAlreadyExistsException', + 'DecrypterAlreadyExistsException', + 'InvalidCipherTextException' + ] + for ex in dir(E): + exp = getattr(sys.modules['couchbase.exceptions'], ex) + try: + if issubclass(exp, E.CouchbaseException) and exp.__name__ not in skip_list: + couchbase_exceptions.append(exp) + except TypeError: + pass + + return couchbase_exceptions + + def test_exceptions_create_only_message(self, cb_exceptions): + for ex in cb_exceptions: + new_ex = ex('This is a test message.') + ex_str = str(new_ex) + assert ex_str.startswith(f'{ex.__name__}(') + assert 'message=This is a test message.' in ex_str + + +class ClassicExceptionTests(ExceptionTestSuite): + + @pytest.fixture(scope='class', autouse=True) + def manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicExceptionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicExceptionTests) if valid_test_method(meth)] + test_list = set(ExceptionTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest not validated. Missing/extra tests: {test_list}.') + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, request): + cb_base_env.setup(request.param) + yield cb_base_env + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/importer.py b/couchbase/tests/importer.py deleted file mode 100644 index 657c7f5c4..000000000 --- a/couchbase/tests/importer.py +++ /dev/null @@ -1,105 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -""" -File which contains all the test cases. -This should be loaded after all the pre-test configuration has -been done. -""" -from __future__ import print_function -import os -import os.path - -imps = [] -testmods = [] -testclasses = [] - -for name in os.listdir(os.path.join(os.path.dirname(__file__), 'cases')): - if name.startswith('__init__'): - continue - - name, ext = os.path.splitext(name) - if ext.lower() != '.py': - continue - - imps.append(name) - -def _get_packages(): - """ - Returns a dictionary of { name: module_object } for all cases - """ - ret = {} - for modname in imps: - # print(repr(modname)) - - module = __import__('couchbase.tests.cases.'+modname, - fromlist=('couchbase', 'tests', 'cases')) - ret[modname] = module - return ret - -def _get_classes(modules): - """ - Returns an extracted dictionary of { name: test_class } as combined - from all the modules provided - """ - ret = {} - - for module in modules: - for attrname in dir(module): - attrobj = getattr(module, attrname) - - if not isinstance(attrobj, type): - continue - - from couchbase.tests.base import CouchbaseTestCase - if not issubclass(attrobj, CouchbaseTestCase): - continue - - ret[attrname] = attrobj - - return ret - - -def get_configured_classes(implconfig, implstr=None, skiplist=None): - """ - returns a tuple of (module_dict, testcase_dict) - :param implstr: A unique string to be appended to each test case - :param implconfig: An ApiConfigurationMixin to use as the mixin for - the test class. - """ - d_mods = _get_packages() - d_cases = _get_classes(d_mods.values()) - ret = {} - - if not implstr: - implstr = "_" + implconfig.factory.__name__ - - if not skiplist: - skiplist = [] - - for name, case in d_cases.items(): - if name in skiplist: - continue - - cls = type(name+implstr, (case, implconfig), {}) - ret[name+implstr] = cls - - return ret - -if __name__ == "__main__": - mods, classes = get_all() - for cls in classes.values(): - print(cls.__name__) diff --git a/couchbase/tests/kv_range_scan_t.py b/couchbase/tests/kv_range_scan_t.py new file mode 100644 index 000000000..047064f38 --- /dev/null +++ b/couchbase/tests/kv_range_scan_t.py @@ -0,0 +1,363 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from uuid import uuid4 + +import pytest + +from couchbase.exceptions import (AlreadyQueriedException, + DocumentNotFoundException, + FeatureUnavailableException, + InvalidArgumentException) +from couchbase.kv_range_scan import (PrefixScan, + RangeScan, + SamplingScan, + ScanTerm) +from couchbase.mutation_state import MutationState +from couchbase.options import ScanOptions +from couchbase.result import ScanResult, ScanResultIterable +from tests.environments import CollectionType +from tests.test_features import EnvironmentFeatures + + +class RangeScanTestSuite: + TEST_MANIFEST = [ + 'test_range_scan', + 'test_range_scan_exclusive', + 'test_range_scan_ids_only', + 'test_range_scan_default_terms', + 'test_prefix_scan', + 'test_range_scan_cancel', + 'test_sampling_scan', + 'test_sampling_scan_with_seed', + 'test_range_scan_with_batch_byte_limit', + 'test_range_scan_with_batch_item_limit', + 'test_range_scan_with_concurrency', + 'test_prefix_scan_with_batch_byte_limit', + 'test_prefix_scan_with_batch_item_limit', + 'test_prefix_scan_with_concurrency', + 'test_sampling_scan_with_batch_byte_limit', + 'test_sampling_scan_with_batch_item_limit', + 'test_sampling_scan_with_concurrency', + 'test_range_scan_with_zero_concurrency', + 'test_sampling_scan_with_zero_limit', + 'test_sampling_scan_with_negative_limit', + 'test_range_scan_feature_unavailable', + ] + + @pytest.fixture(scope='class') + def check_range_scan_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('kv_range_scan', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_range_scan_not_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_not_supported('kv_range_scan', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def test_id(self): + scan_uuid = str(uuid4()) + return scan_uuid + + @pytest.fixture(scope="class") + def test_ids(self, test_id): + scan_ids = [f'{test_id}-{i}' for i in range(100)] + return scan_ids + + @pytest.fixture(scope="class") + def test_mutation_state(self, test_ids, cb_env): + results = [] + for doc_id in test_ids: + res = cb_env.collection.upsert(doc_id, {'id': doc_id}) + results.append(res) + + return MutationState(*results) + + def _purge_temp_docs(self, cb_env, test_ids): + for doc_id in test_ids: + try: + cb_env.collection.remove(doc_id) + except DocumentNotFoundException: + pass + + def _validate_result(self, result, expected_count=0, ids_only=False, return_rows=False, from_sample=False, + skip_content_validation=False): + assert isinstance(result, ScanResultIterable) + rows = [] + for r in result: + assert isinstance(r, ScanResult) + if ids_only: + with pytest.raises(InvalidArgumentException): + r.expiry_time + with pytest.raises(InvalidArgumentException): + r.cas + with pytest.raises(InvalidArgumentException): + r.content_as[str] + else: + assert r.cas is not None + assert r.id is not None + content = r.content_as[dict] + assert content is not None + if not skip_content_validation: + assert content == {'id': r.id} + rows.append(r) + + if from_sample is True: + assert len(rows) <= expected_count + else: + assert len(rows) >= expected_count + + if return_rows: + return rows + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_range_scan(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + self._validate_result(res, 12) + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_range_scan_exclusive(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1', True), ScanTerm(f'{test_id}-2', True)) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + rows = self._validate_result(res, 10, return_rows=True) + ids = [r.id for r in rows] + assert f'{test_id}-1' not in ids + assert f'{test_id}-2' not in ids + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_range_scan_ids_only(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + self._validate_result(res, 12, ids_only=True) + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_range_scan_default_terms(self, cb_env, test_ids): + scan_type = RangeScan() + res = cb_env.collection.scan(scan_type, ids_only=True) + self._validate_result(res, len(test_ids), ids_only=True) + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_range_scan_cancel(self, cb_env, test_id, test_ids, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}')) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + rows = [] + for idx, r in enumerate(res.rows()): + if idx == 2: + res.cancel_scan() + rows.append(r) + assert len(rows) < len(test_ids) + assert len(rows) == 3 + with pytest.raises(AlreadyQueriedException): + res.rows() + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_sampling_scan(self, cb_env, test_mutation_state): + limit = 10 + scan_type = SamplingScan(limit) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=False, + consistent_with=test_mutation_state)) + self._validate_result(res, limit, ids_only=False, return_rows=False, from_sample=True, + skip_content_validation=(not cb_env.use_named_collections)) + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_sampling_scan_with_seed(self, cb_env, test_ids, test_mutation_state): + limit = 10 + scan_type = SamplingScan(limit, 50) + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + rows = self._validate_result(res, limit, ids_only=True, return_rows=True, from_sample=True, + skip_content_validation=(not cb_env.use_named_collections)) + result_ids = [] + for r in rows: + result_ids.append(r.id) + + # w/ the seed, we should get the same results + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + rows = self._validate_result(res, limit, ids_only=True, return_rows=True, from_sample=True, + skip_content_validation=(not cb_env.use_named_collections)) + compare_ids = list(map(lambda r: r.id, rows)) + assert result_ids == compare_ids + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_prefix_scan(self, cb_env, test_id, test_ids, test_mutation_state): + scan_type = PrefixScan(f'{test_id}') + res = cb_env.collection.scan(scan_type, ScanOptions(timeout=timedelta(seconds=10), + ids_only=True, + consistent_with=test_mutation_state)) + rows = self._validate_result(res, 100, ids_only=True, return_rows=True) + for r in rows: + assert r.id in test_ids + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_byte_limit', [0, 1, 25, 100]) + def test_range_scan_with_batch_byte_limit(self, cb_env, test_id, test_mutation_state, batch_byte_limit): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_byte_limit=batch_byte_limit, + consistent_with=test_mutation_state)) + self._validate_result(res, 12) + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_item_limit', [0, 1, 25, 100]) + def test_range_scan_with_batch_item_limit(self, cb_env, test_id, test_mutation_state, batch_item_limit): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_item_limit=batch_item_limit, + consistent_with=test_mutation_state)) + self._validate_result(res, 12) + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('concurrency', [1, 2, 4, 16, 64, 128]) + def test_range_scan_with_concurrency(self, cb_env, test_id, test_mutation_state, concurrency): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + concurrency=concurrency, + consistent_with=test_mutation_state)) + self._validate_result(res, 12) + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_byte_limit', [0, 1, 25, 100]) + def test_prefix_scan_with_batch_byte_limit(self, cb_env, test_id, test_mutation_state, batch_byte_limit): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_byte_limit=batch_byte_limit, + consistent_with=test_mutation_state)) + self._validate_result(res, 100) + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_item_limit', [0, 1, 25, 100]) + def test_prefix_scan_with_batch_item_limit(self, cb_env, test_id, test_mutation_state, batch_item_limit): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_item_limit=batch_item_limit, + consistent_with=test_mutation_state)) + self._validate_result(res, 100) + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('concurrency', [1, 2, 4, 16, 64, 128]) + def test_prefix_scan_with_concurrency(self, cb_env, test_id, test_mutation_state, concurrency): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + concurrency=concurrency, + consistent_with=test_mutation_state)) + self._validate_result(res, 100) + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_byte_limit', [0, 1, 25, 100]) + def test_sampling_scan_with_batch_byte_limit(self, cb_env, test_id, test_mutation_state, batch_byte_limit): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_byte_limit=batch_byte_limit, + consistent_with=test_mutation_state)) + self._validate_result(res, 100, from_sample=True, skip_content_validation=(not cb_env.use_named_collections)) + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('batch_item_limit', [0, 1, 25, 100]) + def test_sampling_scan_with_batch_item_limit(self, cb_env, test_id, test_mutation_state, batch_item_limit): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + batch_item_limit=batch_item_limit, + consistent_with=test_mutation_state)) + self._validate_result(res, 100, from_sample=True, skip_content_validation=(not cb_env.use_named_collections)) + + @pytest.mark.usefixtures('check_range_scan_supported') + @pytest.mark.parametrize('concurrency', [1, 2, 4, 16, 64, 128]) + def test_sampling_scan_with_concurrency(self, cb_env, test_id, test_mutation_state, concurrency): + scan_type = PrefixScan(test_id) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + concurrency=concurrency, + consistent_with=test_mutation_state)) + self._validate_result(res, 100, from_sample=True, skip_content_validation=(not cb_env.use_named_collections)) + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_range_scan_with_zero_concurrency(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + with pytest.raises(InvalidArgumentException): + cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + concurrency=0, + consistent_with=test_mutation_state)) + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_sampling_scan_with_zero_limit(self, cb_env, test_mutation_state): + scan_type = SamplingScan(0) + with pytest.raises(InvalidArgumentException): + cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + + @pytest.mark.usefixtures('check_range_scan_supported') + def test_sampling_scan_with_negative_limit(self, cb_env, test_mutation_state): + scan_type = SamplingScan(-10) + with pytest.raises(InvalidArgumentException): + cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + + @pytest.mark.usefixtures('check_range_scan_not_supported') + def test_range_scan_feature_unavailable(self, cb_env, test_id, test_mutation_state): + scan_type = RangeScan(ScanTerm(f'{test_id}-1'), ScanTerm(f'{test_id}-2')) + res = cb_env.collection.scan(scan_type, + ScanOptions(timeout=timedelta(seconds=10), + consistent_with=test_mutation_state)) + with pytest.raises(FeatureUnavailableException): + self._validate_result(res) + + +class ClassicRangeScanTests(RangeScanTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicRangeScanTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + + method_list = [meth for meth in dir(ClassicRangeScanTests) if valid_test_method(meth)] + compare = set(ClassicRangeScanTests.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request, test_ids): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.setup(collection_type=request.param, num_docs=0) + yield cb_base_env + self._purge_temp_docs(cb_base_env, test_ids) + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/metrics_t.py b/couchbase/tests/metrics_t.py new file mode 100644 index 000000000..b4806bb25 --- /dev/null +++ b/couchbase/tests/metrics_t.py @@ -0,0 +1,112 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.exceptions import CouchbaseException +from tests.environments.tracing_and_metrics_environment import TracingAndMetricsTestEnvironment + + +class MetricsTestSuite: + + TEST_MANIFEST = [ + 'test_custom_logging_meter_kv', + ] + + @pytest.fixture() + def skip_if_mock(self, cb_env): + if cb_env.is_mock_server: + pytest.skip('Test needs real server') + + # @TODO(jc): CXXCBC-207 + # @pytest.fixture() + # def setup_query(self, cb_env): + # cb_env.check_if_feature_supported('query_index_mgmt') + # ixm = cb_env.cluster.query_indexes() + # cb_env.try_n_times(10, 3, ixm.create_primary_index, + # cb_env.bucket.name, + # timeout=timedelta(seconds=60), + # ignore_if_exists=True) + # yield + # cb_env.try_n_times_till_exception(10, 3, + # ixm.drop_primary_index, + # cb_env.bucket.name, + # expected_exceptions=(QueryIndexNotFoundException)) + + @pytest.mark.parametrize('op', ['get', 'upsert', 'insert', 'replace', 'remove']) + def test_custom_logging_meter_kv(self, cb_env, op): + cb_env.meter.reset() + operation = getattr(cb_env.collection, op) + try: + if op == 'insert': + key, value = cb_env.get_new_doc() + operation(key, value) + elif op in ['get', 'remove']: + key = cb_env.get_existing_doc(key_only=True) + operation(key) + else: + key, value = cb_env.get_existing_doc() + operation(key, value) + except CouchbaseException: + pass + + cb_env.validate_metrics(op) + + # @TODO(jc): CXXCBC-207 + # @pytest.mark.usefixtures('skip_if_mock') + # @pytest.mark.usefixtures("setup_query") + # def test_custom_logging_meter_query(self, cb_env): + # cb_env.meter.reset() + # result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 2").execute() + # self._validate_metrics('n1ql') + + # @TODO(jc): CXXCBC-207 + # @pytest.mark.usefixtures('skip_if_mock') + # def test_custom_logging_meter_analytics(self, cb_env): + # cb_env.meter.reset() + + # @TODO(jc): CXXCBC-207 + # @pytest.mark.usefixtures('skip_if_mock') + # def test_custom_logging_meter_search(self, cb_env, default_kvp, new_kvp, op): + # cb_env.meter.reset() + + # @TODO(jc): CXXCBC-207 + # @pytest.mark.usefixtures('skip_if_mock') + # def test_custom_logging_meter_views(self, cb_env, default_kvp, new_kvp, op): + # cb_env.meter.reset() + + +class ClassicMetricsTests(MetricsTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicMetricsTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicMetricsTests) if valid_test_method(meth)] + compare = set(MetricsTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env, test_manifest_validated): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + # a new environment and cluster is created + cb_env = TracingAndMetricsTestEnvironment.from_environment(cb_base_env, + create_meter=True) + cb_env.setup(num_docs=10) + yield cb_env + cb_env.teardown() + cb_env.cluster.close() diff --git a/couchbase/tests/mutation_tokens_t.py b/couchbase/tests/mutation_tokens_t.py new file mode 100644 index 000000000..f4821fbcd --- /dev/null +++ b/couchbase/tests/mutation_tokens_t.py @@ -0,0 +1,149 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import couchbase.subdocument as SD +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment + + +class MutationTokensDisabledTestSuite: + + TEST_MANIFEST = [ + 'test_mutation_tokens_disabled_insert', + 'test_mutation_tokens_disabled_mutate_in', + 'test_mutation_tokens_disabled_remove', + 'test_mutation_tokens_disabled_replace', + 'test_mutation_tokens_disabled_upsert', + ] + + def test_mutation_tokens_disabled_insert(self, cb_env): + key, value = cb_env.get_new_doc() + result = TestEnvironment.try_n_times(5, 3, cb_env.collection.insert, key, value) + cb_env.verify_mutation_tokens_disabled(cb_env.bucket.name, result) + + def test_mutation_tokens_disabled_mutate_in(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = TestEnvironment.try_n_times(5, + 3, + cb_env.collection.mutate_in, + key, + (SD.upsert('make', 'New Make'), SD.replace('model', 'New Model'),)) + cb_env.verify_mutation_tokens_disabled(cb_env.bucket.name, result) + + def test_mutation_tokens_disabled_remove(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = TestEnvironment.try_n_times(5, 3, cb_env.collection.remove, key) + cb_env.verify_mutation_tokens_disabled(cb_env.bucket.name, result) + + def test_mutation_tokens_disabled_replace(self, cb_env): + key, value = cb_env.get_existing_doc() + result = TestEnvironment.try_n_times(5, 3, cb_env.collection.replace, key, value) + cb_env.verify_mutation_tokens_disabled(cb_env.bucket.name, result) + + def test_mutation_tokens_disabled_upsert(self, cb_env): + key, value = cb_env.get_existing_doc() + result = TestEnvironment.try_n_times(5, 3, cb_env.collection.upsert, key, value) + cb_env.verify_mutation_tokens_disabled(cb_env.bucket.name, result) + + +class MutationTokensEnabledTestSuite: + + TEST_MANIFEST = [ + 'test_mutation_tokens_enabled_insert', + 'test_mutation_tokens_enabled_mutate_in', + 'test_mutation_tokens_enabled_remove', + 'test_mutation_tokens_enabled_replace', + 'test_mutation_tokens_enabled_upsert', + ] + + def test_mutation_tokens_enabled_insert(self, cb_env): + key, value = cb_env.get_new_doc() + result = TestEnvironment.try_n_times(5, 3, cb_env.collection.insert, key, value) + cb_env.verify_mutation_tokens(cb_env.bucket.name, result) + + def test_mutation_tokens_enabled_mutate_in(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = TestEnvironment.try_n_times(5, + 3, + cb_env.collection.mutate_in, + key, + (SD.upsert('make', 'New Make'), SD.replace('model', 'New Model'),)) + cb_env.verify_mutation_tokens(cb_env.bucket.name, result) + + def test_mutation_tokens_enabled_remove(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = TestEnvironment.try_n_times(5, 3, cb_env.collection.remove, key) + cb_env.verify_mutation_tokens(cb_env.bucket.name, result) + + def test_mutation_tokens_enabled_replace(self, cb_env): + key, value = cb_env.get_existing_doc() + result = TestEnvironment.try_n_times(5, 3, cb_env.collection.replace, key, value) + cb_env.verify_mutation_tokens(cb_env.bucket.name, result) + + def test_mutation_tokens_enabled_upsert(self, cb_env): + key, value = cb_env.get_existing_doc() + result = TestEnvironment.try_n_times(5, 3, cb_env.collection.upsert, key, value) + cb_env.verify_mutation_tokens(cb_env.bucket.name, result) + + +class ClassicMutationTokensDisabledTests(MutationTokensDisabledTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicMutationTokensDisabledTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicMutationTokensDisabledTests) if valid_test_method(meth)] + compare = set(MutationTokensDisabledTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + if cb_base_env.is_mock_server: + pytest.skip('Mock server does not support disabled mutation tokens.') + + cb_env = TestEnvironment.get_environment(couchbase_config=cb_base_env.config, + data_provider=cb_base_env.data_provider, + enable_mutation_tokens=False) + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) + cb_env.cluster.close() + + +class ClassicMutationTokensEnabledTests(MutationTokensEnabledTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicMutationTokensEnabledTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicMutationTokensEnabledTests) if valid_test_method(meth)] + compare = set(MutationTokensEnabledTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.setup(request.param) + yield cb_base_env + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/query_params_t.py b/couchbase/tests/query_params_t.py new file mode 100644 index 000000000..611b013ec --- /dev/null +++ b/couchbase/tests/query_params_t.py @@ -0,0 +1,357 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +from couchbase.exceptions import InvalidArgumentException +from couchbase.mutation_state import MutationState +from couchbase.n1ql import (N1QLQuery, + QueryProfile, + QueryScanConsistency) +from couchbase.options import QueryOptions +from couchbase.result import MutationToken +from tests.environments import CollectionType + + +class QueryParamTestSuite: + TEST_MANIFEST = [ + 'test_consistent_with', + 'test_encoded_consistency', + 'test_params_adhoc', + 'test_params_base', + 'test_params_client_context_id', + 'test_params_flex_index', + 'test_params_max_parallelism', + 'test_params_metrics', + 'test_params_pipeline_batch', + 'test_params_pipeline_cap', + 'test_params_preserve_expiry', + 'test_params_profile', + 'test_params_query_context', + 'test_params_readonly', + 'test_params_scan_cap', + 'test_params_scan_consistency', + 'test_params_scan_wait', + 'test_params_serializer', + 'test_params_timeout', + 'test_params_use_replica', + ] + + @pytest.fixture(scope='class') + def base_opts(self): + return {'statement': 'SELECT * FROM default', + 'metrics': False} + + def test_consistent_with(self): + + q_str = 'SELECT * FROM default' + ms = MutationState() + mt = MutationToken(token={ + 'partition_id': 42, + 'partition_uuid': 3004, + 'sequence_number': 3, + 'bucket_name': 'default' + }) + ms._add_scanvec(mt) + q_opts = QueryOptions(consistent_with=ms) + query = N1QLQuery.create_query_object(q_str, q_opts) + + # couchbase++ will set scan_consistency, so params should be + # None, but the prop should return AT_PLUS + assert query.params.get('scan_consistency', None) is None + assert query.consistency == QueryScanConsistency.AT_PLUS + + q_mt = query.params.get('mutation_state', None) + assert isinstance(q_mt, list) + assert len(q_mt) == 1 + assert q_mt.pop() == mt.as_dict() + + # Ensure no dups + ms = MutationState() + mt1 = MutationToken(token={ + 'partition_id': 42, + 'partition_uuid': 3004, + 'sequence_number': 3, + 'bucket_name': 'default' + }) + ms._add_scanvec(mt) + ms._add_scanvec(mt1) + q_opts = QueryOptions(consistent_with=ms) + query = N1QLQuery.create_query_object(q_str, q_opts) + + assert query.params.get('scan_consistency', None) is None + assert query.consistency == QueryScanConsistency.AT_PLUS + + q_mt = query.params.get('mutation_state', None) + assert isinstance(q_mt, list) + assert len(q_mt) == 1 + assert q_mt.pop() == mt.as_dict() + + # Try with a second bucket + ms = MutationState() + mt2 = MutationToken(token={ + 'partition_id': 42, + 'partition_uuid': 3004, + 'sequence_number': 3, + 'bucket_name': 'default1' + }) + ms._add_scanvec(mt) + ms._add_scanvec(mt2) + q_opts = QueryOptions(consistent_with=ms) + query = N1QLQuery.create_query_object(q_str, q_opts) + + assert query.params.get('scan_consistency', None) is None + assert query.consistency == QueryScanConsistency.AT_PLUS + + q_mt = query.params.get('mutation_state', None) + assert isinstance(q_mt, list) + assert len(q_mt) == 2 + assert next((m for m in q_mt if m == mt2.as_dict()), None) is not None + + def test_encoded_consistency(self): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(scan_consistency=QueryScanConsistency.REQUEST_PLUS) + query = N1QLQuery.create_query_object(q_str, q_opts) + + assert query.params.get('scan_consistency', None) == QueryScanConsistency.REQUEST_PLUS.value + assert query.consistency == QueryScanConsistency.REQUEST_PLUS + + q_opts = QueryOptions(scan_consistency=QueryScanConsistency.NOT_BOUNDED) + query = N1QLQuery.create_query_object(q_str, q_opts) + + assert query.params.get('scan_consistency', None) == QueryScanConsistency.NOT_BOUNDED.value + assert query.consistency == QueryScanConsistency.NOT_BOUNDED + + # cannot set scan_consistency to AT_PLUS, need to use consistent_with to do that + with pytest.raises(InvalidArgumentException): + q_opts = QueryOptions(scan_consistency=QueryScanConsistency.AT_PLUS) + query = N1QLQuery.create_query_object(q_str, q_opts) + + def test_params_adhoc(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(adhoc=False) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['adhoc'] = False + assert query.params == exp_opts + + def test_params_base(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions() + query = N1QLQuery.create_query_object(q_str, q_opts) + assert query.params == base_opts + + def test_params_client_context_id(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(client_context_id='test-string-id') + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['client_context_id'] = 'test-string-id' + assert query.params == exp_opts + + def test_params_flex_index(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(flex_index=True) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['flex_index'] = True + assert query.params == exp_opts + + def test_params_max_parallelism(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(max_parallelism=5) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['max_parallelism'] = 5 + assert query.params == exp_opts + + def test_params_metrics(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(metrics=True) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['metrics'] = True + assert query.params == exp_opts + + def test_params_pipeline_batch(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(pipeline_batch=5) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['pipeline_batch'] = 5 + assert query.params == exp_opts + + def test_params_pipeline_cap(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(pipeline_cap=5) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['pipeline_cap'] = 5 + assert query.params == exp_opts + + def test_params_preserve_expiry(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(preserve_expiry=True) + query = N1QLQuery.create_query_object(q_str, q_opts) + assert query.params.get('preserve_expiry', None) is True + assert query.preserve_expiry is True + + q_opts = QueryOptions(preserve_expiry=False) + query = N1QLQuery.create_query_object(q_str, q_opts) + assert query.params.get('preserve_expiry', None) is False + assert query.preserve_expiry is False + + # if not set, the prop will return False, but preserve_expiry should + # not be in the params + query = N1QLQuery.create_query_object(q_str) + assert query.params.get('preserve_expiry', None) is None + assert query.preserve_expiry is False + + def test_params_use_replica(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(use_replica=True) + query = N1QLQuery.create_query_object(q_str, q_opts) + assert query.params.get('use_replica', None) is True + assert query.use_replica is True + + q_opts = QueryOptions(use_replica=False) + query = N1QLQuery.create_query_object(q_str, q_opts) + assert query.params.get('use_replica', None) is False + assert query.use_replica is False + + query = N1QLQuery.create_query_object(q_str) + assert query.params.get('use_replica', None) is None + assert query.use_replica is None + + def test_params_profile(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(profile=QueryProfile.PHASES) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['profile_mode'] = QueryProfile.PHASES.value + assert query.params == exp_opts + assert query.profile == QueryProfile.PHASES + + def test_params_query_context(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(query_context='bucket.scope') + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['query_context'] = 'bucket.scope' + assert query.params == exp_opts + + def test_params_readonly(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(read_only=True) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['readonly'] = True + assert query.params == exp_opts + + def test_params_scan_cap(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(scan_cap=5) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['scan_cap'] = 5 + assert query.params == exp_opts + + def test_params_scan_wait(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(scan_wait=timedelta(seconds=30)) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['scan_wait'] = 30000000 + assert query.params == exp_opts + + def test_params_scan_consistency(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(scan_consistency=QueryScanConsistency.REQUEST_PLUS) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['scan_consistency'] = QueryScanConsistency.REQUEST_PLUS.value + assert query.params == exp_opts + assert query.consistency == QueryScanConsistency.REQUEST_PLUS + + def test_params_serializer(self, base_opts): + q_str = 'SELECT * FROM default' + from couchbase.serializer import DefaultJsonSerializer + + # serializer + serializer = DefaultJsonSerializer() + q_opts = QueryOptions(serializer=serializer) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['serializer'] = serializer + assert query.params == exp_opts + + def test_params_timeout(self, base_opts): + q_str = 'SELECT * FROM default' + q_opts = QueryOptions(timeout=timedelta(seconds=20)) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 20000000 + assert query.params == exp_opts + + q_opts = QueryOptions(timeout=20) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 20000000 + assert query.params == exp_opts + + q_opts = QueryOptions(timeout=25.5) + query = N1QLQuery.create_query_object(q_str, q_opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 25500000 + assert query.params == exp_opts + + +class ClassicQueryParamTests(QueryParamTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicQueryParamTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicQueryParamTests) if valid_test_method(meth)] + compare = set(QueryParamTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_base_env.setup(request.param) + yield cb_base_env + cb_base_env.teardown(request.param) diff --git a/couchbase/tests/query_t.py b/couchbase/tests/query_t.py new file mode 100644 index 000000000..51fabd39e --- /dev/null +++ b/couchbase/tests/query_t.py @@ -0,0 +1,521 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading +from datetime import datetime, timedelta + +import pytest + +import couchbase.subdocument as SD +from couchbase.exceptions import (AmbiguousTimeoutException, + CouchbaseException, + KeyspaceNotFoundException, + ParsingFailedException, + QueryErrorContext, + ScopeNotFoundException) +from couchbase.mutation_state import MutationState +from couchbase.n1ql import (QueryMetaData, + QueryMetrics, + QueryProfile, + QueryStatus, + QueryWarning) +from couchbase.options import (QueryOptions, + UnsignedInt64, + UpsertOptions) +from tests.environments import CollectionType +from tests.environments.query_environment import QueryTestEnvironment +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class QueryCollectionTestSuite: + TEST_MANIFEST = [ + 'test_bad_query_context', + 'test_bad_scope_query', + 'test_cluster_query_context', + 'test_query_fully_qualified', + 'test_query_in_thread', + 'test_query_metadata', + 'test_query_ryow', + 'test_query_with_metrics', + 'test_scope_query', + 'test_scope_query_with_named_params_in_options', + 'test_scope_query_with_positional_params_in_options', + ] + + def test_bad_query_context(self, cb_env): + q_str = f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2" + # test w/ no context + with pytest.raises(KeyspaceNotFoundException): + cb_env.cluster.query(q_str).execute() + + # test w/ bad scope + q_context = f'{cb_env.bucket.name}.`fake-scope`' + with pytest.raises(ScopeNotFoundException): + cb_env.cluster.query(q_str, QueryOptions(query_context=q_context)).execute() + + def test_bad_scope_query(self, cb_env): + q_str = f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2" + q_context = f'{cb_env.bucket.name}.`fake-scope`' + with pytest.raises(ScopeNotFoundException): + cb_env.scope.query(q_str, QueryOptions(query_context=q_context)).execute() + + q_context = f'`fake-bucket`.`{cb_env.scope.name}`' + with pytest.raises(KeyspaceNotFoundException): + cb_env.scope.query(q_str, query_context=q_context).execute() + + def test_cluster_query_context(self, cb_env): + q_context = f'{cb_env.bucket.name}.{cb_env.scope.name}' + # test with QueryOptions + q_opts = QueryOptions(query_context=q_context, adhoc=True) + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2", q_opts) + cb_env.assert_rows(result, 2) + + # test with kwargs + result = cb_env.cluster.query( + f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2", query_context=q_context) + cb_env.assert_rows(result, 2) + + def test_query_fully_qualified(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM {cb_env.fqdn} LIMIT 2") + cb_env.assert_rows(result, 2) + assert result.metadata() is not None + # if adhoc is not set, it should be None + assert result._request.params.get('adhoc', None) is None + + def test_query_in_thread(self, cb_env): + results = [None] + + def run_test(scope, collection_name, assert_fn, results): + try: + result = scope.query(f"SELECT * FROM `{collection_name}` LIMIT 2") + assert_fn(result, 2) + assert result.metadata() is not None + except AssertionError: + results[0] = False + except Exception as ex: + results[0] = ex + else: + results[0] = True + + t = threading.Thread(target=run_test, args=(cb_env.scope, cb_env.collection.name, cb_env.assert_rows, results)) + t.start() + t.join() + + assert len(results) == 1 + assert results[0] is True + + def test_query_metadata(self, cb_env): + result = cb_env.scope.query(f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2") + cb_env.assert_rows(result, 2) + metadata = result.metadata() # type: QueryMetaData + for id_meth in (metadata.client_context_id, metadata.request_id): + id_res = id_meth() + fail_msg = "{} failed".format(id_meth) + assert isinstance(id_res, str), fail_msg + assert metadata.status() == QueryStatus.SUCCESS + assert isinstance(metadata.signature(), (str, dict)) + assert isinstance(metadata.warnings(), list) + + for warning in metadata.warnings(): + assert isinstance(warning, QueryWarning) + assert isinstance(warning.message(), str) + assert isinstance(warning.code(), int) + + def test_query_ryow(self, cb_env): + key, value = cb_env.get_new_doc() + result = cb_env.scope.query(f'SELECT * FROM `{cb_env.collection.name}` USE KEYS "{key}"') + cb_env.assert_rows(result, 0) + res = cb_env.collection.insert(key, value) + ms = MutationState().add_mutation_token(res.mutation_token()) + result = cb_env.scope.query(f'SELECT * FROM `{cb_env.collection.name}` USE KEYS "{key}"', + QueryOptions(consistent_with=ms)) + cb_env.assert_rows(result, 1) + + def test_query_with_metrics(self, cb_env): + initial = datetime.now() + result = cb_env.scope.query( + f"SELECT * FROM `{cb_env.collection.name}` LIMIT 1", QueryOptions(metrics=True)) + cb_env.assert_rows(result, 1) + taken = datetime.now() - initial + metadata = result.metadata() # type: QueryMetaData + assert isinstance(metadata, QueryMetaData) + metrics = metadata.metrics() + assert isinstance(metrics, QueryMetrics) + assert isinstance(metrics.elapsed_time(), timedelta) + assert metrics.elapsed_time() < taken + assert metrics.elapsed_time() > timedelta(milliseconds=0) + assert isinstance(metrics.execution_time(), timedelta) + assert metrics.execution_time() < taken + assert metrics.execution_time() > timedelta(milliseconds=0) + + expected_counts = {metrics.mutation_count: 0, + metrics.result_count: 1, + metrics.sort_count: 0, + metrics.warning_count: 0} + for method, expected in expected_counts.items(): + count_result = method() + fail_msg = "{} failed".format(method) + assert isinstance(count_result, UnsignedInt64), fail_msg + assert UnsignedInt64(expected) == count_result, fail_msg + assert metrics.result_size() > UnsignedInt64(0) + assert metrics.error_count() == UnsignedInt64(0) + assert metadata.profile() is None + + def test_scope_query(self, cb_env): + result = cb_env.scope.query(f"SELECT * FROM `{cb_env.collection.name}` LIMIT 2") + cb_env.assert_rows(result, 2) + result = cb_env.scope.query(f"SELECT * FROM {cb_env.fqdn} LIMIT 2") + + def test_scope_query_with_named_params_in_options(self, cb_env): + result = cb_env.scope.query(f"SELECT * FROM `{cb_env.collection.name}` WHERE batch LIKE $batch LIMIT 1", + QueryOptions(named_parameters={'batch': f'{cb_env.get_batch_id()}%'})) + cb_env.assert_rows(result, 1) + + def test_scope_query_with_positional_params_in_options(self, cb_env): + result = cb_env.scope.query(f"SELECT * FROM `{cb_env.collection.name}` WHERE batch LIKE $1 LIMIT 1", + QueryOptions(positional_parameters=[f'{cb_env.get_batch_id()}%'])) + cb_env.assert_rows(result, 1) + + +class QueryTestSuite: + TEST_MANIFEST = [ + 'test_bad_query', + 'test_mixed_named_parameters', + 'test_mixed_positional_parameters', + 'test_preserve_expiry', + 'test_query_error_context', + 'test_query_in_thread', + 'test_query_metadata', + 'test_query_raw_options', + 'test_query_ryow', + 'test_query_timeout', + 'test_query_with_metrics', + 'test_query_with_profile', + 'test_simple_query', + 'test_simple_query_explain', + 'test_simple_query_prepared', + 'test_simple_query_with_named_params', + 'test_simple_query_with_named_params_in_options', + 'test_simple_query_with_positional_params', + 'test_simple_query_with_positional_params_in_options', + 'test_simple_query_without_options_with_kwargs_named_params', + 'test_simple_query_without_options_with_kwargs_positional_params', + ] + + @pytest.fixture(scope='class') + def check_preserve_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('preserve_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + def test_bad_query(self, cb_env): + with pytest.raises(ParsingFailedException): + cb_env.cluster.query("I'm not N1QL!").execute() + + def test_mixed_named_parameters(self, cb_env): + batch_id = cb_env.get_batch_id() + result = cb_env.cluster.query(f'SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT 1', + QueryOptions(named_parameters={'batch': 'xgfflq'}), batch=batch_id) + cb_env.assert_rows(result, 1) + + def test_mixed_positional_parameters(self, cb_env): + # we assume that positional overrides one in the Options + result = cb_env.cluster.query(f'SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 1', + QueryOptions(positional_parameters=['xgfflq']), f'{cb_env.get_batch_id()}') + cb_env.assert_rows(result, 1) + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_preserve_expiry(self, cb_env): + key = "imakey" + content = {"a": "aaa", "b": "bbb"} + + cb_env.collection.upsert(key, content, UpsertOptions(expiry=timedelta(days=1))) + + expiry_path = "$document.exptime" + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + q_str = f"UPDATE `{cb_env.bucket.name}` AS content USE KEYS '{key}' SET content.a = 'aaaa'" + cb_env.cluster.query(q_str, QueryOptions(preserve_expiry=True)).execute() + + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + + def test_query_error_context(self, cb_env): + try: + cb_env.cluster.query("SELECT * FROM no_such_bucket").execute() + except CouchbaseException as ex: + assert isinstance(ex.context, QueryErrorContext) + assert ex.context.statement is not None + assert ex.context.first_error_code is not None + assert ex.context.first_error_message is not None + assert ex.context.client_context_id is not None + assert ex.context.response_body is not None + # @TODO: these are diff from 3.x -> 4.x + # self.assertIsNotNone(ex.context.endpoint) + # self.assertIsNotNone(ex.context.error_response_body) + + def test_query_in_thread(self, cb_env): + results = [None] + + def run_test(cluster, assert_fn, results): + try: + result = cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 2") + assert_fn(result, 2) + assert result.metadata() is not None + except AssertionError: + results[0] = False + except Exception as ex: + results[0] = ex + else: + results[0] = True + + t = threading.Thread(target=run_test, args=(cb_env.cluster, cb_env.assert_rows, results)) + t.start() + t.join() + + assert len(results) == 1 + assert results[0] is True + + def test_query_metadata(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 2") + cb_env.assert_rows(result, 2) + metadata = result.metadata() # type: QueryMetaData + for id_meth in (metadata.client_context_id, metadata.request_id): + id_res = id_meth() + fail_msg = "{} failed".format(id_meth) + assert isinstance(id_res, str), fail_msg + assert metadata.status() == QueryStatus.SUCCESS + assert isinstance(metadata.signature(), (str, dict)) + assert isinstance(metadata.warnings(), list) + + for warning in metadata.warnings(): + assert isinstance(warning, QueryWarning) + assert isinstance(warning.message(), str) + assert isinstance(warning.code(), int) + + def test_query_raw_options(self, cb_env): + # via raw, we should be able to pass any option + # if using named params, need to match full name param in query + # which is different for when we pass in name_parameters via their specific + # query option (i.e. include the $ when using raw) + batch_id = cb_env.get_batch_id() + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT $1", + QueryOptions(raw={'$batch': f'{batch_id}%', 'args': [2]})) + cb_env.assert_rows(result, 2) + + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 1", + QueryOptions(raw={'args': [f'{batch_id}%']})) + cb_env.assert_rows(result, 1) + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_query_timeout(self, cb_env): + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + # Prior to PYCBC-1521, this test would fail as each request would override the cluster level query_timeout. + # If a timeout was not provided in the request, the default 75s timeout would be used. PYCBC-1521 corrects + # this behavior so this test will pass as we are essentially forcing an AmbiguousTimeoutException because + # we are setting the cluster level query_timeout such a small value. + timeout_opts = ClusterTimeoutOptions(query_timeout=timedelta(milliseconds=1)) + cluster = Cluster.connect(f'{conn_string}', ClusterOptions(auth, timeout_options=timeout_opts)) + # don't need to do this except for older server versions + _ = cluster.bucket(f'{cb_env.bucket.name}') + q_str = f'SELECT * FROM `{cb_env.bucket.name}` LIMIT 10;' + with pytest.raises(AmbiguousTimeoutException): + cluster.query(q_str).execute() + + # If we override the timeout w/in the request the query should succeed. + rows = cluster.query(q_str, timeout=timedelta(seconds=10)).execute() + assert len(rows) > 0 + + def test_query_ryow(self, cb_env): + key, value = cb_env.get_new_doc() + q_str = f'SELECT * FROM `{cb_env.bucket.name}` USE KEYS "{key}"' + result = cb_env.cluster.query(q_str) + cb_env.assert_rows(result, 0) + res = cb_env.collection.insert(key, value) + ms = MutationState().add_mutation_token(res.mutation_token()) + result = cb_env.cluster.query(q_str, QueryOptions(consistent_with=ms)) + cb_env.assert_rows(result, 1) + + # prior to PYCBC-1477 the SDK _could_ crash w/ this this sort of MS creation + key, value = cb_env.get_new_doc() + result = cb_env.cluster.query(q_str) + cb_env.assert_rows(result, 0) + res = cb_env.collection.insert(key, value) + ms = MutationState(res) + result = cb_env.cluster.query(q_str, QueryOptions(consistent_with=ms)) + cb_env.assert_rows(result, 1) + + def test_query_with_metrics(self, cb_env): + initial = datetime.now() + result = cb_env.cluster.query( + f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 1", QueryOptions(metrics=True)) + cb_env.assert_rows(result, 1) + taken = datetime.now() - initial + metadata = result.metadata() # type: QueryMetaData + assert isinstance(metadata, QueryMetaData) + metrics = metadata.metrics() + assert isinstance(metrics, QueryMetrics) + assert isinstance(metrics.elapsed_time(), timedelta) + assert metrics.elapsed_time() < taken + assert metrics.elapsed_time() > timedelta(milliseconds=0) + assert isinstance(metrics.execution_time(), timedelta) + assert metrics.execution_time() < taken + assert metrics.execution_time() > timedelta(milliseconds=0) + + expected_counts = {metrics.mutation_count: 0, + metrics.result_count: 1, + metrics.sort_count: 0, + metrics.warning_count: 0} + for method, expected in expected_counts.items(): + count_result = method() + fail_msg = "{} failed".format(method) + assert isinstance(count_result, UnsignedInt64), fail_msg + assert UnsignedInt64(expected) == count_result, fail_msg + assert metrics.result_size() > UnsignedInt64(0) + assert metrics.error_count() == UnsignedInt64(0) + assert metadata.profile() is None + + def test_query_with_profile(self, cb_env): + result = cb_env.cluster.query( + f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 1", QueryOptions(profile=QueryProfile.TIMINGS)) + cb_env.assert_rows(result, 1) + assert result.metadata().profile() is not None + + def test_simple_query(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 2") + cb_env.assert_rows(result, 2) + assert result.metadata() is not None + # if adhoc is not set, it should be None + assert result._request.params.get('adhoc', None) is None + + def test_simple_query_explain(self, cb_env): + result = cb_env.cluster.query(f"EXPLAIN SELECT * FROM `{cb_env.bucket.name}` LIMIT 2", + QueryOptions(metrics=True)) + rows = [] + for r in result.rows(): + rows.append(r) + + assert len(rows) == 1 + assert 'plan' in rows[0] + assert result.metadata() is not None + assert result.metadata().metrics() is not None + + def test_simple_query_prepared(self, cb_env): + # @TODO(CXXCBC-174) + if cb_env.server_version_short < 6.5: + pytest.skip(f'Skipped on server versions < 6.5 (using {cb_env.server_version_short}). Pending CXXCBC-174') + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` LIMIT 2", + QueryOptions(adhoc=False, metrics=True)) + cb_env.assert_rows(result, 2) + assert result.metadata() is not None + assert result.metadata().metrics() is not None + assert result._request.params.get('adhoc', None) is False + + def test_simple_query_with_named_params(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT 2", + batch=f'{cb_env.get_batch_id()}%') + cb_env.assert_rows(result, 2) + + def test_simple_query_with_named_params_in_options(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT 1", + QueryOptions(named_parameters={'batch': f'{cb_env.get_batch_id()}%'})) + cb_env.assert_rows(result, 1) + + def test_simple_query_with_positional_params(self, cb_env): + result = cb_env.cluster.query( + f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 2", f'{cb_env.get_batch_id()}%') + cb_env.assert_rows(result, 2) + + def test_simple_query_with_positional_params_in_options(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 2", + QueryOptions(positional_parameters=[f'{cb_env.get_batch_id()}%'])) + cb_env.assert_rows(result, 2) + + def test_simple_query_without_options_with_kwargs_named_params(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $batch LIMIT 1", + named_parameters={'batch': f'{cb_env.get_batch_id()}%'}) + cb_env.assert_rows(result, 1) + + # NOTE: Ideally I'd notice a set of positional parameters in the query call, and assume they were the positional + # parameters for the query (once popping off the options if it is in there). But this seems a bit tricky so for + # now, kwargs override the corresponding value in the options, only. + def test_simple_query_without_options_with_kwargs_positional_params(self, cb_env): + result = cb_env.cluster.query(f"SELECT * FROM `{cb_env.bucket.name}` WHERE batch LIKE $1 LIMIT 1", + positional_parameters=[f'{cb_env.get_batch_id()}%']) + cb_env.assert_rows(result, 1) + + +class ClassicQueryCollectionTests(QueryCollectionTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicQueryCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicQueryCollectionTests) if valid_test_method(meth)] + compare = set(QueryCollectionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = QueryTestEnvironment.from_environment(cb_base_env) + cb_env.enable_query_mgmt() + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) + + +class ClassicQueryTests(QueryTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicQueryTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicQueryTests) if valid_test_method(meth)] + compare = set(QueryTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = QueryTestEnvironment.from_environment(cb_base_env) + cb_env.enable_query_mgmt() + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) diff --git a/couchbase/tests/querymgmt_t.py b/couchbase/tests/querymgmt_t.py new file mode 100644 index 000000000..a902a5fc7 --- /dev/null +++ b/couchbase/tests/querymgmt_t.py @@ -0,0 +1,1099 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +from couchbase.exceptions import (ParsingFailedException, + QueryIndexAlreadyExistsException, + QueryIndexNotFoundException, + WatchQueryIndexTimeoutException) +from couchbase.management.options import (CreatePrimaryQueryIndexOptions, + CreateQueryIndexOptions, + DropPrimaryQueryIndexOptions, + DropQueryIndexOptions, + WatchQueryIndexOptions) +from tests.environments import CollectionType +from tests.environments.query_index_mgmt_environment import QueryIndexManagementTestEnvironment +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class CollectionQueryIndexManagementTestSuite: + TEST_MANIFEST = [ + 'test_create_index_no_fields', + 'test_create_named_primary', + 'test_create_primary', + 'test_create_primary_ignore_if_exists', + 'test_create_primary_ignore_if_exists_kwargs', + 'test_create_secondary_index_default_coll', + 'test_create_secondary_indexes', + 'test_create_secondary_indexes_condition', + 'test_create_secondary_indexes_fields_kwarg', + 'test_create_secondary_indexes_ignore_if_exists', + 'test_deferred', + 'test_drop_primary', + 'test_drop_primary_ignore_if_not_exists', + 'test_drop_secondary_indexes', + 'test_drop_secondary_indexes_ignore_if_not_exists', + 'test_index_partition_info', + 'test_list_indexes', + 'test_watch', + ] + + @pytest.fixture() + def clear_all_indexes(self, cb_env): + cb_env.clear_all_indexes() + + @staticmethod + def supports_query_without_index(cb_env): + return EnvironmentFeatures.is_feature_supported('query_without_index', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_index_no_fields(self, cb_env): + # raises a TypeError b/c not providing fields means + # create_index() is missing a required positional param + with pytest.raises(TypeError): + cb_env.qixm.create_index('noFields') + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_named_primary(self, cb_env): + ixname = 'namedPrimary' + cb_env.qixm.create_primary_index(CreatePrimaryQueryIndexOptions(index_name=ixname, + timeout=timedelta(seconds=60))) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM {cb_env.get_fqdn()} LIMIT 1' + + cb_env.cluster.query(n1ql).execute() + # Drop the primary index + cb_env.qixm.drop_primary_index(DropPrimaryQueryIndexOptions(index_name=ixname)) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary(self, cb_env): + cb_env.qixm.create_primary_index(CreatePrimaryQueryIndexOptions(timeout=timedelta(seconds=60))) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM {cb_env.get_fqdn()} LIMIT 1' + + cb_env.cluster.query(n1ql).execute() + # Drop the primary index + cb_env.qixm.drop_primary_index() + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary_ignore_if_exists(self, cb_env): + cb_env.qixm.create_primary_index() + cb_env.qixm.create_primary_index(CreatePrimaryQueryIndexOptions(ignore_if_exists=True)) + + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_primary_index() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary_ignore_if_exists_kwargs(self, cb_env): + cb_env.qixm.create_primary_index() + cb_env.qixm.create_primary_index(ignore_if_exists=True) + + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_primary_index() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_index_default_coll(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(ixname, + keys, + timeout=timedelta(seconds=120), + ignore_if_exists=True) + + def check_index(): + indexes = cb_env.qixm.get_all_indexes() + result = next((idx for idx in indexes if idx.name == ixname), None) + assert result is not None + return result + TestEnvironment.try_n_times(10, 5, check_index) + cb_env.qixm.drop_index(ixname, timeout=timedelta(seconds=120)) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(ixname, + keys=keys, + timeout=timedelta(seconds=120)) + n1ql = "SELECT {1}, {2} FROM {0} WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.get_fqdn(), *keys) + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_condition(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + + TestEnvironment.try_n_times_till_exception(10, + 5, + cb_env.qixm.drop_index, + ixname, + expected_exceptions=(QueryIndexNotFoundException,)) + condition = '((`fld1` = 1) and (`fld2` = 2))' + cb_env.qixm.create_index(ixname, + keys, + CreateQueryIndexOptions(timeout=timedelta(days=1), + condition=condition)) + + def check_index(): + indexes = cb_env.qixm.get_all_indexes() + result = next((idx for idx in indexes if idx.name == ixname), None) + assert result is not None + return result + result = TestEnvironment.try_n_times(10, 5, check_index) + assert result.condition == condition + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_fields_kwarg(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(ixname, + fields=keys, + timeout=timedelta(seconds=120)) + n1ql = "SELECT {1}, {2} FROM {0} WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.get_fqdn(), *keys) + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_ignore_if_exists(self, cb_env): + ixname = 'ix2' + cb_env.qixm.create_index(ixname, ['hello']) + cb_env.qixm.create_index(ixname, ['hello'], ignore_if_exists=True) + cb_env.qixm.create_index(ixname, ['hello'], CreateQueryIndexOptions(ignore_if_exists=True)) + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_index(ixname, ['hello']) + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures('clear_all_indexes') + def test_deferred(self, cb_env): + # Create primary index + cb_env.qixm.create_primary_index(deferred=True) + ixs = cb_env.qixm.get_all_indexes() + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + cb_env.qixm.create_index(f'ix{n}', [f'fld{n}'], CreateQueryIndexOptions(deferred=True)) + + ixs = cb_env.qixm.get_all_indexes() + assert len(ixs) == 6 + + ix_names = list(map(lambda i: i.name, ixs)) + + cb_env.qixm.build_deferred_indexes() + cb_env.qixm.watch_indexes(ix_names, WatchQueryIndexOptions(timeout=timedelta(seconds=30))) # Should be OK + cb_env.qixm.watch_indexes(ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30), + watch_primary=True)) # Should be OK again + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.watch_indexes(['idontexist'], WatchQueryIndexOptions(timeout=timedelta(seconds=10))) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_primary(self, cb_env): + # create an index so we can drop + cb_env.qixm.create_primary_index(timeout=timedelta(seconds=60)) + + cb_env.qixm.drop_primary_index(timeout=timedelta(seconds=60)) + # this should fail now + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_primary_index() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_primary_ignore_if_not_exists(self, cb_env): + cb_env.qixm.drop_primary_index(ignore_if_not_exists=True) + cb_env.qixm.drop_primary_index(DropPrimaryQueryIndexOptions(ignore_if_not_exists=True)) + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_primary_index() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_secondary_indexes(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(ixname, keys, timeout=timedelta(seconds=120)) + + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.get_fqdn(), *keys) + + # Drop the index + cb_env.qixm.drop_index(ixname) + # Issue the query again + with pytest.raises((QueryIndexNotFoundException, ParsingFailedException)): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_secondary_indexes_ignore_if_not_exists(self, cb_env): + # Create it + ixname = 'ix2' + cb_env.qixm.create_index(ixname, ['hello']) + # Drop it + cb_env.qixm.drop_index(ixname) + cb_env.qixm.drop_index(ixname, ignore_if_not_exists=True) + cb_env.qixm.drop_index(ixname, DropQueryIndexOptions(ignore_if_not_exists=True)) + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_index(ixname) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_index_partition_info(self, cb_env): + # use query to create index w/ partition, cannot do that via manager ATM + n1ql = f'CREATE INDEX idx_fld1 ON {cb_env.get_fqdn()}(fld1) PARTITION BY HASH(fld1)' + cb_env.cluster.query(n1ql).execute() + ixs = cb_env.qixm.get_all_indexes() + idx = next((ix for ix in ixs if ix.name == "idx_fld1"), None) + assert idx is not None + assert idx.partition is not None + assert idx.partition == 'HASH(`fld1`)' + + @pytest.mark.usefixtures('clear_all_indexes') + def test_list_indexes(self, cb_env): + # start with no indexes + ixs = cb_env.qixm.get_all_indexes() + assert len(ixs) == 0 + + # Create the primary index + cb_env.qixm.create_primary_index() + ixs = cb_env.qixm.get_all_indexes() + assert len(ixs) == 1 + assert ixs[0].is_primary is True + assert ixs[0].name == '#primary' + assert ixs[0].bucket_name == cb_env.bucket.name + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures('clear_all_indexes') + def test_watch(self, cb_env): + # Create primary index + cb_env.qixm.create_primary_index(deferred=True) + ixs = cb_env.qixm.get_all_indexes() + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + defer = False + if n % 2 == 0: + defer = True + cb_env.qixm.create_index(f'ix{n}', [f'fld{n}'], deferred=defer) + + ixs = cb_env.qixm.get_all_indexes() + assert len(ixs) == 6 + # by not building deffered indexes, should timeout + with pytest.raises(WatchQueryIndexTimeoutException): + cb_env.qixm.watch_indexes([i.name for i in ixs], + WatchQueryIndexOptions(timeout=timedelta(seconds=5))) + + +class QueryIndexManagementCollectionTestSuite: + TEST_MANIFEST = [ + 'test_create_index_no_fields', + 'test_create_named_primary', + 'test_create_primary', + 'test_create_primary_ignore_if_exists', + 'test_create_primary_ignore_if_exists_kwargs', + 'test_create_secondary_index_default_coll', + 'test_create_secondary_indexes', + 'test_create_secondary_indexes_condition', + 'test_create_secondary_indexes_fields_kwarg', + 'test_create_secondary_indexes_ignore_if_exists', + 'test_deferred', + 'test_drop_primary', + 'test_drop_primary_ignore_if_not_exists', + 'test_drop_secondary_indexes', + 'test_drop_secondary_indexes_ignore_if_not_exists', + 'test_get_all_correct_collection', + 'test_index_partition_info', + 'test_list_indexes', + 'test_watch', + ] + + @pytest.fixture() + def clear_all_indexes(self, cb_env): + cb_env.clear_all_indexes(CollectionType.NAMED) + + @staticmethod + def supports_query_without_index(cb_env): + return EnvironmentFeatures.is_feature_supported('query_without_index', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_index_no_fields(self, cb_env): + # raises a TypeError b/c not providing fields means + # create_index() is missing a required positional param + with pytest.raises(TypeError): + cb_env.qixm.create_index(cb_env.bucket.name, + 'noFields', + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_named_primary(self, cb_env): + ixname = 'namedPrimary' + cb_env.qixm.create_primary_index(cb_env.bucket.name, + CreatePrimaryQueryIndexOptions(index_name=ixname, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION, + timeout=timedelta(seconds=60))) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM {cb_env.get_fqdn()} LIMIT 1' + + cb_env.cluster.query(n1ql).execute() + # Drop the primary index + cb_env.qixm.drop_primary_index(cb_env.bucket.name, + DropPrimaryQueryIndexOptions(index_name=ixname, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary(self, cb_env): + cb_env.qixm.create_primary_index(cb_env.bucket.name, + CreatePrimaryQueryIndexOptions(scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION, + timeout=timedelta(seconds=60))) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM {cb_env.get_fqdn()} LIMIT 1' + + cb_env.cluster.query(n1ql).execute() + # Drop the primary index + cb_env.qixm.drop_primary_index(cb_env.bucket.name, + DropPrimaryQueryIndexOptions(scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary_ignore_if_exists(self, cb_env): + cb_env.qixm.create_primary_index(cb_env.bucket.name, + CreatePrimaryQueryIndexOptions(scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + cb_env.qixm.create_primary_index(cb_env.bucket.name, + CreatePrimaryQueryIndexOptions(ignore_if_exists=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_primary_index(cb_env.bucket.name, + CreatePrimaryQueryIndexOptions(scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary_ignore_if_exists_kwargs(self, cb_env): + cb_env.qixm.create_primary_index(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + cb_env.qixm.create_primary_index(cb_env.bucket.name, + ignore_if_exists=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_primary_index(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_index_default_coll(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + keys=keys, + timeout=timedelta(seconds=120), + ignore_if_exists=True) + + def check_index(): + indexes = cb_env.qixm.get_all_indexes(cb_env.bucket.name, scope_name='_default') + result = next((idx for idx in indexes if idx.name == ixname), None) + assert result is not None + return result + TestEnvironment.try_n_times(10, 5, check_index) + cb_env.qixm.drop_index(cb_env.bucket.name, ixname, timeout=timedelta(seconds=120)) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + keys=keys, + timeout=timedelta(seconds=120), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + n1ql = "SELECT {1}, {2} FROM {0} WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.get_fqdn(), *keys) + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_condition(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + + TestEnvironment.try_n_times_till_exception(10, + 5, + cb_env.qixm.drop_index, + cb_env.bucket.name, + ixname, + expected_exceptions=(QueryIndexNotFoundException,), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + condition = '((`fld1` = 1) and (`fld2` = 2))' + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + keys, + CreateQueryIndexOptions(timeout=timedelta(days=1), + condition=condition, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + def check_index(): + indexes = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + result = next((idx for idx in indexes if idx.name == ixname), None) + assert result is not None + return result + result = TestEnvironment.try_n_times(10, 5, check_index) + assert result.condition == condition + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_fields_kwarg(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + fields=keys, + timeout=timedelta(seconds=120), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + n1ql = "SELECT {1}, {2} FROM {0} WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.get_fqdn(), *keys) + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_ignore_if_exists(self, cb_env): + ixname = 'ix2' + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + ['hello'], + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + ['hello'], + ignore_if_exists=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + ['hello'], + CreateQueryIndexOptions(ignore_if_exists=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + ['hello'], + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures('clear_all_indexes') + def test_deferred(self, cb_env): + # Create primary index + cb_env.qixm.create_primary_index(cb_env.bucket.name, + deferred=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + cb_env.qixm.create_index(cb_env.bucket.name, + 'ix{0}'.format(n), + ['fld{0}'.format(n)], + CreateQueryIndexOptions(deferred=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + assert len(ixs) == 6 + + ix_names = list(map(lambda i: i.name, ixs)) + + cb_env.qixm.build_deferred_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + cb_env.qixm.watch_indexes(cb_env.bucket.name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) # Should be OK + cb_env.qixm.watch_indexes(cb_env.bucket.name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30), + watch_primary=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) # Should be OK again + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.watch_indexes(cb_env.bucket.name, + ['idontexist'], + WatchQueryIndexOptions(timeout=timedelta(seconds=10), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_primary(self, cb_env): + # create an index so we can drop + cb_env.qixm.create_primary_index(cb_env.bucket.name, + timeout=timedelta(seconds=60), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + cb_env.qixm.drop_primary_index(cb_env.bucket.name, + timeout=timedelta(seconds=60), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + # this should fail now + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_primary_index(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_primary_ignore_if_not_exists(self, cb_env): + cb_env.qixm.drop_primary_index(cb_env.bucket.name, + ignore_if_not_exists=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + cb_env.qixm.drop_primary_index(cb_env.bucket.name, + DropPrimaryQueryIndexOptions(ignore_if_not_exists=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_primary_index(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_secondary_indexes(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + keys, + timeout=timedelta(seconds=120), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.get_fqdn(), *keys) + + # Drop the index + cb_env.qixm.drop_index(cb_env.bucket.name, + ixname, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + # Issue the query again + with pytest.raises((QueryIndexNotFoundException, ParsingFailedException)): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_secondary_indexes_ignore_if_not_exists(self, cb_env): + # Create it + ixname = 'ix2' + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + ['hello'], + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + # Drop it + cb_env.qixm.drop_index(cb_env.bucket.name, + ixname, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + cb_env.qixm.drop_index(cb_env.bucket.name, + ixname, + ignore_if_not_exists=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + cb_env.qixm.drop_index(cb_env.bucket.name, + ixname, + DropQueryIndexOptions(ignore_if_not_exists=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_index(cb_env.bucket.name, + ixname, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures('clear_all_indexes') + def test_get_all_correct_collection(self, cb_env): + # create some indexes in the _default scope & collection + for i in range(2): + cb_env.qixm.create_index(cb_env.bucket.name, + f'ix{i}', + [f'fld{i}'], + CreateQueryIndexOptions(deferred=True)) + + # create some indexes in the test scope & collection + for i in range(2): + cb_env.qixm.create_index(cb_env.bucket.name, + f'ix{i}', + [f'fld{i}'], + CreateQueryIndexOptions(deferred=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + # all indexes in bucket (i.e. in test and _default scopes/collections) + all_ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + + # _should_ be only indexes in test scope & collection + collection_ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + assert len(all_ixs) != len(collection_ixs) + assert len(all_ixs) == 4 + assert len(collection_ixs) == 2 + + @pytest.mark.usefixtures('clear_all_indexes') + def test_index_partition_info(self, cb_env): + # use query to create index w/ partition, cannot do that via manager ATM + n1ql = f'CREATE INDEX idx_fld1 ON {cb_env.get_fqdn()}(fld1) PARTITION BY HASH(fld1)' + cb_env.cluster.query(n1ql).execute() + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + idx = next((ix for ix in ixs if ix.name == "idx_fld1"), None) + assert idx is not None + assert idx.partition is not None + assert idx.partition == 'HASH(`fld1`)' + + @pytest.mark.usefixtures('clear_all_indexes') + def test_list_indexes(self, cb_env): + # start with no indexes + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + assert len(ixs) == 0 + + # Create the primary index + cb_env.qixm.create_primary_index(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + assert len(ixs) == 1 + assert ixs[0].is_primary is True + assert ixs[0].name == '#primary' + assert ixs[0].bucket_name == cb_env.bucket.name + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures('clear_all_indexes') + def test_watch(self, cb_env): + # Create primary index + cb_env.qixm.create_primary_index(cb_env.bucket.name, + deferred=True, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + defer = False + if n % 2 == 0: + defer = True + cb_env.qixm.create_index(cb_env.bucket.name, + 'ix{0}'.format(n), + ['fld{0}'.format(n)], + deferred=defer, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name, + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION) + assert len(ixs) == 6 + # by not building deffered indexes, should timeout + with pytest.raises(WatchQueryIndexTimeoutException): + cb_env.qixm.watch_indexes(cb_env.bucket.name, + [i.name for i in ixs], + WatchQueryIndexOptions(timeout=timedelta(seconds=5), + scope_name=cb_env.TEST_SCOPE, + collection_name=cb_env.TEST_COLLECTION)) + + +class QueryIndexManagementTestSuite: + TEST_MANIFEST = [ + 'test_create_index_no_fields', + 'test_create_named_primary', + 'test_create_primary', + 'test_create_primary_ignore_if_exists', + 'test_create_primary_ignore_if_exists_kwargs', + 'test_create_secondary_indexes', + 'test_create_secondary_indexes_condition', + 'test_create_secondary_indexes_fields_kwarg', + 'test_create_secondary_indexes_ignore_if_exists', + 'test_deferred', + 'test_drop_primary', + 'test_drop_primary_ignore_if_not_exists', + 'test_drop_secondary_indexes', + 'test_drop_secondary_indexes_ignore_if_not_exists', + 'test_index_partition_info', + 'test_list_indexes', + 'test_watch', + ] + + @pytest.fixture() + def clear_all_indexes(self, cb_env): + cb_env.clear_all_indexes() + + @staticmethod + def supports_query_without_index(cb_env): + return EnvironmentFeatures.is_feature_supported('query_without_index', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_index_no_fields(self, cb_env): + # raises a TypeError b/c not providing fields means + # create_index() is missing a required positional param + with pytest.raises(TypeError): + cb_env.qixm.create_index(cb_env.bucket.name, 'noFields') + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_named_primary(self, cb_env): + ixname = 'namedPrimary' + # Try to create a _named_ primary index + cb_env.qixm.create_primary_index(cb_env.bucket.name, + CreatePrimaryQueryIndexOptions(index_name=ixname, + timeout=timedelta(seconds=60))) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM `{cb_env.bucket.name}` LIMIT 1' + + cb_env.cluster.query(n1ql).execute() + # Drop the primary index + cb_env.qixm.drop_primary_index(cb_env.bucket.name, DropPrimaryQueryIndexOptions(index_name=ixname)) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary(self, cb_env): + cb_env.qixm.create_primary_index(cb_env.bucket.name, timeout=timedelta(seconds=60)) + + # Ensure we can issue a query + n1ql = f'SELECT * FROM `{cb_env.bucket.name}` LIMIT 1' + + cb_env.cluster.query(n1ql).execute() + # Drop the primary index + cb_env.qixm.drop_primary_index(cb_env.bucket.name) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary_ignore_if_exists(self, cb_env): + cb_env.qixm.create_primary_index(cb_env.bucket.name) + cb_env.qixm.create_primary_index(cb_env.bucket.name, + CreatePrimaryQueryIndexOptions(ignore_if_exists=True)) + + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_primary_index(cb_env.bucket.name) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_primary_ignore_if_exists_kwargs(self, cb_env): + cb_env.qixm.create_primary_index(cb_env.bucket.name) + cb_env.qixm.create_primary_index(cb_env.bucket.name, ignore_if_exists=True) + + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_primary_index(cb_env.bucket.name) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(cb_env.bucket.name, ixname, keys=keys, timeout=timedelta(seconds=120)) + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.bucket.name, *keys) + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_condition(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + + TestEnvironment.try_n_times_till_exception(10, + 5, + cb_env.qixm.drop_index, + cb_env.bucket.name, + ixname, + expected_exceptions=(QueryIndexNotFoundException,)) + condition = '((`fld1` = 1) and (`fld2` = 2))' + cb_env.qixm.create_index(cb_env.bucket.name, + ixname, + keys, + CreateQueryIndexOptions(timeout=timedelta(days=1), condition=condition)) + + def check_index(): + indexes = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + result = next((idx for idx in indexes if idx.name == ixname), None) + assert result is not None + return result + result = TestEnvironment.try_n_times(10, 5, check_index) + assert result.condition == condition + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_fields_kwarg(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(cb_env.bucket.name, ixname, fields=keys, timeout=timedelta(seconds=120)) + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.bucket.name, *keys) + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_create_secondary_indexes_ignore_if_exists(self, cb_env): + ixname = 'ix2' + cb_env.qixm.create_index(cb_env.bucket.name, ixname, ['hello']) + cb_env.qixm.create_index(cb_env.bucket.name, ixname, ['hello'], ignore_if_exists=True) + cb_env.qixm.create_index(cb_env.bucket.name, ixname, ['hello'], CreateQueryIndexOptions(ignore_if_exists=True)) + with pytest.raises(QueryIndexAlreadyExistsException): + cb_env.qixm.create_index(cb_env.bucket.name, ixname, ['hello']) + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures('clear_all_indexes') + def test_deferred(self, cb_env): + # Create primary index + cb_env.qixm.create_primary_index(cb_env.bucket.name, deferred=True) + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + cb_env.qixm.create_index(cb_env.bucket.name, + 'ix{0}'.format(n), + ['fld{0}'.format(n)], + CreateQueryIndexOptions(deferred=True)) + + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + assert len(ixs) == 6 + + ix_names = list(map(lambda i: i.name, ixs)) + + cb_env.qixm.build_deferred_indexes(cb_env.bucket.name) + cb_env.qixm.watch_indexes(cb_env.bucket.name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30))) # Should be OK + cb_env.qixm.watch_indexes(cb_env.bucket.name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=30), + watch_primary=True)) # Should be OK again + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.watch_indexes(cb_env.bucket.name, + ['idontexist'], + WatchQueryIndexOptions(timeout=timedelta(seconds=10))) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_primary(self, cb_env): + # create an index so we can drop + cb_env.qixm.create_primary_index(cb_env.bucket.name, timeout=timedelta(seconds=60)) + + cb_env.qixm.drop_primary_index(cb_env.bucket.name, timeout=timedelta(seconds=60)) + # this should fail now + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_primary_index(cb_env.bucket.name) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_primary_ignore_if_not_exists(self, cb_env): + cb_env.qixm.drop_primary_index(cb_env.bucket.name, ignore_if_not_exists=True) + cb_env.qixm.drop_primary_index(cb_env.bucket.name, DropPrimaryQueryIndexOptions(ignore_if_not_exists=True)) + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_primary_index(cb_env.bucket.name) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_secondary_indexes(self, cb_env): + ixname = 'ix2' + keys = ('fld1', 'fld2') + cb_env.qixm.create_index(cb_env.bucket.name, ixname, keys, timeout=timedelta(seconds=120)) + + n1ql = "SELECT {1}, {2} FROM `{0}` WHERE {1}=1 AND {2}=2 LIMIT 1".format(cb_env.bucket.name, *keys) + + # Drop the index + cb_env.qixm.drop_index(cb_env.bucket.name, ixname) + + if not self.supports_query_without_index(cb_env): + # Ensure we get an error when executing the query + with pytest.raises(QueryIndexNotFoundException): + cb_env.cluster.query(n1ql).execute() + + @pytest.mark.usefixtures('clear_all_indexes') + def test_drop_secondary_indexes_ignore_if_not_exists(self, cb_env): + # Create it + ixname = 'ix2' + cb_env.qixm.create_index(cb_env.bucket.name, ixname, ['hello']) + # Drop it + cb_env.qixm.drop_index(cb_env.bucket.name, ixname) + cb_env.qixm.drop_index(cb_env.bucket.name, ixname, ignore_if_not_exists=True) + cb_env.qixm.drop_index(cb_env.bucket.name, ixname, DropQueryIndexOptions(ignore_if_not_exists=True)) + with pytest.raises(QueryIndexNotFoundException): + cb_env.qixm.drop_index(cb_env.bucket.name, ixname) + + @pytest.mark.usefixtures('clear_all_indexes') + def test_index_partition_info(self, cb_env): + # use query to create index w/ partition, cannot do that via manager ATM + n1ql = 'CREATE INDEX idx_fld1 ON `{0}`(fld1) PARTITION BY HASH(fld1)'.format(cb_env.bucket.name) + cb_env.cluster.query(n1ql).execute() + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + idx = next((ix for ix in ixs if ix.name == "idx_fld1"), None) + assert idx is not None + assert idx.partition is not None + assert idx.partition == 'HASH(`fld1`)' + + @pytest.mark.usefixtures('clear_all_indexes') + def test_list_indexes(self, cb_env): + # start with no indexes + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + assert len(ixs) == 0 + + # Create the primary index + cb_env.qixm.create_primary_index(cb_env.bucket.name) + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + assert len(ixs) == 1 + assert ixs[0].is_primary is True + assert ixs[0].name == '#primary' + assert ixs[0].bucket_name == cb_env.bucket.name + + @pytest.mark.flaky(reruns=5, reruns_delay=2) + @pytest.mark.usefixtures('clear_all_indexes') + def test_watch(self, cb_env): + # Create primary index + cb_env.qixm.create_primary_index(cb_env.bucket.name, deferred=True) + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + assert len(ixs) == 1 + assert ixs[0].state == 'deferred' + + # Create a bunch of other indexes + for n in range(5): + defer = False + if n % 2 == 0: + defer = True + cb_env.qixm.create_index(cb_env.bucket.name, + 'ix{0}'.format(n), + ['fld{0}'.format(n)], + deferred=defer) + + ixs = cb_env.qixm.get_all_indexes(cb_env.bucket.name) + assert len(ixs) == 6 + # by not building deffered indexes, should timeout + with pytest.raises(WatchQueryIndexTimeoutException): + cb_env.qixm.watch_indexes(cb_env.bucket.name, + [i.name for i in ixs], + WatchQueryIndexOptions(timeout=timedelta(seconds=5))) + + +class ClassicQueryIndexManagementCollectionTests(QueryIndexManagementCollectionTestSuite): + @pytest.fixture(scope='class', autouse=True) + def manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicQueryIndexManagementCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicQueryIndexManagementCollectionTests) if valid_test_method(meth)] + test_list = set(QueryIndexManagementCollectionTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest invalid. Missing/extra test(s): {test_list}.') + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, request): + cb_env = QueryIndexManagementTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) + + +class ClassicQueryIndexManagementTests(QueryIndexManagementTestSuite): + @pytest.fixture(scope='class', autouse=True) + def manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicQueryIndexManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicQueryIndexManagementTests) if valid_test_method(meth)] + test_list = set(QueryIndexManagementTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest invalid. Missing/extra test(s): {test_list}.') + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, request): + cb_env = QueryIndexManagementTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) + + +class ClassicCollectionQueryIndexManagementTests(CollectionQueryIndexManagementTestSuite): + @pytest.fixture(scope='class', autouse=True) + def manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicCollectionQueryIndexManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicCollectionQueryIndexManagementTests) if valid_test_method(meth)] + test_list = set(CollectionQueryIndexManagementTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest invalid. Missing/extra test(s): {test_list}.') + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, request): + cb_env = QueryIndexManagementTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param, test_suite=self.__class__.__name__) + yield cb_env + cb_env.teardown(request.param) diff --git a/couchbase/tests/rate_limit_t.py b/couchbase/tests/rate_limit_t.py new file mode 100644 index 000000000..200049fa1 --- /dev/null +++ b/couchbase/tests/rate_limit_t.py @@ -0,0 +1,433 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import random +from datetime import timedelta + +import pytest + +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.exceptions import (CollectionAlreadyExistsException, + CouchbaseException, + QuotaLimitedException, + RateLimitedException) +from couchbase.management.collections import CollectionSpec +from couchbase.management.search import SearchIndex +from couchbase.options import ClusterOptions, GetOptions +from couchbase.search import SearchOptions, TermQuery +from tests.environments.rate_limit_environment import RateLimitTestEnvironment +from tests.environments.test_environment import TestEnvironment + + +class RateLimitTestSuite: + TEST_MANIFEST = [ + 'test_rate_limits', + 'test_rate_limits_collections_scopes_limits', + 'test_rate_limits_egress', + 'test_rate_limits_fts', + 'test_rate_limits_fts_scopes', + 'test_rate_limits_index_scopes', + 'test_rate_limits_ingress', + 'test_rate_limits_kv_scopes_data_size', + 'test_rate_limits_max_conns', + 'test_rate_limits_query', + ] + + @pytest.fixture() + def remove_docs(self, cb_env): + cb_env.remove_docs() + + @pytest.fixture() + def cleanup_scope_and_collection(self, cb_env): + cb_env.drop_scope() + yield + cb_env.drop_scope() + + def test_rate_limits(self, couchbase_config, cb_env): + cb_env.create_rate_limit_user(cb_env.USERNAME, + {'kv_limits': {'num_connections': 10, + 'num_ops_per_min': 10, + 'ingress_mib_per_min': 1, + 'egress_mib_per_min': 10 + } + }) + conn_string = couchbase_config.get_connection_string() + cluster = None + try: + cluster = Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(cb_env.USERNAME, 'password'))) + bucket = cluster.bucket('default') + collection = bucket.default_collection() + + cb_env.try_until_timeout(5, 10, collection.upsert, 'ratelimit', "test") + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + @pytest.mark.usefixtures('cleanup_scope_and_collection') + def test_rate_limits_collections_scopes_limits(self, cb_env): + scope_name = cb_env.RATE_LIMIT_SCOPE_NAME + cb_env.create_rate_limit_scope(scope_name, {'cluster_mgr_limits': {'num_collections': 1}}) + + collection_spec = CollectionSpec('rate-limit-collection', scope_name=scope_name) + cb_env.cm.create_collection(collection_spec) + + # verify collection exists + TestEnvironment.try_n_times_till_exception(5, + 3, + cb_env.cm.create_collection, + collection_spec, + expected_exceptions=(CollectionAlreadyExistsException,)) + + with pytest.raises(QuotaLimitedException): + collection_spec = CollectionSpec('rate-limit-collection-1', scope_name=scope_name) + cb_env.cm.create_collection(collection_spec) + + @pytest.mark.usefixtures('remove_docs') + def test_rate_limits_egress(self, couchbase_config, cb_env): + cb_env.create_rate_limit_user(cb_env.USERNAME, {'kv_limits': {'num_connections': 10, + 'num_ops_per_min': 100, + 'ingress_mib_per_min': 10, + 'egress_mib_per_min': 2} + }) + conn_string = couchbase_config.get_connection_string() + cluster = None + try: + cluster = Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(cb_env.USERNAME, 'password'))) + bucket = cluster.bucket('default') + collection = bucket.default_collection() + + doc = cb_env.random_doc_by_size(1024*512) + key = 'ratelimit-egress' + collection.upsert(key, doc) + for _ in range(3): + collection.get(key, GetOptions(timeout=timedelta(seconds=10))) + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + def test_rate_limits_fts(self, couchbase_config, cb_env): + cb_env.create_rate_limit_user(cb_env.USERNAME, { + 'fts_limits': { + 'num_queries_per_min': 1, + 'num_concurrent_requests': 10, + 'ingress_mib_per_min': 10, + 'egress_mib_per_min': 10 + } + }) + + conn_string = couchbase_config.get_connection_string() + sixm = cb_env.cluster.search_indexes() + sixm.upsert_index(SearchIndex(name='ratelimit-idx', source_name='default')) + if not cb_env.rate_limit_params.fts_indexes: + cb_env.rate_limit_params.fts_indexes = [] + cb_env.rate_limit_params.fts_indexes.append('ratelimit-idx') + + try: + cluster = Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(cb_env.USERNAME, 'password'))) + + cb_env.try_until_timeout(5, + 50, + cluster.search_query, + 'ratelimit-idx', + TermQuery('auto'), + SearchOptions(limit=1), + fts=True) + + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + finally: + sixm.drop_index('ratelimit-idx') + + @pytest.mark.usefixtures('cleanup_scope_and_collection') # noqa: C901 + def test_rate_limits_fts_scopes(self, cb_env): # noqa: C901 + scope_name = cb_env.RATE_LIMIT_SCOPE_NAME + cb_env.create_rate_limit_scope(scope_name, {'fts_limits': {'num_fts_indexes': 1}}) + + collection_spec = CollectionSpec('rate-limit-collection', scope_name=scope_name) + cb_env.cm.create_collection(collection_spec) + + # verify collection exists + TestEnvironment.try_n_times_till_exception(5, + 3, + cb_env.cm.create_collection, + collection_spec, + expected_exceptions=(CollectionAlreadyExistsException,)) + + # see beer-search-coll-index-params.json for ref + idx_name = "{}.{}".format(scope_name, collection_spec.name) + idx_params = { + 'doc_config': { + 'mode': 'scope.collection.type_field', + 'type_field': 'type' + }, + 'mapping': { + 'default_analyzer': 'standard', + 'default_datetime_parser': 'dateTimeOptional', + 'default_field': '_all', + 'default_mapping': { + 'dynamic': True, + 'enabled': False + }, + 'default_type': '_default', + 'docvalues_dynamic': True, + 'index_dynamic': True, + 'store_dynamic': False, + 'type_field': '_type', + 'types': { + idx_name: { + 'dynamic': False, + 'enabled': True + } + } + } + } + + ixm = cb_env.cluster.search_indexes() + if not cb_env.rate_limit_params.fts_indexes: + cb_env.rate_limit_params.fts_indexes = [] + with pytest.raises(QuotaLimitedException): + # random helps to avoid "Index already exist" failure + new_idx = SearchIndex(name='rate-limit-idx-{}'.format(random.randrange(0, 50)), + idx_type='fulltext-index', + source_name='default', + source_type='couchbase', + params=json.loads(json.dumps(idx_params))) + cb_env.rate_limit_params.fts_indexes.append(new_idx.name) + # try multiple times to avoid scope not w/in bucket failure + num_tries = 10 + success = False + for i in range(num_tries): + try: + ixm.upsert_index(new_idx) + success = True + except CouchbaseException: + if i < (num_tries - 1): + cb_env.sleep(3) + except Exception: + raise + if not success: + ixm.upsert_index(new_idx) + + # random helps to avoid "Index already exist" failure + new_idx = SearchIndex(name='rate-limit-idx-{}'.format(random.randrange(51, 100)), + idx_type='fulltext-index', + source_name='default', + source_type='couchbase', + params=json.loads(json.dumps(idx_params))) + cb_env.rate_limit_params.fts_indexes.append(new_idx.name) + ixm.upsert_index(new_idx) + + @pytest.mark.usefixtures('cleanup_scope_and_collection') # noqa: C901 + def test_rate_limits_index_scopes(self, cb_env): # noqa: C901 + scope_name = cb_env.RATE_LIMIT_SCOPE_NAME + cb_env.create_rate_limit_scope(scope_name, {'index_limits': {'num_indexes': 1}}) + + collection_spec = CollectionSpec('rate-limit-collection', scope_name=scope_name) + cb_env.cm.create_collection(collection_spec) + + # verify collection exists + TestEnvironment.try_n_times_till_exception(5, + 3, + cb_env.cm.create_collection, + collection_spec, + expected_exceptions=(CollectionAlreadyExistsException,)) + + # make sure query service sees the new keyspace + # drop the index and then re-create + ixm = cb_env.cluster.query_indexes() + + def create_primary_index(): + try: + ixm.create_primary_index('default', scope_name=scope_name, collection_name=collection_spec.name) + indexes = ixm.get_all_indexes('default', scope_name=scope_name, collection_name=collection_spec.name) + if len(indexes) == 0: + return False + except CouchbaseException: + return False + + return True + + count = 1 + while not create_primary_index(): + if count == 5: + raise pytest.skip('Unable to create primary index.') + + count += 1 + indexes = ixm.get_all_indexes('default', scope_name=scope_name, collection_name=collection_spec.name) + TestEnvironment.sleep(1) + if len(indexes) > 0: + break + + TestEnvironment.try_n_times(10, + 3, + ixm.drop_primary_index, + 'default', + scope_name=scope_name, + collection_name=collection_spec.name) + + scope = cb_env.bucket.scope(scope_name) + + with pytest.raises(QuotaLimitedException): + TestEnvironment.try_n_times(10, + 3, + ixm.create_primary_index, + 'default', + scope_name=scope_name, + collection_name=collection_spec.name) + + indexes = ixm.get_all_indexes('default', scope_name=scope_name, collection_name=collection_spec.name) + assert len(indexes) >= 1 + assert indexes[0].is_primary is True + assert '#primary' == indexes[0].name + assert collection_spec.name == indexes[0].collection_name + # helps to avoid "Index already exist" failure + idx_name = 'rate-limit-idx-{}'.format(random.randrange(0, 100)) + scope.query("CREATE INDEX `{}` ON `{}`(testField)".format(idx_name, collection_spec.name)).execute() + + @pytest.mark.usefixtures('remove_docs') + def test_rate_limits_ingress(self, couchbase_config, cb_env): + cb_env.create_rate_limit_user(cb_env.USERNAME, { + 'kv_limits': { + 'num_connections': 10, + 'num_ops_per_min': 100, + 'ingress_mib_per_min': 1, + 'egress_mib_per_min': 10 + } + }) + conn_string = couchbase_config.get_connection_string() + cluster = None + try: + cluster = Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(cb_env.USERNAME, 'password'))) + bucket = cluster.bucket('default') + collection = bucket.default_collection() + + doc = cb_env.random_doc_by_size(1024*512) + for _ in range(3): + collection.upsert('ratelimit-ingress', doc) + + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + @pytest.mark.usefixtures('cleanup_scope_and_collection') + def test_rate_limits_kv_scopes_data_size(self, cb_env): + scope_name = cb_env.RATE_LIMIT_SCOPE_NAME + cb_env.create_rate_limit_scope(scope_name, {'kv_limits': {'data_size': 1024*1024}}) + + collection_spec = CollectionSpec('rate-limit-collection', scope_name=scope_name) + cb_env.cm.create_collection(collection_spec) + + # verify collection exists + TestEnvironment.try_n_times_till_exception(5, + 3, + cb_env.cm.create_collection, + collection_spec, + expected_exceptions=(CollectionAlreadyExistsException,)) + + scope = cb_env.bucket.scope(scope_name) + collection = scope.collection(collection_spec.name) + + doc = cb_env.random_doc_by_size(1024*512) + with pytest.raises(QuotaLimitedException): + for _ in range(5): + collection.upsert('ratelimit-datasize', doc) + + def test_rate_limits_max_conns(self, couchbase_config, cb_env): + cb_env.create_rate_limit_user(cb_env.USERNAME, { + 'kv_limits': { + 'num_connections': 1, + 'num_ops_per_min': 100, + 'ingress_mib_per_min': 10, + 'egress_mib_per_min': 10 + } + }) + cluster = None + cluster1 = None + conn_string = couchbase_config.get_connection_string() + try: + cluster = Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(cb_env.USERNAME, 'password'))) + bucket = cluster.bucket('default') + collection = bucket.default_collection() + collection.exists('some-key') + + cluster1 = Cluster(conn_string, + ClusterOptions(PasswordAuthenticator(cb_env.USERNAME, 'password'))) + bucket1 = cluster1.bucket('default') + collection1 = bucket1.default_collection() + collection1.exists('some-key') + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + def test_rate_limits_query(self, couchbase_config, cb_env): + cb_env.create_rate_limit_user(cb_env.USERNAME, { + 'query_limits': { + 'num_queries_per_min': 1, + 'num_concurrent_requests': 10, + 'ingress_mib_per_min': 10, + 'egress_mib_per_min': 10 + } + }) + + conn_string = couchbase_config.get_connection_string() + cluster = None + qm = None + try: + cluster = Cluster.connect(conn_string, + ClusterOptions(PasswordAuthenticator(cb_env.USERNAME, 'password'))) + + qm = cluster.query_indexes() + qm.create_primary_index('default', ignore_if_exists=True) + cb_env.try_until_timeout(5, 50, cluster.query, "SELECT 'Hi there!'", query=True) + + except RateLimitedException: + pass + except Exception: + pytest.fail('Expected RateLimitedException') + + +@pytest.mark.flaky(reruns=5, reruns_delay=1) +class ClassicRateLimitTests(RateLimitTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicRateLimitTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicRateLimitTests) if valid_test_method(meth)] + compare = set(RateLimitTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env, test_manifest_validated): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = RateLimitTestEnvironment.from_environment(cb_base_env) + cb_env.setup() + yield cb_env + cb_env.teardown() diff --git a/couchbase/tests/search_params_t.py b/couchbase/tests/search_params_t.py new file mode 100644 index 000000000..fa88436ae --- /dev/null +++ b/couchbase/tests/search_params_t.py @@ -0,0 +1,1207 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from datetime import timedelta + +import pytest + +import couchbase.search as search +from couchbase.exceptions import InvalidArgumentException +from couchbase.mutation_state import MutationState +from couchbase.options import SearchOptions, VectorSearchOptions +from couchbase.result import MutationToken +from couchbase.search import (HighlightStyle, + MatchOperator, + SearchRequest) +from couchbase.vector_search import (VectorQuery, + VectorQueryCombination, + VectorSearch) +from tests.environments import CollectionType +from tests.environments.search_environment import SearchTestEnvironment + + +class SearchParamTestSuite: + TEST_MANIFEST = [ + 'test_boolean_query', + 'test_booleanfield_query', + 'test_conjunction_query', + 'test_consistent_with', + 'test_daterange_query', + 'test_disjunction_query', + 'test_docid_query', + 'test_facets', + 'test_match_all_query', + 'test_match_none_query', + 'test_match_phrase', + 'test_match_query', + 'test_numrange_query', + 'test_params_base', + 'test_params_client_context_id', + 'test_params_disable_scoring', + 'test_params_explain', + 'test_params_facets', + 'test_params_fields', + 'test_params_highlight_style', + 'test_params_highlight_style_fields', + 'test_params_include_locations', + 'test_params_limit', + 'test_params_logging', + 'test_params_scan_consistency', + 'test_params_scope_collections', + 'test_params_serializer', + 'test_params_show_request', + 'test_params_skip', + 'test_params_sort', + 'test_params_timeout', + 'test_phrase_query', + 'test_prefix_query', + 'test_raw_query', + 'test_regexp_query', + 'test_string_query', + 'test_term_search', + 'test_termrange_query', + 'test_wildcard_query', + ] + + @pytest.fixture(scope='class') + def base_query_opts(self): + return search.TermQuery('someterm'), {'metrics': True} + + def test_boolean_query(self, cb_env): + prefix_q = search.PrefixQuery('someterm', boost=2) + bool_q = search.BooleanQuery( + must=prefix_q, must_not=prefix_q, should=prefix_q) + exp = {'prefix': 'someterm', 'boost': 2.0} + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, bool_q + ) + encoded_q = cb_env.get_encoded_query(search_query) + + conjuncts = { + 'conjuncts': [exp] + } + disjuncts = { + 'disjuncts': [exp], + 'min': 1 + } + assert encoded_q['query']['must'] == conjuncts + assert encoded_q['query']['must_not'] == disjuncts + assert encoded_q['query']['should'] == disjuncts + + # Test multiple criteria in must and must_not + pq_1 = search.PrefixQuery('someterm', boost=2) + pq_2 = search.PrefixQuery('otherterm') + bool_q = search.BooleanQuery(must=[pq_1, pq_2]) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, bool_q + ) + encoded_q = cb_env.get_encoded_query(search_query) + conjuncts = { + 'conjuncts': [ + {'prefix': 'someterm', 'boost': 2.0}, + {'prefix': 'otherterm'} + ] + } + assert encoded_q['query']['must'] == conjuncts + + def test_booleanfield_query(self, cb_env): + exp_json = { + 'query': { + 'bool': True + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.BooleanFieldQuery(True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_conjunction_query(self, cb_env): + q = search.ConjunctionQuery() + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + with pytest.raises(search.NoChildrenException): + _ = cb_env.get_encoded_query(search_query) + + conjuncts = { + 'conjuncts': [{'prefix': 'somePrefix'}], + } + q.conjuncts.append(search.PrefixQuery('somePrefix')) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == conjuncts + + def test_consistent_with(self, cb_env): + q = search.TermQuery('someterm') + + ms = MutationState() + mt = MutationToken(token={ + 'partition_id': 42, + 'partition_uuid': 3004, + 'sequence_number': 3, + 'bucket_name': 'default' + }) + ms._add_scanvec(mt) + opts = SearchOptions(consistent_with=ms) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + + # couchbase++ will set scan_consistency, so params should be + # None, but the prop should return AT_PLUS + assert search_query.params.get('scan_consistency', None) is None + assert search_query.consistency == search.SearchScanConsistency.AT_PLUS + + q_mt = search_query.params.get('mutation_state', None) + assert isinstance(q_mt, list) + assert len(q_mt) == 1 + assert q_mt[0] == mt.as_dict() + + def test_daterange_query(self, cb_env): + with pytest.raises(TypeError): + q = search.DateRangeQuery() + + q = search.DateRangeQuery(end='2024-12-01') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'end': '2024-12-01'} + + q = search.DateRangeQuery(start='2024-01-01') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'start': '2024-01-01'} + + q = search.DateRangeQuery(start='2024-01-01', end='2024-12-01') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'start': '2024-01-01', 'end': '2024-12-01'} + + q = search.DateRangeQuery('', '') # Empty strings should be ok + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'start': '', 'end': ''} + + # deprecated start_inclusive & end_inclusive + q = search.DateRangeQuery('2024-01-01', '2024-12-01', start_inclusive=True, end_inclusive=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'start': '2024-01-01', + 'end': '2024-12-01', + 'inclusive_start': True, + 'inclusive_end': True} + + q = search.DateRangeQuery('2024-01-01', '2024-12-01', inclusive_start=True, inclusive_end=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'start': '2024-01-01', + 'end': '2024-12-01', + 'inclusive_start': True, + 'inclusive_end': True} + + def test_disjunction_query(self, cb_env): + q = search.DisjunctionQuery() + assert q.min == 1 + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + with pytest.raises(search.NoChildrenException): + _ = cb_env.get_encoded_query(search_query) + + disjuncts = { + 'disjuncts': [{'prefix': 'somePrefix'}], + 'min': 1 + } + q.disjuncts.append(search.PrefixQuery('somePrefix')) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == disjuncts + + with pytest.raises(InvalidArgumentException): + q.min = 0 + + q.min = 2 + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + with pytest.raises(InvalidArgumentException): + _ = cb_env.get_encoded_query(search_query) + + def test_docid_query(self, cb_env): + q = search.DocIdQuery([]) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + with pytest.raises(search.NoChildrenException): + _ = cb_env.get_encoded_query(search_query) + + exp_json = { + 'query': { + 'ids': ['foo', 'bar', 'baz'] + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + + q.ids = ['foo', 'bar', 'baz'] + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_facets(self, cb_env): + q = search.TermQuery('someterm') + + f = search.NumericFacet('numfield') + with pytest.raises(InvalidArgumentException): + f.add_range('range1') + + opts = SearchOptions() + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + with pytest.raises(InvalidArgumentException): + search_query.facets['facetName'] = f + + search_query.facets['facetName'] = f.add_range('range1', min=123, max=321) + assert 'facetName' in search_query.facets + + f = search.DateFacet('datefield') + f.add_range('r1', start='2012', end='2013') + f.add_range('r2', start='2014') + f.add_range('r3', end='2015') + opts = SearchOptions(facets={'facetName': f}) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + + exp = { + 'field': 'datefield', + 'date_ranges': [ + {'name': 'r1', 'start': '2012', 'end': '2013'}, + {'name': 'r2', 'start': '2014'}, + {'name': 'r3', 'end': '2015'} + ] + } + encoded_facets = {} + for name, facet in search_query.facets.items(): + encoded_facets[name] = facet.encodable + + assert encoded_facets['facetName'] == exp + # self.assertEqual(exp, f.encodable) + + f = search.TermFacet('termfield') + f.limit = 10 + opts = SearchOptions(facets={'facetName': f}) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + encoded_facets = {} + for name, facet in search_query.facets.items(): + encoded_facets[name] = facet.encodable + + assert encoded_facets['facetName'] == {'field': 'termfield', 'size': 10} + + def test_match_all_query(self, cb_env): + exp_json = { + 'query': { + 'match_all': None + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.MatchAllQuery() + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_match_none_query(self, cb_env): + exp_json = { + 'query': { + 'match_none': None + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.MatchNoneQuery() + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_match_phrase(self, cb_env): + exp_json = { + 'query': { + 'match_phrase': 'salty beers', + 'analyzer': 'analyzer', + 'boost': 1.5, + 'field': 'field' + }, + 'limit': 10, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + + q = search.MatchPhraseQuery('salty beers', boost=1.5, analyzer='analyzer', + field='field') + opts = search.SearchOptions(limit=10) + + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + + encoded_q = cb_env.get_encoded_query(search_query) + + assert exp_json == encoded_q + + def test_match_query(self, cb_env): + exp_json = { + 'query': { + 'match': 'salty beers', + 'analyzer': 'analyzer', + 'boost': 1.5, + 'field': 'field', + 'fuzziness': 1234, + 'prefix_length': 4, + 'operator': 'or' + }, + 'limit': 10, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + + q = search.MatchQuery('salty beers', boost=1.5, analyzer='analyzer', + field='field', fuzziness=1234, prefix_length=4, match_operator=MatchOperator.OR) + opts = search.SearchOptions(limit=10) + + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + exp_json["query"]["operator"] = "and" + + q = search.MatchQuery('salty beers', boost=1.5, analyzer='analyzer', + field='field', fuzziness=1234, prefix_length=4, match_operator=MatchOperator.AND) + opts = search.SearchOptions(limit=10) + + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_numrange_query(self, cb_env): + with pytest.raises(TypeError): + q = search.NumericRangeQuery() + + q = search.NumericRangeQuery(0, 0) # Should be OK + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 0, 'max': 0} + + q = search.NumericRangeQuery(0.1, 0.9) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 0.1, 'max': 0.9} + + q = search.NumericRangeQuery(max=0.9) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'max': 0.9} + + q = search.NumericRangeQuery(min=0.1) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 0.1} + + # deprecated min_inclusive & max_inclusive + q = search.NumericRangeQuery(0.1, 0.9, min_inclusive=True, max_inclusive=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 0.1, + 'max': 0.9, + 'inclusive_min': True, + 'inclusive_max': True} + + q = search.NumericRangeQuery(0.1, 0.9, inclusive_min=True, inclusive_max=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 0.1, + 'max': 0.9, + 'inclusive_min': True, + 'inclusive_max': True} + + def test_params_base(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions() + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + + assert search_query.params == base_opts + + def test_params_client_context_id(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(client_context_id='test-id') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['client_context_id'] = 'test-id' + assert search_query.params == exp_opts + + def test_params_disable_scoring(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(disable_scoring=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['disable_scoring'] = True + assert search_query.params == exp_opts + + def test_params_explain(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(explain=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['explain'] = True + assert search_query.params == exp_opts + + def test_params_facets(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(facets={ + 'term': search.TermFacet('somefield', limit=10), + 'dr': search.DateFacet('datefield').add_range('name', 'start', 'end'), + 'nr': search.NumericFacet('numfield').add_range('name2', 0.0, 99.99) + }) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['facets'] = { + 'term': { + 'field': 'somefield', + 'size': 10 + }, + 'dr': { + 'field': 'datefield', + 'date_ranges': [{ + 'name': 'name', + 'start': 'start', + 'end': 'end' + }] + }, + 'nr': { + 'field': 'numfield', + 'numeric_ranges': [{ + 'name': 'name2', + 'min': 0.0, + 'max': 99.99 + }] + }, + } + + params = search_query.params + # handle encoded here + encoded_facets = {} + for name, facet in search_query.facets.items(): + encoded_facets[name] = facet.encodable + params['facets'] = encoded_facets + assert params == exp_opts + + def test_params_fields(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(fields=['foo', 'bar', 'baz']) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['fields'] = ['foo', 'bar', 'baz'] + assert search_query.params == exp_opts + + def test_params_highlight_style(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(highlight_style=HighlightStyle.Html) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['highlight_style'] = HighlightStyle.Html.value + assert search_query.params == exp_opts + assert search_query.highlight_style == HighlightStyle.Html + + def test_params_highlight_style_fields(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(highlight_style=HighlightStyle.Ansi, highlight_fields=['foo', 'bar', 'baz']) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['highlight_style'] = HighlightStyle.Ansi.value + exp_opts['highlight_fields'] = ['foo', 'bar', 'baz'] + assert search_query.params == exp_opts + assert search_query.highlight_style == HighlightStyle.Ansi + + def test_params_include_locations(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(include_locations=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['include_locations'] = True + assert search_query.params == exp_opts + + def test_params_limit(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(limit=10) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['limit'] = 10 + assert search_query.params == exp_opts + + def test_params_logging(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(log_request=True, log_response=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['log_request'] = True + exp_opts['log_response'] = True + assert search_query.params == exp_opts + + def test_params_scan_consistency(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(scan_consistency=search.SearchScanConsistency.REQUEST_PLUS) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['scan_consistency'] = search.SearchScanConsistency.REQUEST_PLUS.value + assert search_query.params == exp_opts + assert search_query.consistency == search.SearchScanConsistency.REQUEST_PLUS + + def test_params_scope_collections(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(scope_name='test-scope', collections=['test-collection-1', 'test-collection-2']) + caught_warnings = [] + warnings.resetwarnings() + with warnings.catch_warnings(record=True) as ws: + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + caught_warnings = ws + exp_opts = base_opts.copy() + # We still need to check for the scope_name option until we actually remove the option. + # The option is deprecated and not sent to the C++ client, but it still in the search_query.params + # until we send the params over to the C++ client. + exp_opts['scope_name'] = 'test-scope' + exp_opts['collections'] = ['test-collection-1', 'test-collection-2'] + assert search_query.params == exp_opts + assert len(caught_warnings) == 1 + assert 'The scope_name option is not used by the search API.' in caught_warnings[0].message.args[0] + + def test_params_serializer(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + from couchbase.serializer import DefaultJsonSerializer + + # serializer + serializer = DefaultJsonSerializer() + opts = SearchOptions(serializer=serializer) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + + exp_opts = base_opts.copy() + exp_opts['serializer'] = serializer + assert search_query.params == exp_opts + + def test_params_show_request(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(show_request=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['show_request'] = True + assert search_query.params == exp_opts + + def test_params_skip(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(skip=10) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['skip'] = 10 + assert search_query.params == exp_opts + + def test_params_sort(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(sort=['f1', 'f2', '-_score']) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['sort'] = ['f1', 'f2', '-_score'] + params = search_query.params + params['sort'] = search_query.sort + assert params == exp_opts + + def test_params_timeout(self, cb_env, base_query_opts): + q, base_opts = base_query_opts + opts = SearchOptions(timeout=timedelta(seconds=20)) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + exp_opts = base_opts.copy() + exp_opts['timeout'] = int(timedelta(seconds=20).total_seconds() * 1e6) + assert search_query.params == exp_opts + + opts = SearchOptions(timeout=20) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 20000000 + assert search_query.params == exp_opts + + opts = SearchOptions(timeout=25.5) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 25500000 + assert search_query.params == exp_opts + + def test_phrase_query(self, cb_env): + exp_json = { + 'query': { + 'terms': ['salty', 'beers'] + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.PhraseQuery('salty', 'beers') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + q = search.PhraseQuery() + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + with pytest.raises(search.NoChildrenException): + _ = cb_env.get_encoded_query(search_query) + + q.terms.append('salty') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + exp_json['query']['terms'] = ['salty'] + assert exp_json == encoded_q + + def test_prefix_query(self, cb_env): + exp_json = { + 'query': { + 'prefix': 'someterm', + 'boost': 1.5 + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.PrefixQuery('someterm', boost=1.5) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_raw_query(self, cb_env): + exp_json = { + 'query': { + 'foo': 'bar' + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.RawQuery({'foo': 'bar'}) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_regexp_query(self, cb_env): + exp_json = { + 'query': { + 'regexp': 'some?regex' + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.RegexQuery('some?regex') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_string_query(self, cb_env): + exp_json = { + 'query': { + 'query': 'q*ry', + 'boost': 2.0, + }, + 'explain': True, + 'limit': 10, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.QueryStringQuery('q*ry', boost=2.0) + opts = search.SearchOptions(limit=10, explain=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_term_search(self, cb_env): + q = search.TermQuery('someterm', field='field', boost=1.5, + prefix_length=23, fuzziness=12) + opts = search.SearchOptions(explain=True) + + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q, opts + ) + + encoded_q = cb_env.get_encoded_query(search_query) + + exp_json = { + 'query': { + 'term': 'someterm', + 'boost': 1.5, + 'fuzziness': 12, + 'prefix_length': 23, + 'field': 'field' + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'explain': True, + 'metrics': True, + } + + assert exp_json == encoded_q + + def test_termrange_query(self, cb_env): + with pytest.raises(TypeError): + q = search.TermRangeQuery() + + q = search.TermRangeQuery('', '') # Should be OK + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': '', 'max': ''} + + q = search.TermRangeQuery('startTerm', 'endTerm') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 'startTerm', 'max': 'endTerm'} + + # deprecated end + q = search.TermRangeQuery(end='endTerm') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'max': 'endTerm'} + + q = search.TermRangeQuery(max='endTerm') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'max': 'endTerm'} + + # deprecated start + q = search.TermRangeQuery(start='startTerm') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 'startTerm'} + + q = search.TermRangeQuery(min='startTerm') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 'startTerm'} + + # deprecated start_inclusive & end_inclusive + q = search.TermRangeQuery('startTerm', 'endTerm', start_inclusive=True, end_inclusive=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 'startTerm', + 'max': 'endTerm', + 'inclusive_min': True, + 'inclusive_max': True} + + q = search.TermRangeQuery('startTerm', 'endTerm', inclusive_min=True, inclusive_max=True) + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert encoded_q['query'] == {'min': 'startTerm', + 'max': 'endTerm', + 'inclusive_min': True, + 'inclusive_max': True} + + def test_wildcard_query(self, cb_env): + exp_json = { + 'query': { + 'wildcard': 'f*o', + 'field': 'wc', + }, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + } + q = search.WildcardQuery('f*o', field='wc') + search_query = search.SearchQueryBuilder.create_search_query_object( + cb_env.TEST_INDEX_NAME, q + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + +class VectorSearchParamTestSuite: + TEST_VECTOR = [-0.014653487130999565, + -0.008658270351588726, + 0.017129190266132355, + -0.015563474968075752, + -0.02059517428278923, + 0.019551364704966545] + + TEST_VECTOR_BASE64 = 'SSdtIGp1c3QgYSB0ZXN0IHN0cmluZw==' + + TEST_MANIFEST = [ + 'test_search_request_invalid', + 'test_vector_query_invalid_boost', + 'test_vector_query_invalid_num_candidates', + 'test_vector_query_invalid_vector', + 'test_vector_search', + 'test_vector_search_with_prefilter', + 'test_vector_search_base64', + 'test_vector_search_invalid', + 'test_vector_search_multiple_queries' + ] + + def test_search_request_invalid(self): + with pytest.raises(InvalidArgumentException): + SearchRequest(None) + with pytest.raises(InvalidArgumentException): + SearchRequest.create(None) + + vector_search = VectorSearch.from_vector_query(VectorQuery('vector_field', self.TEST_VECTOR)) + search_query = search.MatchAllQuery() + with pytest.raises(InvalidArgumentException): + req = SearchRequest(vector_search) + req.with_search_query(vector_search) + with pytest.raises(InvalidArgumentException): + req = SearchRequest(vector_search) + req.with_vector_search(search_query) + with pytest.raises(InvalidArgumentException): + req = SearchRequest(vector_search) + req.with_vector_search(vector_search) + + with pytest.raises(InvalidArgumentException): + req = SearchRequest(search_query) + req.with_vector_search(search_query) + with pytest.raises(InvalidArgumentException): + req = SearchRequest(search_query) + req.with_search_query(search_query) + with pytest.raises(InvalidArgumentException): + req = SearchRequest(search_query) + req.with_search_query(vector_search) + + def test_vector_query_invalid_boost(self): + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', self.TEST_VECTOR, boost=1) + + def test_vector_query_invalid_num_candidates(self): + # cannot be < 1 + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', self.TEST_VECTOR, 0) + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', self.TEST_VECTOR, num_candidates=0) + # value should be int + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', self.TEST_VECTOR, 3.14159) + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', self.TEST_VECTOR, num_candidates=3.14159) + + def test_vector_query_invalid_vector(self): + # cannot be None/empty + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', []) + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', None) + # values should all be floats + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', [1]) + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', [1.111, 2, 3.14159]) + # if not a list, should be a str + with pytest.raises(InvalidArgumentException): + VectorQuery('vector_field', {}) + with pytest.raises(InvalidArgumentException): + VectorQuery(None, self.TEST_VECTOR) + with pytest.raises(InvalidArgumentException): + VectorQuery('', self.TEST_VECTOR) + + def test_vector_search(self, cb_env): + exp_json = { + 'query': {'match_none': None}, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + 'show_request': False, + 'vector_search': [ + { + 'field': 'vector_field', + 'vector': self.TEST_VECTOR, + 'k': 3 + } + ] + } + + vector_search = VectorSearch.from_vector_query(VectorQuery('vector_field', self.TEST_VECTOR)) + req = SearchRequest.create(vector_search) + search_query = search.SearchQueryBuilder.create_search_query_from_request( + cb_env.TEST_INDEX_NAME, + req + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_vector_search_with_prefilter(self, cb_env): + exp_json = { + 'query': {'match_none': None}, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + 'show_request': False, + 'vector_search': [ + { + 'field': 'vector_field', + 'vector': self.TEST_VECTOR, + 'k': 3, + 'filter': { + 'match': 'salty beers', + 'analyzer': 'analyzer', + 'boost': 1.5, + 'field': 'field', + 'fuzziness': 1234, + 'prefix_length': 4, + 'operator': 'or' + } + } + ] + } + + q = search.MatchQuery('salty beers', boost=1.5, analyzer='analyzer', + field='field', fuzziness=1234, prefix_length=4, match_operator=MatchOperator.OR) + vector_search = VectorSearch.from_vector_query(VectorQuery('vector_field', + self.TEST_VECTOR, + prefilter=q)) + req = SearchRequest.create(vector_search) + search_query = search.SearchQueryBuilder.create_search_query_from_request( + cb_env.TEST_INDEX_NAME, + req + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_vector_search_base64(self, cb_env): + exp_json = { + 'query': {'match_none': None}, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + 'show_request': False, + 'vector_search': [ + { + 'field': 'vector_field', + 'vector_base64': self.TEST_VECTOR_BASE64, + 'k': 3 + } + ] + } + + vector_search = VectorSearch.from_vector_query(VectorQuery('vector_field', self.TEST_VECTOR_BASE64)) + req = SearchRequest.create(vector_search) + search_query = search.SearchQueryBuilder.create_search_query_from_request( + cb_env.TEST_INDEX_NAME, + req + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + def test_vector_search_invalid(self): + with pytest.raises(InvalidArgumentException): + VectorSearch([]) + with pytest.raises(InvalidArgumentException): + VectorSearch(None) + with pytest.raises(InvalidArgumentException): + VectorSearch([1, VectorQuery('vector_field', self.TEST_VECTOR)]) + with pytest.raises(InvalidArgumentException): + VectorSearch.from_vector_query(None) + with pytest.raises(InvalidArgumentException): + VectorSearch.from_vector_query(1) + + def test_vector_search_multiple_queries(self, cb_env): + exp_json = { + 'query': {'match_none': None}, + 'index_name': cb_env.TEST_INDEX_NAME, + 'metrics': True, + 'show_request': False, + 'vector_search': [ + { + 'field': 'vector_field', + 'vector': self.TEST_VECTOR, + 'k': 3 + }, + { + 'field': 'vector_field', + 'vector': self.TEST_VECTOR, + 'k': 3 + }, + ], + 'vector_query_combination': 'and' + } + + vector_queries = [ + VectorQuery('vector_field', self.TEST_VECTOR), + VectorQuery('vector_field', self.TEST_VECTOR) + ] + vector_search = VectorSearch(vector_queries, + VectorSearchOptions(vector_query_combination=VectorQueryCombination.AND)) + req = SearchRequest.create(vector_search) + search_query = search.SearchQueryBuilder.create_search_query_from_request( + cb_env.TEST_INDEX_NAME, + req + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + exp_json['vector_query_combination'] = 'or' + + vector_search = VectorSearch(vector_queries, + VectorSearchOptions(vector_query_combination=VectorQueryCombination.OR)) + req = SearchRequest.create(vector_search) + search_query = search.SearchQueryBuilder.create_search_query_from_request( + cb_env.TEST_INDEX_NAME, + req + ) + encoded_q = cb_env.get_encoded_query(search_query) + assert exp_json == encoded_q + + +class ClassicSearchParamTests(SearchParamTestSuite): + @pytest.fixture(scope='class', autouse=True) + def validate_test_manifest(self): + def valid_test_method(meth): + attr = getattr(ClassicSearchParamTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicSearchParamTests) if valid_test_method(meth)] + manifest_invalid = set(SearchParamTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if manifest_invalid: + pytest.fail(f'Test manifest not validated. Missing/extra tests: {manifest_invalid}.') + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, request): + cb_env = SearchTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param, test_suite=self.__class__.__name__) + yield cb_env + cb_env.teardown(request.param, test_suite=self.__class__.__name__) + + +class ClassicVectorSearchParamTests(VectorSearchParamTestSuite): + @pytest.fixture(scope='class', autouse=True) + def validate_test_manifest(self): + def valid_test_method(meth): + attr = getattr(ClassicVectorSearchParamTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicVectorSearchParamTests) if valid_test_method(meth)] + manifest_invalid = set(VectorSearchParamTestSuite.TEST_MANIFEST).symmetric_difference(method_list) + if manifest_invalid: + pytest.fail(f'Test manifest not validated. Missing/extra tests: {manifest_invalid}.') + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, request): + cb_env = SearchTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param, test_suite=self.__class__.__name__) + yield cb_env + cb_env.teardown(request.param, test_suite=self.__class__.__name__) diff --git a/couchbase/tests/search_t.py b/couchbase/tests/search_t.py new file mode 100644 index 000000000..0490d334f --- /dev/null +++ b/couchbase/tests/search_t.py @@ -0,0 +1,843 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import threading +import uuid +from copy import copy +from datetime import datetime, timedelta + +import pytest + +import couchbase.search as search +from couchbase.exceptions import (AmbiguousTimeoutException, + InvalidArgumentException, + QueryIndexNotFoundException) +from couchbase.mutation_state import MutationState +from couchbase.options import SearchOptions +from couchbase.search import (HighlightStyle, + SearchDateRangeFacet, + SearchFacetResult, + SearchNumericRangeFacet, + SearchRow, + SearchTermFacet) +from tests.environments import CollectionType +from tests.environments.search_environment import SearchTestEnvironment +from tests.test_features import EnvironmentFeatures + + +class SearchCollectionTestSuite: + TEST_MANIFEST = [ + 'test_cluster_query_collections', + 'test_scope_query_collections', + 'test_scope_search_fields', + 'test_scope_search_highlight', + 'test_search_query_in_thread', + ] + + def test_cluster_query_collections(self, cb_env): + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10, + scope_name=cb_env.scope.name, + collections=[cb_env.collection.name])) + rows = cb_env.assert_rows(res, 2, return_rows=True) + + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + def test_scope_query_collections(self, cb_env): + q = search.TermQuery('auto') + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10, collections=[cb_env.collection.name])) + rows = cb_env.assert_rows(res, 2, return_rows=True) + + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, q, SearchOptions(limit=10)) + rows = cb_env.assert_rows(res, 2, return_rows=True) + + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c in [cb_env.collection.name, cb_env.OTHER_COLLECTION]]) is True + + def test_scope_search_fields(self, cb_env): + test_fields = ['make', 'model'] + q = search.TermQuery('auto') + # verify fields works w/in kwargs + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10), + fields=test_fields, + collections=[cb_env.collection.name]) + + fields_with_col = copy(test_fields) + fields_with_col.append('_$c') + rows = cb_env.assert_rows(res, 1, return_rows=True) + first_entry = rows[0] + assert isinstance(first_entry, SearchRow) + assert isinstance(first_entry.fields, dict) + assert first_entry.fields != {} + assert all(map(lambda f: f in fields_with_col, first_entry.fields.keys())) is True + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + # verify fields works w/in options + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10, + fields=test_fields, + collections=[cb_env.collection.name])) + + rows = cb_env.assert_rows(res, 1, return_rows=True) + first_entry = rows[0] + assert isinstance(first_entry, SearchRow) + assert isinstance(first_entry.fields, dict) + assert first_entry.fields != {} + assert all(map(lambda f: f in fields_with_col, first_entry.fields.keys())) is True + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + def test_scope_search_highlight(self, cb_env): + + q = search.TermQuery('auto') + # check w/in options + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, + q, + SearchOptions(limit=10, highlight_style=HighlightStyle.Html)) + rows = cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + fragments = rows[0].fragments + assert isinstance(locations, search.SearchRowLocations) + assert isinstance(fragments, dict) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + # check w/in kwargs + res = cb_env.scope.search_query(cb_env.TEST_COLLECTION_INDEX_NAME, q, SearchOptions( + limit=10), highlight_style=HighlightStyle.Html, collections=[cb_env.collection.name]) + rows = cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + fragments = rows[0].fragments + assert isinstance(locations, search.SearchRowLocations) + assert isinstance(fragments, dict) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + collections = list(map(lambda r: r.fields['_$c'], rows)) + assert all([c for c in collections if c == cb_env.collection.name]) is True + + def test_search_query_in_thread(self, cb_env): + results = [None] + + def run_test(scope, search_idx, assert_fn, results): + try: + q = search.TermQuery('auto') + result = scope.search_query(search_idx, q, SearchOptions(limit=10)) + assert_fn(result, 2) + assert result.metadata() is not None + except AssertionError: + results[0] = False + except Exception as ex: + results[0] = ex + else: + results[0] = True + + t = threading.Thread(target=run_test, + args=(cb_env.scope, cb_env.TEST_COLLECTION_INDEX_NAME, cb_env.assert_rows, results)) + t.start() + t.join() + + assert len(results) == 1 + assert results[0] is True + + +class SearchTestSuite: + TEST_MANIFEST = [ + 'test_bad_search_query', + 'test_cluster_search', + 'test_cluster_search_date_facets', + 'test_cluster_search_disable_scoring', + 'test_cluster_search_facets_fail', + 'test_cluster_search_fields', + 'test_cluster_search_highlight', + 'test_cluster_search_numeric_facets', + 'test_cluster_search_ryow', + 'test_cluster_search_scan_consistency', + 'test_cluster_search_term_facets', + 'test_cluster_search_top_level_facets', + 'test_cluster_search_top_level_facets_kwargs', + 'test_cluster_sort_field', + 'test_cluster_sort_field_multi', + 'test_cluster_sort_geo', + 'test_cluster_sort_id', + 'test_cluster_sort_score', + 'test_cluster_sort_str', + 'test_search_include_locations', + 'test_search_match_operator', + 'test_search_match_operator_fail', + 'test_search_no_include_locations', + 'test_search_query_in_thread', + 'test_search_raw_query', + 'test_search_timeout' + ] + + @pytest.fixture(scope="class") + def check_disable_scoring_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('search_disable_scoring', + cb_env.server_version_short, + cb_env.mock_server_type) + + def test_bad_search_query(self, cb_env): + res = cb_env.cluster.search_query('not-an-index', search.TermQuery('auto')) + with pytest.raises(QueryIndexNotFoundException): + [r for r in res] + + def test_cluster_search(self, cb_env): + q = search.TermQuery('auto') + # PYCBC-1572, add show_request, log_request and log_response to options. + # to validate run at the commandline: + # PYCBC_LOG_LEVEL=info python \ + # -m pytest couchbase/tests/search_t.py::ClassicSearchTests::test_cluster_search \ + # -p no:asyncio -p no:warnings -vv -rA --capture=tee-sys + # output should have lines that contain w/ SEARCH: and SEARCH RESPONSE: and the JSON + # in the SEARCH RESPONSE line should have the request JSON + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, SearchOptions(limit=10, + show_request=True, + log_request=True, + log_response=True)) + cb_env.assert_rows(res, 2) + + def test_cluster_search_date_facets(self, cb_env): + facet_name = 'last_updated' + facet = search.DateFacet('last_updated', limit=3) + now = datetime.now() + today = datetime(year=now.year, month=now.month, day=now.day) + early_end = (today - timedelta(days=241)) + mid_start = (today - timedelta(days=240)) + mid_end = (today - timedelta(days=121)) + late_start = (today - timedelta(days=120)) + facet.add_range('early', end=f"{early_end.strftime('%Y-%m-%dT%H:%M:%SZ')}") + facet.add_range('mid', start=f"{mid_start.strftime('%Y-%m-%dT%H:%M:%SZ')}", + end=f"{mid_end.strftime('%Y-%m-%dT%H:%M:%SZ')}") + facet.add_range('late', start=f"{late_start.strftime('%Y-%m-%dT%H:%M:%SZ')}") + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert all(map(lambda ft: isinstance(ft, SearchDateRangeFacet), result_facet.date_ranges)) is True + assert len(result_facet.date_ranges) <= facet.limit + + @pytest.mark.usefixtures('check_disable_scoring_supported') + def test_cluster_search_disable_scoring(self, cb_env): + + # verify disable scoring works w/in SearchOptions + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, disable_scoring=True)) + rows = cb_env.assert_rows(res, 1, return_rows=True) + assert all(map(lambda r: r.score == 0, rows)) is True + + # verify disable scoring works w/in kwargs + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), disable_scoring=True) + rows = cb_env.assert_rows(res, 1, return_rows=True) + assert all(map(lambda r: r.score == 0, rows)) is True + + # verify setting disable_scoring to False works + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, disable_scoring=False)) + rows = cb_env.assert_rows(res, 1, return_rows=True) + assert all(map(lambda r: r.score != 0, rows)) is True + + # verify default disable_scoring is False + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10)) + rows = cb_env.assert_rows(res, 1, return_rows=True) + assert all(map(lambda r: r.score != 0, rows)) is True + + # @TODO: 3.x raises a SearchException... + def test_cluster_search_facets_fail(self, cb_env): + q = search.TermQuery('auto') + with pytest.raises(ValueError): + cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={'test-facet': None})) + + def test_cluster_search_fields(self, cb_env): + test_fields = ['make', 'mode'] + q = search.TermQuery('auto') + # verify fields works w/in kwargs + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), fields=test_fields) + + rows = cb_env.assert_rows(res, 1, return_rows=True) + first_entry = rows[0] + assert isinstance(first_entry, SearchRow) + assert isinstance(first_entry.fields, dict) + assert first_entry.fields != {} + assert all(map(lambda f: f in test_fields, first_entry.fields.keys())) is True + + # verify fields works w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, fields=test_fields)) + + rows = cb_env.assert_rows(res, 1, return_rows=True) + first_entry = rows[0] + assert isinstance(first_entry, SearchRow) + assert isinstance(first_entry.fields, dict) + assert first_entry.fields != {} + assert all(map(lambda f: f in test_fields, first_entry.fields.keys())) is True + + def test_cluster_search_highlight(self, cb_env): + + q = search.TermQuery('auto') + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, highlight_style=HighlightStyle.Html)) + rows = cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + fragments = rows[0].fragments + assert isinstance(locations, search.SearchRowLocations) + assert isinstance(fragments, dict) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, SearchOptions( + limit=10), highlight_style=HighlightStyle.Html) + rows = cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + fragments = rows[0].fragments + assert isinstance(locations, search.SearchRowLocations) + assert isinstance(fragments, dict) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + + def test_cluster_search_numeric_facets(self, cb_env): + + facet_name = 'rating' + facet = search.NumericFacet('rating', limit=3) + facet.add_range('low', max=4) + facet.add_range('med', min=4, max=7) + facet.add_range('high', min=7) + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert all(map(lambda ft: isinstance(ft, SearchNumericRangeFacet), result_facet.numeric_ranges)) is True + assert len(result_facet.numeric_ranges) <= facet.limit + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_cluster_search_ryow(self, cb_env): + key, value = cb_env.get_new_doc() + # need to make sure content is unique + content = str(uuid.uuid4())[:8] + value['description'] = content + q = search.TermQuery(content) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10)) + cb_env.assert_rows(res, 0) + res = cb_env.collection.insert(key, value) + ms = MutationState() + ms.add_mutation_token(res.mutation_token()) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, consistent_with=ms)) + cb_env.assert_rows(res, 1) + + # prior to PYCBC-1477 the SDK _could_ crash w/ this this sort of MS creation + key, value = cb_env.get_new_doc() + # need to make sure content is unique + content = str(uuid.uuid4())[:8] + value['description'] = content + q = search.TermQuery(content) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10)) + cb_env.assert_rows(res, 0) + res = cb_env.collection.insert(key, value) + ms = MutationState(res) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, consistent_with=ms)) + cb_env.assert_rows(res, 1) + + def test_cluster_search_term_facets(self, cb_env): + + facet_name = 'model' + facet = search.TermFacet('model', 5) + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert all(map(lambda ft: isinstance(ft, SearchTermFacet), result_facet.terms)) is True + assert len(result_facet.terms) <= facet.limit + + def test_cluster_search_scan_consistency(self, cb_env): + q = search.TermQuery('auto') + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, + scan_consistency=search.SearchScanConsistency.NOT_BOUNDED)) + cb_env.assert_rows(res, 1) + + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, + scan_consistency=search.SearchScanConsistency.REQUEST_PLUS)) + cb_env.assert_rows(res, 1) + + with pytest.raises(InvalidArgumentException): + cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, + scan_consistency=search.SearchScanConsistency.AT_PLUS)) + + def test_cluster_search_top_level_facets(self, cb_env): + # if the facet limit is omitted, the details of the facets will not be provided + # (i.e. SearchFacetResult.terms is None, + # SearchFacetResult.numeric_ranges is None and SearchFacetResult.date_ranges is None) + facet_name = 'model' + facet = search.TermFacet('model') + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert result_facet.terms is None + assert result_facet.numeric_ranges is None + assert result_facet.date_ranges is None + + facet_name = 'rating' + facet = search.NumericFacet('rating') + facet.add_range('low', max=4) + facet.add_range('med', min=4, max=7) + facet.add_range('high', min=7) + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, facets={facet_name: facet})) + + cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert result_facet.terms is None + assert result_facet.numeric_ranges is None + assert result_facet.date_ranges is None + + def test_cluster_search_top_level_facets_kwargs(self, cb_env): + # if the facet limit is omitted, the details of the facets will not be provided + # (i.e. SearchFacetResult.terms is None, + # SearchFacetResult.numeric_ranges is None and SearchFacetResult.date_ranges is None) + facet_name = 'model' + facet = search.TermFacet('model') + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), facets={facet_name: facet}) + + cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert result_facet.terms is None + assert result_facet.numeric_ranges is None + assert result_facet.date_ranges is None + + facet_name = 'rating' + facet = search.NumericFacet('rating') + facet.add_range('low', max=4) + facet.add_range('med', min=4, max=7) + facet.add_range('high', min=7) + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), facets={facet_name: facet}) + + cb_env.assert_rows(res, 1) + facets = res.facets() + assert isinstance(facets, dict) + result_facet = facets[facet_name] + assert isinstance(result_facet, SearchFacetResult) + assert result_facet.name == facet_name + assert result_facet.field == facet_name + assert result_facet.terms is None + assert result_facet.numeric_ranges is None + assert result_facet.date_ranges is None + + def test_cluster_sort_field(self, cb_env): + sort_field = "rating" + q = search.TermQuery('auto') + # field - ascending + sort = search.SortField(field=sort_field, type="number", mode="min", missing="last") + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[sort], fields=[sort_field])) + + rows = cb_env.assert_rows(res, 1, return_rows=True) + rating = rows[0].fields[sort_field] + for row in rows[1:]: + assert row.fields[sort_field] >= rating + rating = row.fields[sort_field] + + # field - descending + sort.desc = True + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[sort], fields=[sort_field])) + + rows = cb_env.assert_rows(res, 1, return_rows=True) + rating = rows[0].fields[sort_field] + for row in rows[1:]: + assert rating >= row.fields[sort_field] + rating = row.fields[sort_field] + + def test_cluster_sort_field_multi(self, cb_env): + sort_fields = [ + search.SortField(field="rating", type="number", + mode="min", missing="last"), + search.SortField(field="last_updated", type="number", + mode="min", missing="last"), + search.SortScore(), + ] + sort_field_names = ["rating", "last_updated"] + q = search.TermQuery('auto') + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=sort_fields, fields=sort_field_names)) + cb_env.assert_rows(res, 1) + + sort_fields = [ + search.SortField(field="rating", type="number", + mode="min", missing="last", desc=True), + search.SortField(field="last_updated", type="number", + mode="min", missing="last"), + search.SortScore(desc=True), + ] + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=sort_fields, fields=sort_field_names)) + cb_env.assert_rows(res, 1) + + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=["rating", "last_udpated", "-_score"])) + cb_env.assert_rows(res, 1) + + def test_cluster_sort_geo(self, cb_env): + # @TODO: better confirmation on results? + sort_field = "geo" + q = search.TermQuery('auto') + # geo - ascending + sort = search.SortGeoDistance(field=sort_field, location=(37.7749, 122.4194), unit="meters") + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[sort], fields=[sort_field])) + cb_env.assert_rows(res, 1) + + # geo - descending + sort.desc = True + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[sort], fields=[sort_field])) + cb_env.assert_rows(res, 1) + + def test_cluster_sort_id(self, cb_env): + q = search.TermQuery('auto') + # score - ascending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[search.SortID()])) + rows = cb_env.assert_rows(res, 1, return_rows=True) + + id = rows[0].id + for row in rows[1:]: + assert row.id >= id + id = row.id + + # score - descending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[search.SortID(desc=True)])) + rows = cb_env.assert_rows(res, 1, return_rows=True) + + id = rows[0].id + for row in rows[1:]: + assert id >= row.id + id = row.id + + def test_cluster_sort_score(self, cb_env): + q = search.TermQuery('auto') + # score - ascending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[search.SortScore()])) + rows = cb_env.assert_rows(res, 1, return_rows=True) + + score = rows[0].score + for row in rows[1:]: + assert row.score >= score + score = row.score + + # score - descending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=[search.SortScore(desc=True)])) + rows = cb_env.assert_rows(res, 1, return_rows=True) + + score = rows[0].score + for row in rows[1:]: + assert score >= row.score + score = row.score + + def test_cluster_sort_str(self, cb_env): + q = search.TermQuery('auto') + # score - ascending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=['_score'])) + rows = cb_env.assert_rows(res, 1, return_rows=True) + score = rows[0].score + for row in rows[1:]: + assert row.score >= score + score = row.score + + # score - descending + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, sort=['-_score'])) + rows = cb_env.assert_rows(res, 1, return_rows=True) + score = rows[0].score + for row in rows[1:]: + assert score >= row.score + score = row.score + + def test_search_include_locations(self, cb_env): + q = search.TermQuery('auto') + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, include_locations=True)) + rows = cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + assert isinstance(locations, search.SearchRowLocations) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + + # check w/in kwargs + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), include_locations=True) + rows = cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + assert isinstance(locations, search.SearchRowLocations) + assert all(map(lambda l: isinstance(l, search.SearchRowLocation), locations.get_all())) is True + + @pytest.mark.parametrize('operator, query_terms, expect_rows', + [(search.MatchOperator.AND, "auto deal", True), + (search.MatchOperator.AND, "auto :random:", False), + (search.MatchOperator.OR, "auto deal", True), + (search.MatchOperator.OR, "auto :random:", True)]) + def test_search_match_operator(self, cb_env, operator, query_terms, expect_rows): + import random + import string + + random_query_term = "".join(random.choice(string.ascii_letters) + for _ in range(10)) + + if ':random:' in query_terms: + query_terms.replace(':random:', random_query_term) + + q = search.MatchQuery(query_terms, match_operator=operator) + + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, limit=10) + rows = cb_env.assert_rows(res, 0, return_rows=True) + + if expect_rows: + assert len(rows) > 0 + else: + assert len(rows) == 0 + + def test_search_match_operator_fail(self, cb_env): + with pytest.raises(ValueError): + q = search.MatchQuery('auto deal', match_operator='NOT') + cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, limit=10) + + # @TODO(PYCBC-1296): DIFF between 3.x and 4.x, locations returns None + def test_search_no_include_locations(self, cb_env): + q = search.TermQuery('auto') + # check w/in options + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10, include_locations=False)) + rows = cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + assert locations is None + + # check w/in kwargs + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, + q, + SearchOptions(limit=10), include_locations=False) + rows = cb_env.assert_rows(res, 1, return_rows=True) + locations = rows[0].locations + assert locations is None + + def test_search_query_in_thread(self, cb_env): + results = [None] + + def run_test(cluster, search_idx, assert_fn, results): + try: + q = search.TermQuery('auto') + result = cluster.search_query(search_idx, q, SearchOptions(limit=10)) + assert_fn(result, 2) + assert result.metadata() is not None + except AssertionError: + results[0] = False + except Exception as ex: + results[0] = ex + else: + results[0] = True + + t = threading.Thread(target=run_test, + args=(cb_env.cluster, cb_env.TEST_INDEX_NAME, cb_env.assert_rows, results)) + t.start() + t.join() + + assert len(results) == 1 + assert results[0] is True + + def test_search_raw_query(self, cb_env): + query_args = {"match": "auto deal", + "fuzziness": 2, "operator": "and"} + q = search.RawQuery(query_args) + res = cb_env.cluster.search_query(cb_env.TEST_INDEX_NAME, q, limit=10) + cb_env.assert_rows(res, 1) + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_search_timeout(self, cb_env): + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + # Prior to PYCBC-1521, this test would fail as each request would override the cluster level search_timeout. + # If a timeout was not provided in the request, the default 75s timeout would be used. PYCBC-1521 corrects + # this behavior so this test will pass as we are essentially forcing an AmbiguousTimeoutException because + # we are setting the cluster level search_timeout such a small value. + timeout_opts = ClusterTimeoutOptions(search_timeout=timedelta(milliseconds=1)) + cluster = Cluster.connect(f'{conn_string}', ClusterOptions(auth, timeout_options=timeout_opts)) + # don't need to do this except for older server versions + _ = cluster.bucket(f'{cb_env.bucket.name}') + q = search.TermQuery('auto') + with pytest.raises(AmbiguousTimeoutException): + res = cluster.search_query(cb_env.TEST_INDEX_NAME, q, SearchOptions(limit=10)) + [r for r in res.rows()] + + # if we override the timeout w/in the request the query should succeed. + res = cluster.search_query(cb_env.TEST_INDEX_NAME, q, SearchOptions(limit=10, timeout=timedelta(seconds=10))) + rows = [r for r in res.rows()] + assert len(rows) > 0 + + +class ClassicSearchCollectionTests(SearchCollectionTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicSearchCollectionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicSearchCollectionTests) if valid_test_method(meth)] + compare = set(SearchCollectionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = SearchTestEnvironment.from_environment(cb_base_env) + cb_env.enable_search_mgmt() + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) + + +class ClassicSearchTests(SearchTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicSearchTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicSearchTests) if valid_test_method(meth)] + compare = set(SearchTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = SearchTestEnvironment.from_environment(cb_base_env) + cb_env.enable_search_mgmt() + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) diff --git a/couchbase/tests/searchmgmt_t.py b/couchbase/tests/searchmgmt_t.py new file mode 100644 index 000000000..7bf7e52fd --- /dev/null +++ b/couchbase/tests/searchmgmt_t.py @@ -0,0 +1,331 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import pathlib +from os import path + +import pytest + +from couchbase.exceptions import (FeatureUnavailableException, + InvalidArgumentException, + QueryIndexAlreadyExistsException, + SearchIndexNotFoundException) +from couchbase.management.search import SearchIndex +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment + + +class SearchIndexManagementTestSuite: + IDX_NAME = 'test-fts-index' + + TEST_COLLECTION_INDEX_NAME = 'test-search-coll-index' + TEST_COLLECTION_INDEX_PATH = path.join(pathlib.Path(__file__).parent.parent.parent, + 'tests', + 'test_cases', + f'{TEST_COLLECTION_INDEX_NAME}-params-new.json') + + TEST_UI_INDEX_NAME = 'test-search-index-full' + TEST_UI_INDEX_PATH = path.join(pathlib.Path(__file__).parent.parent.parent, + 'tests', + 'test_cases', + f'{TEST_UI_INDEX_NAME}.json') + + TEST_UI_SCOPE_INDEX_NAME = 'test-scope-search-index-full' + TEST_UI_SCOPE_INDEX_PATH = path.join(pathlib.Path(__file__).parent.parent.parent, + 'tests', + 'test_cases', + f'{TEST_UI_SCOPE_INDEX_NAME}.json') + + TEST_MANIFEST = [ + 'test_analyze_doc', + 'test_drop_index', + 'test_drop_index_fail', + 'test_get_all_index_stats', + 'test_get_all_indexes', + 'test_get_index', + 'test_get_index_doc_count', + 'test_get_index_fail', + 'test_get_index_fail_no_index_name', + 'test_get_index_stats', + 'test_ingestion_control', + 'test_plan_freeze_control', + 'test_query_control', + 'test_upsert_index', + 'test_upsert_index_from_json' + ] + + @pytest.fixture(scope='class', name='test_idx') + def get_test_index(self): + return SearchIndex(name=self.IDX_NAME, source_name='default') + + @pytest.fixture() + def create_test_index(self, cb_env, test_idx): + if cb_env.use_scope_search_mgmt: + with open(self.TEST_COLLECTION_INDEX_PATH) as params_file: + input = params_file.read() + params_json = json.loads(input) + mapping_types = params_json.get('mapping', {}).get('types', {}) + if mapping_types and 'test-scope.other-collection' in mapping_types: + del params_json['mapping']['types']['test-scope.other-collection'] + test_idx.params = params_json + TestEnvironment.try_n_times_till_exception(10, + 3, + cb_env.sixm.upsert_index, + test_idx, + expected_exceptions=(QueryIndexAlreadyExistsException, )) + + @pytest.fixture() + def drop_test_index(self, cb_env, test_idx): + yield + TestEnvironment.try_n_times_till_exception(10, + 3, + cb_env.sixm.drop_index, + test_idx.name, + expected_exceptions=(SearchIndexNotFoundException, )) + + @pytest.fixture() + def drop_test_index_from_json(self, cb_env): + yield + if cb_env.use_scope_search_mgmt: + idx_name = 'test-scope-search-index-from-ui' + else: + idx_name = 'test-search-index-from-ui' + TestEnvironment.try_n_times_till_exception(10, + 3, + cb_env.sixm.drop_index, + idx_name, + expected_exceptions=(SearchIndexNotFoundException, )) + + @pytest.fixture(scope='class', name='test_idx_json') + def load_index_json(self, cb_env): + if cb_env.use_scope_search_mgmt: + with open(self.TEST_UI_SCOPE_INDEX_PATH) as index_json: + json_obj = json.load(index_json) + else: + with open(self.TEST_UI_INDEX_PATH) as index_json: + json_obj = json.load(index_json) + # >= 7.0 sourceType = gocbcore + if cb_env.server_version_short <= 6.6: + json_obj['sourceType'] = 'couchbase' + return json_obj + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_analyze_doc(self, cb_env, test_idx): + if cb_env.server_version_short < 6.5: + pytest.skip((f'FTS analyzeDoc only supported on server versions >= 6.5. ' + f'Using server version: {cb_env.server_version}.')) + doc = {"field": "I got text in here"} + if cb_env.use_scope_search_mgmt: + with pytest.raises(FeatureUnavailableException): + cb_env.sixm.get_index_stats(test_idx.name, doc) + else: + # like getting the doc count, this can fail immediately after index creation + analysis = TestEnvironment.try_n_times(5, + 2, + cb_env.sixm.analyze_document, + test_idx.name, + doc) + + assert analysis.get('analysis', None) is not None + assert isinstance(analysis.get('analysis'), (list, dict)) + assert analysis.get('status', None) == 'ok' + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_drop_index(self, cb_env, test_idx): + res = TestEnvironment.try_n_times(3, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + cb_env.sixm.drop_index(test_idx.name) + with pytest.raises(SearchIndexNotFoundException): + cb_env.sixm.drop_index(test_idx.name) + + def test_drop_index_fail(self, cb_env, test_idx): + with pytest.raises(SearchIndexNotFoundException): + cb_env.sixm.drop_index(test_idx.name) + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_get_all_index_stats(self, cb_env): + # like getting the doc count, this can fail immediately after index + # creation + stats = TestEnvironment.try_n_times(5, 2, cb_env.sixm.get_all_index_stats) + + assert stats is not None + assert isinstance(stats, dict) + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_get_all_indexes(self, cb_env, test_idx): + # lets add one more + new_idx = SearchIndex(name='new-search-idx', source_name='default') + res = cb_env.sixm.upsert_index(new_idx) + assert res is None + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.get_index, new_idx.name) + assert isinstance(res, SearchIndex) + + indexes = cb_env.sixm.get_all_indexes() + assert isinstance(indexes, list) + assert len(indexes) >= 2 + assert next((idx for idx in indexes if idx.name == test_idx.name), None) is not None + assert next((idx for idx in indexes if idx.name == new_idx.name), None) is not None + + cb_env.sixm.drop_index(new_idx.name) + TestEnvironment.try_n_times_till_exception(10, + 3, + cb_env.sixm.get_index, + new_idx.name, + expected_exceptions=(SearchIndexNotFoundException,)) + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_get_index(self, cb_env, test_idx): + res = TestEnvironment.try_n_times(3, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + assert res.name == test_idx.name + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_get_index_doc_count(self, cb_env, test_idx): + # like getting the doc count, this can fail immediately after index + # creation + doc_count = TestEnvironment.try_n_times(5, 2, cb_env.sixm.get_indexed_documents_count, test_idx.name) + + assert doc_count is not None + assert isinstance(doc_count, int) + + def test_get_index_fail(self, cb_env): + with pytest.raises(SearchIndexNotFoundException): + cb_env.sixm.get_index('not-an-index') + + def test_get_index_fail_no_index_name(self, cb_env): + with pytest.raises(InvalidArgumentException): + cb_env.sixm.get_index('') + with pytest.raises(InvalidArgumentException): + cb_env.sixm.get_index(None) + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_get_index_stats(self, cb_env, test_idx): + if cb_env.use_scope_search_mgmt: + with pytest.raises(FeatureUnavailableException): + cb_env.sixm.get_index_stats(test_idx.name) + else: + # like getting the doc count, this can fail immediately after index + # creation + stats = TestEnvironment.try_n_times(5, 2, cb_env.sixm.get_index_stats, test_idx.name) + + assert stats is not None + assert isinstance(stats, dict) + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_ingestion_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.pause_ingest, test_idx.name) + assert res is None + + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.resume_ingest, test_idx.name) + assert res is None + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_plan_freeze_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.freeze_plan, test_idx.name) + assert res is None + + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.unfreeze_plan, test_idx.name) + assert res is None + + @pytest.mark.usefixtures('create_test_index') + @pytest.mark.usefixtures('drop_test_index') + def test_query_control(self, cb_env, test_idx): + # can't easily test this, but lets at least call them and insure we get no + # exceptions + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.disallow_querying, test_idx.name) + assert res is None + + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.allow_querying, test_idx.name) + assert res is None + + @pytest.mark.usefixtures('drop_test_index') + def test_upsert_index(self, cb_env, test_idx): + res = cb_env.sixm.upsert_index(test_idx) + assert res is None + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.get_index, test_idx.name) + assert isinstance(res, SearchIndex) + + @pytest.mark.usefixtures('drop_test_index_from_json') + @pytest.mark.parametrize("idx_json_type", [str, dict]) + def test_upsert_index_from_json(self, cb_env, test_idx_json, idx_json_type): + if idx_json_type.__name__ == 'str': + search_idx = SearchIndex.from_json(json.dumps(test_idx_json)) + else: + search_idx = SearchIndex.from_json(test_idx_json) + res = cb_env.sixm.upsert_index(search_idx) + assert res is None + res = TestEnvironment.try_n_times(10, 3, cb_env.sixm.get_index, search_idx.name) + assert isinstance(res, SearchIndex) + assert res.name == search_idx.name + assert res.source_type == search_idx.source_type + assert res.idx_type == search_idx.idx_type + assert res.source_name == search_idx.source_name + assert res.params == search_idx.params + assert res.plan_params == search_idx.plan_params + assert res.source_params == search_idx.source_params + + +@pytest.mark.flaky(reruns=5, reruns_delay=1) +class ClassicSearchIndexManagementTests(SearchIndexManagementTestSuite): + @pytest.fixture(scope='class', autouse=True) + def validate_test_manifest(self): + def valid_test_method(meth): + attr = getattr(ClassicSearchIndexManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicSearchIndexManagementTests) if valid_test_method(meth)] + test_list = set(ClassicSearchIndexManagementTests.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest invalid. Missing/extra tests: {test_list}.') + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env): + cb_base_env.enable_search_mgmt() + yield cb_base_env + cb_base_env.disable_search_mgmt() + + +# @pytest.mark.flaky(reruns=5, reruns_delay=1) +class ClassicScopeSearchIndexManagementTests(SearchIndexManagementTestSuite): + @pytest.fixture(scope='class', autouse=True) + def validate_test_manifest(self): + def valid_test_method(meth): + attr = getattr(ClassicScopeSearchIndexManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicScopeSearchIndexManagementTests) if valid_test_method(meth)] + test_list = set(ClassicScopeSearchIndexManagementTests.TEST_MANIFEST).symmetric_difference(method_list) + if test_list: + pytest.fail(f'Test manifest invalid. Missing/extra tests: {test_list}.') + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env): + cb_base_env.setup(CollectionType.NAMED, num_docs=100) + cb_base_env.enable_scope_search_mgmt().enable_search_mgmt() + yield cb_base_env + cb_base_env.disable_search_mgmt().disable_scope_search_mgmt() + cb_base_env.teardown(CollectionType.NAMED) diff --git a/couchbase/tests/subdoc_t.py b/couchbase/tests/subdoc_t.py new file mode 100644 index 000000000..81688fa75 --- /dev/null +++ b/couchbase/tests/subdoc_t.py @@ -0,0 +1,1125 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import types +from datetime import datetime, timedelta + +import pytest + +import couchbase.subdocument as SD +from couchbase.exceptions import (DocumentExistsException, + DocumentNotFoundException, + DocumentUnretrievableException, + InvalidArgumentException, + InvalidValueException, + PathExistsException, + PathMismatchException, + PathNotFoundException) +from couchbase.options import (GetOptions, + LookupInAllReplicasOptions, + LookupInAnyReplicaOptions, + LookupInOptions, + MutateInOptions) +from couchbase.replica_reads import ReadPreference +from couchbase.result import (GetResult, + LookupInReplicaResult, + LookupInResult, + MutateInResult) +from tests.environments import CollectionType +from tests.environments.subdoc_environment import SubdocTestEnvironment +from tests.environments.test_environment import TestEnvironment +from tests.mock_server import MockServerType +from tests.test_features import EnvironmentFeatures + + +class SubDocumentTestSuite: + + TEST_MANIFEST = [ + 'test_array_add_unique', + 'test_array_add_unique_create_parents', + 'test_array_add_unique_fail', + 'test_array_add_unique_mutate_in_macros', + 'test_array_append', + 'test_array_append_create_parents', + 'test_array_append_multi_insert', + 'test_array_add_append_mutate_in_macros', + 'test_array_as_document', + 'test_array_insert', + 'test_array_insert_multi_insert', + 'test_array_insert_mutate_in_macros', + 'test_array_prepend', + 'test_array_prepend_create_parents', + 'test_array_prepend_multi_insert', + 'test_array_add_prepend_mutate_in_macros', + 'test_count', + 'test_decrement', + 'test_decrement_create_parents', + 'test_increment', + 'test_increment_create_parents', + 'test_insert_create_parents', + 'test_lookup_in_all_replicas_bad_key', + 'test_lookup_in_all_replicas_exists', + 'test_lookup_in_all_replicas_exists_bad_path', + 'test_lookup_in_all_replicas_get', + 'test_lookup_in_all_replicas_get_bad_path', + 'test_lookup_in_all_replicas_get_full', + 'test_lookup_in_all_replicas_multiple_specs', + 'test_lookup_in_all_replicas_with_timeout', + 'test_lookup_in_all_replicas_read_preference', + 'test_lookup_in_any_replica_bad_key', + 'test_lookup_in_any_replica_exists', + 'test_lookup_in_any_replica_exists_bad_path', + 'test_lookup_in_any_replica_get', + 'test_lookup_in_any_replica_get_bad_path', + 'test_lookup_in_any_replica_get_full', + 'test_lookup_in_any_replica_multiple_specs', + 'test_lookup_in_any_replica_with_timeout', + 'test_lookup_in_any_replica_read_preference', + 'test_lookup_in_macros', + 'test_lookup_in_multiple_specs', + 'test_lookup_in_one_path_not_found', + 'test_lookup_in_simple_exists', + 'test_lookup_in_simple_exists_bad_path', + 'test_lookup_in_simple_get', + 'test_lookup_in_simple_get_bad_path', + 'test_lookup_in_simple_get_spec_as_list', + 'test_lookup_in_simple_long_path', + 'test_lookup_in_simple_with_timeout', + 'test_lookup_in_valid_path_null_content', + 'test_mutate_in_expiry', + 'test_mutate_in_insert_semantics', + 'test_mutate_in_insert_semantics_fail', + 'test_mutate_in_insert_semantics_kwargs', + 'test_mutate_in_macros_insert', + 'test_mutate_in_macros_replace_upsert', + 'test_mutate_in_preserve_expiry', + 'test_mutate_in_preserve_expiry_fails', + 'test_mutate_in_preserve_expiry_not_used', + 'test_mutate_in_remove', + 'test_mutate_in_remove_blank_path', + 'test_mutate_in_replace_semantics', + 'test_mutate_in_replace_semantics_fail', + 'test_mutate_in_replace_semantics_kwargs', + 'test_mutate_in_replace_full_document', + 'test_mutate_in_simple', + 'test_mutate_in_simple_spec_as_list', + 'test_mutate_in_store_semantics_fail', + 'test_mutate_in_upsert_semantics', + 'test_mutate_in_upsert_semantics_kwargs', + 'test_upsert_create_parents', + ] + + @pytest.fixture(scope="class") + def check_preserve_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('preserve_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_xattr_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('xattr', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def skip_if_go_caves(self, cb_env): + if cb_env.is_mock_server and cb_env.mock_server_type == MockServerType.GoCAVES: + pytest.skip("GoCAVES does not like this operation.") + + @pytest.fixture(scope="class") + def check_replica_read_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('subdoc_replica_read', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_server_groups_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('server_groups', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.mark.usefixtures('skip_if_go_caves') + @pytest.mark.parametrize('value', [True, + False, + 3.14159, + 13, + 'foo', + None]) + def test_array_add_unique(self, cb_env, value): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in(key, (SD.array_addunique('array', value),)) + assert isinstance(result, MutateInResult) + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + assert value in val['array'] + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_array_add_unique_create_parents(self, cb_env): + key, value = cb_env.get_new_doc_by_type('array') + cb_env.collection.upsert(key, value) + TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + result = cb_env.collection.mutate_in(key, ( + SD.array_addunique("new.set", "new", create_parents=True), + SD.array_addunique("new.set", "unique"), + SD.array_addunique("new.set", "set"))) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + new_set = result.content_as[dict]["new"]["set"] + assert isinstance(new_set, list) + assert "new" in new_set + assert "unique" in new_set + assert "set" in new_set + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_array_add_unique_fail(self, cb_env): + key = "simple-key" + value = { + "a": "aaa", + "b": [0, 1, 2, 3], + "c": [1.25, 1.5, {"nested": ["str", "array"]}], + } + cb_env.collection.upsert(key, value) + TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + + with pytest.raises(PathExistsException): + cb_env.collection.mutate_in(key, (SD.array_addunique("b", 3),)) + + with pytest.raises(InvalidValueException): + cb_env.collection.mutate_in(key, (SD.array_addunique("b", [4, 5, 6]),)) + + with pytest.raises(InvalidValueException): + cb_env.collection.mutate_in(key, (SD.array_addunique("b", {"c": "d"}),)) + + with pytest.raises(PathMismatchException): + cb_env.collection.mutate_in(key, (SD.array_addunique("c", 2),)) + + @pytest.mark.usefixtures('skip_if_go_caves') + @pytest.mark.usefixtures('check_xattr_supported') + def test_array_add_unique_mutate_in_macros(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + xattr_array = 'xattr_array' + specs = [ + SD.upsert(xattr_array, [], xattr=True), + SD.array_addunique(xattr_array, SD.MutationMacro.cas()), + SD.array_addunique(xattr_array, SD.MutationMacro.seq_no()), + SD.array_addunique(xattr_array, SD.MutationMacro.value_crc32c()), + ] + # TODO: PYCBC-1557: Server raises invalid argument when using mutate-in macro w/ arrayaddunique. + # Update test if associated MB is addressed in the future. + with pytest.raises(InvalidArgumentException): + cb_env.collection.mutate_in(key, specs) + + def test_array_append(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.array_append('array', 6),)) + assert isinstance(result, MutateInResult) + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + assert len(val['array']) == 6 + assert val['array'][5] == 6 + + def test_array_append_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in(key, ( + SD.array_append('new.array', 'Hello,', create_parents=True), + SD.array_append('new.array', 'World!'),)) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + assert result.content_as[dict]['new']['array'] == [ + 'Hello,', 'World!'] + + def test_array_append_multi_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.array_append('array', 8, 9, 10),)) + assert isinstance(result, MutateInResult) + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + app_res = val['array'][5:] + assert len(app_res) == 3 + assert app_res == [8, 9, 10] + + @pytest.mark.usefixtures('skip_if_go_caves') + @pytest.mark.usefixtures('check_xattr_supported') + def test_array_add_append_mutate_in_macros(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + xattr_array = 'xattr_array' + specs = [ + SD.array_append(xattr_array, SD.MutationMacro.cas(), create_parents=True), + SD.array_append(xattr_array, SD.MutationMacro.seq_no()), + SD.array_append(xattr_array, SD.MutationMacro.value_crc32c()), + ] + result = cb_env.collection.mutate_in(key, specs) + assert isinstance(result, MutateInResult) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(xattr_array, xattr=True),)) + + res_array = res.content_as[list](0) + assert len(res_array) == 3 + assert all(map(lambda x: isinstance(x, str), res_array)) is True + res_cas = res_array[0] + res_seqno = res_array[1] + res_value = res_array[2] + + assert res_cas.startswith('0x') is True + assert res.cas == SD.convert_macro_cas_to_cas(res_cas) + assert res_seqno.startswith('0x') is True + assert res_value.startswith('0x') is True + + def test_array_as_document(self, cb_env): + key = cb_env.get_existing_doc_by_type('array_only', key_only=True) + result = cb_env.collection.mutate_in(key, (SD.array_append( + '', 2), SD.array_prepend('', 0), SD.array_insert('[1]', 1),)) + assert isinstance(result, MutateInResult) + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[list] + assert isinstance(val, list) + assert len(val) == 3 + assert val[0] == 0 + assert val[1] == 1 + assert val[2] == 2 + + def test_array_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.array_insert('array.[2]', 10),)) + assert isinstance(result, MutateInResult) + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + assert len(val['array']) == 6 + assert val['array'][2] == 10 + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_array_insert_multi_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.array_insert('array.[3]', 6, 7, 8),)) + assert isinstance(result, MutateInResult) + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + ins_res = val['array'][3:6] + assert len(ins_res) == 3 + assert ins_res == [6, 7, 8] + + @pytest.mark.usefixtures('skip_if_go_caves') + @pytest.mark.usefixtures('check_xattr_supported') + def test_array_insert_mutate_in_macros(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + xattr_array = 'xattr_array' + specs = [ + SD.upsert(xattr_array, [], xattr=True), + SD.array_insert(f'{xattr_array}.[0]', SD.MutationMacro.cas()), + SD.array_insert(f'{xattr_array}.[0]', SD.MutationMacro.seq_no()), + SD.array_insert(f'{xattr_array}.[0]', SD.MutationMacro.value_crc32c()), + ] + result = cb_env.collection.mutate_in(key, specs) + assert isinstance(result, MutateInResult) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(xattr_array, xattr=True),)) + + res_array = res.content_as[list](0) + assert len(res_array) == 3 + assert all(map(lambda x: isinstance(x, str), res_array)) is True + # prepending -- order is reversed + res_cas = res_array[2] + res_seqno = res_array[1] + res_value = res_array[0] + + assert res_cas.startswith('0x') is True + assert res.cas == SD.convert_macro_cas_to_cas(res_cas) + assert res_seqno.startswith('0x') is True + assert res_value.startswith('0x') is True + + def test_array_prepend(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.array_prepend('array', 0),)) + assert isinstance(result, MutateInResult) + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + assert len(val['array']) == 6 + assert val['array'][0] == 0 + + def test_array_prepend_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in(key, ( + SD.array_prepend('new.array', 'World!', create_parents=True), + SD.array_prepend('new.array', 'Hello,'),)) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + assert result.content_as[dict]['new']['array'] == [ + 'Hello,', 'World!'] + + def test_array_prepend_multi_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.array_prepend('array', -2, -1, 0),)) + assert isinstance(result, MutateInResult) + result = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + val = result.content_as[dict] + assert isinstance(val['array'], list) + pre_res = val['array'][:3] + assert len(pre_res) == 3 + assert pre_res == [-2, -1, 0] + + @pytest.mark.usefixtures('skip_if_go_caves') + @pytest.mark.usefixtures('check_xattr_supported') + def test_array_add_prepend_mutate_in_macros(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + xattr_array = 'xattr_array' + specs = [ + SD.array_prepend(xattr_array, SD.MutationMacro.cas(), create_parents=True), + SD.array_prepend(xattr_array, SD.MutationMacro.seq_no()), + SD.array_prepend(xattr_array, SD.MutationMacro.value_crc32c()), + ] + result = cb_env.collection.mutate_in(key, specs) + assert isinstance(result, MutateInResult) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(xattr_array, xattr=True),)) + + res_array = res.content_as[list](0) + assert len(res_array) == 3 + assert all(map(lambda x: isinstance(x, str), res_array)) is True + # prepending -- order is reversed + res_cas = res_array[2] + res_seqno = res_array[1] + res_value = res_array[0] + + assert res_cas.startswith('0x') is True + assert res.cas == SD.convert_macro_cas_to_cas(res_cas) + assert res_seqno.startswith('0x') is True + assert res_value.startswith('0x') is True + + def test_count(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.lookup_in(key, (SD.count('array'),)) + assert isinstance(result, LookupInResult) + assert result.content_as[int](0) == 5 + + def test_decrement(self, cb_env): + key = cb_env.get_existing_doc_by_type('count', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.decrement('count', 50),)) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + assert result.content_as[dict]['count'] == 50 + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_decrement_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.decrement('new.counter', 100, create_parents=True),)) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + assert result.content_as[dict]['new']['counter'] == -100 + + def test_increment(self, cb_env): + key = cb_env.get_existing_doc_by_type('count', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.increment('count', 50),)) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + assert result.content_as[dict]['count'] == 150 + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_increment_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('count', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.increment('new.counter', 100, create_parents=True),)) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + assert result.content_as[dict]['new']['counter'] == 100 + + def test_insert_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('array', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.insert('new.path', 'parents created', create_parents=True),)) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + assert result.content_as[dict]['new']['path'] == 'parents created' + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_bad_key(self, cb_env): + with pytest.raises(DocumentNotFoundException): + cb_env.collection.lookup_in_all_replicas('asdfgh', [SD.exists('batch')]) + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_exists(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + results = cb_env.collection.lookup_in_all_replicas(key, [SD.exists('batch')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.exists(0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_exists_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + results = cb_env.collection.lookup_in_all_replicas(key, [SD.exists('qzzxy')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert not result.exists(0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_get(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + results = cb_env.collection.lookup_in_all_replicas(key, [SD.get('batch')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_get_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + results = cb_env.collection.lookup_in_all_replicas(key, [SD.get('qzzxy')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + with pytest.raises(PathNotFoundException): + result.content_as[str](0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_get_full(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + results = cb_env.collection.lookup_in_all_replicas(key, [SD.get_full()]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[dict](0) == value + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_multiple_specs(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + results = cb_env.collection.lookup_in_all_replicas(key, [SD.get('batch'), SD.exists('manufacturer.city')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.exists(1) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_with_timeout(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + opts = LookupInAllReplicasOptions(timeout=timedelta(milliseconds=5000)) + results = cb_env.collection.lookup_in_all_replicas(key, [SD.get('batch')], opts) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + @pytest.mark.usefixtures('check_server_groups_supported') + def test_lookup_in_all_replicas_read_preference(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + # No preferred server group was specified in the cluster options so this should raise + # DocumentUnretrievableException + with pytest.raises(DocumentUnretrievableException): + cb_env.collection.lookup_in_all_replicas( + key, + [SD.get('batch')], + LookupInAllReplicasOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP)) + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_bad_key(self, cb_env): + with pytest.raises(DocumentUnretrievableException): + cb_env.collection.lookup_in_any_replica('asdfgh', [SD.exists('batch')]) + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_exists(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.lookup_in_any_replica(key, [SD.exists('batch')]) + assert isinstance(result, LookupInReplicaResult) + assert result.exists(0) + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_exists_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.lookup_in_any_replica(key, [SD.exists('qzzxy')]) + assert isinstance(result, LookupInReplicaResult) + assert not result.exists(0) + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_get(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.lookup_in_any_replica(key, [SD.get('batch')]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_get_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.lookup_in_any_replica(key, [SD.get('qzzxy')]) + assert isinstance(result, LookupInReplicaResult) + with pytest.raises(PathNotFoundException): + result.content_as[str](0) + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_get_full(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.lookup_in_any_replica(key, [SD.get_full()]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[dict](0) == value + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_multiple_specs(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.lookup_in_any_replica(key, [SD.get('batch'), SD.exists('manufacturer.city')]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.exists(1) + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_with_timeout(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + opts = LookupInAnyReplicaOptions(timeout=timedelta(milliseconds=5000)) + result = cb_env.collection.lookup_in_any_replica(key, [SD.get('batch')], opts) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['batch'] + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + @pytest.mark.usefixtures('check_server_groups_supported') + def test_lookup_in_any_replica_read_preference(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + # No preferred server group was specified in the cluster options so this should raise + # DocumentUnretrievableException + with pytest.raises(DocumentUnretrievableException): + cb_env.collection.lookup_in_any_replica( + key, [SD.get('batch')], LookupInAnyReplicaOptions(read_preference=ReadPreference.SELECTED_SERVER_GROUP)) + + @pytest.mark.usefixtures('check_xattr_supported') + @pytest.mark.parametrize('macro', + [getattr(SD.LookupInMacro, m)() for m in SD.LookupInMacro.__dict__.keys() if isinstance(getattr(SD.LookupInMacro, m), # noqa: E501 + types.FunctionType)]) # noqa: E501 + def test_lookup_in_macros(self, cb_env, macro): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + macro_res = cb_env.collection.lookup_in(key, (SD.get(macro, xattr=True),)) + macro_res_value = macro_res.content_as[lambda x: x](0) + document_res = cb_env.collection.lookup_in(key, (SD.get(SD.LookupInMacro.document(), xattr=True),)) + macro_key = macro.replace('$document.', '') + document_value = document_res.content_as[dict](0) + + assert isinstance(macro_res, LookupInResult) + + if macro_key == '$document': + assert document_value == macro_res_value + else: + assert document_value[macro_key] == macro_res_value + # we expect these to be hex values + if macro_key in ['CAS', 'seqno', 'vbucket_uuid', 'value_crc32c']: + assert str(macro_res_value).startswith('0x') + + @pytest.mark.usefixtures("check_xattr_supported") + def test_lookup_in_multiple_specs(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.lookup_in(key, (SD.get('$document.exptime', xattr=True), + SD.exists('manufacturer'), + SD.get('manufacturer'), + SD.get('manufacturer.geo.accuracy'),)) + assert isinstance(result, LookupInResult) + assert result.content_as[int](0) == 0 + assert result.exists(1) + assert result.content_as[dict](2) == value['manufacturer'] + assert result.content_as[str](3) == value['manufacturer']['geo']['accuracy'] + + def test_lookup_in_one_path_not_found(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.lookup_in( + key, (SD.exists('batch'), SD.exists('qzzxy'),)) + assert isinstance(result, LookupInResult) + assert result.exists(0) + assert not result.exists(1) + # PYCBC-1480, update exists to follow RFC + assert result.content_as[bool](0) is True + assert result.content_as[bool](1) is False + + def test_lookup_in_simple_exists(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.lookup_in(key, (SD.exists('batch'),)) + assert isinstance(result, LookupInResult) + assert result.exists(0) + # PYCBC-1480, update exists to follow RFC + assert result.content_as[bool](0) is True + + def test_lookup_in_simple_exists_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.lookup_in(key, (SD.exists('qzzxy'),)) + assert isinstance(result, LookupInResult) + assert not result.exists(0) + assert result.content_as[bool](0) is False + + def test_lookup_in_simple_get(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.lookup_in(key, (SD.get('batch'),)) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value['batch'] + + def test_lookup_in_simple_get_bad_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.lookup_in(key, (SD.get('qzzxy'),)) + assert isinstance(result, LookupInResult) + with pytest.raises(PathNotFoundException): + result.content_as[str](0) + + def test_lookup_in_simple_get_spec_as_list(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.lookup_in(key, [SD.get('batch')]) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value['batch'] + + def test_lookup_in_simple_long_path(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + result = cb_env.collection.lookup_in( + key, (SD.get('manufacturer.geo.location.tz'),)) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value['manufacturer']['geo']['location']['tz'] + + def test_lookup_in_simple_with_timeout(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.lookup_in(key, + (SD.get('batch'),), + LookupInOptions(timeout=timedelta(milliseconds=5000))) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value['batch'] + + def test_lookup_in_valid_path_null_content(self, cb_env): + key, value = cb_env.get_new_doc_by_type('vehicle') + value['empty_field'] = None + cb_env.collection.upsert(key, value) + res = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert 'empty_field' in res.content_as[dict] + result = cb_env.collection.lookup_in(key, (SD.get('empty_field'), SD.get('batch'))) + assert isinstance(result, LookupInResult) + assert result.content_as[lambda x: x](0) is None + + @pytest.mark.usefixtures("check_xattr_supported") + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_expiry(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.mutate_in(key, + (SD.upsert("make", "New Make"), + SD.replace("model", "New Model")), + MutateInOptions(expiry=timedelta(seconds=1000))) + + def cas_matches(cb, new_cas): + r = cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + TestEnvironment.try_n_times(10, 3, cas_matches, cb_env.collection, result.cas) + + result = cb_env.collection.get(key, GetOptions(with_expiry=True)) + expires_in = (result.expiry_time - datetime.now()).total_seconds() + assert expires_in > 0 and expires_in < 1021 + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_insert_semantics(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + cb_env.collection.mutate_in(key, + (SD.insert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.INSERT)) + + res = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_insert_semantics_kwargs(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + cb_env.collection.mutate_in(key, + (SD.insert('new_path', 'im new'),), + insert_doc=True) + + res = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_insert_semantics_fail(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentExistsException): + cb_env.collection.mutate_in(key, + (SD.insert('new_path', 'im new'),), + insert_doc=True) + + @pytest.mark.usefixtures('skip_if_go_caves') + @pytest.mark.usefixtures('check_xattr_supported') + @pytest.mark.parametrize('macro', [SD.MutationMacro.cas(), + SD.MutationMacro.seq_no(), + SD.MutationMacro.value_crc32c()]) + def test_mutate_in_macros_insert(self, cb_env, macro): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + xattr_key = 'xattr_key' + cb_env.collection.mutate_in(key, (SD.insert(xattr_key, macro),)) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(xattr_key, xattr=True),)) + macro_res = res.content_as[str](0) + assert macro_res.startswith('0x') is True + if 'CAS' in macro.value: + assert res.cas == SD.convert_macro_cas_to_cas(macro_res) + + @pytest.mark.usefixtures('skip_if_go_caves') + @pytest.mark.usefixtures('check_xattr_supported') + @pytest.mark.parametrize('macro', [SD.MutationMacro.cas(), + SD.MutationMacro.seq_no(), + SD.MutationMacro.value_crc32c()]) + def test_mutate_in_macros_replace_upsert(self, cb_env, macro): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + xattr_key = 'xattr_key' + cb_env.collection.mutate_in(key, (SD.upsert(xattr_key, macro),)) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(xattr_key, xattr=True),)) + upsert_macro_res = res.content_as[str](0) + assert upsert_macro_res.startswith('0x') is True + if 'CAS' in macro.value: + assert res.cas == SD.convert_macro_cas_to_cas(upsert_macro_res) + + cb_env.collection.mutate_in(key, (SD.replace(xattr_key, macro),)) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(xattr_key, xattr=True),)) + replace_macro_res = res.content_as[str](0) + assert replace_macro_res.startswith('0x') is True + if 'CAS' in macro.value: + assert res.cas == SD.convert_macro_cas_to_cas(replace_macro_res) + if 'value_crc32c' in macro.value: + assert replace_macro_res == upsert_macro_res + else: + assert replace_macro_res != upsert_macro_res + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_mutate_in_preserve_expiry(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + cb_env.collection.mutate_in(key, + (SD.upsert('make', 'New Make'), + SD.replace('model', 'New Model')), + MutateInOptions(expiry=timedelta(seconds=2))) + + expiry_path = '$document.exptime' + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + cb_env.collection.mutate_in(key, + (SD.upsert('make', 'Updated Make'),), + MutateInOptions(preserve_expiry=True)) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + TestEnvironment.sleep(3) + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_mutate_in_preserve_expiry_fails(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + with pytest.raises(InvalidArgumentException): + cb_env.collection.mutate_in( + key, + (SD.insert('c', 'ccc'),), + MutateInOptions(preserve_expiry=True), + ) + + with pytest.raises(InvalidArgumentException): + cb_env.collection.mutate_in( + key, + (SD.replace('c', 'ccc'),), + MutateInOptions( + expiry=timedelta( + seconds=5), + preserve_expiry=True), + ) + + @pytest.mark.usefixtures('check_preserve_expiry_supported') + def test_mutate_in_preserve_expiry_not_used(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + cb_env.collection.mutate_in(key, + (SD.upsert('make', 'New Make'), + SD.replace('model', 'New Model')), + MutateInOptions(expiry=timedelta(seconds=5))) + + expiry_path = '$document.exptime' + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + cb_env.collection.mutate_in(key, (SD.upsert('make', 'Updated Make'),)) + res = TestEnvironment.try_n_times(10, + 3, + cb_env.collection.lookup_in, + key, + (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + TestEnvironment.sleep(3) + result = cb_env.collection.get(key) + assert isinstance(result, GetResult) + assert result.content_as[dict]['make'] == 'Updated Make' + + def test_mutate_in_remove(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + cb_env.collection.mutate_in(key, [SD.remove('manufacturer.geo')]) + result = cb_env.collection.get(key) + assert 'geo' not in result.content_as[dict]['manufacturer'] + + def test_mutate_in_remove_blank_path(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + cb_env.collection.mutate_in(key, [SD.remove('')]) + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_replace_semantics(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.REPLACE)) + + res = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict]['new_path'] == 'im new' + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_replace_semantics_fail(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + replace_doc=True) + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_replace_semantics_kwargs(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + replace_doc=True) + + res = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict]['new_path'] == 'im new' + + def test_mutate_in_replace_full_document(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + + cb_env.collection.mutate_in(key, + (SD.replace('', {'make': 'New Make', 'model': 'New Model'}),)) + + res = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict]['make'] == 'New Make' + assert res.content_as[dict]['model'] == 'New Model' + + def test_mutate_in_simple(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.mutate_in(key, + (SD.upsert('make', 'New Make'), + SD.replace('model', 'New Model'))) + + value['make'] = 'New Make' + value['model'] = 'New Model' + + def cas_matches(cb, new_cas): + r = cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + TestEnvironment.try_n_times(10, 3, cas_matches, cb_env.collection, result.cas) + + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + def test_mutate_in_simple_spec_as_list(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('vehicle') + result = cb_env.collection.mutate_in(key, + [SD.upsert('make', 'New Make'), + SD.replace('model', 'New Model')]) + + value['make'] = 'New Make' + value['model'] = 'New Model' + + def cas_matches(cb, new_cas): + r = cb.get(key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + TestEnvironment.try_n_times(10, 3, cas_matches, cb_env.collection, result.cas) + + result = cb_env.collection.get(key) + assert value == result.content_as[dict] + + def test_mutate_in_store_semantics_fail(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(InvalidArgumentException): + cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + insert_doc=True, upsert_doc=True) + + with pytest.raises(InvalidArgumentException): + cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + insert_doc=True, replace_doc=True) + + with pytest.raises(InvalidArgumentException): + cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + upsert_doc=True, replace_doc=True) + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_upsert_semantics(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.UPSERT)) + + res = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_mutate_in_upsert_semantics_kwargs(self, cb_env): + key = cb_env.get_new_doc_by_type('vehicle', key_only=True) + + with pytest.raises(DocumentNotFoundException): + cb_env.collection.get(key) + + cb_env.collection.mutate_in(key, + (SD.upsert('new_path', 'im new'),), + upsert_doc=True) + + res = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + def test_upsert_create_parents(self, cb_env): + key = cb_env.get_existing_doc_by_type('vehicle', key_only=True) + result = cb_env.collection.mutate_in( + key, (SD.upsert('new.path', 'parents created', create_parents=True),)) + assert isinstance(result, MutateInResult) + result = cb_env.collection.get(key) + assert result.content_as[dict]['new']['path'] == 'parents created' + + +# For whatever reason GoCAVES is rather flaky w/ subdocument tests +@pytest.mark.flaky(reruns=3, reruns_delay=1) +class ClassicSubDocumentTests(SubDocumentTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicSubDocumentTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicSubDocumentTests) if valid_test_method(meth)] + compare = set(SubDocumentTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = SubdocTestEnvironment.from_environment(cb_base_env) + cb_env.enable_bucket_mgmt() + cb_env.setup(request.param) + + yield cb_env + + cb_env.teardown(request.param) diff --git a/couchbase/tests/test_sync.py b/couchbase/tests/test_sync.py deleted file mode 100644 index 7cc86a55c..000000000 --- a/couchbase/tests/test_sync.py +++ /dev/null @@ -1,12 +0,0 @@ -from couchbase.bucket import Bucket -from couchbase.views.iterator import View -from couchbase.tests.base import ApiImplementationMixin -from couchbase.tests.importer import get_configured_classes - -class SyncImplMixin(ApiImplementationMixin): - factory = Bucket - viewfactory = View - should_check_refcount = True - -configured_cases = get_configured_classes(SyncImplMixin) -globals().update(configured_cases) diff --git a/couchbase/tests/tracing_t.py b/couchbase/tests/tracing_t.py new file mode 100644 index 000000000..c19bba9dc --- /dev/null +++ b/couchbase/tests/tracing_t.py @@ -0,0 +1,140 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.exceptions import CouchbaseException +from couchbase.options import (AnalyticsOptions, + GetOptions, + InsertOptions, + QueryOptions, + RemoveOptions, + ReplaceOptions, + SearchOptions, + UpsertOptions, + ViewOptions) +from couchbase.search import TermQuery +from tests.environments.tracing_and_metrics_environment import TracingAndMetricsTestEnvironment + + +class TracerTestsSuite: + TEST_MANIFEST = [ + 'test_http', + 'test_kv', + ] + + @pytest.fixture(scope='class') + def skip_if_mock(self, cb_env): + if cb_env.is_mock_server: + pytest.skip('Test needs real server') + + @pytest.mark.parametrize('op, span_name, opts, value', [ + ('get', 'cb.get', GetOptions, None), + ('upsert', 'cb.upsert', UpsertOptions, {'some': 'thing'}), + ('insert', 'cb.insert', InsertOptions, {'some': 'thing'}), + ('replace', 'cb.replace', ReplaceOptions, {'some': 'thing'}), + ('remove', 'cb.remove', RemoveOptions, None), + ]) + @pytest.mark.parametrize("with_parent", [True, False]) + def test_kv(self, cb_env, op, span_name, opts, value, with_parent): + # have to reset between parameterized runs + cb_env.tracer.reset() + cb = cb_env.collection + parent = None + if with_parent: + parent = cb_env.tracer.start_span(f'parent_{op}') + key = cb_env.get_existing_doc(key_only=True) + options = opts(span=parent) + operation = getattr(cb, op) + try: + if value: + operation(key, value, options) + else: + operation(key, options) + except CouchbaseException: + pass # insert will fail, who cares. + + spans = cb_env.tracer.spans() + if with_parent: + assert len(spans) == 2 + span = spans.pop(0) + assert span == parent + assert span.is_finished() is False + + assert len(spans) == 1 + assert spans[0].is_finished() is True + assert spans[0].get_name() == span_name + assert spans[0].get_parent() == parent + + @pytest.mark.parametrize('http_op, http_span_name, http_opts, query, extra', [ + ('query', 'cb.query', QueryOptions, 'Select 1', None), + ('analytics_query', 'cb.analytics', AnalyticsOptions, "Select 1", None), + ('search_query', 'cb.search', SearchOptions, 'whatever', TermQuery('foo')), + ('view_query', 'cb.views', ViewOptions, 'whatever', 'whatever_else') + ]) + @pytest.mark.parametrize('http_with_parent', [True, False]) + @pytest.mark.usefixtures('skip_if_mock') + def test_http(self, cb_env, http_op, http_span_name, http_opts, query, extra, http_with_parent): + cb_env.tracer.reset() + cb = cb_env.bucket if http_op == 'view_query' else cb_env.cluster + parent = None + if http_with_parent: + parent = cb_env.tracer.start_span(f'parent_{http_op}') + options = http_opts(span=parent) + operation = getattr(cb, http_op) + result = None + try: + if extra: + result = operation(query, extra, options).rows() + else: + result = operation(query, options).rows() + for r in result: + assert r is not None + except CouchbaseException: + pass + spans = cb_env.tracer.spans() + if http_with_parent: + assert len(spans) == 2 + span = spans.pop(0) + assert span == parent + assert span.is_finished() is False + assert len(spans) == 1 + assert spans[0].is_finished() is True + assert spans[0].get_name() == http_span_name + assert spans[0].get_parent() == parent + + +class ClassicTracerTests(TracerTestsSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicTracerTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicTracerTests) if valid_test_method(meth)] + compare = set(TracerTestsSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env, test_manifest_validated): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + # a new environment and cluster is created + cb_env = TracingAndMetricsTestEnvironment.from_environment(cb_base_env, + create_tracer=True) + cb_env.setup(num_docs=10) + yield cb_env + cb_env.teardown() + cb_env.cluster.close() diff --git a/couchbase/tests/transactions_t.py b/couchbase/tests/transactions_t.py new file mode 100644 index 000000000..d1f4e4149 --- /dev/null +++ b/couchbase/tests/transactions_t.py @@ -0,0 +1,799 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta + +import pytest + +from couchbase.durability import DurabilityLevel, ServerDurability +from couchbase.exceptions import (BucketNotFoundException, + DocumentExistsException, + DocumentNotFoundException, + DocumentUnretrievableException, + FeatureUnavailableException, + ParsingFailedException, + TransactionExpired, + TransactionFailed, + TransactionOperationFailed) +from couchbase.n1ql import QueryProfile, QueryScanConsistency +from couchbase.options import (TransactionConfig, + TransactionGetOptions, + TransactionInsertOptions, + TransactionOptions, + TransactionQueryOptions, + TransactionReplaceOptions) +from couchbase.transactions import TransactionKeyspace, TransactionResult +from couchbase.transcoder import RawBinaryTranscoder +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class TransactionTestSuite: + TEST_MANIFEST = [ + 'test_adhoc', + 'test_bad_query', + 'test_binary', + 'test_binary_kwargs', + 'test_binary_not_supported', + 'test_cleanup_client_attempts', + 'test_cleanup_lost_attempts', + 'test_cleanup_window', + 'test_client_context_id', + 'test_expiration_time', + 'test_get', + 'test_get_lambda_raises_doc_not_found', + 'test_get_inner_exc_doc_not_found', + 'test_get_replica_from_preferred_server_group_unretrievable', + 'test_get_replica_from_preferred_server_group_propagate_unretrievable_exc', + 'test_insert', + 'test_insert_lambda_raises_doc_exists', + 'test_insert_inner_exc_doc_exists', + 'test_max_parallelism', + 'test_metadata_collection', + 'test_metadata_collection_not_found', + 'test_metrics', + 'test_named_params', + 'test_per_txn_config', + 'test_pipeline_batch', + 'test_pipeline_cap', + 'test_positional_params', + 'test_profile_mode', + 'test_query', + 'test_query_lambda_raises_parsing_failure', + 'test_query_inner_exc_parsing_failure', + 'test_query_mode_insert', + 'test_query_mode_remove', + 'test_raw', + 'test_read_only', + 'test_remove', + 'test_remove_fail_bad_cas', + 'test_replace', + 'test_replace_fail_bad_cas', + 'test_rollback', + 'test_rollback_eating_exceptions', + 'test_scan_consistency', + 'test_scope_qualifier', + 'test_timeout', + 'test_transaction_config_durability', + 'test_transaction_result', + ] + + @pytest.fixture(scope='class') + def check_txn_queries_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('txn_queries', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_binary_txns_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('binary_txns', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.fixture(scope='class') + def check_binary_txns_not_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_not_supported('binary_txns', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.fixture(scope='class') + def check_server_groups_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('server_groups', + cb_env.server_version_short, + cb_env.mock_server_type, + cb_env.server_version_patch) + + @pytest.mark.parametrize('adhoc', [True, False]) + def test_adhoc(self, adhoc): + cfg = TransactionQueryOptions(adhoc=adhoc) + cfg_adhoc = cfg._base.to_dict().get('adhoc', None) + assert cfg_adhoc is not None + assert cfg_adhoc == adhoc + + @pytest.mark.usefixtures('check_txn_queries_supported') + def test_bad_query(self, cb_env): + + def txn_logic(ctx): + try: + ctx.query('this wont parse') + pytest.fail('expected bad query to raise exception') + except ParsingFailedException: + pass + except Exception as e: + pytest.fail(f"Expected bad query to raise ParsingFailedException, not {e.__class__.__name__}") + + cb_env.cluster.transactions.run(txn_logic) + + @pytest.mark.usefixtures('check_binary_txns_supported') + def test_binary(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + tc = RawBinaryTranscoder() + value = 'bytes content'.encode('utf-8') + new_content = b'\xFF' + + def txn_logic(ctx): + ctx.insert(cb_env.collection, key, value, TransactionInsertOptions(transcoder=tc)) + get_res = ctx.get(cb_env.collection, key, TransactionGetOptions(transcoder=tc)) + assert get_res.content_as[bytes] == value + replace_res = ctx.replace(get_res, new_content, TransactionReplaceOptions(transcoder=tc)) + # there's a bug where we don't return the correct content in the replace, so comment this out for now + # assert replace_res.content_as[bytes] == new_content + assert get_res.cas != replace_res.cas + + cb_env.cluster.transactions.run(txn_logic) + result = cb_env.collection.get(key, transcoder=tc) + assert result.content_as[bytes] == new_content + + @pytest.mark.usefixtures('check_binary_txns_supported') + def test_binary_kwargs(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + tc = RawBinaryTranscoder() + value = 'bytes content'.encode('utf-8') + new_content = b'\xFF' + + def txn_logic(ctx): + ctx.insert(cb_env.collection, key, value, transcoder=tc) + get_res = ctx.get(cb_env.collection, key, transcoder=tc) + assert get_res.content_as[bytes] == value + replace_res = ctx.replace(get_res, new_content, transcoder=tc) + # there's a bug where we don't return the correct content in the replace, so comment this out for now + # assert replace_res.content_as[bytes] == new_content + assert get_res.cas != replace_res.cas + + cb_env.cluster.transactions.run(txn_logic) + result = cb_env.collection.get(key, transcoder=tc) + assert result.content_as[bytes] == new_content + + @pytest.mark.usefixtures('check_binary_txns_not_supported') + def test_binary_not_supported(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + tc = RawBinaryTranscoder() + value = 'bytes content'.encode('utf-8') + + def txn_logic(ctx): + ctx.insert(cb_env.collection, key, value, TransactionInsertOptions(transcoder=tc)) + + try: + cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + err_msg = f"Expected to raise FeatureUnavailableException, not {ex.inner_cause.__class__.__name__}" + assert isinstance(ex.inner_cause, FeatureUnavailableException), err_msg + + @pytest.mark.parametrize('cleanup', [False, True]) + def test_cleanup_client_attempts(self, cleanup): + cfg = TransactionConfig(cleanup_client_attempts=cleanup) + cfg_cleanup = cfg._base.to_dict().get('cleanup_client_attempts', None) + assert cfg_cleanup is not None + assert cfg_cleanup is cleanup + + @pytest.mark.parametrize('cleanup', [False, True]) + def test_cleanup_lost_attempts(self, cleanup): + cfg = TransactionConfig(cleanup_lost_attempts=cleanup) + cfg_cleanup = cfg._base.to_dict().get('cleanup_lost_attempts', None) + assert cfg_cleanup is not None + assert cfg_cleanup is cleanup + + @pytest.mark.parametrize('window', [timedelta(seconds=30), timedelta(milliseconds=500)]) + def test_cleanup_window(self, window): + cfg = TransactionConfig(cleanup_window=window) + cfg_window = cfg._base.to_dict().get('cleanup_window', None) + assert cfg_window is not None + assert cfg_window == window.total_seconds() * 1000 # milliseconds + + def test_client_context_id(self): + ctxid = "somestring" + cfg = TransactionQueryOptions(client_context_id=ctxid) + cfg_ctxid = cfg._base.to_dict().get('client_context_id', None) + assert cfg_ctxid is not None + assert cfg_ctxid == ctxid + + @pytest.mark.parametrize('cls', [TransactionConfig, TransactionOptions]) + @pytest.mark.parametrize('exp', [timedelta(seconds=30), timedelta(milliseconds=100)]) + def test_expiration_time(self, cls, exp): + cfg = cls(expiration_time=exp) + # CXXCBC-391, changes make expiration_time an invalid key + cfg_expiry = cfg._base.to_dict().get('expiration_time', None) + assert cfg_expiry is None + cfg_timeout = cfg._base.to_dict().get('timeout', None) + assert cfg_timeout == exp.total_seconds() * 1000*1000*1000 # nanoseconds - and can't use 'is' here + + def test_get(self, cb_env): + key, value = cb_env.get_existing_doc() + + def txn_logic(ctx): + res = ctx.get(cb_env.collection, key) + assert res.cas > 0 + assert res.id == key + assert res.content_as[dict] == value + + cb_env.cluster.transactions.run(txn_logic) + + def test_get_lambda_raises_doc_not_found(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + num_attempts = 0 + + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + try: + ctx.get(cb_env.collection, key) + except Exception as ex: + err_msg = f"Expected to raise DocumentNotFoundException, not {ex.__class__.__name__}" + assert isinstance(ex, DocumentNotFoundException), err_msg + + raise Exception('User raised exception.') + + try: + cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert 'User raised exception.' in str(ex) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + def test_get_inner_exc_doc_not_found(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + num_attempts = 0 + + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + ctx.get(cb_env.collection, key) + + try: + cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + assert isinstance(ex.inner_cause, DocumentNotFoundException) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.usefixtures('check_server_groups_supported') + def test_get_replica_from_preferred_server_group_unretrievable(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + num_attempts = 0 + + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + with pytest.raises(DocumentUnretrievableException): + ctx.get_replica_from_preferred_server_group(cb_env.collection, key) + + cb_env.cluster.transactions.run(txn_logic) + + assert num_attempts == 1 + + @pytest.mark.usefixtures('check_server_groups_supported') + def test_get_replica_from_preferred_server_group_propagate_unretrievable_exc(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + num_attempts = 0 + + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + ctx.get_replica_from_preferred_server_group(cb_env.collection, key) + + try: + cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + assert isinstance(ex.inner_cause, DocumentUnretrievableException) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + def test_insert(self, cb_env): + key, value = cb_env.get_new_doc() + + def txn_logic(ctx): + ctx.insert(cb_env.collection, key, value) + + cb_env.cluster.transactions.run(txn_logic) + get_result = cb_env.collection.get(key) + assert get_result.content_as[dict] == value + + def test_insert_lambda_raises_doc_exists(self, cb_env): + key, value = cb_env.get_existing_doc() + num_attempts = 0 + + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + try: + ctx.insert(cb_env.collection, key, value) + except Exception as ex: + err_msg = f"Expected to raise DocumentExistsException, not {ex.__class__.__name__}" + assert isinstance(ex, DocumentExistsException), err_msg + + raise Exception('User raised exception.') + + try: + cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert 'User raised exception.' in str(ex) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + def test_insert_inner_exc_doc_exists(self, cb_env): + key, value = cb_env.get_existing_doc() + num_attempts = 0 + + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + ctx.insert(cb_env.collection, key, value) + + try: + cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + assert isinstance(ex.inner_cause, DocumentExistsException) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + def test_max_parallelism(self): + max = 100 + cfg = TransactionQueryOptions(max_parallelism=max) + cfg_max = cfg._base.to_dict().get('max_parallelism', None) + assert cfg_max is not None + assert cfg_max == max + + @pytest.mark.parametrize('cls', [TransactionOptions, TransactionConfig]) + def test_metadata_collection(self, cls, cb_env): + coll = cb_env.collection + cfg = cls(metadata_collection=TransactionKeyspace(coll=coll)) + cfg_coll = cfg._base.to_dict().get('metadata_collection', None) + assert cfg_coll is not None + assert cfg_coll == f'{coll._scope.bucket_name}.{coll._scope.name}.{coll.name}' + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_metadata_collection_not_found(self, cb_env): + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.options import ClusterOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + metadata = TransactionKeyspace(bucket='no-bucket', scope='_default', collection='_default') + cluster = Cluster.connect(f'{conn_string}', + ClusterOptions(auth, + transaction_config=TransactionConfig(metadata_collection=metadata))) + collection = cluster.bucket(cb_env.bucket.name).default_collection() + + def txn_logic(ctx): + # key should not matter as we should fail when creating the + # transactions object and not actually get to this point + ctx.get(collection, 'test-key') + + with pytest.raises(BucketNotFoundException): + cluster.transactions.run(txn_logic) + + @pytest.mark.parametrize('metrics', [True, False]) + def test_metrics(self, metrics): + cfg = TransactionQueryOptions(metrics=metrics) + cfg_metrics = cfg._base.to_dict().get('metrics', None) + assert cfg_metrics is not None + assert cfg_metrics == metrics + + @pytest.mark.parametrize('params', [{'key1': 'thing'}, + {'key1': ['an', 'array']}, + {'key1': 10, 'key2': 'something else'}]) + def test_named_params(self, params): + cfg = TransactionQueryOptions(named_parameters=params) + cfg_params = cfg._base.to_dict().get('named_parameters', None) + assert cfg_params is not None + assert isinstance(cfg_params, dict) + for k, v in params.items(): + assert json.loads(cfg_params[k]) == v + + def test_per_txn_config(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + + def txn_logic(ctx): + ctx.insert(cb_env.collection, key, {'some': 'thing'}) + TestEnvironment.sleep(0.001) + ctx.get(cb_env.collection, key) + + with pytest.raises(TransactionExpired): + cb_env.cluster.transactions.run(txn_logic, + TransactionOptions(timeout=timedelta(microseconds=1))) + res = cb_env.collection.exists(key) + assert res.exists is False + + def test_pipeline_batch(self): + batch = 100 + cfg = TransactionQueryOptions(pipeline_batch=batch) + cfg_batch = cfg._base.to_dict().get('pipeline_batch', None) + assert cfg_batch is not None + assert cfg_batch == batch + + def test_pipeline_cap(self): + cap = 100 + cfg = TransactionQueryOptions(pipeline_cap=cap) + cfg_cap = cfg._base.to_dict().get('pipeline_cap', None) + assert cfg_cap is not None + assert cfg_cap == cap + + @pytest.mark.parametrize('params', [['a', 'b', 'c']]) # , [[1, 2, 3], ['a', 'b', 'c']]]) + def test_positional_params(self, params): + cfg = TransactionQueryOptions(positional_parameters=params) + cfg_params = cfg._base.to_dict().get('positional_parameters', None) + assert cfg_params is not None + assert isinstance(cfg_params, list) + for idx, p in enumerate(cfg_params): + assert params[idx] == json.loads(p) + + @pytest.mark.parametrize('profile', [QueryProfile.OFF, QueryProfile.PHASES, QueryProfile.TIMINGS]) + def test_profile_mode(self, profile): + cfg = TransactionQueryOptions(profile=profile) + cfg_profile = cfg._base.to_dict().get('profile', None) + assert cfg_profile is not None + assert cfg_profile == profile.value + + @pytest.mark.usefixtures('check_txn_queries_supported') + def test_query(self, cb_env): + coll = cb_env.collection + key, value = cb_env.get_new_doc() + + def txn_logic(ctx): + location = f"default:`{coll._scope.bucket_name}`.`{coll._scope.name}`.`{coll.name}`" + ctx.query( + f'INSERT INTO {location} VALUES("{key}", {json.dumps(value)})', + TransactionQueryOptions(metrics=False)) + + cb_env.cluster.transactions.run(txn_logic) + res = cb_env.collection.exists(key) + assert res.exists is True + + @pytest.mark.usefixtures('check_txn_queries_supported') + def test_query_lambda_raises_parsing_failure(self, cb_env): + num_attempts = 0 + + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + try: + ctx.query('This is not N1QL!', TransactionQueryOptions(metrics=False)) + except Exception as ex: + err_msg = f"Expected to raise ParsingFailedException, not {ex.__class__.__name__}" + assert isinstance(ex, ParsingFailedException), err_msg + + raise Exception('User raised exception.') + + try: + cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert 'User raised exception.' in str(ex) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.usefixtures('check_txn_queries_supported') + def test_query_inner_exc_parsing_failure(self, cb_env): + num_attempts = 0 + + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + ctx.query('This is not N1QL!', TransactionQueryOptions(metrics=False)) + + try: + cb_env.cluster.transactions.run(txn_logic) + except TransactionFailed as ex: + assert ex.inner_cause is not None + assert isinstance(ex.inner_cause, ParsingFailedException) + except Exception as ex: + pytest.fail(f"Expected to raise TransactionFailed, not {ex.__class__.__name__}") + + assert num_attempts == 1 + + @pytest.mark.usefixtures('check_txn_queries_supported') + def test_query_mode_insert(self, cb_env): + coll = cb_env.collection + key, value = cb_env.get_new_doc() + key1, value1 = cb_env.get_new_doc() + coll.insert(key, value) + + def txn_logic(ctx): + fdqn = f"`{coll._scope.bucket_name}`.`{coll._scope.name}`.`{coll.name}`" + statement = f'SELECT * FROM {fdqn} WHERE META().id IN $1 ORDER BY META().id ASC' + res = ctx.query(statement, TransactionQueryOptions(positional_parameters=[[key]])) + assert len(res.rows()) == 1 + assert res.rows()[0].get(f'{coll.name}', {}).get('id') == value.get('id') + ctx.insert(coll, key1, value1) + + cb_env.cluster.transactions.run(txn_logic) + get_res = coll.get(key1) + assert get_res is not None + assert get_res.content_as[dict] == value1 + + @pytest.mark.usefixtures('check_txn_queries_supported') + def test_query_mode_remove(self, cb_env): + coll = cb_env.collection + key, value = cb_env.get_new_doc() + key1, value1 = cb_env.get_new_doc() + coll.insert(key, value) + + def txn_logic(ctx): + ctx.insert(coll, key1, value1) + fdqn = f"`{coll._scope.bucket_name}`.`{coll._scope.name}`.`{coll.name}`" + statement = f'SELECT * FROM {fdqn} WHERE META().id IN $1 ORDER BY META().id ASC' + res = ctx.query(statement, TransactionQueryOptions(positional_parameters=[[key, key1]])) + assert len(res.rows()) == 2 + getRes = ctx.get(coll, key) + ctx.remove(getRes) + + cb_env.cluster.transactions.run(txn_logic) + with pytest.raises(DocumentNotFoundException): + coll.get(key) + + @pytest.mark.parametrize('raw', [{'key1': 'yo'}, {'key1': 5, 'key2': 'foo'}, {'key': [1, 2, 3]}]) + def test_raw(self, raw): + cfg = TransactionQueryOptions(raw=raw) + cfg_raw = cfg._base.to_dict().get('raw', None) + assert cfg_raw is not None + assert isinstance(cfg_raw, dict) + for k, v in cfg_raw.items(): + assert json.loads(cfg_raw[k]) == raw[k] + + @pytest.mark.parametrize('read_only', [True, False]) + def test_read_only(self, read_only): + cfg = TransactionQueryOptions(read_only=read_only) + cfg_read_only = cfg._base.to_dict().get('read_only', None) + assert cfg_read_only is not None + assert cfg_read_only == read_only + + def test_remove(self, cb_env): + key, value = cb_env.get_new_doc() + cb_env.collection.insert(key, value) + + def txn_logic(ctx): + get_res = ctx.get(cb_env.collection, key) + ctx.remove(get_res) + + cb_env.cluster.transactions.run(txn_logic) + result = cb_env.collection.exists(key) + assert result.exists is False + + def test_remove_fail_bad_cas(self, cb_env): + key, value = cb_env.get_existing_doc() + num_attempts = 0 + + # txn will retry until timeout + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + rem_res = ctx.get(cb_env.collection, key) + ctx.replace(rem_res, {'what': 'new content!'}) + try: + ctx.remove(rem_res) + except Exception as ex: + assert isinstance(ex, TransactionOperationFailed) + assert 'transaction expired' in ex.message or 'cas_mismatch' in ex.message + + try: + cb_env.cluster.transactions.run(txn_logic, TransactionOptions(timeout=timedelta(seconds=2))) + except Exception as ex: + assert isinstance(ex, TransactionExpired) + assert 'transaction expired' in ex.message or 'expired in auto' in ex.message + + assert num_attempts > 1 + # txn should fail, so doc should exist + res = cb_env.collection.get(key) + assert res.content_as[dict] == value + + def test_replace(self, cb_env): + key, value = cb_env.get_existing_doc() + new_value = {'some': 'thing else'} + + cb_env.collection.upsert(key, value) + + def txn_logic(ctx): + get_res = ctx.get(cb_env.collection, key) + assert get_res.content_as[dict] == value + replace_res = ctx.replace(get_res, new_value) + # there's a bug where we don't return the correct content in the replace, so comment this out for now + # assert replace_res.content_as[str] == new_value + assert get_res.cas != replace_res.cas + + cb_env.cluster.transactions.run(txn_logic) + result = cb_env.collection.get(key) + assert result.content_as[dict] == new_value + + def test_replace_fail_bad_cas(self, cb_env): + key, value = cb_env.get_existing_doc() + num_attempts = 0 + + # txn will retry until timeout + def txn_logic(ctx): + nonlocal num_attempts + num_attempts += 1 + rem_res = ctx.get(cb_env.collection, key) + ctx.replace(rem_res, {'foo': 'bar'}) + try: + ctx.replace(rem_res, {'foo': 'baz'}) + except Exception as ex: + assert isinstance(ex, TransactionOperationFailed) + assert 'transaction expired' in ex.message or 'cas_mismatch' in ex.message + + try: + cb_env.cluster.transactions.run(txn_logic, TransactionOptions(timeout=timedelta(seconds=2))) + except Exception as ex: + assert isinstance(ex, TransactionExpired) + assert 'transaction expired' in ex.message or 'expired in auto' in ex.message + + assert num_attempts > 1 + # txn should fail, so doc should have original content + res = cb_env.collection.get(key) + assert res.content_as[dict] == value + + def test_rollback(self, cb_env): + key, value = cb_env.get_new_doc() + + def txn_logic(ctx): + res = ctx.insert(cb_env.collection, key, value) + assert res.id == key + assert res.cas > 0 + raise RuntimeError('this should rollback txn') + + with pytest.raises(TransactionFailed): + cb_env.cluster.transactions.run(txn_logic) + + result = cb_env.collection.exists(key) + result.exists is False + + def test_rollback_eating_exceptions(self, cb_env): + key = cb_env.get_existing_doc(key_only=True) + result = cb_env.collection.get(key) + cas = result.cas + + def txn_logic(ctx): + try: + ctx.insert(cb_env.collection, key, {'this': 'should fail'}) + pytest.fail("insert of existing key should have failed") + except DocumentExistsException: + # just eat the exception + pass + except Exception as e2: + pytest.fail(f"Expected insert to raise TransactionOperationFailed, not {e2.__class__.__name__}") + + try: + cb_env.cluster.transactions.run(txn_logic) + except Exception as ex: + assert isinstance(ex, TransactionFailed) + # the inner cause should be a DocumentExistsException for this example + # if pytest.fail() occurred this will not be the case, thus failing the test + assert isinstance(ex.inner_cause, DocumentExistsException) + + result = cb_env.collection.get(key) + assert result.cas == cas + + @pytest.mark.parametrize('cls', [TransactionQueryOptions, TransactionConfig, TransactionOptions]) + @pytest.mark.parametrize('consistency', [QueryScanConsistency.REQUEST_PLUS, + QueryScanConsistency.NOT_BOUNDED, + QueryScanConsistency.AT_PLUS]) + def test_scan_consistency(self, cls, consistency): + cfg = None + try: + cfg = cls(scan_consistency=consistency) + except Exception: + if consistency != QueryScanConsistency.AT_PLUS: + pytest.fail("got unexpected exception creating TransactionConfig", True) + if cfg: + cfg_consistency = cfg._base.to_dict().get('scan_consistency', None) + assert cfg_consistency is not None + assert cfg_consistency == consistency.value + + def test_scope_qualifier(self, cb_env): + pytest.skip('CBD-5091: Pending Transactions changes') + cfg = TransactionQueryOptions(scope=cb_env.collection._scope) + cfg_scope_qualifier = cfg._base.to_dict().get('scope_qualifier', None) + expected = f'default:`{cb_env.collection._scope.bucket_name}`.`{cb_env.collection._scope.name}`' + assert cfg_scope_qualifier is not None + assert cfg_scope_qualifier == expected + bucket, scope = cfg.split_scope_qualifier() + assert bucket == cb_env.collection._scope.bucket_name + assert scope == cb_env.collection._scope.name + + @pytest.mark.parametrize('cls', [TransactionConfig, TransactionOptions]) + @pytest.mark.parametrize('exp', [timedelta(seconds=30), timedelta(milliseconds=100)]) + def test_timeout(self, cls, exp): + cfg = cls(timeout=exp) + cfg_timeout = cfg._base.to_dict().get('timeout', None) + assert cfg_timeout is not None + assert cfg_timeout == exp.total_seconds() * 1000*1000*1000 # nanoseconds - and can't use 'is' here + + @pytest.mark.parametrize('cls', [TransactionConfig, TransactionOptions]) + @pytest.mark.parametrize('level', [DurabilityLevel.NONE, + DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE, + DurabilityLevel.MAJORITY, + DurabilityLevel.PERSIST_TO_MAJORITY]) + def test_transaction_config_durability(self, cls, level): + cfg = cls(durability=ServerDurability(level)) + cfg_level = cfg._base.to_dict().get('durability_level', None) + assert cfg_level is not None + assert DurabilityLevel(cfg_level) is level + + def test_transaction_result(self, cb_env): + key = cb_env.get_new_doc(key_only=True) + + def txn_logic(ctx): + ctx.insert(cb_env.collection, key, {'some': 'thing'}) + doc = ctx.get(cb_env.collection, key) + ctx.replace(doc, {'some': 'thing else'}) + + result = cb_env.cluster.transactions.run(txn_logic) + assert isinstance(result, TransactionResult) is True + assert result.transaction_id is not None + assert result.unstaging_complete is True + + +class ClassicTransactionTests(TransactionTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicTransactionTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicTransactionTests) if valid_test_method(meth)] + compare = set(TransactionTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_txn_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + EnvironmentFeatures.check_if_feature_supported('txns', + cb_base_txn_env.server_version_short, + cb_base_txn_env.mock_server_type) + + cb_base_txn_env.setup(request.param, __name__) + yield cb_base_txn_env + cb_base_txn_env.teardown(request.param, __name__) diff --git a/couchbase/tests/transcoder_t.py b/couchbase/tests/transcoder_t.py new file mode 100644 index 000000000..190c560bb --- /dev/null +++ b/couchbase/tests/transcoder_t.py @@ -0,0 +1,764 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta +from typing import Any, Tuple + +import pytest + +from couchbase.constants import FMT_JSON +from couchbase.exceptions import (DocumentLockedException, + DocumentNotFoundException, + ValueFormatException) +from couchbase.options import (GetAndLockOptions, + GetAndTouchOptions, + GetOptions, + ReplaceOptions) +from couchbase.transcoder import (JSONTranscoder, + LegacyTranscoder, + RawBinaryTranscoder, + RawJSONTranscoder, + RawStringTranscoder, + Transcoder) +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment +from tests.environments.transcoder_environment import FakeTestObj, TranscoderTestEnvironment + + +class ZeroFlagsTranscoder(Transcoder): + def encode_value(self, + value, # type: Any + ) -> Tuple[bytes, int]: + return json.dumps(value, ensure_ascii=False).encode('utf-8'), 0 + + def decode_value(self, + value, # type: bytes + flags # type: int + ) -> Any: + # ignoring flags...only for test purposes + return json.loads(value.decode('utf-8')) + + +class DefaultTranscoderTestSuite: + TEST_MANIFEST = [ + 'test_default_tc_binary_insert', + 'test_default_tc_binary_replace', + 'test_default_tc_binary_upsert', + 'test_default_tc_bytearray_upsert', + 'test_default_tc_decoding', + 'test_default_tc_flags_zero', + 'test_default_tc_json_insert', + 'test_default_tc_json_replace', + 'test_default_tc_json_upsert', + 'test_default_tc_string_insert', + 'test_default_tc_string_replace', + 'test_default_tc_string_upsert', + ] + + def test_default_tc_binary_insert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + with pytest.raises(ValueFormatException): + cb_env.collection.insert(key, value) + + def test_default_tc_binary_replace(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + with pytest.raises(ValueFormatException): + cb_env.collection.replace(key, value) + + def test_default_tc_binary_upsert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + with pytest.raises(ValueFormatException): + cb_env.collection.upsert(key, value) + + def test_default_tc_bytearray_upsert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + with pytest.raises(ValueFormatException): + cb_env.collection.upsert(key, bytearray(value)) + + def test_default_tc_decoding(self): + tc = JSONTranscoder() + content = {'foo': 'bar'} + value, flags = tc.encode_value(content) + assert flags == FMT_JSON + decoded = tc.decode_value(value, None) + assert content == decoded + decoded = tc.decode_value(value, 0) + assert content == decoded + + def test_default_tc_flags_zero(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('json') + cb_env.collection.upsert(key, value, transcoder=ZeroFlagsTranscoder()) + res = cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + def test_default_tc_json_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('json') + cb_env.collection.insert(key, value) + + res = cb_env.collection.get(key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + def test_default_tc_json_replace(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('json') + value['new_content'] = 'new content!' + cb_env.collection.replace(key, value) + res = cb_env.collection.get(key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + def test_default_tc_json_upsert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('json') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + def test_default_tc_string_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('utf8') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == value + + def test_default_tc_string_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8', key_only=True) + new_content = "new string content" + cb_env.collection.replace(key, new_content) + res = cb_env.collection.get(key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == new_content + + def test_default_tc_string_upsert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('utf8') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == value + + +class KeyValueOpTranscoderTestSuite: + TEST_MANIFEST = [ + 'test_get', + 'test_get_and_lock', + 'test_get_and_touch', + 'test_insert', + 'test_replace', + 'test_upsert', + ] + + def test_get(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + tc = RawBinaryTranscoder() + with pytest.raises(ValueFormatException): + cb_env.collection.get(key) + res = cb_env.collection.get(key, GetOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + + def test_get_and_touch(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + tc = RawBinaryTranscoder() + with pytest.raises(ValueFormatException): + cb_env.collection.get_and_touch(key, timedelta(seconds=30)) + + res = cb_env.collection.get_and_touch(key, + timedelta(seconds=3), + GetAndTouchOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + TestEnvironment.try_n_times_till_exception(10, + 3, + cb_env.collection.get, + key, + GetOptions(transcoder=tc), DocumentNotFoundException) + + def test_get_and_lock(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + tc = RawBinaryTranscoder() + with pytest.raises(ValueFormatException): + cb_env.collection.get_and_lock(key, timedelta(seconds=1)) + + # lets get another doc + key, value = cb_env.get_existing_doc_by_type('bytes') + res = cb_env.collection.get_and_lock(key, + timedelta(seconds=3), + GetAndLockOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + # upsert should definitely fail + with pytest.raises(DocumentLockedException): + cb_env.collection.upsert(key, value, transcoder=tc) + # but succeed eventually + TestEnvironment.try_n_times(10, 1, cb_env.collection.upsert, key, value, transcoder=tc) + + def test_insert(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8', key_only=True) + # use RawStringTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + with pytest.raises(ValueFormatException): + cb_env.collection.get(key) + + def test_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes', key_only=True) + # use RawBinaryTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + tc = RawBinaryTranscoder() + new_content = 'some new bytes content'.encode('utf-8') + cb_env.collection.replace(key, new_content, ReplaceOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + cb_env.collection.get(key) + + def test_upsert(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes', key_only=True) + # use RawBinaryTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + with pytest.raises(ValueFormatException): + cb_env.collection.get(key) + + +class LegacyTranscoderTestSuite: + TEST_MANIFEST = [ + 'test_legacy_tc_bytes_insert', + 'test_legacy_tc_bytes_replace', + 'test_legacy_tc_bytes_upsert', + 'test_legacy_tc_decoding', + 'test_legacy_tc_flags_zero', + 'test_legacy_tc_json_insert', + 'test_legacy_tc_json_replace', + 'test_legacy_tc_json_upsert', + 'test_legacy_tc_obj_insert', + 'test_legacy_tc_obj_replace', + 'test_legacy_tc_obj_upsert', + 'test_legacy_tc_string_insert', + 'test_legacy_tc_string_replace', + 'test_legacy_tc_string_upsert', + ] + + def test_legacy_tc_bytes_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('bytes') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_legacy_tc_bytes_replace(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + cb_env.collection.upsert(key, value) + new_content = 'new string content'.encode('utf-8') + cb_env.collection.replace(key, new_content) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + def test_legacy_tc_bytes_upsert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('bytes') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_legacy_tc_decoding(self): + tc = LegacyTranscoder() + content = {'foo': 'bar'} + value, flags = tc.encode_value(content) + assert flags == FMT_JSON + decoded = tc.decode_value(value, None) + assert content == decoded + decoded = tc.decode_value(value, 0) + assert content == decoded + + def test_legacy_tc_flags_zero(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('json') + cb_env.collection.upsert(key, value, transcoder=ZeroFlagsTranscoder()) + res = cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + def test_legacy_tc_json_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('json') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + def test_legacy_tc_json_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes', key_only=True) + _, value = cb_env.get_new_doc_by_type('json') + cb_env.collection.replace(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + def test_legacy_tc_json_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('json') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + def test_legacy_tc_obj_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('obj') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, FakeTestObj) + assert value.PROP == res.value.PROP + assert value.PROP1 == res.value.PROP1 + + def test_legacy_tc_obj_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes', key_only=True) + _, value = cb_env.get_new_doc_by_type('obj') + cb_env.collection.replace(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, FakeTestObj) + assert value.PROP == res.value.PROP + assert value.PROP1 == res.value.PROP1 + + def test_legacy_tc_obj_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('obj') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, FakeTestObj) + assert value.PROP == res.value.PROP + assert value.PROP1 == res.value.PROP1 + + def test_legacy_tc_string_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('utf8') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + def test_legacy_tc_string_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes', key_only=True) + _, value = cb_env.get_new_doc_by_type('utf8') + cb_env.collection.replace(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + def test_legacy_tc_string_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('utf8') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + +class RawBinaryTranscoderTestSuite: + TEST_MANIFEST = [ + 'test_raw_binary_tc_bytes_insert', + 'test_raw_binary_tc_bytes_replace', + 'test_raw_binary_tc_bytes_upsert', + 'test_raw_binary_tc_hex_insert', + 'test_raw_binary_tc_hex_replace', + 'test_raw_binary_tc_hex_upsert', + 'test_raw_binary_tc_json_insert', + 'test_raw_binary_tc_json_replace', + 'test_raw_binary_tc_json_upsert', + 'test_raw_binary_tc_string_insert', + 'test_raw_binary_tc_string_replace', + 'test_raw_binary_tc_string_upsert', + ] + + def test_raw_binary_tc_bytes_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('bytes') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_binary_tc_bytes_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes', key_only=True) + new_content = 'new string content'.encode('utf-8') + cb_env.collection.replace(key, new_content) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + def test_raw_binary_tc_bytes_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('bytes') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_binary_tc_hex_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('hex') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_binary_tc_hex_replace(self, cb_env): + key, value = cb_env.get_new_doc_by_type('hex') + cb_env.collection.upsert(key, value) + new_content = b'\xFF' + cb_env.collection.replace(key, new_content) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + def test_raw_binary_tc_hex_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('hex') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_binary_tc_json_insert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.insert(key, value) + + def test_raw_binary_tc_json_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes', key_only=True) + _, value = cb_env.get_new_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.replace(key, value) + + def test_raw_binary_tc_json_upsert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.upsert(key, value) + + def test_raw_binary_tc_string_upsert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('utf8') + with pytest.raises(ValueFormatException): + cb_env.collection.upsert(key, value) + + def test_raw_binary_tc_string_insert(self, cb_env): + key, value = cb_env.get_existing_doc_by_type('utf8') + with pytest.raises(ValueFormatException): + cb_env.collection.insert(key, value) + + def test_raw_binary_tc_string_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('bytes', key_only=True) + _, value = cb_env.get_new_doc_by_type('utf8') + with pytest.raises(ValueFormatException): + cb_env.collection.replace(key, value) + + +class RawJsonTranscoderTestSuite: + TEST_MANIFEST = [ + 'test_pass_through', + 'test_raw_json_tc_bytes_insert', + 'test_raw_json_tc_bytes_replace', + 'test_raw_json_tc_bytes_upsert', + 'test_raw_json_tc_json_insert', + 'test_raw_json_tc_json_replace', + 'test_raw_json_tc_json_upsert', + 'test_raw_json_tc_string_insert', + 'test_raw_json_tc_string_replace', + 'test_raw_json_tc_string_upsert', + ] + + def test_pass_through(self, cb_env): + key, value = cb_env.get_new_doc_by_type('json') + json_str = json.dumps(value) + cb_env.collection.upsert(key, json_str) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + # should not be equal + assert res.content_as[bytes] != value + decoded = json.loads(res.content_as[bytes].decode('utf-8')) + # should be good now + assert decoded == value + + def test_raw_json_tc_bytes_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('bytes') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_json_tc_bytes_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('json', key_only=True) + new_content = 'new string content'.encode('utf-8') + cb_env.collection.replace(key, new_content) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + def test_raw_json_tc_bytes_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('bytes') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_json_tc_json_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.insert(key, value) + + def test_raw_json_tc_json_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8', key_only=True) + _, value = cb_env.get_new_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.replace(key, value) + + def test_raw_json_tc_json_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.upsert(key, value) + + def test_raw_json_tc_string_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('utf8') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes].decode('utf-8') + + def test_raw_json_tc_string_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8', key_only=True) + new_content = "new string content" + cb_env.collection.replace(key, new_content) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes].decode('utf-8') + + def test_raw_json_tc_string_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('utf8') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes].decode('utf-8') + + +class RawStringTranscoderTestSuite: + TEST_MANIFEST = [ + 'test_raw_string_tc_bytes_insert', + 'test_raw_string_tc_bytes_replace', + 'test_raw_string_tc_bytes_upsert', + 'test_raw_string_tc_json_insert', + 'test_raw_string_tc_json_replace', + 'test_raw_string_tc_json_upsert', + 'test_raw_string_tc_string_insert', + 'test_raw_string_tc_string_replace', + 'test_raw_string_tc_string_upsert', + ] + + def test_raw_string_tc_bytes_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('bytes') + with pytest.raises(ValueFormatException): + cb_env.collection.insert(key, value) + + def test_raw_string_tc_bytes_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8', key_only=True) + _, value = cb_env.get_new_doc_by_type('bytes') + with pytest.raises(ValueFormatException): + cb_env.collection.replace(key, value) + + def test_raw_string_tc_bytes_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('bytes') + with pytest.raises(ValueFormatException): + cb_env.collection.upsert(key, value) + + def test_raw_string_tc_json_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.insert(key, value) + + def test_raw_string_tc_json_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8', key_only=True) + _, value = cb_env.get_new_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.replace(key, value) + + def test_raw_string_tc_json_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('json') + with pytest.raises(ValueFormatException): + cb_env.collection.upsert(key, value) + + def test_raw_string_tc_string_insert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('utf8') + cb_env.collection.insert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + def test_raw_string_tc_string_replace(self, cb_env): + key = cb_env.get_existing_doc_by_type('utf8', key_only=True) + new_content = "new string content" + cb_env.collection.replace(key, new_content) + res = cb_env.collection.get(key) + assert isinstance(res.value, str) + assert new_content == res.content_as[str] + + def test_raw_string_tc_string_upsert(self, cb_env): + key, value = cb_env.get_new_doc_by_type('utf8') + cb_env.collection.upsert(key, value) + res = cb_env.collection.get(key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + +class ClassicDefaultTranscoderTests(DefaultTranscoderTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicDefaultTranscoderTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicDefaultTranscoderTests) if valid_test_method(meth)] + compare = set(DefaultTranscoderTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = TranscoderTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) + + +class KeyValueOpTranscoderTests(KeyValueOpTranscoderTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(KeyValueOpTranscoderTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(KeyValueOpTranscoderTests) if valid_test_method(meth)] + compare = set(KeyValueOpTranscoderTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = TranscoderTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) + + +class ClassicLegacyTranscoderTests(LegacyTranscoderTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicLegacyTranscoderTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(LegacyTranscoderTestSuite) if valid_test_method(meth)] + compare = set(ClassicLegacyTranscoderTests.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = TranscoderTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param) + cb_env.cluster.default_transcoder = LegacyTranscoder() + yield cb_env + cb_env.teardown(request.param) + # reset the transcoder + cb_env.cluster.default_transcoder = JSONTranscoder() + + +class ClassicRawBinaryTranscoderTests(RawBinaryTranscoderTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicRawBinaryTranscoderTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(RawBinaryTranscoderTestSuite) if valid_test_method(meth)] + compare = set(ClassicRawBinaryTranscoderTests.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = TranscoderTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param) + cb_env.cluster.default_transcoder = RawBinaryTranscoder() + yield cb_env + cb_env.teardown(request.param) + # reset the transcoder + cb_env.cluster.default_transcoder = JSONTranscoder() + + +class ClassicRawJsonTranscoderTests(RawJsonTranscoderTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicRawJsonTranscoderTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(RawJsonTranscoderTestSuite) if valid_test_method(meth)] + compare = set(ClassicRawJsonTranscoderTests.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = TranscoderTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param) + cb_env.cluster.default_transcoder = RawJSONTranscoder() + yield cb_env + cb_env.teardown(request.param) + # reset the transcoder + cb_env.cluster.default_transcoder = JSONTranscoder() + + +class ClassicRawStringTranscoderTests(RawStringTranscoderTestSuite): + + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicRawStringTranscoderTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(RawStringTranscoderTestSuite) if valid_test_method(meth)] + compare = set(ClassicRawStringTranscoderTests.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = TranscoderTestEnvironment.from_environment(cb_base_env) + cb_env.setup(request.param) + cb_env.cluster.default_transcoder = RawStringTranscoder() + yield cb_env + cb_env.teardown(request.param) + # reset the transcoder + cb_env.cluster.default_transcoder = JSONTranscoder() diff --git a/couchbase/tests/usermgmt_t.py b/couchbase/tests/usermgmt_t.py new file mode 100644 index 000000000..a720d65d5 --- /dev/null +++ b/couchbase/tests/usermgmt_t.py @@ -0,0 +1,717 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +import pytest + +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.exceptions import (AuthenticationException, + FeatureUnavailableException, + GroupNotFoundException, + InvalidArgumentException, + UserNotFoundException) +from couchbase.management.collections import CollectionSpec +from couchbase.management.options import (DropUserOptions, + GetUserOptions, + UpsertUserOptions) +from couchbase.management.users import (Group, + Role, + User) +from couchbase.options import ClusterOptions +from tests.environments.test_environment import TestEnvironment +from tests.environments.user_mgmt_environment import UserManagementTestEnvironment +from tests.test_features import EnvironmentFeatures + + +class UserManagementTestSuite: + TEST_MANIFEST = [ + 'test_default_domain', + 'test_external_nopassword', + 'test_external_user', + 'test_get_all_groups', + 'test_get_roles', + 'test_get_roles_all_valid', + 'test_group', + 'test_group_feature_not_found', + 'test_internal_user', + 'test_internal_user_fail', + 'test_internal_user_kwargs', + 'test_invalid_domain_raises_argument_error', + 'test_missing_group', + 'test_missing_user', + 'test_user_and_groups', + 'test_user_change_password', + 'test_user_display_name', + 'test_user_scopes_collections', + ] + + @pytest.fixture(scope='class') + def check_collections_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('collections', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope='class') + def check_user_groups_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('user_group_mgmt', + cb_env.server_version_short, + cb_env.mock_server_type) + + def test_default_domain(self, cb_env): + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + + cb_env.um.upsert_user(User(username=username, password=password, roles=roles)) + + cb_env.consistency.wait_until_user_present(username) + user_metadata = cb_env.um.get_user(username) + assert user_metadata is not None + + # handle 7.0 roles w/ scopes/collections + test_roles = roles + collections_supported = EnvironmentFeatures.is_feature_supported('collections', + cb_env.server_version_short, + cb_env.mock_server_type) + if collections_supported: + test_roles = [] + for r in roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + users_metadata = cb_env.um.get_all_users() + assert users_metadata is not None + result = all( + map(lambda um: cb_env.validate_user_and_metadata(um), + users_metadata)) + assert result is True + + cb_env.um.drop_user(username) + + def test_external_nopassword(self, cb_env): + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + + # password with external generates argument error + with pytest.raises(InvalidArgumentException): + cb_env.um.upsert_user(User(username=username, + password=password, + roles=roles), + domain_name='external') + + with pytest.raises(InvalidArgumentException): + cb_env.um.upsert_user(User(username=username, + password=password, + roles=None), + domain_name='external') + + with pytest.raises(InvalidArgumentException): + cb_env.um.upsert_user(User(username=username, password=password, roles=[]), + domain_name='external') + + # Should not raise + cb_env.um.upsert_user( + User(username=username, password=None, roles=roles), + UpsertUserOptions(domain_name='external')) + + cb_env.consistency.wait_until_user_present(username, domain='external') + cb_env.um.drop_user(username, domain_name='external') + cb_env.consistency.wait_until_user_dropped(username, domain='external') + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_external_user(self, cb_env): + """ + test_external_user() + Tests create, retrieve, update and removal + of external (domain_name='external') + Uses *UserOptions() for options + """ + + username = 'custom-user' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + initial_user = User(username=username, roles=roles) + # create user + cb_env.um.upsert_user(initial_user, UpsertUserOptions(domain_name='external')) + cb_env.consistency.wait_until_user_present(username, 'external') + + # get user + user_metadata = cb_env.um.get_user(username, GetUserOptions(domain_name='external')) + + # handle 7.0 roles w/ scopes/collections + test_roles = roles + collections_supported = EnvironmentFeatures.is_feature_supported('collections', + cb_env.server_version_short, + cb_env.mock_server_type) + if collections_supported: + test_roles = [] + for r in roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + # update user + user = user_metadata.user + user.roles = Role('admin') + + cb_env.um.upsert_user(user, UpsertUserOptions(domain_name='external')) + + # get user and verify updates + user_metadata = TestEnvironment.try_n_times(5, + 1, + cb_env.um.get_user, + username, + GetUserOptions(domain_name='external')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=user.roles) + user_update = user_metadata.user + assert initial_user != user_update + + # remove user + cb_env.um.drop_user(username, DropUserOptions(domain_name='external')) + cb_env.consistency.wait_until_user_dropped(username, 'external') + with pytest.raises(UserNotFoundException): + cb_env.um.get_user(username, GetUserOptions(domain_name='external')) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.mark.usefixtures('check_user_groups_supported') + def test_get_all_groups(self, cb_env): + roles = [ + Role(name='data_reader', bucket='*'), + Role(name='data_writer', bucket='*') + ] + fresh_group = Group(name='my-test-group', + roles=roles, + description='test group description') + cb_env.um.upsert_group(fresh_group) + cb_env.consistency.wait_until_group_present(fresh_group.name) + admin_group = Group('admin-test-group', roles=[Role(name='admin')]) + cb_env.um.upsert_group(admin_group) + cb_env.consistency.wait_until_group_present(admin_group.name) + all_groups = cb_env.um.get_all_groups() + # NOTE: we could well have other groups on this server, apart from the one we added, so + # lets be ok with there being more of them. However, the one we added + # _MUST_ be there. + assert len(all_groups) >= 2 + admin_group_dict = admin_group.as_dict() + found = False + for g in all_groups: + if admin_group_dict == g.as_dict(): + found = True + + cb_env.um.drop_group('my-test-group') + cb_env.um.drop_group('admin-test-group') + assert found is True + + # @TODO: is this test useful? + # @pytest.mark.usefixtures('check_user_groups_supported') + # def test_timeout(self, cb_env): + # with pytest.raises(AmbiguousTimeoutException): + # cb_env.um.get_all_groups(timeout=timedelta(seconds=0.1)) + + def test_get_roles(self, cb_env): + roles = cb_env.um.get_roles() + admin_desc = re.compile( + r'.*all cluster features.*web console.*read and write all data.*$') + for rad in reversed(roles): + desc_matches = admin_desc.match(rad.description) + if desc_matches: + assert rad.role.name == 'admin' + assert rad.display_name == 'Full Admin' + return + pytest.fail('No admin role found') + + # see PYCBC-1030 + def test_get_roles_all_valid(self, cb_env): + roles = cb_env.um.get_roles() + for r in roles: + assert r is not None + + @pytest.mark.usefixtures('check_user_groups_supported') + def test_group(self, cb_env): + roles = Role(name='admin') + test_group = Group(name='my-test-group', + roles=roles, + description='test group description') + # add group + cb_env.um.upsert_group(test_group) + cb_env.consistency.wait_until_group_present(test_group.name) + + # get group + result = cb_env.um.get_group(test_group.name) + cb_env.validate_group(result, test_group.roles) + + # remove group + cb_env.um.drop_group(test_group.name) + cb_env.consistency.wait_until_group_dropped(test_group.name) + with pytest.raises(GroupNotFoundException): + cb_env.um.get_group(test_group.name) + + def test_group_feature_not_found(self, cb_env): + groups_supported = EnvironmentFeatures.is_feature_supported('user_group_mgmt', + cb_env.server_version_short, + cb_env.mock_server_type) + if groups_supported: + pytest.skip(f'Only test on server versions < 6.5. Using server version: {cb_env.server_version}') + + roles = Role(name='admin') + test_group = Group(name='my-test-group', + roles=roles, + description='test group description') + + with pytest.raises(FeatureUnavailableException): + cb_env.um.upsert_group(test_group) + with pytest.raises(FeatureUnavailableException): + cb_env.um.get_all_groups() + with pytest.raises(FeatureUnavailableException): + cb_env.um.get_group(test_group.name) + with pytest.raises(FeatureUnavailableException): + cb_env.um.drop_group(test_group.name) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_internal_user(self, cb_env): + """ + test_internal_user() + Tests create, retrieve, update and removal + of internal (domain_name='local') + Uses *UserOptions() for options + """ + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + initial_user = User(username=username, roles=roles, password=password) + + # create user + cb_env.um.upsert_user(User(username=username, roles=roles, password=password), + UpsertUserOptions(domain_name='local')) + cb_env.consistency.wait_until_user_present(username) + + # get user + user_metadata = cb_env.um.get_user(username, GetUserOptions(domain_name='local')) + + # handle 7.0 roles w/ scopes/collections + test_roles = roles + collections_supported = EnvironmentFeatures.is_feature_supported('collections', + cb_env.server_version_short, + cb_env.mock_server_type) + if collections_supported: + test_roles = [] + for r in roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + # update user + user = user_metadata.user + user.roles = Role('admin') + user.password = 's3cr3t_pa33w0rd' + + cb_env.um.upsert_user(user, UpsertUserOptions(domain_name='local')) + + # get user and verify updates + user_metadata = TestEnvironment.try_n_times(5, + 1, + cb_env.um.get_user, + username, + GetUserOptions(domain_name='local')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=user.roles) + user_update = user_metadata.user + assert initial_user != user_update + + # remove user + cb_env.um.drop_user(username, DropUserOptions(domain_name='local')) + cb_env.consistency.wait_until_user_dropped(username) + with pytest.raises((UserNotFoundException)): + cb_env.um.get_user(username, GetUserOptions(domain_name='local')) + + def test_internal_user_fail(self, cb_env): + """ + test_internal_user() + Tests create, retrieve, update and removal + of internal (domain_name='local') + Uses *UserOptions() for options + """ + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='not-a-bucket'), + Role(name='data_writer', bucket='not-a-bucket') + ] + + # create user + with pytest.raises(InvalidArgumentException): + cb_env.um.upsert_user(User(username=username, roles=roles, password=password), + UpsertUserOptions(domain_name='local')) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_internal_user_kwargs(self, cb_env): + """ + test_internal_user_kwargs() + Tests create, retrieve, update and removal + of internal (domain_name='local') + Uses kwargs for options + """ + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + initial_user = User(username=username, roles=roles, password=password) + + # create user + cb_env.um.upsert_user(initial_user, domain_name='local') + cb_env.consistency.wait_until_user_present(username) + + # get single user + user_metadata = cb_env.um.get_user(username, domain_name='local') + + # handle 7.0 roles w/ scopes/collections + test_roles = roles + collections_supported = EnvironmentFeatures.is_feature_supported('collections', + cb_env.server_version_short, + cb_env.mock_server_type) + if collections_supported: + test_roles = [] + for r in roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + # update user + user = user_metadata.user + user.roles = Role('admin') + user.password = 's3cr3t_pa33w0rd' + + cb_env.um.upsert_user(user, domain_name='local') + + # get user and verify updates + user_metadata = TestEnvironment.try_n_times(5, + 1, + cb_env.um.get_user, + username, + domain_name='local') + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=user.roles) + user_update = user_metadata.user + assert initial_user != user_update + + # remove user + cb_env.um.drop_user(username, domain_name='local') + cb_env.consistency.wait_until_user_dropped(username) + with pytest.raises(UserNotFoundException): + cb_env.um.get_user(username, domain_name='local') + + def test_invalid_domain_raises_argument_error(self, cb_env): + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + + # invalid domain generates argument error + with pytest.raises(InvalidArgumentException): + cb_env.um.get_all_users(domain_name='fake-domain') + + with pytest.raises(InvalidArgumentException): + cb_env.um.get_user(username, domain_name='fake-domain') + + with pytest.raises(InvalidArgumentException): + cb_env.um.upsert_user(User(username=username, + password=password, + roles=roles), + domain_name='fake-domain') + + with pytest.raises(InvalidArgumentException): + cb_env.um.drop_user(username, domain_name='fake-domain') + + @pytest.mark.usefixtures('check_user_groups_supported') + def test_missing_group(self, cb_env): + with pytest.raises(GroupNotFoundException): + cb_env.um.get_group('fred') + + def test_missing_user(self, cb_env): + with pytest.raises(UserNotFoundException): + cb_env.um.get_user('keith') + + @pytest.mark.usefixtures('check_user_groups_supported') + def test_user_and_groups(self, cb_env): + user_roles = [ + Role(name='query_select', bucket='default'), + Role(name='fts_searcher', bucket='default') + ] + group_roles = [ + Role(name='data_reader', bucket='*'), + Role(name='data_writer', bucket='*') + ] + groups = [ + Group(name='my-test-group', + roles=group_roles, + description='test group description'), + Group(name='my-test-group-1', + roles=Role(name='admin'), + description='test group description') + ] + + # add groups + for group in groups: + cb_env.um.upsert_group(group) + cb_env.consistency.wait_until_group_present(group.name) + user_groups = list(map(lambda g: g.name, groups)) + + # add user + test_user = User(username='custom-user', + roles=user_roles, + groups=user_groups, + password='s3cr3t') + cb_env.um.upsert_user(test_user, domain_name='local') + cb_env.consistency.wait_until_user_present(test_user.username) + + # get user + user_metadata = cb_env.um.get_user(test_user.username, domain_name='local') + + # handle 7.0 roles w/ scopes/collections + test_roles = user_roles + collections_supported = EnvironmentFeatures.is_feature_supported('collections', + cb_env.server_version_short, + cb_env.mock_server_type) + if collections_supported: + test_roles = [] + for r in user_roles: + test_roles.append( + Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, + user_roles=test_roles, + groups=groups) + + # remove group + remove_group = groups.pop() + cb_env.um.drop_group(remove_group.name) + cb_env.consistency.wait_until_group_dropped(remove_group.name) + with pytest.raises(GroupNotFoundException): + cb_env.um.get_group(remove_group.name) + + # get user to verify roles from removed group are removed + user_metadata = cb_env.um.get_user(test_user.username, domain_name='local') + + # handle 7.0 roles w/ scopes/collections + if collections_supported: + test_roles = [] + for r in user_roles: + test_roles.append(Role(name=r.name, + bucket=r.bucket, + scope='*', + collection='*')) + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, + user_roles=test_roles, + groups=groups) + + # cleanup + cb_env.um.drop_user(test_user.username, domain_name='local') + for group in groups: + cb_env.um.drop_group(group.name) + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_user_change_password(self, cb_env): + username = 'change-password-user' + admin_role = Role(name='admin') + original_password = 'original_password' + new_password = 'new_password' + + change_password_user = User(username=username, + display_name="Change Password User", + roles=admin_role, + password=original_password) + # Upsert user + cb_env.um.upsert_user(change_password_user, UpsertUserOptions(domain_name="local")) + cb_env.consistency.wait_until_user_present(username) + + # Authenticate as change-password-user. + # Done in a while loop to emulate retry + auth = PasswordAuthenticator(username, original_password) + new_cluster = None + while True: + try: + new_cluster = Cluster.connect(cb_env.cluster._connstr, ClusterOptions(auth)) + except AuthenticationException: + continue + break + + # Change password + new_cluster.users().change_password(new_password) + + # Assert can authenticate using new password + success_auth = PasswordAuthenticator(username, new_password) + while True: + try: + success_cluster = Cluster.connect(cb_env.cluster._connstr, ClusterOptions(success_auth)) + except AuthenticationException: + continue + success_cluster.close() + break + + # Assert cannot authenticate using old password + fail_auth = PasswordAuthenticator(username, original_password) + with pytest.raises(AuthenticationException): + Cluster.connect(cb_env.cluster._connstr, ClusterOptions(fail_auth)) + + new_cluster.close() + + def test_user_display_name(self, cb_env): + roles = [ + Role(name='data_reader', bucket='default'), + Role(name='data_writer', bucket='default') + ] + user = User(username='custom-user', + display_name="Custom User", + roles=roles, + password='s3cr3t') + + # create user + cb_env.um.upsert_user(user, UpsertUserOptions(domain_name='local')) + cb_env.consistency.wait_until_user_present(user.username) + + # get user + user_metadata = cb_env.um.get_user(user.username, GetUserOptions(domain_name='local')) + + assert user_metadata.user.display_name == user.display_name + + cb_env.um.drop_user(user.username, DropUserOptions(domain_name='local')) + + @pytest.mark.usefixtures('check_collections_supported') + def test_user_scopes_collections(self, cb_env): + + if cb_env.cm is None: + cm = cb_env.bucket.collections() + else: + cm = cb_env.cm + + scope_name = 'um-test-scope' + collection_name = 'test-collection' + + cm.create_scope(scope_name) + cb_env.consistency.wait_until_scope_present(cb_env.bucket.name, scope_name) + + collection = CollectionSpec(collection_name, scope_name=scope_name) + cm.create_collection(collection) + cb_env.consistency.wait_until_collection_present(cb_env.bucket.name, scope_name, collection_name) + + username = 'custom-user' + password = 's3cr3t' + roles = [ + Role(name='data_reader', bucket='default', scope='um-test-scope'), + Role(name='data_writer', + bucket='default', + scope='um-test-scope', + collection='test-collection') + ] + initial_user = User(username=username, roles=roles, password=password) + + # create user + cb_env.um.upsert_user(initial_user, domain_name='local') + cb_env.consistency.wait_until_user_present(username) + + # get single user + user_metadata = cb_env.um.get_user(username, domain_name='local') + + test_roles = [] + for r in roles: + if not r.collection: + test_roles.append(Role(name=r.name, + bucket=r.bucket, + scope=r.scope, + collection='*')) + else: + test_roles.append(r) + + assert user_metadata is not None + cb_env.validate_user_and_metadata(user_metadata, user_roles=test_roles) + + cb_env.um.drop_user(username) + cm.drop_collection(collection) + cm.drop_scope('um-test-scope') + + +class ClassicUserManagementTests(UserManagementTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicUserManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicUserManagementTests) if valid_test_method(meth)] + compare = set(UserManagementTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env') + def couchbase_test_environment(self, cb_base_env, test_manifest_validated): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = UserManagementTestEnvironment.from_environment(cb_base_env) + cb_env.setup() + yield cb_env + cb_env.teardown() diff --git a/couchbase/tests/viewmgmt_t.py b/couchbase/tests/viewmgmt_t.py new file mode 100644 index 000000000..6420d4665 --- /dev/null +++ b/couchbase/tests/viewmgmt_t.py @@ -0,0 +1,170 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta + +import pytest + +from couchbase.exceptions import DesignDocumentNotFoundException +from couchbase.management.options import (GetAllDesignDocumentsOptions, + GetDesignDocumentOptions, + PublishDesignDocumentOptions) +from couchbase.management.views import DesignDocumentNamespace +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment +from tests.environments.views_environment import ViewsTestEnvironment + + +class ViewIndexManagementTestSuite: + TEST_MANIFEST = [ + 'test_drop_design_doc', + 'test_drop_design_doc_fail', + 'test_get_all_design_documents', + 'test_get_all_design_documents_excludes_namespaces', + 'test_get_design_document', + 'test_get_design_document_fail', + 'test_publish_design_doc', + 'test_upsert_design_doc', + ] + + @pytest.fixture() + def create_test_view(self, cb_env): + cb_env.add_test_ddoc() + + @pytest.fixture() + def drop_test_view(self, cb_env): + yield + cb_env.drop_ddoc() + + @pytest.fixture() + def drop_test_view_from_prod(self, cb_env): + yield + cb_env.drop_ddoc(from_prod=True) + + @pytest.mark.usefixtures('create_test_view') + def test_drop_design_doc(self, cb_env): + cb_env.add_test_ddoc() + cb_env.vixm.drop_design_document(cb_env.test_ddoc.name, DesignDocumentNamespace.DEVELOPMENT) + + def test_drop_design_doc_fail(self, cb_env): + with pytest.raises(DesignDocumentNotFoundException): + cb_env.vixm.drop_design_document(cb_env.test_ddoc.name, DesignDocumentNamespace.PRODUCTION) + + @pytest.mark.usefixtures('create_test_view') + @pytest.mark.usefixtures('drop_test_view') + def test_get_all_design_documents(self, cb_env): + # should start out in _some_ state. Since we don't know for sure, but we + # do know it does have self.DOCNAME in it in development ONLY, lets assert on that and that + # it succeeds, meaning we didn't get an exception. + # make sure it is there + ddoc = TestEnvironment.try_n_times(10, + 3, + cb_env.vixm.get_design_document, + cb_env.test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + GetDesignDocumentOptions(timeout=timedelta(seconds=5))) + assert ddoc is not None + # should also be in the get_all response + result = cb_env.vixm.get_all_design_documents(DesignDocumentNamespace.DEVELOPMENT, + GetAllDesignDocumentsOptions(timeout=timedelta(seconds=10))) + names = [doc.name for doc in result if doc.name == cb_env.test_ddoc.name] + assert names.count(cb_env.test_ddoc.name) > 0 + + @pytest.mark.usefixtures('create_test_view') + @pytest.mark.usefixtures('drop_test_view') + def test_get_all_design_documents_excludes_namespaces(self, cb_env): + # we know the test_ddoc.name is _only_ in development, so... + result = cb_env.vixm.get_all_design_documents(DesignDocumentNamespace.PRODUCTION) + names = [doc.name for doc in result if doc.name == cb_env.test_ddoc.name] + assert names.count(cb_env.test_ddoc.name) == 0 + + @pytest.mark.usefixtures('create_test_view') + @pytest.mark.usefixtures('drop_test_view') + def test_get_design_document_fail(self, cb_env): + with pytest.raises(DesignDocumentNotFoundException): + cb_env.vixm.get_design_document(cb_env.test_ddoc.name, + DesignDocumentNamespace.PRODUCTION, + GetDesignDocumentOptions(timeout=timedelta(seconds=5))) + + @pytest.mark.usefixtures('create_test_view') + @pytest.mark.usefixtures('drop_test_view') + def test_get_design_document(self, cb_env): + ddoc = TestEnvironment.try_n_times(10, + 3, + cb_env.vixm.get_design_document, + cb_env.test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + GetDesignDocumentOptions(timeout=timedelta(seconds=5))) + assert ddoc is not None + assert ddoc.name == cb_env.test_ddoc.name + + @pytest.mark.usefixtures('create_test_view') + @pytest.mark.usefixtures('drop_test_view_from_prod') + def test_publish_design_doc(self, cb_env): + # make sure we have the ddoc + ddoc = TestEnvironment.try_n_times(10, + 3, + cb_env.vixm.get_design_document, + cb_env.test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + GetDesignDocumentOptions(timeout=timedelta(seconds=5))) + assert ddoc is not None + + # starts off not in prod + with pytest.raises(DesignDocumentNotFoundException): + cb_env.vixm.get_design_document(cb_env.test_ddoc.name, DesignDocumentNamespace.PRODUCTION) + + cb_env.vixm.publish_design_document(cb_env.test_ddoc.name, + PublishDesignDocumentOptions(timeout=timedelta(seconds=10))) + # should be in prod now + TestEnvironment.try_n_times(10, + 3, + cb_env.vixm.get_design_document, + cb_env.test_ddoc.name, + DesignDocumentNamespace.PRODUCTION) + # and still in dev + TestEnvironment.try_n_times(10, + 3, + cb_env.vixm.get_design_document, + cb_env.test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT) + + @pytest.mark.usefixtures('drop_test_view') + def test_upsert_design_doc(self, cb_env): + # we started with this already in here, so this isn't really + # necessary...` + cb_env.vixm.upsert_design_document(cb_env.test_ddoc, DesignDocumentNamespace.DEVELOPMENT) + + +class ClassicViewIndexManagementTests(ViewIndexManagementTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicViewIndexManagementTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicViewIndexManagementTests) if valid_test_method(meth)] + compare = set(ViewIndexManagementTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = ViewsTestEnvironment.from_environment(cb_base_env) + cb_env.enable_views_mgmt() + cb_env.setup(request.param, test_suite=self.__class__.__name__, num_docs=10) + yield cb_env + cb_env.teardown(request.param, test_suite=self.__class__.__name__) diff --git a/couchbase/tests/views_params_t.py b/couchbase/tests/views_params_t.py new file mode 100644 index 000000000..9ea56e632 --- /dev/null +++ b/couchbase/tests/views_params_t.py @@ -0,0 +1,296 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta + +import pytest + +from couchbase.management.views import DesignDocumentNamespace +from couchbase.options import ViewOptions +from couchbase.serializer import DefaultJsonSerializer +from couchbase.views import (ViewErrorMode, + ViewOrdering, + ViewQuery, + ViewScanConsistency) +from tests.environments import CollectionType +from tests.environments.views_environment import ViewsTestEnvironment + + +class ViewsParamSuite: + TEST_MANIFEST = [ + 'test_params_base', + 'test_params_client_context_id', + 'test_params_debug', + 'test_params_endkey', + 'test_params_endkey_docid', + 'test_params_group', + 'test_params_group_level', + 'test_params_inclusive_end', + 'test_params_key', + 'test_params_keys', + 'test_params_limit', + 'test_params_namespace', + 'test_params_on_error', + 'test_params_order', + 'test_params_reduce', + 'test_params_scan_consistency', + 'test_params_serializer', + 'test_params_skip', + 'test_params_startkey', + 'test_params_startkey_docid', + 'test_params_timeout', + ] + + @pytest.fixture(scope='class') + def base_opts(self, cb_env): + return {'bucket_name': 'default', + 'document_name': cb_env.DOCNAME, + 'view_name': cb_env.TEST_VIEW_NAME + } + + def test_params_base(self, cb_env, base_opts): + opts = ViewOptions() + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + params = query.as_encodable() + assert params == base_opts + + def test_params_client_context_id(self, cb_env, base_opts): + opts = ViewOptions(client_context_id='test-context-id') + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['client_context_id'] = 'test-context-id' + params = query.as_encodable() + assert params == exp_opts + + def test_params_debug(self, cb_env, base_opts): + opts = ViewOptions(debug=True) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['debug'] = True + params = query.as_encodable() + assert params == exp_opts + + def test_params_endkey(self, cb_env, base_opts): + key = ['101 Coffee Shop', 'landmark_11769'] + opts = ViewOptions(endkey=key) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['end_key'] = json.dumps(key) + params = query.as_encodable() + assert params == exp_opts + + def test_params_endkey_docid(self, cb_env, base_opts): + key = 'landmark_11769' + opts = ViewOptions(endkey_docid=key) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['end_key_doc_id'] = key + params = query.as_encodable() + assert params == exp_opts + + def test_params_group(self, cb_env, base_opts): + opts = ViewOptions(group=True) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['group'] = True + params = query.as_encodable() + assert params == exp_opts + + def test_params_group_level(self, cb_env, base_opts): + opts = ViewOptions(group_level=10) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['group_level'] = 10 + params = query.as_encodable() + assert params == exp_opts + + def test_params_inclusive_end(self, cb_env, base_opts): + opts = ViewOptions(inclusive_end=True) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['inclusive_end'] = True + params = query.as_encodable() + assert params == exp_opts + + def test_params_key(self, cb_env, base_opts): + opts = ViewOptions(key='test-key') + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['key'] = json.dumps('test-key') + params = query.as_encodable() + assert params == exp_opts + + def test_params_keys(self, cb_env, base_opts): + test_keys = ['test-key1', 'test-key2'] + opts = ViewOptions(keys=test_keys) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['keys'] = list(map(lambda k: json.dumps(k), test_keys)) + params = query.as_encodable() + assert params == exp_opts + + def test_params_limit(self, cb_env, base_opts): + opts = ViewOptions(limit=10) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['limit'] = 10 + params = query.as_encodable() + assert params == exp_opts + + def test_params_namespace(self, cb_env, base_opts): + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['namespace'] = DesignDocumentNamespace.DEVELOPMENT.value + params = query.as_encodable() + assert params == exp_opts + assert query.namespace == DesignDocumentNamespace.DEVELOPMENT + + def test_params_on_error(self, cb_env, base_opts): + opts = ViewOptions(on_error=ViewErrorMode.CONTINUE) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['on_error'] = ViewErrorMode.CONTINUE.value + params = query.as_encodable() + assert params == exp_opts + assert query.on_error == ViewErrorMode.CONTINUE + + def test_params_order(self, cb_env, base_opts): + opts = ViewOptions(order=ViewOrdering.ASCENDING) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['order'] = ViewOrdering.ASCENDING.value + params = query.as_encodable() + assert params == exp_opts + assert query.order == ViewOrdering.ASCENDING + + def test_params_reduce(self, cb_env, base_opts): + opts = ViewOptions(reduce=True) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['reduce'] = True + params = query.as_encodable() + assert params == exp_opts + + def test_params_scan_consistency(self, cb_env, base_opts): + opts = ViewOptions(scan_consistency=ViewScanConsistency.REQUEST_PLUS) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['scan_consistency'] = ViewScanConsistency.REQUEST_PLUS.value + params = query.as_encodable() + assert params == exp_opts + assert query.consistency == ViewScanConsistency.REQUEST_PLUS + + def test_params_serializer(self, cb_env, base_opts): + serializer = DefaultJsonSerializer() + opts = ViewOptions(serializer=serializer) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['serializer'] = serializer + params = query.as_encodable() + assert params == exp_opts + + def test_params_skip(self, cb_env, base_opts): + opts = ViewOptions(skip=10) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['skip'] = 10 + params = query.as_encodable() + assert params == exp_opts + + def test_params_startkey(self, cb_env, base_opts): + key = ['101 Coffee Shop', 'landmark_11769'] + opts = ViewOptions(startkey=key) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['start_key'] = json.dumps(key) + params = query.as_encodable() + assert params == exp_opts + + def test_params_startkey_docid(self, cb_env, base_opts): + key = 'landmark_11769' + opts = ViewOptions(startkey_docid=key) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['start_key_doc_id'] = key + params = query.as_encodable() + assert params == exp_opts + + def test_params_timeout(self, cb_env, base_opts): + opts = ViewOptions(timeout=timedelta(seconds=20)) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 20000000 + params = query.as_encodable() + assert params == exp_opts + + opts = ViewOptions(timeout=20) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 20000000 + params = query.as_encodable() + assert params == exp_opts + + opts = ViewOptions(timeout=25.5) + query = ViewQuery.create_view_query_object('default', cb_env.DOCNAME, cb_env.TEST_VIEW_NAME, opts) + + exp_opts = base_opts.copy() + exp_opts['timeout'] = 25500000 + params = query.as_encodable() + assert params == exp_opts + + +class ClassicViewsParamTests(ViewsParamSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicViewsParamTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicViewsParamTests) if valid_test_method(meth)] + compare = set(ViewsParamSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = ViewsTestEnvironment.from_environment(cb_base_env) + cb_env.enable_views_mgmt() + cb_env.setup(request.param, test_suite=self.__class__.__name__) + yield cb_env + cb_env.teardown(request.param, test_suite=self.__class__.__name__) diff --git a/couchbase/tests/views_t.py b/couchbase/tests/views_t.py new file mode 100644 index 000000000..e1ab056de --- /dev/null +++ b/couchbase/tests/views_t.py @@ -0,0 +1,350 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import threading +from datetime import timedelta + +import pytest + +from couchbase.exceptions import (AmbiguousTimeoutException, + DesignDocumentNotFoundException, + InvalidArgumentException) +from couchbase.management.views import DesignDocumentNamespace +from couchbase.options import ViewOptions +from couchbase.views import (ViewMetaData, + ViewOrdering, + ViewRow) +from tests.environments import CollectionType +from tests.environments.views_environment import ViewsTestEnvironment + + +class ViewsTestSuite: + TEST_MANIFEST = [ + 'test_bad_view_query', + 'test_view_query', + 'test_view_query_ascending', + 'test_view_query_descending', + 'test_view_query_endkey_docid', + 'test_view_query_in_thread', + 'test_view_query_key', + 'test_view_query_keys', + 'test_view_query_raw', + 'test_view_query_raw_fail', + 'test_view_query_startkey_docid', + 'test_view_query_timeout', + ] + + def test_bad_view_query(self, cb_env): + view_result = cb_env.bucket.view_query('fake-ddoc', + 'fake-view', + limit=10, + namespace=DesignDocumentNamespace.DEVELOPMENT) + + with pytest.raises(DesignDocumentNotFoundException): + [r for r in view_result] + + def test_view_query(self, cb_env): + expected_count = 10 + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + full_set=True, + namespace=DesignDocumentNamespace.DEVELOPMENT) + + cb_env.assert_rows(view_result, expected_count) + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + def test_view_query_ascending(self, cb_env): + # set the batch id + cb_env.get_batch_id() + keys = cb_env.get_keys() + expected_docids = [i for k in sorted(keys) + for i in sorted(cb_env.get_docids_by_key(k))] + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + namespace=DesignDocumentNamespace.DEVELOPMENT, + order=ViewOrdering.ASCENDING) + + rows = cb_env.assert_rows(view_result, cb_env.num_docs, return_rows=True) + row_ids = list(map(lambda r: r.id, rows)) + assert row_ids == expected_docids + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= cb_env.num_docs + + def test_view_query_descending(self, cb_env): + # set the batch id + cb_env.get_batch_id() + keys = cb_env.get_keys() + expected_docids = [i for k in sorted(keys, reverse=True) + for i in sorted(cb_env.get_docids_by_key(k), reverse=True)] + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + namespace=DesignDocumentNamespace.DEVELOPMENT, + order=ViewOrdering.DESCENDING) + + rows = cb_env.assert_rows(view_result, cb_env.num_docs, return_rows=True) + row_ids = list(map(lambda r: r.id, rows)) + assert row_ids == expected_docids + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= cb_env.num_docs + + def test_view_query_endkey_docid(self, cb_env): + # set the batch id + cb_env.get_batch_id() + keys = cb_env.get_keys() + # take the 2nd-smallest key so that we can purposefully select results of previous key group + key_idx = keys.index(sorted(keys)[1]) + key = keys[key_idx] + key_docids = cb_env.get_docids_by_key(key) + # purposefully select a docid in the middle of key group + endkey_docid = key_docids[4] + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, + endkey=key, + endkey_docid=endkey_docid) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # all docs w/in first key (10 docs) + half of docs w/in next key (5 docs) + expected_count = 15 + rows = cb_env.assert_rows(view_result, expected_count, True) + # last doc in results should be the endkey and endkey_docid + assert rows[-1].key == key + assert rows[-1].id == endkey_docid + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + def test_view_query_in_thread(self, cb_env): + results = [None] + + def run_test(bucket, doc_name, view_name, opts, assert_fn, results): + try: + result = bucket.view_query(doc_name, view_name, opts) + assert_fn(result, opts['limit']) + assert result.metadata() is not None + except AssertionError: + results[0] = False + except Exception as ex: + results[0] = ex + else: + results[0] = True + + expected_count = 5 + opts = ViewOptions(limit=expected_count, + namespace=DesignDocumentNamespace.DEVELOPMENT) + t = threading.Thread(target=run_test, + args=(cb_env.bucket, + cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts, + cb_env.assert_rows, + results)) + t.start() + t.join() + + assert len(results) == 1 + assert results[0] is True + + def test_view_query_key(self, cb_env): + batch_id = cb_env.get_batch_id() + expected_count = 10 + keys = cb_env.get_keys() + key = keys[0] + docids = cb_env.get_docids_by_key(key) + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, + key=key) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + + rows = cb_env.assert_rows(view_result, expected_count, True) + for row in rows: + assert isinstance(row, ViewRow) + assert isinstance(row.id, str) + assert isinstance(row.key, str) + assert isinstance(row.value, dict) + assert row.key == key + assert row.value['batch'] == batch_id + assert row.value['id'] in docids + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + def test_view_query_keys(self, cb_env): + keys = cb_env.get_keys() + expected_keys = keys[:2] + expected_count = 20 + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, + keys=expected_keys) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + + rows = cb_env.assert_rows(view_result, expected_count, True) + assert all(map(lambda r: r.key in expected_keys, rows)) is True + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + def test_view_query_raw(self, cb_env): + # set the batch id + cb_env.get_batch_id() + keys = cb_env.get_keys() + # take the largest key so that we can purposefully narrow the result to 1 record + key_idx = keys.index(max(keys)) + key = keys[key_idx] + key_docids = cb_env.get_docids_by_key(key) + # purposefully use the last doc w/in the key + startkey_docid = key_docids[-1] + # execute a query so we can have pagination + opts = ViewOptions(limit=5, + namespace=DesignDocumentNamespace.DEVELOPMENT) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # need to iterate over the result to execute the query + [r for r in view_result] + raw = { + 'limit': '5', + 'startkey': json.dumps(key), + 'startkey_docid': startkey_docid, + 'full_set': 'true' + } + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, raw=raw) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # expect only a single record to be returned + expected_count = 1 + rows = cb_env.assert_rows(view_result, expected_count, True) + assert rows[0].key == key + assert rows[0].id == startkey_docid + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + def test_view_query_raw_fail(self, cb_env): + raw = { + 'limit': '5', + # this will fail as it is not encoded JSON + 'startkey': 'view-key', + 'startkey_docid': 'fake-doc-id' + } + opts = ViewOptions(namespace=DesignDocumentNamespace.DEVELOPMENT, raw=raw) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + with pytest.raises(InvalidArgumentException): + [r for r in view_result] + + def test_view_query_startkey_docid(self, cb_env): + # set the batch id + cb_env.get_batch_id() + keys = cb_env.get_keys() + # take the largest key so that we can purposefully narrow the result to 1 record + key_idx = keys.index(max(keys)) + key = keys[key_idx] + key_docids = cb_env.get_docids_by_key(key) + # purposefully use the last doc w/in the key + startkey_docid = key_docids[-1] + # execute a query so we can have pagination + opts = ViewOptions(limit=5, + namespace=DesignDocumentNamespace.DEVELOPMENT) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # need to iterate over the result to execute the query + [r for r in view_result] + opts = ViewOptions(limit=5, + namespace=DesignDocumentNamespace.DEVELOPMENT, + startkey=key, + startkey_docid=startkey_docid) + view_result = cb_env.bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + opts) + # expect only a single record to be returned + expected_count = 1 + rows = cb_env.assert_rows(view_result, expected_count, True) + assert rows[0].key == key + assert rows[0].id == startkey_docid + + metadata = view_result.metadata() + assert isinstance(metadata, ViewMetaData) + assert metadata.total_rows() >= expected_count + + # creating a new connection, allow retries + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_view_query_timeout(self, cb_env): + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + from couchbase.options import ClusterOptions, ClusterTimeoutOptions + conn_string = cb_env.config.get_connection_string() + username, pw = cb_env.config.get_username_and_pw() + auth = PasswordAuthenticator(username, pw) + # Prior to PYCBC-1521, this test would fail as each request would override the cluster level views_timeout. + # If a timeout was not provided in the request, the default 75s timeout would be used. PYCBC-1521 corrects + # this behavior so this test will pass as we are essentially forcing an AmbiguousTimeoutException because + # we are setting the cluster level views_timeout such a small value. + timeout_opts = ClusterTimeoutOptions(views_timeout=timedelta(milliseconds=1)) + cluster = Cluster.connect(f'{conn_string}', ClusterOptions(auth, timeout_options=timeout_opts)) + # don't need to do this except for older server versions + bucket = cluster.bucket(f'{cb_env.bucket.name}') + with pytest.raises(AmbiguousTimeoutException): + res = bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + limit=10, + namespace=DesignDocumentNamespace.DEVELOPMENT) + [r for r in res.rows()] + + # if we override the timeout w/in the request the query should succeed. + res = bucket.view_query(cb_env.DOCNAME, + cb_env.TEST_VIEW_NAME, + limit=10, + namespace=DesignDocumentNamespace.DEVELOPMENT, + timeout=timedelta(seconds=10)) + rows = [r for r in res.rows()] + assert len(rows) > 0 + + +class ClassicViewsTests(ViewsTestSuite): + @pytest.fixture(scope='class') + def test_manifest_validated(self): + def valid_test_method(meth): + attr = getattr(ClassicViewsTests, meth) + return callable(attr) and not meth.startswith('__') and meth.startswith('test') + method_list = [meth for meth in dir(ClassicViewsTests) if valid_test_method(meth)] + compare = set(ViewsTestSuite.TEST_MANIFEST).difference(method_list) + return compare + + @pytest.fixture(scope='class', name='cb_env', params=[CollectionType.DEFAULT]) + def couchbase_test_environment(self, cb_base_env, test_manifest_validated, request): + if test_manifest_validated: + pytest.fail(f'Test manifest not validated. Missing tests: {test_manifest_validated}.') + + cb_env = ViewsTestEnvironment.from_environment(cb_base_env) + cb_env.enable_views_mgmt() + cb_env.setup(request.param) + yield cb_env + cb_env.teardown(request.param) diff --git a/couchbase/tracing.py b/couchbase/tracing.py new file mode 100644 index 000000000..9cb57dd66 --- /dev/null +++ b/couchbase/tracing.py @@ -0,0 +1,106 @@ +# Copyright 2021, Couchbase, Inc. +# All Rights Reserved +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +from abc import ABC, abstractmethod +from typing import Any + + +class CouchbaseSpan(ABC): + """ + This is an abstract base class intended to wrap an external span implementation, to use withing the + python client. You will want to create a class deriving from this, which implements the :meth:`set_attribute` + and :meth:`finish` methods. + """ + + def __init__(self, + span # type: Any + ): + # type: (...) -> CouchbaseSpan + """ + Construct a new :class:`~CouchbaseSpan`. + :param: Any span: The concrete span this class will wrap. + """ + self._span = span + super().__init__() + + @property + def span(self): + # type: (...) -> Any + """ + Return the underlying wrapped span object. + :return: The underlying span object this class wrapped. + """ + return self._span + + @abstractmethod + def set_attribute(self, + key, # type: str + value # type: Any + ): + # type: (...) -> None + """ + This method will need to be implemented in derived classes. Given a key, and a value, use the underlying + `self._span` to set an attribute on the span. + + :param: str key: Key for the attribute. + :param: Any value: Value of the attribute. + """ + pass + + @abstractmethod + def finish(self): + # type: (...) -> None + """ + This method will need to be implemented in derived classes. Using `self._span`, the intent is to finish the + span. + """ + pass + + +class CouchbaseTracer(ABC): + """ + This is an abstract base class, intended to wrap a concrete tracer implementation. There is a single method, + :meth:`start_span`, which must be implemented in the derived class. + """ + + def __init__(self, + external_tracer # type: Any + ): + # type: (...) -> CouchbaseTracer + """ + Construct a new :class:`~.CouchbaseTracer`. + :param: Any external_tracer: The concrete tracer which this class will wrap. + """ + self._external_tracer = external_tracer + super().__init__() + + @abstractmethod + def start_span(self, + name, # type: str + parent=None # type: CouchbaseSpan + ): + # type: (...) -> CouchbaseSpan + """ + This method must be implemented in derived classes. The intent is to use the underlying `self._tracer` to + actually start a span, and return it wrapped in a :class:`CouchbaseSpan`. + for and example. + + :param: str name: Name of the span. + :param CouchbaseSpan parent: Parent span, if any. Will + be None when this is to be a top-level span. + :return: A new CouchbaseSpan, wrapping a span created by the + wrapped tracer. + """ + pass diff --git a/couchbase/transactions/__init__.py b/couchbase/transactions/__init__.py new file mode 100644 index 000000000..341b13611 --- /dev/null +++ b/couchbase/transactions/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .transaction_get_result import TransactionGetResult # noqa: F401 +from .transaction_keyspace import TransactionKeyspace # noqa: F401 +from .transaction_query_results import TransactionQueryResults # noqa: F401 +from .transaction_result import TransactionResult # noqa: F401 +from .transactions import AttemptContext # noqa: F401 +from .transactions import Transactions # noqa: F401 diff --git a/couchbase/transactions/logic/__init__.py b/couchbase/transactions/logic/__init__.py new file mode 100644 index 000000000..a000ce380 --- /dev/null +++ b/couchbase/transactions/logic/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .attempt_context_logic import AttemptContextLogic # noqa: F401 +from .transactions_logic import TransactionsLogic # noqa: F401 diff --git a/couchbase/transactions/logic/attempt_context_logic.py b/couchbase/transactions/logic/attempt_context_logic.py new file mode 100644 index 000000000..5111e363a --- /dev/null +++ b/couchbase/transactions/logic/attempt_context_logic.py @@ -0,0 +1,146 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import (TYPE_CHECKING, + Any, + Dict, + Optional) + +from couchbase.exceptions import ErrorMapper +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.options import TransactionOptions +from couchbase.pycbc_core import (create_new_attempt_context, + create_transaction_context, + transaction_commit, + transaction_op, + transaction_operations, + transaction_query_op, + transaction_rollback) + +if TYPE_CHECKING: + from asyncio import AbstractEventLoop + + from couchbase._utils import PyCapsuleType + from couchbase.transcoder import Transcoder + +log = logging.getLogger(__name__) + + +class AttemptContextLogic: + def __init__(self, + txns, # type: PyCapsuleType + transcoder, # type: Transcoder + loop, # type: Optional[AbstractEventLoop] + opts # type: Optional[PyCapsuleType] + ) -> None: + if opts is None: + opts = TransactionOptions()._base + self._txnctx = create_transaction_context(txns=txns, transaction_options=opts) + self._loop = loop + self._transcoder = transcoder + + def _handle_exception(self, ex: Any) -> None: + if isinstance(ex, Exception): + raise ex + if isinstance(ex, CouchbaseBaseException): + raise ErrorMapper.build_exception(ex) + + def _new_attempt(self) -> None: + new_attempt_ctx = create_new_attempt_context(ctx=self._txnctx) + self._handle_exception(new_attempt_ctx) + + def _new_attempt_async(self, + **kwargs, # type: Dict[str, Any] + ) -> None: + create_new_attempt_context(ctx=self._txnctx, **kwargs) + + def _rollback(self) -> None: + rollback_res = transaction_rollback(ctx=self._txnctx) + self._handle_exception(rollback_res) + + def _rollback_async(self, + **kwargs, # type: Dict[str, Any] + ) -> None: + transaction_rollback(ctx=self._txnctx, **kwargs) + + def _commit(self) -> Optional[Dict[str, Any]]: + commit_res = transaction_commit(ctx=self._txnctx) + self._handle_exception(commit_res) + return commit_res + + def _commit_async(self, + **kwargs, # type: Dict[str, Any] + ) -> Optional[Dict[str, Any]]: + return transaction_commit(ctx=self._txnctx, **kwargs) + + def get(self, coll, key, **kwargs): + # make sure we don't pass the transcoder along + kwargs.pop('transcoder', None) + kwargs.update(coll._get_connection_args()) + kwargs.pop("conn") + kwargs.update({ + 'key': key, + 'ctx': self._txnctx, + 'op': transaction_operations.GET.value + }) + log.debug('get calling transaction op with %s', kwargs) + return transaction_op(**kwargs) + + def get_replica_from_preferred_server_group(self, coll, key, **kwargs): + kwargs.pop('transcoder', None) + kwargs.update(coll._get_connection_args()) + kwargs.pop("conn") + kwargs["key"] = key + kwargs["ctx"] = self._txnctx + kwargs["op"] = transaction_operations.GET_REPLICA_FROM_PREFERRED_SERVER_GROUP.value + log.debug('get_replica_from_preferred_server_group calling transaction op with %s', kwargs) + return transaction_op(**kwargs) + + def insert(self, coll, key, value, **kwargs): + transcoder = kwargs.pop('transcoder', self._transcoder) + kwargs.update(coll._get_connection_args()) + kwargs.pop("conn") + kwargs.update({ + 'key': key, + 'ctx': self._txnctx, + 'op': transaction_operations.INSERT.value, + 'value': transcoder.encode_value(value) + }) + log.debug('insert calling transaction op with %s', kwargs) + return transaction_op(**kwargs) + + def replace(self, txn_get_result, value, **kwargs): + transcoder = kwargs.pop('transcoder', self._transcoder) + kwargs.update({ + 'ctx': self._txnctx, + 'op': transaction_operations.REPLACE.value, + 'value': transcoder.encode_value(value), + 'txn_get_result': txn_get_result._res + }) + log.debug('replace calling transaction op with %s', kwargs) + return transaction_op(**kwargs) + + def remove(self, txn_get_result, **kwargs): + kwargs.update({'ctx': self._txnctx, + 'op': transaction_operations.REMOVE.value, + 'txn_get_result': txn_get_result._res}) + log.debug('remove calling transaction op with %s', kwargs) + return transaction_op(**kwargs) + + def query(self, query, options, **kwargs): + kwargs.update({'ctx': self._txnctx, 'statement': query, 'options': options._base}) + log.debug('query calling transaction_op with %s', kwargs) + return transaction_query_op(**kwargs) diff --git a/couchbase/transactions/logic/transactions_logic.py b/couchbase/transactions/logic/transactions_logic.py new file mode 100644 index 000000000..fc48af013 --- /dev/null +++ b/couchbase/transactions/logic/transactions_logic.py @@ -0,0 +1,102 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import (TYPE_CHECKING, + Any, + Callable, + Coroutine) + +from couchbase.exceptions import TransactionExpired, TransactionFailed +from couchbase.pycbc_core import create_transactions, destroy_transactions +from couchbase.transactions.transaction_result import TransactionResult +from couchbase.transcoder import JSONTranscoder + +if TYPE_CHECKING: + from acouchbase.transactions import AttemptContext as AsyncAttemptContext + from couchbase.logic.cluster import ClusterLogic + from couchbase.options import TransactionConfig + from couchbase.transactions import AttemptContext as BlockingAttemptContext + +log = logging.getLogger(__name__) + + +class TransactionsLogic: + def __init__(self, + cluster, # type: ClusterLogic + config # type: TransactionConfig + ): + self._config = config + self._loop = None + # while the cluster has a default transcoder, it might not be a JSONTranscoder + self._transcoder = JSONTranscoder(cluster.default_serializer) + if hasattr(cluster, "loop"): + self._loop = cluster.loop + self._txns = create_transactions(cluster.connection, self._config._base) + log.info('created transactions object using config=%s, transcoder=%s', self._config, self._transcoder) + + def run(self, + logic, # type: Callable[[BlockingAttemptContext], None] + attempt_ctx # type: BlockingAttemptContext + ) -> TransactionResult: + + while True: + attempt_ctx._new_attempt() + try: + logic(attempt_ctx) + except Exception as ex: + attempt_ctx._rollback() + if isinstance(ex, TransactionExpired): + raise ex from None + raise TransactionFailed(exc_info={'inner_cause': ex}) from None + + try: + # calls finalize internally + res = attempt_ctx._commit() + if not res: + continue + return TransactionResult(**res) + except Exception: + # commit failed, retrying... + pass # nosec + + async def run_async(self, + logic, # type: Callable[[AsyncAttemptContext], Coroutine[Any, Any, None]] + attempt_ctx # type: AsyncAttemptContext + ) -> TransactionResult: + + while True: + await attempt_ctx._new_attempt() + try: + await logic(attempt_ctx) + except Exception as ex: + await attempt_ctx._rollback() + if isinstance(ex, TransactionExpired): + raise ex from None + raise TransactionFailed(exc_info={'inner_cause': ex}) from None + + try: + # calls finalize internally + res = await attempt_ctx._commit() + if not res: + continue + return res + except Exception: + # commit failed, retrying... + pass # nosec + + def close(self, **kwargs): + log.info('shutting down transactions...') + return destroy_transactions(txns=self._txns, **kwargs) diff --git a/couchbase/transactions/transaction_get_result.py b/couchbase/transactions/transaction_get_result.py new file mode 100644 index 000000000..b29e59e75 --- /dev/null +++ b/couchbase/transactions/transaction_get_result.py @@ -0,0 +1,58 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from typing import TYPE_CHECKING + +from couchbase.result import ContentProxy +from couchbase.transcoder import Transcoder + +if TYPE_CHECKING: + from couchbase.pycbc_core import transaction_get_result + +log = logging.getLogger(__name__) + + +class TransactionGetResult: + def __init__(self, + res, # type: transaction_get_result + transcoder # type: Transcoder + ): + self._res = res + self._transcoder = transcoder + self._decoded_value = None + + @property + def id(self): + return self._res.get("id") + + @property + def cas(self): + return self._res.get("cas") + + @property + def content_as(self): + if not self._decoded_value: + val, flags = self._res.get('value') + if val: + self._decoded_value = self._transcoder.decode_value(val, flags) + log.debug(f'Result has decoded value {self._decoded_value}') + return ContentProxy(self._decoded_value) + + log.debug('Result is missing decoded value ') + return ContentProxy('') + + def __str__(self): + return f'TransactionGetResult{{id={self.id}, cas={self.cas}, value={self.content_as[str]} }}' diff --git a/couchbase/transactions/transaction_keyspace.py b/couchbase/transactions/transaction_keyspace.py new file mode 100644 index 000000000..295749f7e --- /dev/null +++ b/couchbase/transactions/transaction_keyspace.py @@ -0,0 +1,55 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING, overload + +if TYPE_CHECKING: + from couchbase.logic.collection import CollectionLogic + + +class TransactionKeyspace: + @overload + def __init__(self, + coll=None, # type: CollectionLogic + bucket=None, # type: str + scope=None, # type: str + collection=None # type: str + ): + pass + + def __init__(self, **kwargs): + self._bucket = None + self._scope = None + self._collection = None + if 'coll' in kwargs: + self._bucket = kwargs['coll']._scope._bucket.name + self._scope = kwargs['coll']._scope.name + self._collection = kwargs['coll'].name + else: + self._bucket = kwargs['bucket'] + self._scope = kwargs['scope'] + self._collection = kwargs['collection'] + + @property + def bucket(self): + return self._bucket + + @property + def scope(self): + return self._scope + + @property + def collection(self): + return self._collection diff --git a/couchbase/transactions/transaction_query_results.py b/couchbase/transactions/transaction_query_results.py new file mode 100644 index 000000000..d891a55c9 --- /dev/null +++ b/couchbase/transactions/transaction_query_results.py @@ -0,0 +1,60 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import logging + +from couchbase.transcoder import DefaultJsonSerializer + +log = logging.getLogger(__name__) + + +class TransactionQueryResults: + # we could fake an async rows with something like this, and wrap + # the rows with it. But, this seems pointless for now, since the + # transactions lib doesn't stream the results. However, if we decide + # to update the transactions to support it, maybe we should have the + # correct interface? + class RowIter: + def __init__(self, rows): + self._loop = asyncio.get_event_loop() + self._queue = asyncio.Queue(loop=self._loop) + for r in rows: + self._queue.put_nowait(r) + + def __aiter__(self): + return self + + async def __anext__(self): + if self._queue.empty(): + log.debug('done iterating over rows') + raise StopAsyncIteration + row = await self._queue.get() + return row + + def __init__(self, + res # type: str + ): + + self._res = DefaultJsonSerializer().deserialize(res) + + def rows(self): + return self._res.get("results") + + def execute(self): + return + + def __str__(self): + return f'TransactionQueryResult{{res={self._res}}}' diff --git a/couchbase/transactions/transaction_result.py b/couchbase/transactions/transaction_result.py new file mode 100644 index 000000000..945f76e50 --- /dev/null +++ b/couchbase/transactions/transaction_result.py @@ -0,0 +1,31 @@ + +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +class TransactionResult(dict): + + @property + def transaction_id(self) -> str: + return self.get("transaction_id", None) + + @property + def unstaging_complete(self) -> bool: + return self.get("unstaging_complete", None) + + def __repr__(self): + return f"{type(self).__name__}({super().__repr__()})" + + def __str__(self): + return f"{type(self).__name__}({super().__str__()})" diff --git a/couchbase/transactions/transactions.py b/couchbase/transactions/transactions.py new file mode 100644 index 000000000..f715eea20 --- /dev/null +++ b/couchbase/transactions/transactions.py @@ -0,0 +1,335 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +from functools import wraps +from typing import (TYPE_CHECKING, + Any, + Callable, + Dict, + Optional) + +from couchbase.exceptions import (CouchbaseException, + ErrorMapper, + TransactionsErrorContext) +from couchbase.exceptions import exception as BaseCouchbaseException +from couchbase.logic.supportability import Supportability +from couchbase.options import (TransactionGetOptions, + TransactionGetReplicaFromPreferredServerGroupOptions, + TransactionInsertOptions, + TransactionQueryOptions, + TransactionReplaceOptions) +from couchbase.transactions.logic import AttemptContextLogic, TransactionsLogic + +from .transaction_get_result import TransactionGetResult +from .transaction_query_results import TransactionQueryResults +from .transaction_result import TransactionResult + +if TYPE_CHECKING: + from couchbase._utils import JSONType, PyCapsuleType + from couchbase.collection import Collection + from couchbase.options import TransactionOptions + from couchbase.transcoder import Transcoder + +log = logging.getLogger(__name__) + + +class BlockingWrapper: + @classmethod + def block(cls, return_cls): + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + try: + tc = kwargs.get('transcoder', self._transcoder) + ret = fn(self, *args, **kwargs) + log.debug('%s returned %s', fn.__name__, ret) + if isinstance(ret, Exception): + raise ret + if isinstance(ret, BaseCouchbaseException): + raise ErrorMapper.build_exception(ret) + if return_cls is None: + return None + if return_cls is TransactionGetResult: + retval = return_cls(ret, tc) + else: + retval = return_cls(ret) + return retval + except CouchbaseException as cb_exc: + raise cb_exc + except Exception as e: + raise CouchbaseException(message=str(e), context=TransactionsErrorContext()) + + return wrapped_fn + return decorator + + +class Transactions(TransactionsLogic): + + def run(self, + txn_logic, # type: Callable[[AttemptContext], None] + transaction_options=None, # type: Optional[TransactionOptions] + **kwargs # type: Dict[str, Any] + ) -> TransactionResult: + """ Run a set of operations within a transaction. + + Args: + txn_logic (Callable[:class:`~couchbase.transactions.AttemptContext`]): The transaction logic to perform. + transaction_options (:class:``): Options to override those in the :class:`couchbase.options.TransactionConfig` + for this transaction only. ** DEPRECATED ** Use transaction_config instead. + **kwargs (Dict[str, Any]): Override options for this transaction only - currently unimplemented. + + Returns: + :class:`~couchbase.transactions.TransactionResult`: Results of the transaction. + + Raises: + :class:`~couchbase.exceptions.TransactionFailed`: If the transaction failed. + :class:`~couchbase.exceptions.TransactionExpired`: If the transaction expired. + :class:`~couchbase.exceptions.TransactionCommitAmbiguous`: If the transaction's commit was ambiguous. See + :class:`~couchbase.exceptions.TransactionCommitAmbiguous` for a detailed description. + + Examples: + Transactional update of 2 documents, using :class:`couchbase.options.TransactionConfig` in the cluster:: + + doc1_id = "doc1-key" + doc2_id = "doc2-key" + coll = cluster.bucket("default").default_collection() + + def txn_logic(ctx): + doc1 = ctx.get(coll, doc1_id) + doc2 = ctx.get(coll, doc2_id) + ctx.update(doc1, {"some_key": f"I'm in {doc1_id}, and updated"}) + ctx.update(doc2, {"some_key": f"I'm in {doc2_id}, and updated"}) + + cluster.transactions.run(txn_logic + + """ # noqa: E501 + + opts = None + if transaction_options: + opts = transaction_options._base + if 'per_txn_config' in kwargs: + Supportability.method_param_deprecated('per_txn_config', 'transaction_options') + opts = kwargs.pop('per_txn_config', None) + + return super().run(txn_logic, AttemptContext(self._txns, self._transcoder, opts)) + + def close(self): + super().close() + log.info("transactions closed") + + +class AttemptContext(AttemptContextLogic): + + def __init__(self, + txns, # type: PyCapsuleType + transcoder, # type: Transcoder + opts # type: Optional[PyCapsuleType] + ): + super().__init__(txns, transcoder, None, opts) + + @BlockingWrapper.block(TransactionGetResult) + def _get(self, + coll, # type: Collection + key, # type: str + **kwargs # type: Dict[str, Any] + ) -> TransactionGetResult: + return super().get(coll, key, **kwargs) + + def get(self, + coll, # type: Collection + key, # type: str + options=None, # type: Optional[TransactionGetOptions] + **kwargs # type: Dict[str, Any] + ) -> TransactionGetResult: + """ + Get a document within this transaction. + + Args: + coll (:class:`couchbase.collection.Collection`): Collection to use to find the document. + key (str): document key. + options (:class:`~couchbase.options.TransactionGetOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TransactionGetOptions` + + Returns: + :class:`couchbase.transactions.TransactionGetResult`: Document in collection, in a form useful for passing + to other transaction operations. Or `None` if the document was not found. + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + """ + if 'transcoder' not in kwargs and isinstance(options, TransactionGetOptions): + kwargs['transcoder'] = options.get('transcoder', None) + return self._get(coll, key, **kwargs) + + @BlockingWrapper.block(TransactionGetResult) + def _get_replica_from_preferred_server_group(self, + coll, # type: Collection + key, # type: str + **kwargs # type: Dict[str, Any] + ) -> TransactionGetResult: + return super().get_replica_from_preferred_server_group(coll, key, **kwargs) + + def get_replica_from_preferred_server_group(self, + coll, # type: Collection + key, # type: str + options=None, # type: Optional[TransactionGetReplicaFromPreferredServerGroupOptions] # noqa: E501 + **kwargs # type: Dict[str, Any] + ) -> TransactionGetResult: + """ + Get a document within this transaction from any replica in the preferred server group that has been specified in + the :class:`couchbase.options.ClusterOptions`. + + Args: + coll (:class:`couchbase.collection.Collection`): Collection to use to find the document. + key (str): document key. + options (:class:`~couchbase.options.TransactionGetReplicaFromPreferredServerGroupOptions`): Optional + parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TransactionGetReplicaFromPreferredServerGroupOptions` + + Returns: + :class:`couchbase.transactions.TransactionGetResult`: Document in collection, in a form useful for passing + to other transaction operations. Or `None` if the document was not found. + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + :class:`couchbase.exceptions.DocumentUnretrievableException`: If the document could not be retrieved from + any replica in the preferred server group. The transaction will not rollback if this exception is + caught. + """ + if 'transcoder' not in kwargs and isinstance(options, TransactionGetReplicaFromPreferredServerGroupOptions): + kwargs['transcoder'] = options.get('transcoder', None) + return self._get_replica_from_preferred_server_group(coll, key, **kwargs) + + @BlockingWrapper.block(TransactionGetResult) + def _insert(self, + coll, # type: Collection + key, # type: str + value, # type: JSONType + **kwargs # type: Dict[str, Any] + ) -> Optional[TransactionGetResult]: + return super().insert(coll, key, value, **kwargs) + + def insert(self, + coll, # type: Collection + key, # type: str + value, # type: JSONType + options=None, # type: Optional[TransactionInsertOptions] + **kwargs # type: Dict[str, Any] + ) -> Optional[TransactionGetResult]: + """ + Insert a new document within a transaction. + + Args: + coll (:class:`couchbase.collection.Collection`): Collection to use to find the document. + key (str): document key. + value (:class:`couchbase._utils.JSONType): Contents of the document. + options (:class:`~couchbase.options.TransactionInsertOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TransactionInsertOptions` + + Returns: + :class:`couchbase.transactions.TransactionGetResult`: Document in collection, in a form useful for passing + to other transaction operations. + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + """ + if 'transcoder' not in kwargs and isinstance(options, TransactionInsertOptions): + kwargs['transcoder'] = options.get('transcoder', None) + return self._insert(coll, key, value, **kwargs) + + @BlockingWrapper.block(TransactionGetResult) + def _replace(self, + txn_get_result, # type: TransactionGetResult + value, # type: JSONType + **kwargs, # type: Dict[str, Any] + ) -> TransactionGetResult: + return super().replace(txn_get_result, value, **kwargs) + + def replace(self, + txn_get_result, # type: TransactionGetResult + value, # type: JSONType + options=None, # type: Optional[TransactionReplaceOptions] + **kwargs, # type: Dict[str, Any] + ) -> TransactionGetResult: + """ + Replace the contents of a document within a transaction. + + Args: + txn_get_result (:class:`couchbase.transactions.TransactionGetResult`): Document to replace, gotten from a + previous call to another transaction operation. + value (:class:`couchbase._utils.JSONType): The new contents of the document. + options (:class:`~couchbase.options.TransactionReplaceOptions`): Optional parameters for this operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used in place or to + override provided :class:`~couchbase.options.TransactionReplaceOptions` + + Returns: + :class:`couchbase.transactions.TransactionGetResult`: Document in collection, in a form useful for passing + to other transaction operations. + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + + """ + if 'transcoder' not in kwargs and isinstance(options, TransactionReplaceOptions): + kwargs['transcoder'] = options.get('transcoder', None) + return self._replace(txn_get_result, value, **kwargs) + + @BlockingWrapper.block(None) + def remove(self, + txn_get_result, # type: TransactionGetResult + **kwargs # type: Dict[str, Any] + ) -> None: + """ + Remove a document in a transaction. + + Args: + txn_get_result (:class:`couchbase.transactions.TransactionGetResult`): Document to delete. + **kwargs (Dict[str, Any]): currently unused. + Returns: + None + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + """ + return super().remove(txn_get_result, **kwargs) + + @BlockingWrapper.block(TransactionQueryResults) + def query(self, + query, # type: str + options=TransactionQueryOptions(), # type: TransactionQueryOptions + **kwargs # type: Dict[str, Any] + ) -> TransactionQueryResults: + """ + Perform a query within a transaction. + + Args: + query (str): Query to perform. + options (:class:`couchbase.transactions.TransactionQueryOptions`): Query options to use, if any. + **kwargs (Dict[str, Any]): currently unused. + + Returns: + :class:`couchbase.transactions.TransactionQueryResult` + Raises: + :class:`couchbase.exceptions.TransactionOperationFailed`: If the operation failed. In practice, there is + no need to handle the exception, as the transaction will rollback regardless. + :class:`couchbase.exceptions.CouchbaseException`: If the operation failed, but the transaction will not + necessarily be rolled back, a CouchbaseException other than TransactionOperationFailed will be raised. + If handled, the transaction will not rollback. + """ + return super().query(query, options, **kwargs) diff --git a/couchbase/transcoder.py b/couchbase/transcoder.py old mode 100755 new mode 100644 index a12abe0ed..719771842 --- a/couchbase/transcoder.py +++ b/couchbase/transcoder.py @@ -1,32 +1,40 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. # -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -import warnings -import json -import pickle +from __future__ import annotations -from couchbase import (FMT_JSON, FMT_AUTO, - FMT_BYTES, FMT_UTF8, FMT_PICKLE, - FMT_LEGACY_MASK, FMT_COMMON_MASK) -from couchbase.exceptions import ValueFormatError -from couchbase._libcouchbase import Transcoder -from couchbase._pyport import unicode - -# Initialize our dictionary +import json +import pickle # nosec +from abc import ABC, abstractmethod +from typing import (TYPE_CHECKING, + Any, + Optional, + Tuple, + Union) + +from couchbase.constants import (FMT_BYTES, + FMT_COMMON_MASK, + FMT_JSON, + FMT_LEGACY_MASK, + FMT_PICKLE, + FMT_UTF8) +from couchbase.exceptions import ValueFormatException +from couchbase.serializer import DefaultJsonSerializer + +if TYPE_CHECKING: + from couchbase.serializer import Serializer UNIFIED_FORMATS = (FMT_JSON, FMT_BYTES, FMT_UTF8, FMT_PICKLE) LEGACY_FORMATS = tuple([x & FMT_LEGACY_MASK for x in UNIFIED_FORMATS]) @@ -40,133 +48,254 @@ LEGACY2UNIFIED[fl & FMT_LEGACY_MASK] = fl -def get_decode_format(flags): - """ - Returns a tuple of format, recognized +def get_decode_format(flags, # type: Optional[int] + ) -> Optional[int]: + """ Decode the flags value for transcoding. + + Args: + flags (int, optional): The flags to decode. + + Returns: + Optional[int]: The common flags or legacy flags format. If None + is returned the format is UNKNOWN. """ + # return None for unknown format + if flags is None: + return flags + c_flags = flags & FMT_COMMON_MASK l_flags = flags & FMT_LEGACY_MASK if c_flags: - if c_flags not in COMMON_FORMATS: - return FMT_BYTES, False - else: - return COMMON2UNIFIED[c_flags], True + # if unknown format, default to None + return COMMON2UNIFIED.get(c_flags, None) else: - if not l_flags in LEGACY_FORMATS: - return FMT_BYTES, False - else: - return LEGACY2UNIFIED[l_flags], True + # if unknown format, default to None + return LEGACY2UNIFIED.get(l_flags, None) -class TranscoderPP(object): - """ - This is a pure-python Transcoder class. It is here only to show a reference - implementation. It is recommended that you subclass from - the :class:`Transcoder` object instead if all the methods are not - implemented. +class Transcoder(ABC): + """Interface a Custom Transcoder must implement """ - def encode_key(self, key): - ret = (self.encode_value(key, FMT_UTF8))[0] - return ret + @abstractmethod + def encode_value(self, + value # type: Any + ) -> Tuple[bytes, int]: + raise NotImplementedError() + + @abstractmethod + def decode_value(self, + value, # type: bytes + flags # type: int + ) -> Any: + raise NotImplementedError() + + @classmethod + def __subclasshook__(cls, subclass): + return (hasattr(subclass, 'encode_value') and + callable(subclass.encode_value) and + hasattr(subclass, 'decode_value') and + callable(subclass.decode_value)) + - def decode_key(self, key): - return self.decode_value(key, FMT_UTF8) +class JSONTranscoder(Transcoder): - def encode_value(self, value, format): - if format == 0: + def __init__(self, serializer=None # type: Serializer + ): + + if not serializer: + self._serializer = DefaultJsonSerializer() + else: + self._serializer = serializer + + def encode_value(self, + value, # type: Any + ) -> Tuple[bytes, int]: + + if isinstance(value, str): format = FMT_JSON - elif format == FMT_AUTO: - if isinstance(value, unicode): - format = FMT_UTF8 - elif isinstance(value, (bytes, bytearray)): - format = FMT_BYTES - elif isinstance(value, (list, tuple, dict, bool)) or value is None: - format = FMT_JSON - else: - format = FMT_PICKLE + elif isinstance(value, (bytes, bytearray)): + raise ValueFormatException( + "The JSONTranscoder (default transcoder) does not support binary data.") + elif isinstance(value, (list, tuple, dict, bool, int, float)) or value is None: + format = FMT_JSON + else: + raise ValueFormatException( + "Unrecognized value type {}".format(type(value))) + + if format != FMT_JSON: + raise ValueFormatException(f"Unrecognized format {format}") + + return self._serializer.serialize(value), FMT_JSON + + def decode_value(self, + value, # type: bytes + flags # type: int + ) -> Any: + + format = get_decode_format(flags) + + # flags=[0 | None] special case, attempt JSON deserialize + if format in [FMT_JSON, 0, None]: + try: + return self._serializer.deserialize(value) + except Exception: + # if error encountered, assume return bytes + return value + elif format == FMT_BYTES: + raise ValueFormatException("The JSONTranscoder (default transcoder) does not support binary format") + elif format == FMT_UTF8: + raise ValueFormatException("The JSONTranscoder (default transcoder) does not support string format") + else: + raise ValueFormatException(f"Unrecognized format provided: {format}") - if format not in (FMT_PICKLE, FMT_JSON, FMT_BYTES, FMT_UTF8): - raise ValueError("Unrecognized format") - if format == FMT_BYTES: - if isinstance(value, bytes): - pass +class RawJSONTranscoder(Transcoder): - elif isinstance(value, bytearray): + def encode_value(self, + value # type: Union[str,bytes,bytearray] + ) -> Tuple[bytes, int]: + + if isinstance(value, str): + return value.encode('utf-8'), FMT_JSON + elif isinstance(value, (bytes, bytearray)): + if isinstance(value, bytearray): value = bytes(value) + return value, FMT_JSON + else: + raise ValueFormatException("Only binary and string data supported by RawJSONTranscoder") - else: - raise TypeError("Expected bytes") + def decode_value(self, + value, # type: bytes + flags # type: int + ) -> Union[str, bytes]: - return value, format + format = get_decode_format(flags) + if format == FMT_BYTES: + raise ValueFormatException("Binary format type not supported by RawJSONTranscoder") elif format == FMT_UTF8: - return value.encode('utf-8'), format + raise ValueFormatException("String format type not supported by RawJSONTranscoder") + elif format == FMT_JSON: + if isinstance(value, str): + value = value.decode('utf-8') + elif isinstance(value, bytearray): + value = bytes(value) + return value + else: + raise ValueFormatException(f"Unrecognized format provided: {format}") - elif format == FMT_PICKLE: - return self._do_pickle_encode(value), FMT_PICKLE +class RawStringTranscoder(Transcoder): + + def encode_value(self, + value # type: str + ) -> Tuple[bytes, int]: + + if isinstance(value, str): + return value.encode('utf-8'), FMT_UTF8 + else: + raise ValueFormatException("Only string data supported by RawStringTranscoder") + + def decode_value(self, + value, # type: bytes + flags # type: int + ) -> Union[str, bytes]: + + format = get_decode_format(flags) + + if format == FMT_BYTES: + raise ValueFormatException("Binary format type not supported by RawStringTranscoder") + elif format == FMT_UTF8: + return value.decode('utf-8') elif format == FMT_JSON: - return self._do_json_encode(value).encode('utf-8'), FMT_JSON + raise ValueFormatException("JSON format type not supported by RawStringTranscoder") + else: + raise ValueFormatException(f"Unrecognized format provided: {format}") + +class RawBinaryTranscoder(Transcoder): + def encode_value(self, + value # type: Union[bytes,bytearray] + ) -> Tuple[bytes, int]: + + if isinstance(value, (bytes, bytearray)): + if isinstance(value, bytearray): + value = bytes(value) + return value, FMT_BYTES else: - raise ValueError("Unrecognized format '%r'" % (format,)) + raise ValueFormatException("Only binary data supported by RawBinaryTranscoder") - def decode_value(self, value, flags): - format, is_recognized = get_decode_format(flags) + def decode_value(self, + value, # type: bytes + flags # type: int + ) -> bytes: + + format = get_decode_format(flags) if format == FMT_BYTES: - if not is_recognized: - warnings.warn("Received unrecognized flags %d" % (flags,)) + if isinstance(value, bytearray): + value = bytes(value) return value - elif format == FMT_UTF8: - return value.decode("utf-8") - + raise ValueFormatException("String format type not supported by RawBinaryTranscoder") elif format == FMT_JSON: - return self._do_json_decode(value.decode('utf-8')) + raise ValueFormatException("JSON format type not supported by RawBinaryTranscoder") + else: + raise ValueFormatException(f"Unrecognized format provided: {format}") + +class LegacyTranscoder(Transcoder): + + def encode_value(self, + value # type: Any + ) -> Tuple[bytes, int]: + + if isinstance(value, str): + format = FMT_UTF8 + elif isinstance(value, (bytes, bytearray)): + format = FMT_BYTES + elif isinstance(value, (list, tuple, dict, bool, int, float)) or value is None: + format = FMT_JSON + else: + format = FMT_PICKLE + + if format == FMT_BYTES: + if isinstance(value, bytes): + pass + elif isinstance(value, bytearray): + value = bytes(value) + else: + raise ValueFormatException('Expected bytes') + return value, format + elif format == FMT_UTF8: + return value.encode('utf-8'), format elif format == FMT_PICKLE: - return self._do_pickle_decode(value) - - def _do_json_encode(self, value): - """ - Can be overidden by subclasses. This should do the same as `json.dumps` - :param value: Python object - :return: JSON string - """ - return json.dumps(value, ensure_ascii=False) - - def _do_json_decode(self, value): - """ - Can be overidden by subclasses. This should do the same as `json.loads` - :param value: The JSON string - :return: The decoded Python value - """ - return json.loads(value) - - def _do_pickle_encode(self, value): - """ - Can be overidden by subclasses. This should do the same as - `pickle.dumps` - :param value: The value to pickle - :return: The pickled buffer - """ - return pickle.dumps(value) - - def _do_pickle_decode(self, value): - """ - Can be overidden by subclasses. This should do the same as - `pickle.loads` - :param value: The pickled buffer - :return: The unpickled python object - """ - return pickle.loads(value) - - -class LegacyTranscoderPP(TranscoderPP): - def encode_value(self, value, format): - encoded, flags = super(LegacyTranscoderPP, self).encode_value(value, format) - return encoded, flags & FMT_LEGACY_MASK + return pickle.dumps(value), FMT_PICKLE + else: # default to JSON + return json.dumps(value, ensure_ascii=False).encode('utf-8'), FMT_JSON + + def decode_value(self, + value, # type: bytes + flags # type: int + ) -> Any: + + format = get_decode_format(flags) + + # flags=[0 | None] special case, attempt JSON deserialize + if format in [FMT_JSON, 0, None]: + try: + return json.loads(value.decode('utf-8')) + except Exception: + # if error encountered, assume bytes + return value + elif format == FMT_BYTES: + return value + elif format == FMT_UTF8: + return value.decode('utf-8') + elif format == FMT_PICKLE: + return pickle.loads(value) # nosec + else: + # default to returning bytes + return value diff --git a/couchbase/user_constants.py b/couchbase/user_constants.py deleted file mode 100644 index dbf2b74be..000000000 --- a/couchbase/user_constants.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -Constants defined in _libcouchbase module for use by users -""" -import couchbase._bootstrap -from couchbase._libcouchbase import ( - FMT_JSON, - FMT_BYTES, - FMT_UTF8, - FMT_PICKLE, - FMT_AUTO, - FMT_COMMON_MASK, - FMT_LEGACY_MASK, - - OBS_PERSISTED, - OBS_FOUND, - OBS_NOTFOUND, - OBS_LOGICALLY_DELETED, - - OBS_MASK, - - LOCKMODE_WAIT, - LOCKMODE_EXC, - LOCKMODE_NONE -) diff --git a/couchbase/vector_search.py b/couchbase/vector_search.py new file mode 100644 index 000000000..050841d39 --- /dev/null +++ b/couchbase/vector_search.py @@ -0,0 +1,3 @@ +from couchbase.logic.vector_search import VectorQuery # noqa: F401 +from couchbase.logic.vector_search import VectorQueryCombination # noqa: F401 +from couchbase.logic.vector_search import VectorSearch # noqa: F401 diff --git a/couchbase/views.py b/couchbase/views.py new file mode 100644 index 000000000..952b6bcc5 --- /dev/null +++ b/couchbase/views.py @@ -0,0 +1,104 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.views import ViewErrorMode # noqa: F401 +from couchbase.logic.views import ViewMetaData # noqa: F401 +from couchbase.logic.views import ViewOrdering # noqa: F401 +from couchbase.logic.views import ViewQuery # noqa: F401 +from couchbase.logic.views import ViewScanConsistency # noqa: F401 +from couchbase.logic.views import ViewRequestLogic, ViewRow + + +class ViewRequest(ViewRequestLogic): + def __init__(self, + connection, + encoded_query, + **kwargs + ): + super().__init__(connection, encoded_query, **kwargs) + + @classmethod + def generate_view_request(cls, connection, encoded_query, **kwargs): + return cls(connection, encoded_query, **kwargs) + + def execute(self): + return [r for r in list(self)] + + def _get_metadata(self): + try: + # @TODO: PYCBC-1524 + views_response = next(self._streaming_result) + self._set_metadata(views_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def __iter__(self): + if self.done_streaming: + raise AlreadyQueriedException() + + if not self.started_streaming: + self._submit_query() + + return self + + def _get_next_row(self): + if self.done_streaming is True: + return + + try: + row = next(self._streaming_result) + except StopIteration: + # @TODO: PYCBC-1524 + row = next(self._streaming_result) + + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopIteration + + # TODO: until streaming, a dict is returned, no deserializing... + # deserialized_row = self.serializer.deserialize(row) + deserialized_row = row + if issubclass(self.row_factory, ViewRow): + if hasattr(self.row_factory, 'from_json'): + return self.row_factory.from_json(deserialized_row) + return self.row_factory(**deserialized_row) + else: + return deserialized_row + + def __next__(self): + try: + return self._get_next_row() + except StopIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/couchbase/views/__init__.py b/couchbase/views/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/couchbase/views/iterator.py b/couchbase/views/iterator.py deleted file mode 100644 index 53ca56bf6..000000000 --- a/couchbase/views/iterator.py +++ /dev/null @@ -1,386 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from collections import namedtuple -from copy import deepcopy -from warnings import warn - -from couchbase.exceptions import ArgumentError, CouchbaseError, ViewEngineError -from couchbase.views.params import ViewQuery, SpatialQuery, QueryBase -from couchbase._pyport import basestring -import couchbase._libcouchbase as C -from couchbase._bootstrap import MAX_URI_LENGTH - -class AlreadyQueriedError(CouchbaseError): - """Thrown when iterating over a View which was already iterated over""" - - -ViewRow = namedtuple('ViewRow', ['key', 'value', 'docid', 'doc']) -""" -Default class for a single row. -""" - -SpatialRow = namedtuple('SpatialRow', - ['key', 'value', 'geometry', 'docid', 'doc']) -""" -Default class for a spatial row -""" - - -class RowProcessor(object): - """ - This class contains the handling and conversion functions between - multiple rows and the means by which they are returned from the - view iterator. - - This class should be overidden if you are: - - * Using a custom row class - This saves on processing time and memory by converting from the raw - results rather than having to unpack from the default class. This - class returns a :class:`ViewRow` object by default. (This can also - be overridden using the :attr:`rowclass` attribute) - - * Fetching multiple documents for each row - You can use the :meth:`~couchbase.bucket.Bucket.get_multi` - method to efficiently fetch multiple docs beforehand for the entire - page. - - .. attribute:: rowclass - - Class or function to call for each result (row) received. This - is called as ``rowclass(key, value, docid, doc)`` - - * ``key`` is the key as returned by the first argument to the - view function's ``emit``. - * ``value`` is the value returned as the second argument to the - view function's ``emit``, or the value of the ``reduce`` - function - * ``docid`` is the ID of the document itself (as stored by one - of the :meth:`~couchbase.bucket.Bucket.upsert` family of - methods). - If ``reduce`` was set to true for the view, this will always - be None. - * ``doc`` is the document itself - Only valid if ``include_docs`` - is set to true - in which case a - :class:`~couchbase.connection.Result` object is passed. - If ``reduce`` was set to true for the view, this will always - be None - Otherwise, ``None`` is passed instead. - - By default, the :class:`ViewRow` is used. - """ - def __init__(self, rowclass=ViewRow): - self.rowclass = rowclass - - def handle_rows(self, rows, *_): - """ - Preprocesses a page of rows. - - :param list rows: A list of rows. Each row is a JSON object containing - the decoded JSON of the view as returned from the server - :param connection: The connection object (pass to the :class:`View` - constructor) - :param include_docs: Whether to include documents in the return value. - This is ``True`` or ``False`` depending on what was passed to the - :class:`View` constructor - - :return: an iterable. When the iterable is exhausted, this method will - be called again with a new 'page'. - """ - for row in rows: - yield self.rowclass(row['key'], row['value'], - row.get('id'), get_row_doc(row)) - - -class SpatialRowProcessor(RowProcessor): - def __init__(self, rowclass=SpatialRow): - super(SpatialRowProcessor, self).__init__(rowclass) - - def handle_rows(self, rows, *_): - for row in rows: - yield self.rowclass(row['key'], row['value'], row.get('geometry'), - row.get('id'), get_row_doc(row)) - - -def get_row_doc(row_json): - """ - Gets the document for the given parsed JSON row. - - Use this function in custom :class:`~.RowProcessor` - implementations to extract the actual document. The document - itself is stored within a private field of the row itself, and - should only be accessed by this function. - - :param dict row_json: The parsed row (passed to the processor) - :return: The document, or None - """ - return row_json.get('__DOCRESULT__') - - -class View(object): - def __init__(self, parent, design, view, row_processor=None, - include_docs=False, query=None, streaming=True, **params): - """ - Construct a iterable which can be used to iterate over view query - results. - - :param parent: The parent Bucket object - :type parent: :class:`~couchbase.bucket.Bucket` - :param string design: The design document - :param string view: The name of the view within the design document - :param callable row_processor: See :attr:`row_processor` for more - details. - - :param boolean include_docs: If set, the document itself will be - retrieved for each row in the result. The default algorithm - uses :meth:`~couchbase.bucket.Bucket.get_multi` for each - page (i.e. every :attr:`streaming` results). - - The :attr:`~couchbase.views.params.Query.reduce` - family of attributes must not be active, as results fro - ``reduce`` views do not have corresponding - doc IDs (as these are aggregation functions). - - :param query: If set, should be a :class:`~.Query` or - :class:`~.SpatialQuery` object. It is illegal to use - this in conjunction with additional ``params`` - - :param params: Extra view options. This may be used to pass view - arguments (as defined in :class:`~couchbase.views.params.Query`) - without explicitly constructing a - :class:`~couchbase.views.params.Query` object. - It is illegal to use this together with the ``query`` argument. - If you wish to 'inline' additional arguments to the provided - ``query`` object, use the - query's :meth:`~couchbase.views.params.Query.update` method - instead. - - This object is an iterator - it does not send out the request until - the first item from the iterator is request. See :meth:`__iter__` for - more details on what this object returns. - - - Simple view query, with no extra options:: - - # c is the Bucket object. - - for result in View(c, "beer", "brewery_beers"): - print("emitted key: {0}, doc_id: {1}" - .format(result.key, result.docid)) - - - Execute a view with extra query options:: - - # Implicitly creates a Query object - - view = View(c, "beer", "by_location", - limit=4, - reduce=True, - group_level=2) - - Execute a spatial view:: - - from couchbase.views.params import SpatialQuery - # .... - q = SpatialQuery() - q.start_range = [ -119.9556, 38.7056 ] - q.end_range = [ -118.8122, 39.7086 ] - view = View(c, 'geodesign', 'spatialview', query=q) - for row in view: - print('Location is {0}'.format(row.geometry)) - - Pass a Query object:: - - q = Query( - stale=False, - inclusive_end=True, - mapkey_range=[ - ["21st_ammendment_brewery_cafe"], - ["21st_ammendment_brewery_cafe", Query.STRING_RANGE_END] - ] - ) - - view = View(c, "beer", "brewery_beer", query=q) - - Add extra parameters to query object for single call:: - - view = View(c, "beer", "brewery_beer", - query=q.update(debug=True, copy=True)) - - - Include documents with query:: - - view = View(c, "beer", "brewery_beer", - query=q, include_docs=True) - - for result in view: - print("Emitted key: {0}, Document: {1}".format( - result.key, result.doc.value)) - """ - - self._parent = parent - self.design = design - self.view = view - self.errors = [] - self.rows_returned = 0 - self.indexed_rows = 0 - - if query and params: - raise ArgumentError.pyexc( - "Extra parameters are mutually exclusive with the " - "'query' argument. Use query.update() to add extra arguments") - - if query: - if isinstance(query, basestring): - self._query = ViewQuery.from_string(query) - else: - self._query = deepcopy(query) - else: - self._query = QueryBase.from_any(params) - - self._flags = 0 - if include_docs: - self._flags |= C.LCB_CMDVIEWQUERY_F_INCLUDE_DOCS - if isinstance(self._query, SpatialQuery): - self._flags |= C.LCB_CMDVIEWQUERY_F_SPATIAL - - if include_docs and self._query.reduce: - raise ArgumentError.pyexc( - "include_docs is only applicable for map-only views, but " - "'reduce', 'group', or 'group_level' was specified; or " - "this is a spatial query", - self._query) - - # The original 'limit' parameter, passed to the query. - self._do_iter = True - self._mres = None - - if not row_processor: - if self._spatial: - row_processor = SpatialRowProcessor() - else: - row_processor = RowProcessor() - self.row_processor = row_processor - - @property - def _spatial(self): - return self._flags & C.LCB_CMDVIEWQUERY_F_SPATIAL - - def _start(self): - if self._mres: - return - - self._mres = self._parent._view_request( - design=self.design, view=self.view, options=self.query, - _flags=self._flags) - self.__raw = self._mres[None] - - def _clear(self): - """ - Clears references to other internal objects. - - This is useful to break out of a reference cycle - """ - del self._parent - del self._mres - - @property - def query(self): - """ - Returns the :class:`~couchbase.views.params.Query` object associated - with this execution instance. - - Note that is normally a modified version - of the passed object (in the constructor's ``query`` params). It should - not be directly modified. - """ - return self._query - - @property - def raw(self): - return self.__raw - - def _handle_errors(self, errors): - if not errors: - return - - self.errors += [errors] - - if self._query.on_error != 'continue': - raise ViewEngineError.pyexc("Error while executing view.", - self.errors) - else: - warn("Error encountered when executing view. Inspect 'errors' " - "for more information") - - def _handle_meta(self, value): - if not isinstance(value, dict): - return - self.indexed_rows = value.get('total_rows', 0) - self._handle_errors(value.get('errors')) - - def _process_payload(self, rows): - if rows: - self.rows_returned += len(rows) - return self.row_processor.handle_rows(rows, self._parent, False) - - elif self.raw.done: - self._handle_meta(self.raw.value) - self._do_iter = False - - return [] - - def __iter__(self): - """ - Returns a row for each query. - The type of the row depends on the :attr:`row_processor` being used. - - :raise: :exc:`~couchbase.exceptions.ViewEngineError` - - If an error was encountered while processing the view, and the - :attr:`~couchbase.views.params.Query.on_error` - attribute was not set to `continue`. - - If `continue` was specified, a warning message is printed to the - screen (via ``warnings.warn`` and operation continues). To inspect - the error, examine :attr:`errors` - - :raise: :exc:`AlreadyQueriedError` - - If this object was already iterated - over and the last result was already returned. - """ - if not self._do_iter: - raise AlreadyQueriedError.pyexc( - "This object has already been executed. Create a new one to " - "query again") - - self._start() - while self._do_iter: - raw_rows = self.raw.fetch(self._mres) - for row in self._process_payload(raw_rows): - yield row - - def __repr__(self): - details = [] - details.append("Design={0}".format(self.design)) - details.append("View={0}".format(self.view)) - details.append("Query={0}".format(self._query)) - details.append("Rows Fetched={0}".format(self.rows_returned)) - return '{cls}<{details}>'.format(cls=self.__class__.__name__, - details=', '.join(details)) diff --git a/couchbase/views/params.py b/couchbase/views/params.py deleted file mode 100644 index 430b45fe7..000000000 --- a/couchbase/views/params.py +++ /dev/null @@ -1,575 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# This module is largely used by other modules, though it just contains -# simple string utilities :) - -import json -from copy import deepcopy -from collections import namedtuple - -from couchbase._pyport import long, xrange, ulp, basestring, parse_qs -from couchbase.exceptions import ArgumentError - -# Some constants -STALE_UPDATE_BEFORE = "false" -STALE_UPDATE_AFTER = "update_after" -STALE_OK = "ok" -ONERROR_CONTINUE = "continue" -ONERROR_STOP = "stop" - - -class _Unspec(object): - def __nonzero__(self): - return False - - # Py3 - __bool__ = __nonzero__ - - def __str__(self): - return "" - - def __repr__(self): - return "<Placeholder>" - -UNSPEC = _Unspec() - -def _bool_param_handler(input): - if isinstance(input, bool): - if input: - return "true" - else: - return "false" - - if isinstance(input, basestring): - if input not in ("true", "false"): - raise ArgumentError.pyexc("String for boolean must be " - "'true' or 'false'", input) - return input - - try: - input + 0 - if input: - return "true" - else: - return "false" - - except TypeError: - raise ArgumentError.pyexc("Boolean value must be boolean, " - "numeric, or a string of 'true' " - "or 'false'", input) - - -def _num_param_handler(input): - # Don't allow booleans: - if isinstance(input, bool): - raise ArgumentError.pyexc("Cannot use booleans as numeric values", - input) - try: - return str(int(input)) - except Exception as e: - raise ArgumentError.pyexc("Expected a numeric argument", input, e) - - -def _string_param_common(input, do_quote=False): - # TODO, if we pass this to urlencode, do we ever need to quote? - # For the moment, i'm always forcing non-quote behavior - do_quote = False - - s = None - if isinstance(input, basestring): - s = input - - elif isinstance(input, bool): - raise ArgumentError.pyexc("Can't use boolean as string", input) - - elif isinstance(input, (int, long, float)): - # Basic numeric types: - s = str(input) - - else: - raise ArgumentError.pyexc("Expected simple numeric type or string ", - input) - if do_quote: - s = ulp.quote(s) - - return s - - -def _string_param_handler(input): - return _string_param_common(input, do_quote=True) - - -def _generic_param_handler(input): - return _string_param_handler(input, do_quote=False) - - -def _stale_param_handler(input): - if input in (STALE_UPDATE_AFTER, STALE_OK, STALE_UPDATE_BEFORE): - return input - - ret = _bool_param_handler(input) - if ret == "true": - ret = STALE_OK - return ret - - -def _onerror_param_handler(input): - if input not in (ONERROR_CONTINUE, ONERROR_STOP): - raise ArgumentError.pyexc( - "on_error must be 'continue' or 'stop'", input) - - return input - - -def _jval_param_handler(input): - try: - ret = json.dumps(input) - return _string_param_handler(ret) - except Exception as e: - raise ArgumentError.pyexc("Couldn't convert value to JSON", input, e) - - -def _jarry_param_handler(input): - ret = _jval_param_handler(input) - if not ret.startswith('['): - raise ArgumentError.pyexc( - "Value must be converted to JSON array", input) - - return ret - - -# Some more constants. Yippie! -class Params(object): - # Random, unspecified value. - - DESCENDING = "descending" - STARTKEY = "startkey" - STARTKEY_DOCID = "startkey_docid" - ENDKEY = "endkey" - ENDKEY_DOCID = "endkey_docid" - KEY = "key" - KEYS = "keys" - INCLUSIVE_END = "inclusive_end" - - GROUP = "group" - GROUP_LEVEL = "group_level" - REDUCE = "reduce" - - SKIP = "skip" - LIMIT = "limit" - - ON_ERROR = "on_error" - STALE = "stale" - DEBUG = "debug" - CONNECTION_TIMEOUT = "connection_timeout" - FULL_SET = "full_set" - - MAPKEY_SINGLE = "mapkey_single" - MAPKEY_MULTI = "mapkey_multi" - MAPKEY_RANGE = "mapkey_range" - DOCKEY_RANGE = "dockey_range" - START_RANGE = "start_range" - END_RANGE = "end_range" - -_HANDLER_MAP = { - Params.DESCENDING : _bool_param_handler, - - Params.STARTKEY : _jval_param_handler, - Params.STARTKEY_DOCID : _string_param_handler, - Params.ENDKEY : _jval_param_handler, - Params.ENDKEY_DOCID : _string_param_handler, - - Params.FULL_SET : _bool_param_handler, - - Params.GROUP : _bool_param_handler, - Params.GROUP_LEVEL : _num_param_handler, - Params.INCLUSIVE_END : _bool_param_handler, - Params.KEY : _jval_param_handler, - Params.KEYS : _jarry_param_handler, - Params.ON_ERROR : _onerror_param_handler, - Params.REDUCE : _bool_param_handler, - Params.STALE : _stale_param_handler, - Params.SKIP : _num_param_handler, - Params.LIMIT : _num_param_handler, - Params.DEBUG : _bool_param_handler, - Params.CONNECTION_TIMEOUT: _num_param_handler, - Params.START_RANGE : _jarry_param_handler, - Params.END_RANGE : _jarry_param_handler, -} - - -def _gendoc(param): - for k, v in Params.__dict__.items(): - if param == v: - return "\n:data:`Params.{0}`".format(k) - - -def _rangeprop(k_sugar, k_start, k_end): - def getter(self): - return self._user_options.get(k_sugar, UNSPEC) - - def setter(self, value): - self._set_range_common(k_sugar, k_start, k_end, value) - - return property(getter, setter, fdel=None, doc=_gendoc(k_sugar)) - - -def _genprop(p): - def getter(self): - return self._get_common(p) - - def setter(self, value): - self._set_common(p, value) - - return property(getter, setter, fdel=None, doc=_gendoc(p)) - - -class QueryBase(object): - def _set_common(self, param, value, set_user=True): - # Invalidate encoded string - self._encoded = None - - if value is UNSPEC: - self._real_options.pop(param, None) - if set_user: - self._user_options.pop(param, None) - return - - handler = _HANDLER_MAP.get(param) - if not handler: - if not self.unrecognized_ok: - raise ArgumentError.pyexc( - "Unrecognized parameter. To use unrecognized parameters, " - "set 'unrecognized_ok' to True") - - if not handler: - self._extra_options[param] = _string_param_handler(value) - return - - if self.passthrough: - handler = _string_param_handler - - self._real_options[param] = handler(value) - if set_user: - self._user_options[param] = value - - def _get_common(self, param): - if param in self._user_options: - return self._user_options[param] - return self._real_options.get(param, UNSPEC) - - def _set_range_common(self, k_sugar, k_start, k_end, value): - """ - Checks to see if the client-side convenience key is present, and if so - converts the sugar convenience key into its real server-side - equivalents. - - :param string k_sugar: The client-side convenience key - :param string k_start: The server-side key specifying the beginning of - the range - :param string k_end: The server-side key specifying the end of the - range - """ - - if not isinstance(value, (list, tuple, _Unspec)): - raise ArgumentError.pyexc( - "Range specification for {0} must be a list, tuple or UNSPEC" - .format(k_sugar)) - - if self._user_options.get(k_start, UNSPEC) is not UNSPEC or ( - self._user_options.get(k_end, UNSPEC) is not UNSPEC): - - raise ArgumentError.pyexc( - "Cannot specify {0} with either {1} or {2}" - .format(k_sugar, k_start, k_end)) - - if not value: - self._set_common(k_start, UNSPEC, set_user=False) - self._set_common(k_end, UNSPEC, set_user=False) - self._user_options[k_sugar] = UNSPEC - return - - if len(value) not in (1, 2): - raise ArgumentError.pyexc("Range specification " - "must have one or two elements", - value) - - value = value[::] - if len(value) == 1: - value.append(UNSPEC) - - for p, ix in ((k_start, 0), (k_end, 1)): - self._set_common(p, value[ix], set_user=False) - - self._user_options[k_sugar] = value - - STRING_RANGE_END = json.loads('"\u0FFF"') - """ - Highest acceptable unicode value - """ - - def __init__(self, passthrough=False, unrecognized_ok=False, **params): - """ - Create a new Query object. - - A Query object is used as a container for the various view options. - It can be used as a standalone object to encode queries but is typically - passed as the ``query`` value to :class:`~couchbase.views.iterator.View`. - - :param boolean passthrough: - Whether *passthrough* mode is enabled - - :param boolean unrecognized_ok: - Whether unrecognized options are acceptable. See - :ref:`passthrough_values`. - - :param params: - Key-value pairs for view options. See :ref:`view_options` for - a list of acceptable options and their values. - - - :raise: :exc:`couchbase.exceptions.ArgumentError` if a view option - or a combination of view options were deemed invalid. - - """ - self.passthrough = passthrough - self.unrecognized_ok = unrecognized_ok - self._real_options = {} - self._user_options = {} - self._extra_options = {} - - self._encoded = None - - # String literal to pass along with the query - self._base_str = "" - self.update(**params) - - def update(self, copy=False, **params): - """ - Chained assignment operator. - - This may be used to quickly assign extra parameters to the - :class:`Query` object. - - Example:: - - q = Query(reduce=True, full_sec=True) - - # Someplace later - - v = View(design, view, query=q.update(mapkey_range=["foo"])) - - Its primary use is to easily modify the query object (in-place). - - :param boolean copy: - If set to true, the original object is copied before new attributes - are added to it - :param params: Extra arguments. These must be valid query options. - - :return: A :class:`Query` object. If ``copy`` was set to true, this - will be a new instance, otherwise it is the same instance on which - this method was called - - """ - if copy: - self = deepcopy(self) - - for k, v in params.items(): - if not hasattr(self, k): - if not self.unrecognized_ok: - raise ArgumentError.pyexc("Unknown option", k) - self._set_common(k, v) - - else: - setattr(self, k, v) - - return self - - @classmethod - def from_any(cls, params, **ctor_opts): - """ - Creates a new Query object from input. - - :param params: Parameter to convert to query - :type params: dict, string, or :class:`Query` - - If ``params`` is a :class:`Query` object already, a deep copy is made - and a new :class:`Query` object is returned. - - If ``params`` is a string, then a :class:`Query` object is contructed - from it. The string itself is not parsed, but rather prepended to - any additional parameters (defined via the object's methods) - with an additional ``&`` characted. - - If ``params`` is a dictionary, it is passed to the :class:`Query` - constructor. - - :return: a new :class:`Query` object - :raise: :exc:`ArgumentError` if the input is none of the acceptable - types mentioned above. Also raises any exceptions possibly thrown - by the constructor. - - """ - if isinstance(params, cls): - return deepcopy(params) - - elif isinstance(params, dict): - ctor_opts.update(**params) - if cls is QueryBase: - if ('bbox' in params or 'start_range' in params or - 'end_range' in params): - return SpatialQuery(**ctor_opts) - else: - return ViewQuery(**ctor_opts) - - elif isinstance(params, basestring): - ret = cls() - ret._base_str = params - return ret - - else: - raise ArgumentError.pyexc("Params must be Query, dict, or string") - - @classmethod - def from_string(cls, qstr): - """Wrapper for :meth:`from_any`""" - return cls.from_any(qstr) - - def _encode(self, omit_keys=False): - res_d = [] - - for k, v in self._real_options.items(): - if v is UNSPEC: - continue - - if omit_keys and k == "keys": - continue - - if not self.passthrough: - k = ulp.quote(k) - v = ulp.quote(v) - - res_d.append("{0}={1}".format(k, v)) - - for k, v in self._extra_options.items(): - res_d.append("{0}={1}".format(k, v)) - - return '&'.join(res_d) - - @property - def encoded(self): - """ - Returns an encoded form of the query - """ - if not self._encoded: - self._encoded = self._encode() - - if self._base_str: - return '&'.join((self._base_str, self._encoded)) - - else: - return self._encoded - - @property - def _long_query_encoded(self): - """ - Returns the (uri_part, post_data_part) for a long query. - """ - uristr = self._encode(omit_keys=True) - kstr = "{}" - - klist = self._real_options.get('keys', UNSPEC) - if klist != UNSPEC: - kstr = '{{"keys":{0}}}'.format(klist) - - return (uristr, kstr) - - @property - def has_blob(self): - """ - Whether this query object is 'dirty'. - - A 'dirty' object is one which - contains parameters unrecognized by the internal handling methods. - A dirty query may be constructed by using the ``passthrough`` - or ``unrecognized_ok`` options, or by passing a string to - :meth:`from_any` - """ - return self._base_str or self.unrecognized_ok or self.passthrough - - def __repr__(self): - return "Query:'{0}'".format(self.encoded) - - - # Common parameters: - stale = _genprop(Params.STALE) - skip = _genprop(Params.SKIP) - limit = _genprop(Params.LIMIT) - full_set = _genprop(Params.FULL_SET) - connection_timeout = _genprop(Params.CONNECTION_TIMEOUT) - debug = _genprop(Params.DEBUG) - on_error = _genprop(Params.ON_ERROR) - - -class ViewQuery(QueryBase): - descending = _genprop(Params.DESCENDING) - - # Use the range parameters. They're easier - startkey = _genprop(Params.STARTKEY) - endkey = _genprop(Params.ENDKEY) - startkey_docid = _genprop(Params.STARTKEY_DOCID) - endkey_docid = _genprop(Params.ENDKEY_DOCID) - - keys = _genprop(Params.KEYS) - key = _genprop(Params.KEY) - inclusive_end = _genprop(Params.INCLUSIVE_END) - - reduce = _genprop(Params.REDUCE) - group = _genprop(Params.GROUP) - group_level = _genprop(Params.GROUP_LEVEL) - - # Aliases: - mapkey_single = _genprop(Params.KEY) - mapkey_multi = _genprop(Params.KEYS) - - mapkey_range = _rangeprop(Params.MAPKEY_RANGE, - Params.STARTKEY, Params.ENDKEY) - - dockey_range = _rangeprop(Params.DOCKEY_RANGE, - Params.STARTKEY_DOCID, - Params.ENDKEY_DOCID) - - -class SpatialQuery(QueryBase): - start_range = _genprop(Params.START_RANGE) - end_range = _genprop(Params.END_RANGE) - - -class Query(ViewQuery): - pass - - -def make_options_string(input, unrecognized_ok=False, passthrough=False): - if not isinstance(input, QueryBase): - input = QueryBase.from_any(input, unrecognized_ok=unrecognized_ok, - passthrough=passthrough) - return input.encoded - - -def make_dvpath(doc, view): - return "_design/{0}/_view/{1}?".format(doc, view) diff --git a/couchbase_version.py b/couchbase_version.py old mode 100755 new mode 100644 index b82749446..7aa15915f --- a/couchbase_version.py +++ b/couchbase_version.py @@ -1,11 +1,27 @@ #!/usr/bin/env python +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + from __future__ import print_function -import subprocess + import datetime import os.path -import warnings import re +import subprocess +import warnings class CantInvokeGit(Exception): @@ -16,17 +32,23 @@ class VersionNotFound(Exception): pass -RE_XYZ = re.compile(r'(\d+)\.(\d+)\.(\d)(?:-(.*))?') +class MalformedGitTag(Exception): + pass + + +RE_XYZ = re.compile(r'(\d+)\.(\d+)\.(\d+)(?:-(.*))?') -VERSION_FILE = os.path.join( - os.path.dirname(__file__), - os.path.join("couchbase", "_version.py")) +VERSION_FILE = os.path.join(os.path.dirname(__file__), 'couchbase', '_version.py') class VersionInfo(object): def __init__(self, rawtext): self.rawtext = rawtext - vinfo, self.ncommits, self.sha = rawtext.rsplit('-', 2) + t = self.rawtext.rsplit('-', 2) + if len(t) != 3: + raise MalformedGitTag(self.rawtext) + + vinfo, self.ncommits, self.sha = t self.ncommits = int(self.ncommits) # Split up the X.Y.Z @@ -37,8 +59,11 @@ def __init__(self, rawtext): # Per PEP-440, replace any 'DP' with an 'a', and any beta with 'b' if self.ver_extra: self.ver_extra = re.sub(r'^dp', 'a', self.ver_extra, count=1) + self.ver_extra = re.sub(r'^alpha', 'a', self.ver_extra, count=1) self.ver_extra = re.sub(r'^beta', 'b', self.ver_extra, count=1) - m = re.search(r'^([ab])(\d+)?', self.ver_extra) + m = re.search(r'^([ab]|dev|rc|post)(\d+)?', self.ver_extra) + if m.group(1) in ["dev", "post"]: + self.ver_extra = "." + self.ver_extra if m.group(2) is None: # No suffix, then add the number first = self.ver_extra[0] @@ -129,9 +154,9 @@ def gen_version(do_write=True, txt=None): try: info = VersionInfo(txt) vstr = info.package_version - except IndexError: + except MalformedGitTag: warnings.warn("Malformed input '{0}'".format(txt)) - vstr = '0.0.0'+txt + vstr = '0.0.0' + txt if not do_write: print(vstr) @@ -147,6 +172,7 @@ def gen_version(do_write=True, txt=None): with open(VERSION_FILE, "w") as fp: fp.write("\n".join(lines)) + if __name__ == '__main__': from argparse import ArgumentParser ap = ArgumentParser(description='Parse git version to PEP-440 version') diff --git a/deps/couchbase-cxx-client b/deps/couchbase-cxx-client new file mode 160000 index 000000000..eb5fc7653 --- /dev/null +++ b/deps/couchbase-cxx-client @@ -0,0 +1 @@ +Subproject commit eb5fc765322c6ed501464f7029861d5ad3a6c100 diff --git a/dev_requirements.txt b/dev_requirements.txt index 6f181a027..ff30595f0 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,4 +1,9 @@ -nose==1.3.0 -numpydoc==0.4 -sphinx==1.2b1 -testresources>=0.2.7 +autopep8~=1.6 +pre-commit~=2.17 +Faker~=16.6 +faker-vehicle~=0.2 +pytest~=6.2,>=6.2.5 +pytest-asyncio~=0.17,>=0.17.2 +pytest-rerunfailures~=10.2 +requests~=2.26 +Twisted~=22.2,>=22.2.1 diff --git a/docs/Makefile b/docs/Makefile index 914456c88..d4bb2cbb9 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,153 +1,20 @@ -# Makefile for Sphinx documentation +# Minimal makefile for Sphinx documentation # -# You can set these variables from the command line. -SPHINXOPTS = -SPHINXBUILD = sphinx-build -PAPER = -BUILDDIR = build - -# Internal variables. -PAPEROPT_a4 = -D latex_paper_size=a4 -PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source -# the i18n builder cannot share the environment and doctrees with the others -I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) source - -.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest gettext +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = _build +# Put it first so that "make" without argument is like "make help". help: - @echo "Please use \`make <target>' where <target> is one of" - @echo " html to make standalone HTML files" - @echo " dirhtml to make HTML files named index.html in directories" - @echo " singlehtml to make a single large HTML file" - @echo " pickle to make pickle files" - @echo " json to make JSON files" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " qthelp to make HTML files and a qthelp project" - @echo " devhelp to make HTML files and a Devhelp project" - @echo " epub to make an epub" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " latexpdf to make LaTeX files and run them through pdflatex" - @echo " text to make text files" - @echo " man to make manual pages" - @echo " texinfo to make Texinfo files" - @echo " info to make Texinfo files and run them through makeinfo" - @echo " gettext to make PO message catalogs" - @echo " changes to make an overview of all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " doctest to run all doctests embedded in the documentation (if enabled)" - -clean: - -rm -rf $(BUILDDIR)/* - -html: - $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." - -dirhtml: - $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml - @echo - @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." - -singlehtml: - $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml - @echo - @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." - -pickle: - $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle - @echo - @echo "Build finished; now you can process the pickle files." - -json: - $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json - @echo - @echo "Build finished; now you can process the JSON files." - -htmlhelp: - $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp - @echo - @echo "Build finished; now you can run HTML Help Workshop with the" \ - ".hhp project file in $(BUILDDIR)/htmlhelp." - -qthelp: - $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp - @echo - @echo "Build finished; now you can run "qcollectiongenerator" with the" \ - ".qhcp project file in $(BUILDDIR)/qthelp, like this:" - @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/CouchbasePythonClientLibrary.qhcp" - @echo "To view the help file:" - @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/CouchbasePythonClientLibrary.qhc" - -devhelp: - $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp - @echo - @echo "Build finished." - @echo "To view the help file:" - @echo "# mkdir -p $$HOME/.local/share/devhelp/CouchbasePythonClientLibrary" - @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/CouchbasePythonClientLibrary" - @echo "# devhelp" - -epub: - $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub - @echo - @echo "Build finished. The epub file is in $(BUILDDIR)/epub." - -latex: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo - @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." - @echo "Run \`make' in that directory to run these through (pdf)latex" \ - "(use \`make latexpdf' here to do that automatically)." - -latexpdf: - $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex - @echo "Running LaTeX files through pdflatex..." - $(MAKE) -C $(BUILDDIR)/latex all-pdf - @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." - -text: - $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text - @echo - @echo "Build finished. The text files are in $(BUILDDIR)/text." - -man: - $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man - @echo - @echo "Build finished. The manual pages are in $(BUILDDIR)/man." - -texinfo: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo - @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." - @echo "Run \`make' in that directory to run these through makeinfo" \ - "(use \`make info' here to do that automatically)." - -info: - $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo - @echo "Running Texinfo files through makeinfo..." - make -C $(BUILDDIR)/texinfo info - @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." - -gettext: - $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale - @echo - @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." - -changes: - $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes - @echo - @echo "The overview file is in $(BUILDDIR)/changes." + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -linkcheck: - $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck - @echo - @echo "Link check complete; look for any errors in the above output " \ - "or in $(BUILDDIR)/linkcheck/output.txt." +.PHONY: help Makefile -doctest: - $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest - @echo "Testing of doctests in the sources finished, look at the " \ - "results in $(BUILDDIR)/doctest/output.txt." +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/acouchbase_api/acouchbase.rst b/docs/acouchbase_api/acouchbase.rst new file mode 100644 index 000000000..53439896c --- /dev/null +++ b/docs/acouchbase_api/acouchbase.rst @@ -0,0 +1,13 @@ +========== +acouchbase +========== + +.. note:: + Further updates to the acouchbase docs will come with future 4.x releases. In the meantime, + check out the provided examples in the :acouchbase_examples:`Github repo <>`. + +.. toctree:: + :maxdepth: 2 + + acouchbase_core + acouchbase_binary_collection diff --git a/docs/acouchbase_api/acouchbase_binary_collection.rst b/docs/acouchbase_api/acouchbase_binary_collection.rst new file mode 100644 index 000000000..b622a2d03 --- /dev/null +++ b/docs/acouchbase_api/acouchbase_binary_collection.rst @@ -0,0 +1,11 @@ +================= +BinaryCollection +================= + +.. module:: acouchbase.binary_collection +.. autoclass:: BinaryCollection + + .. automethod:: append + .. automethod:: prepend + .. automethod:: increment + .. automethod:: decrement diff --git a/docs/acouchbase_api/acouchbase_core.rst b/docs/acouchbase_api/acouchbase_core.rst new file mode 100644 index 000000000..829d6091d --- /dev/null +++ b/docs/acouchbase_api/acouchbase_core.rst @@ -0,0 +1,105 @@ +============================== +Asyncio (acouchbase) Core API +============================== + +.. note:: + Further updates to the acouchbase docs will come with future 4.x releases. In the meantime, + check out the provided examples in the :acouchbase_examples:`Github repo <>`. + + +.. contents:: + :local: + +Cluster +============== + +.. module:: acouchbase.cluster +.. autoclass:: Cluster + +.. autoclass:: AsyncCluster + + .. automethod:: connect + .. automethod:: on_connect + .. autoproperty:: connected + .. automethod:: bucket + .. automethod:: cluster_info + .. automethod:: ping + .. automethod:: diagnostics + .. automethod:: wait_until_ready + .. automethod:: query + .. automethod:: search_query + .. automethod:: analytics_query + .. autoproperty:: transactions + .. automethod:: buckets + .. automethod:: users + .. automethod:: query_indexes + .. automethod:: analytics_indexes + .. automethod:: search_indexes + .. automethod:: eventing_functions + .. automethod:: close + +Authentication +================ + +See :ref:`Global API Authentication<authentication-ref>` + +Bucket +============== + +.. module:: acouchbase.bucket +.. autoclass:: Bucket + +.. autoclass:: AsyncBucket + + .. autoproperty:: name + .. autoproperty:: connected + .. automethod:: scope + .. automethod:: collection + .. automethod:: default_collection + .. automethod:: ping + .. automethod:: view_query + .. automethod:: collections + .. automethod:: view_indexes + +Scope +============== + +.. module:: acouchbase.scope +.. autoclass:: Scope + +.. autoclass:: AsyncScope + + .. autoproperty:: name + .. autoproperty:: bucket_name + .. automethod:: query + .. automethod:: search_query + .. automethod:: analytics_query + +Collection +============== + +.. module:: acouchbase.collection + +.. autoclass:: Collection + +.. class:: AsyncCollection + + .. autoproperty:: name + .. automethod:: exists + .. automethod:: get + .. automethod:: get_and_lock + .. automethod:: get_and_touch + .. automethod:: insert + .. automethod:: lookup_in + .. automethod:: mutate_in + .. automethod:: remove + .. automethod:: replace + .. automethod:: touch + .. automethod:: unlock + .. automethod:: upsert + .. automethod:: scan + .. automethod:: binary + .. automethod:: couchbase_list + .. automethod:: couchbase_map + .. automethod:: couchbase_set + .. automethod:: couchbase_queue diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 000000000..d48b48b3c --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,122 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + +import os +import sys + +sys.path.insert(0, os.path.abspath('..')) +sys.path.insert(0, os.path.dirname(__file__)) + +# -- Project information ----------------------------------------------------- + +project = 'Couchbase Python Client Library' +copyright = '2022, Couchbase, Inc.' +author = 'Couchbase, Inc.' + +# from .. import couchbase_version +import couchbase_version # nopep8 # isort:skip # noqa: E402 + +try: + from datetime import datetime + year = f'{datetime.today():%Y}' +except BaseException: + year = "2022" +copyright = "2013-{}, Couchbase, Inc.".format(year) + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The full version, including alpha/beta/rc tags. +sdk_version = couchbase_version.get_version() +version = sdk_version +release = sdk_version + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx_rtd_theme', + 'sphinx.ext.autodoc', + 'sphinx.ext.napoleon', + 'sphinx.ext.extlinks', + 'sphinx_copybutton', + 'enum_tools.autoenum' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' +# html_theme = 'classic' + +html_theme_options = { + 'display_version': True, +} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] +html_static_path = [] + +# The docs are unclear, but adding the `%s` to the end of the URL prevents: +# TypeError: not all arguments converted during string formatting +extlinks = { + 'analytics_intro': ('https://docs.couchbase.com/server/current/analytics/introduction.html%s', None), + 'search_create_idx': ('https://docs.couchbase.com/server/current/fts/fts-creating-indexes.html%s', None), + 'couchbase_dev_portal': ('https://developer.couchbase.com/%s', None), + 'couchbase_discord': ('https://discord.com/invite/sQ5qbPZuTh%s', None), + 'python_sdk_github': ('https://github.com/couchbase/couchbase-python-client%s', None), + 'acouchbase_examples': + ('https://github.com/couchbase/couchbase-python-client/tree/master/examples/acouchbase%s', None), + 'couchbase_examples': + ('https://github.com/couchbase/couchbase-python-client/tree/master/examples/couchbase%s', None), + 'txcouchbase_examples': + ('https://github.com/couchbase/couchbase-python-client/tree/master/examples/txcouchbase%s', None), + 'txns_examples': + ('https://docs.couchbase.com/python-sdk/current/howtos/distributed-acid-transactions-from-the-sdk.html%s', None), # noqa: E501 + 'python_sdk_jira': ('https://issues.couchbase.com/projects/PYCBC/issues/%s', None), + 'python_sdk_docs': ('https://docs.couchbase.com/python-sdk/current/hello-world/overview.html%s', None), + 'python_sdk_release_notes': + ('https://docs.couchbase.com/python-sdk/current/project-docs/sdk-release-notes.html%s', None), + 'python_sdk_compatibility': + ('https://docs.couchbase.com/python-sdk/current/project-docs/compatibility.html%s', None), + 'python_sdk_3x_migration': + ('https://docs.couchbase.com/python-sdk/current/project-docs/migrating-sdk-code-to-3.n.html%s', None), + 'python_sdk_api_version': + ('https://docs.couchbase.com/python-sdk/current/project-docs/compatibility.html#api-version%s', None), + 'python_sdk_forums': ('https://forums.couchbase.com/c/python-sdk/10%s', None), + 'python_sdk_license': ('https://github.com/couchbase/couchbase-python-client/blob/master/LICENSE%s', None), + 'python_sdk_contribute': + ('https://github.com/couchbase/couchbase-python-client/blob/master/CONTRIBUTING.md%s', None), + 'python_sdk_version_compat': + ('https://docs.couchbase.com/python-sdk/current/project-docs/compatibility.html#python-version-compat%s', None), +} diff --git a/docs/couchbase_api/authentication.rst b/docs/couchbase_api/authentication.rst new file mode 100644 index 000000000..6e6ba74c3 --- /dev/null +++ b/docs/couchbase_api/authentication.rst @@ -0,0 +1,15 @@ +================ +Authentication +================ + +.. _authentication-ref: + +.. module:: couchbase.auth + +PasswordAuthenticator +=========================== +.. autoclass:: PasswordAuthenticator + +CertificateAuthenticator +=========================== +.. autoclass:: CertificateAuthenticator diff --git a/docs/couchbase_api/couchbase_analytics.rst b/docs/couchbase_api/couchbase_analytics.rst new file mode 100644 index 000000000..1831a781b --- /dev/null +++ b/docs/couchbase_api/couchbase_analytics.rst @@ -0,0 +1,52 @@ +============== +Analytics +============== + +.. note:: + Further updates to the analytics docs will come with future 4.0.x releases. + +.. contents:: + :local: + :depth: 2 + +Enumerations +=============== + +.. module:: couchbase.analytics +.. autoenum:: AnalyticsScanConsistency + +Options +=============== + +.. module:: couchbase.options + :noindex: +.. autoclass:: AnalyticsOptions + :noindex: + +Results +=============== +.. module:: couchbase.analytics + :noindex: + +AnalyticsMetaData ++++++++++++++++++++ +.. autoclass:: AnalyticsMetaData + :members: + +AnalyticsMetrics ++++++++++++++++++++ +.. autoclass:: AnalyticsMetrics + :members: + +AnalyticsResult ++++++++++++++++++++ +.. module:: couchbase.result + :noindex: + +.. class:: AnalyticsResult + :noindex: + + .. automethod:: rows + :noindex: + .. automethod:: metadata + :noindex: diff --git a/docs/couchbase_api/couchbase_binary_collection.rst b/docs/couchbase_api/couchbase_binary_collection.rst new file mode 100644 index 000000000..e8afde31d --- /dev/null +++ b/docs/couchbase_api/couchbase_binary_collection.rst @@ -0,0 +1,15 @@ +================= +BinaryCollection +================= + +.. module:: couchbase.binary_collection +.. autoclass:: BinaryCollection + + .. automethod:: append + .. automethod:: prepend + .. automethod:: increment + .. automethod:: decrement + .. automethod:: append_multi + .. automethod:: prepend_multi + .. automethod:: increment_multi + .. automethod:: decrement_multi diff --git a/docs/couchbase_api/couchbase_core.rst b/docs/couchbase_api/couchbase_core.rst new file mode 100644 index 000000000..b598e00eb --- /dev/null +++ b/docs/couchbase_api/couchbase_core.rst @@ -0,0 +1,126 @@ +=================== +Core couchbase API +=================== + +.. contents:: + :local: + +Cluster +============== + +.. module:: couchbase.cluster +.. autoclass:: Cluster + + .. automethod:: connect + .. autoproperty:: connected + .. automethod:: bucket + .. automethod:: cluster_info + .. automethod:: ping + .. automethod:: diagnostics + .. automethod:: wait_until_ready + .. automethod:: query + .. automethod:: search_query + .. automethod:: search + .. automethod:: analytics_query + .. autoproperty:: transactions + .. automethod:: buckets + .. automethod:: users + .. automethod:: query_indexes + .. automethod:: analytics_indexes + .. automethod:: search_indexes + .. automethod:: eventing_functions + .. automethod:: close + +Authentication +================ + +See :ref:`Global API Authentication<authentication-ref>` + +Bucket +============== + +.. module:: couchbase.bucket +.. autoclass:: Bucket + + .. autoproperty:: name + .. autoproperty:: connected + .. automethod:: scope + .. automethod:: collection + .. automethod:: default_collection + .. automethod:: ping + .. automethod:: view_query + .. automethod:: collections + .. automethod:: view_indexes + +Scope +============== + +.. module:: couchbase.scope + +.. autoclass:: Scope + + .. autoproperty:: name + .. autoproperty:: bucket_name + .. automethod:: query + .. automethod:: search_query + .. automethod:: search + .. automethod:: analytics_query + .. automethod:: search_indexes + +Collection +============== + +.. module:: couchbase.collection + +.. class:: Collection + + .. autoproperty:: name + .. automethod:: exists + .. automethod:: get + .. automethod:: get_all_replicas + .. automethod:: get_and_lock + .. automethod:: get_and_touch + .. automethod:: get_any_replica + .. automethod:: insert + .. automethod:: lookup_in + .. automethod:: mutate_in + .. automethod:: remove + .. automethod:: replace + .. automethod:: touch + .. automethod:: unlock + .. automethod:: upsert + .. automethod:: scan + .. automethod:: binary + .. automethod:: couchbase_list + .. automethod:: list_append + .. automethod:: list_prepend + .. automethod:: list_set + .. automethod:: list_get + .. automethod:: list_remove + .. automethod:: list_size + .. automethod:: couchbase_map + .. automethod:: map_add + .. automethod:: map_get + .. automethod:: map_remove + .. automethod:: map_size + .. automethod:: couchbase_set + .. automethod:: set_add + .. automethod:: set_remove + .. automethod:: set_size + .. automethod:: set_contains + .. automethod:: couchbase_queue + .. automethod:: queue_push + .. automethod:: queue_pop + .. automethod:: queue_size + .. automethod:: get_multi + .. automethod:: get_and_lock_multi + .. automethod:: get_all_replicas_multi + .. automethod:: get_any_replica_multi + .. automethod:: exists_multi + .. automethod:: insert_multi + .. automethod:: lock_multi + .. automethod:: remove_multi + .. automethod:: replace_multi + .. automethod:: touch_multi + .. automethod:: unlock_multi + .. automethod:: upsert_multi diff --git a/docs/couchbase_api/couchbase_datastructures.rst b/docs/couchbase_api/couchbase_datastructures.rst new file mode 100644 index 000000000..3a3c47aca --- /dev/null +++ b/docs/couchbase_api/couchbase_datastructures.rst @@ -0,0 +1,61 @@ +================= +Datastructures +================= + +.. contents:: + :local: + :depth: 2 + +CouchbaseList +=============== +.. module:: couchbase.datastructures +.. autoclass:: CouchbaseList + + .. automethod:: append + .. automethod:: clear + .. automethod:: get_all + .. automethod:: get_at + .. automethod:: index_of + .. automethod:: prepend + .. automethod:: remove_at + .. automethod:: set_at + .. automethod:: size + + +CouchbaseMap +=============== + +.. autoclass:: CouchbaseMap + + .. automethod:: add + .. automethod:: clear + .. automethod:: exists + .. automethod:: get + .. automethod:: get_all + .. automethod:: items + .. automethod:: keys + .. automethod:: remove + .. automethod:: size + .. automethod:: values + +CouchbaseSet +=============== + +.. autoclass:: CouchbaseSet + + .. automethod:: add + .. automethod:: clear + .. automethod:: contains + .. automethod:: remove + .. automethod:: size + .. automethod:: values + +CouchbaseQueue +=============== + +.. autoclass:: CouchbaseQueue + + .. automethod:: clear + .. automethod:: pop + .. automethod:: push + .. automethod:: size diff --git a/docs/couchbase_api/couchbase_diagnostics.rst b/docs/couchbase_api/couchbase_diagnostics.rst new file mode 100644 index 000000000..3d83b4c8c --- /dev/null +++ b/docs/couchbase_api/couchbase_diagnostics.rst @@ -0,0 +1,105 @@ +============== +Diagnostics +============== + +.. contents:: + :local: + :depth: 2 + +Ping +================== + +Bucket +++++++++++ + +.. module:: couchbase.bucket + :noindex: +.. class:: Bucket + :noindex: + + .. automethod:: ping + :noindex: + +Cluster +++++++++++ + +.. module:: couchbase.cluster + :noindex: +.. class:: Cluster + :noindex: + + .. automethod:: ping + :noindex: + +Options +++++++++++ + +.. module:: couchbase.options + :noindex: +.. autoclass:: PingOptions + :noindex: + +Results +++++++++++ + +.. module:: couchbase.result + :noindex: + +.. class:: PingResult + :noindex: + + .. autoproperty:: id + :noindex: + .. autoproperty:: endpoints + :noindex: + .. autoproperty:: sdk + :noindex: + .. autoproperty:: version + :noindex: + .. automethod:: as_json + :noindex: + + +Diagnostics +===================== + +Cluster +++++++++++ + +.. module:: couchbase.cluster + :noindex: +.. class:: Cluster + :noindex: + + .. automethod:: diagnostics + :noindex: + +Options +++++++++++ + +.. module:: couchbase.options + :noindex: +.. autoclass:: DiagnosticsOptions + :noindex: + +Results +++++++++++ + +.. module:: couchbase.result + :noindex: + +.. class:: DiagnosticsResult + :noindex: + + .. autoproperty:: id + :noindex: + .. autoproperty:: endpoints + :noindex: + .. autoproperty:: state + :noindex: + .. autoproperty:: sdk + :noindex: + .. autoproperty:: version + :noindex: + .. automethod:: as_json + :noindex: diff --git a/docs/couchbase_api/couchbase_management.rst b/docs/couchbase_api/couchbase_management.rst new file mode 100644 index 000000000..15ac72a4a --- /dev/null +++ b/docs/couchbase_api/couchbase_management.rst @@ -0,0 +1,128 @@ +============== +Management +============== + +.. note:: + Further updates to the management API docs will come with future 4.0.x releases. + +.. contents:: + :local: + +Bucket Management +================================= + +.. module:: couchbase.management.buckets +.. autoclass:: BucketManager + + .. automethod:: create_bucket + .. automethod:: drop_bucket + .. automethod:: flush_bucket + .. automethod:: get_all_buckets + .. automethod:: get_bucket + .. automethod:: update_bucket + +Collection Management +================================= + +.. module:: couchbase.management.collections + +.. autoclass:: CollectionManager + + .. automethod:: create_collection + + .. py:method:: create_collection(collection: CollectionSpec, *options: CreateCollectionOptions, **kwargs: Dict[str, Any]) -> None + :noindex: + + .. deprecated:: 4.1.9 + Use ``create_collection(scope_name, collection_name, settings=None, *options, **kwargs)`` instead. + + Creates a new collection in a specified scope. + + :param collection: The collection details. + :type collection: :class:`.CollectionSpec` + :param \*options: Optional parameters for this operation. + :type \*options: :class:`~couchbase.management.options.CreateCollectionOptions` + :param \*\*kwargs: keyword arguments that can be used as optional parameters for this operation. + :type \*\*kwargs: Dict[str, Any] + :raises `~couchbase.exceptions.CollectionAlreadyExistsException`: If the collection already exists. + :raises `~couchbase.exceptions.ScopeNotFoundException`: If the scope does not exist. + + .. automethod:: create_scope + + .. automethod:: update_collection + + .. automethod:: drop_collection + + .. py:method:: drop_collection(collection: CollectionSpec, *options: DropCollectionOptions, **kwargs: Dict[str, Any]) -> None + :noindex: + + .. deprecated:: 4.1.9 + Use ``drop_collection(scope_name, collection_name, *options, **kwargs)`` instead. + + Drops a collection from a specified scope. + + :param collection: The collection details. + :type collection: :class:`.CollectionSpec` + :param \*options: Optional parameters for this operation. + :type \*options: :class:`~couchbase.management.options.DropCollectionOptions` + :param \*\*kwargs: keyword arguments that can be used as optional parameters for this operation. + :type \*\*kwargs: Dict[str, Any] + :raises `~couchbase.exceptions.CollectionNotFoundException`: If the collection does not exist. + + .. automethod:: drop_scope + .. automethod:: get_all_scopes + +.. autoclass:: CreateCollectionSettings + + .. autoproperty:: max_expiry + .. autoproperty:: history + +.. autoclass:: UpdateCollectionSettings + + .. autoproperty:: max_expiry + .. autoproperty:: history + +.. autoclass:: CollectionSpec + + .. autoproperty:: name + .. autoproperty:: scope_name + .. autoproperty:: max_expiry + .. autoproperty:: max_ttl + .. autoproperty:: history + +.. autoclass:: ScopeSpec + + .. autoproperty:: name + .. autoproperty:: collections + +Query Index Management +================================= + +.. module:: couchbase.management.queries + +.. autoclass:: QueryIndexManager + + .. automethod:: build_deferred_indexes + .. automethod:: create_index + .. automethod:: create_primary_index + .. automethod:: drop_index + .. automethod:: drop_primary_index + .. automethod:: get_all_indexes + .. automethod:: watch_indexes + +User Management +================================= + +.. module:: couchbase.management.users + +.. autoclass:: UserManager + + .. automethod:: drop_user + .. automethod:: drop_group + .. automethod:: get_all_groups + .. automethod:: get_all_users + .. automethod:: get_group + .. automethod:: get_roles + .. automethod:: get_user + .. automethod:: upsert_group + .. automethod:: upsert_user diff --git a/docs/couchbase_api/couchbase_n1ql.rst b/docs/couchbase_api/couchbase_n1ql.rst new file mode 100644 index 000000000..a6af420a8 --- /dev/null +++ b/docs/couchbase_api/couchbase_n1ql.rst @@ -0,0 +1,51 @@ +============== +Query (SQL++) +============== + +.. contents:: + :local: + :depth: 2 + +Enumerations +=============== + +.. module:: couchbase.n1ql +.. autoenum:: QueryProfile +.. autoenum:: QueryScanConsistency +.. autoenum:: QueryStatus + +Options +=============== + +.. module:: couchbase.options + :noindex: +.. autoclass:: QueryOptions + :noindex: + +Results +=============== +.. module:: couchbase.n1ql + :noindex: + +QueryMetaData ++++++++++++++++++++ +.. autoclass:: QueryMetaData + :members: + +QueryMetrics ++++++++++++++++++++ +.. autoclass:: QueryMetrics + :members: + +QueryResult ++++++++++++++++++++ +.. module:: couchbase.result + :noindex: + +.. class:: QueryResult + :noindex: + + .. automethod:: rows + :noindex: + .. automethod:: metadata + :noindex: diff --git a/docs/couchbase_api/couchbase_rangescan.rst b/docs/couchbase_api/couchbase_rangescan.rst new file mode 100644 index 000000000..d1bc62510 --- /dev/null +++ b/docs/couchbase_api/couchbase_rangescan.rst @@ -0,0 +1,93 @@ +================= +Range Scan +================= + +.. note:: + Use this API for low concurrency batch queries where latency is not a critical as the system may have to scan a lot of documents to find the matching documents. + For low latency range queries, it is recommended that you use SQL++ with the necessary indexes. + +.. contents:: + :local: + :depth: 2 + +Scan Types +=============== + +.. module:: couchbase.kv_range_scan + :noindex: + +ScanTerm ++++++++++++++++++++ + +.. autoclass:: ScanTerm + :members: + +RangeScan ++++++++++++++++++++ + +.. class:: RangeScan + + .. autoproperty:: start + :noindex: + .. autoproperty:: end + :noindex: + +PrefixScan ++++++++++++++++++++ + +.. class:: PrefixScan + + .. autoproperty:: prefix + :noindex: + +SamplingScan ++++++++++++++++++++ + +.. class:: SamplingScan + + .. autoproperty:: limit + :noindex: + .. autoproperty:: seed + :noindex: + +Options +=============== + +.. module:: couchbase.options + :noindex: +.. autoclass:: ScanOptions + :noindex: + +Results +=============== + +.. module:: couchbase.result + :noindex: + +ScanResult ++++++++++++++++++++ + +.. class:: ScanResult + :noindex: + + .. autoproperty:: id + :noindex: + .. autoproperty:: ids_only + :noindex: + .. autoproperty:: cas + :noindex: + .. autoproperty:: content_as + :noindex: + .. autoproperty:: expiry_time + :noindex: + +ScanResultIterable ++++++++++++++++++++ + +.. class:: ScanResultIterable + :noindex: + + .. automethod:: rows + :noindex: + .. automethod:: cancel_scan + :noindex: diff --git a/docs/couchbase_api/couchbase_search.rst b/docs/couchbase_api/couchbase_search.rst new file mode 100644 index 000000000..c66474494 --- /dev/null +++ b/docs/couchbase_api/couchbase_search.rst @@ -0,0 +1,90 @@ +================= +Full Text Search +================= + +.. note:: + Further updates to the search docs will come with future 4.x releases. + +.. contents:: + :local: + :depth: 2 + +Enumerations +=============== + +.. module:: couchbase.search +.. autoenum:: SearchScanConsistency + +Options +=============== + +.. module:: couchbase.options + :noindex: +.. autoclass:: SearchOptions + :noindex: + +SearchRequest +=============== + +.. module:: couchbase.search + :noindex: +.. autoclass:: SearchRequest + :members: + +Results +=============== + +.. module:: couchbase.search + :noindex: + +SearchMetaData ++++++++++++++++++++ + +.. autoclass:: SearchMetaData + :members: + +SearchMetrics ++++++++++++++++++++ + +.. autoclass:: SearchMetrics + :members: + +SearchResult ++++++++++++++++++++ + +.. module:: couchbase.result + :noindex: + +.. class:: SearchResult + :noindex: + + .. automethod:: rows + :noindex: + .. automethod:: metadata + :noindex: + +Vector Search +=============== + +.. module:: couchbase.vector_search + :noindex: + +.. autoclass:: VectorQuery + :members: + +.. autoclass:: VectorSearch + :members: + +Enumerations ++++++++++++++++++++ + +.. module:: couchbase.vector_search + :noindex: +.. autoenum:: VectorQueryCombination + +Options ++++++++++++++++++++ + +.. module:: couchbase.options + :noindex: +.. autoclass:: VectorSearchOptions diff --git a/docs/couchbase_api/couchbase_transactions.rst b/docs/couchbase_api/couchbase_transactions.rst new file mode 100644 index 000000000..76e5b3bf0 --- /dev/null +++ b/docs/couchbase_api/couchbase_transactions.rst @@ -0,0 +1,7 @@ +================= +Transactions +================= + +.. note:: + Further updates to the transactions docs will come with a future 4.0.x release. In the meantime, + checkout the provided examples on the :txns_examples:`Couchbase Python SDK docs page <>`. diff --git a/docs/couchbase_api/couchbase_views.rst b/docs/couchbase_api/couchbase_views.rst new file mode 100644 index 000000000..a7b30ac0b --- /dev/null +++ b/docs/couchbase_api/couchbase_views.rst @@ -0,0 +1,47 @@ +================= +Views +================= + +.. note:: + Further updates to the views docs will come with future 4.0.x releases. + +.. contents:: + :local: + :depth: 2 + +Enumerations +=============== + +.. module:: couchbase.views +.. autoenum:: ViewScanConsistency + +Options +=============== + +.. module:: couchbase.options + :noindex: +.. autoclass:: ViewOptions + :noindex: + +Results +=============== +.. module:: couchbase.views + :noindex: + +ViewMetaData ++++++++++++++++++++ +.. autoclass:: ViewMetaData + :members: + +ViewResult ++++++++++++++++++++ +.. module:: couchbase.result + :noindex: + +.. class:: ViewResult + :noindex: + + .. automethod:: rows + :noindex: + .. automethod:: metadata + :noindex: diff --git a/docs/couchbase_api/exceptions.rst b/docs/couchbase_api/exceptions.rst new file mode 100644 index 000000000..9b571ee34 --- /dev/null +++ b/docs/couchbase_api/exceptions.rst @@ -0,0 +1,112 @@ +============== +Exceptions +============== + +.. contents:: + :local: + +.. module:: couchbase.exceptions + +Base +================== + +CouchbaseException +++++++++++++++++++++++++++++++++ +.. autoclass:: CouchbaseException + +Common +================== + +AmbiguousTimeoutException +++++++++++++++++++++++++++++++++ +.. autoclass:: AmbiguousTimeoutException + +InternalServerFailureException +++++++++++++++++++++++++++++++++ +.. autoclass:: InternalServerFailureException + +InvalidArgumentException +++++++++++++++++++++++++++++++++ +.. autoclass:: InvalidArgumentException + +UnAmbiguousTimeoutException +++++++++++++++++++++++++++++++++ +.. autoclass:: UnAmbiguousTimeoutException + +Authentication +================== + +AuthenticationException +++++++++++++++++++++++++++++++++ +.. autoclass:: AuthenticationException + +Bucket +================== + +BucketNotFoundException +++++++++++++++++++++++++++++++++ +.. autoclass:: BucketNotFoundException + +Key-Value +================== + +DocumentExistsException +++++++++++++++++++++++++++++++++ +.. autoclass:: DocumentExistsException + +DocumentLockedException +++++++++++++++++++++++++++++++++ +.. autoclass:: DocumentLockedException + +DocumentNotFoundException +++++++++++++++++++++++++++++++++ +.. autoclass:: DocumentNotFoundException + +DurabilityImpossibleException +++++++++++++++++++++++++++++++++ +.. autoclass:: DurabilityImpossibleException + +DurabilityInvalidLevelException +++++++++++++++++++++++++++++++++ +.. autoclass:: DurabilityInvalidLevelException + +DurabilitySyncWriteAmbiguousException +++++++++++++++++++++++++++++++++++++++++ +.. autoclass:: DurabilitySyncWriteAmbiguousException + +DurabilitySyncWriteInProgressException +++++++++++++++++++++++++++++++++++++++++ +.. autoclass:: DurabilitySyncWriteInProgressException + +Subdocument +================== + +InvalidValueException +++++++++++++++++++++++++++++++++ +.. autoclass:: InvalidValueException + +PathExistsException +++++++++++++++++++++++++++++++++ +.. autoclass:: PathExistsException + +PathMismatchException +++++++++++++++++++++++++++++++++ +.. autoclass:: PathMismatchException + +PathNotFoundException +++++++++++++++++++++++++++++++++ +.. autoclass:: PathNotFoundException + +.. warning:: + This is a *deprecated* class, use :class:`.InvalidValueException` instead. + +SubdocCantInsertValueException +++++++++++++++++++++++++++++++++ +.. autoclass:: SubdocCantInsertValueException + +.. warning:: + This is a *deprecated* class, use :class:`.PathMismatchException` instead. + +SubdocPathMismatchException +++++++++++++++++++++++++++++++++ +.. autoclass:: SubdocPathMismatchException diff --git a/docs/couchbase_api/management_options.rst b/docs/couchbase_api/management_options.rst new file mode 100644 index 000000000..ec718d738 --- /dev/null +++ b/docs/couchbase_api/management_options.rst @@ -0,0 +1,135 @@ +=================== +Management Options +=================== + +.. note:: + Further updates to the management options docs will come with future 4.0.x releases. + +.. contents:: + :local: + +.. module:: couchbase.management.options + +Bucket Management +======================== + +CreateBucketOptions +++++++++++++++++++++++++ +.. autoclass:: CreateBucketOptions + +DropBucketOptions +++++++++++++++++++++++++ +.. autoclass:: DropBucketOptions + +FlushBucketOptions +++++++++++++++++++++++++ +.. autoclass:: FlushBucketOptions + +GetAllBucketOptions +++++++++++++++++++++++++ +.. autoclass:: GetAllBucketOptions + +GetBucketOptions +++++++++++++++++++++++++ +.. autoclass:: GetBucketOptions + +UpdateBucketOptions +++++++++++++++++++++++++ +.. autoclass:: UpdateBucketOptions + +Collection Management +======================== + +CreateCollectionOptions +++++++++++++++++++++++++ +.. autoclass:: CreateCollectionOptions + +CreateScopeOptions +++++++++++++++++++++++++ +.. autoclass:: CreateScopeOptions + +UpdateCollectionOptions +++++++++++++++++++++++++ +.. autoclass:: UpdateCollectionOptions + +DropCollectionOptions +++++++++++++++++++++++++ +.. autoclass:: DropCollectionOptions + +DropScopeOptions +++++++++++++++++++++++++ +.. autoclass:: DropScopeOptions + +GetAllScopesOptions +++++++++++++++++++++++++ +.. autoclass:: GetAllScopesOptions + +Query Index Management +======================== + +BuildDeferredQueryIndexOptions ++++++++++++++++++++++++++++++++++ +.. autoclass:: BuildDeferredQueryIndexOptions + +CreatePrimaryQueryIndexOptions ++++++++++++++++++++++++++++++++++ +.. autoclass:: CreatePrimaryQueryIndexOptions + +CreateQueryIndexOptions ++++++++++++++++++++++++++++++++++ +.. autoclass:: CreateQueryIndexOptions + +DropPrimaryQueryIndexOptions ++++++++++++++++++++++++++++++++++ +.. autoclass:: DropPrimaryQueryIndexOptions + +DropQueryIndexOptions ++++++++++++++++++++++++++++++++++ +.. autoclass:: DropQueryIndexOptions + +GetAllQueryIndexOptions ++++++++++++++++++++++++++++++++++ +.. autoclass:: GetAllQueryIndexOptions + +WatchQueryIndexOptions ++++++++++++++++++++++++++++++++++ +.. autoclass:: WatchQueryIndexOptions + +User Management +======================== + +DropGroupOptions +++++++++++++++++++++++++ +.. autoclass:: DropGroupOptions + +DropUserOptions +++++++++++++++++++++++++ +.. autoclass:: DropUserOptions + +GetAllGroupsOptions +++++++++++++++++++++++++ +.. autoclass:: GetAllGroupsOptions + +GetAllUsersOptions +++++++++++++++++++++++++ +.. autoclass:: GetAllUsersOptions + +GetGroupOptions +++++++++++++++++++++++++ +.. autoclass:: GetGroupOptions + +GetRolesOptions +++++++++++++++++++++++++ +.. autoclass:: GetRolesOptions + +GetUserOptions +++++++++++++++++++++++++ +.. autoclass:: GetUserOptions + +UpsertGroupOptions +++++++++++++++++++++++++ +.. autoclass:: UpsertGroupOptions + +UpsertUserOptions +++++++++++++++++++++++++ +.. autoclass:: UpsertUserOptions diff --git a/docs/couchbase_api/options.rst b/docs/couchbase_api/options.rst new file mode 100644 index 000000000..e6badc69b --- /dev/null +++ b/docs/couchbase_api/options.rst @@ -0,0 +1,236 @@ +============== +Options +============== + +.. contents:: + :local: + +.. module:: couchbase.options + +Analytics +================= + +Binary +================= + +AppendOptions +++++++++++++++++++++++ + +.. autoclass:: AppendOptions + +DecrementOptions +++++++++++++++++++++++ + +.. autoclass:: DecrementOptions + +IncrementOptions +++++++++++++++++++++++ + +.. autoclass:: IncrementOptions + +PrependOptions +++++++++++++++++++++++ + +.. autoclass:: PrependOptions + +Binary Multi +================= + +AppendMultiOptions +++++++++++++++++++++++ + +.. autoclass:: AppendMultiOptions + +DecrementMultiOptions +++++++++++++++++++++++ + +.. autoclass:: DecrementMultiOptions + +IncrementMultiOptions +++++++++++++++++++++++ + +.. autoclass:: IncrementMultiOptions + +PrependMultiOptions +++++++++++++++++++++++ + +.. autoclass:: PrependMultiOptions + +Cluster +================= + +ClusterOptions +++++++++++++++++++++++ + +.. autoclass:: ClusterOptions + +ClusterTimeoutOptions +++++++++++++++++++++++ + +.. autoclass:: ClusterTimeoutOptions + +ClusterTracingOptions +++++++++++++++++++++++ + +.. autoclass:: ClusterTracingOptions + +Diagnostics +================= + +DiagnosticsOptions +++++++++++++++++++++++ + +.. autoclass:: DiagnosticsOptions + +PingOptions +++++++++++++++++++++++ + +.. autoclass:: PingOptions + +WaitUntilReadyOptions +++++++++++++++++++++++ + +.. autoclass:: WaitUntilReadyOptions + +Key-Value +================= + +ExistsOptions +++++++++++++++++++++++ + +.. autoclass:: ExistsOptions + +GetOptions +++++++++++++++++++++++ + +.. autoclass:: GetOptions + +GetAndLockOptions +++++++++++++++++++++++ + +.. autoclass:: GetAndLockOptions + +GetAndTouchOptions +++++++++++++++++++++++ + +.. autoclass:: GetAndTouchOptions + +InsertOptions +++++++++++++++++++++++ + +.. autoclass:: InsertOptions + +RemoveOptions +++++++++++++++++++++++ + +.. autoclass:: RemoveOptions + +ReplaceOptions +++++++++++++++++++++++ + +.. autoclass:: ReplaceOptions + +TouchOptions +++++++++++++++++++++++ + +.. autoclass:: TouchOptions + +UnlockOptions +++++++++++++++++++++++ + +.. autoclass:: UnlockOptions + +UpsertOptions +++++++++++++++++++++++ + +.. autoclass:: UpsertOptions + +ScanOptions +++++++++++++++++++++++ + +.. autoclass:: ScanOptions + +Key-Value Multi +================= + +ExistsMultiOptions +++++++++++++++++++++++ + +.. autoclass:: ExistsMultiOptions + +GetMultiOptions +++++++++++++++++++++++ + +.. autoclass:: GetMultiOptions + +InsertMultiOptions +++++++++++++++++++++++ + +.. autoclass:: InsertMultiOptions + +LockMultiOptions +++++++++++++++++++++++ + +.. autoclass:: LockMultiOptions + +ReplaceMultiOptions +++++++++++++++++++++++ + +.. autoclass:: ReplaceMultiOptions + +RemoveMultiOptions +++++++++++++++++++++++ + +.. autoclass:: RemoveMultiOptions + +TouchMultiOptions +++++++++++++++++++++++ + +.. autoclass:: TouchMultiOptions + +UnlockMultiOptions +++++++++++++++++++++++ + +.. autoclass:: UnlockMultiOptions + +UpsertMultiOptions +++++++++++++++++++++++ + +.. autoclass:: UpsertMultiOptions + +Query +================= + +QueryOptions +++++++++++++++++++++++ + +.. autoclass:: QueryOptions + +Search +================= + +SearchOptions +++++++++++++++++++++++ + +.. autoclass:: SearchOptions + +Subdocument +================= + +LookupInOptions +++++++++++++++++++++++ + +.. autoclass:: LookupInOptions + +MutateInOptions +++++++++++++++++++++++ + +.. autoclass:: MutateInOptions + +Views +================= + +ViewOptions +++++++++++++++++++++++ + +.. autoclass:: ViewOptions diff --git a/docs/couchbase_api/parallelism.rst b/docs/couchbase_api/parallelism.rst new file mode 100644 index 000000000..7f6921869 --- /dev/null +++ b/docs/couchbase_api/parallelism.rst @@ -0,0 +1,33 @@ +============ +Parallelism +============ + +.. contents:: + :local: + +Thread-based Parallelism +========================== + +The 4.x version of the Python SDK significantly improves how the SDK handles the +`Global Interpreter Lock (GIL) <https://docs.python.org/3/glossary.html#term-global-interpreter-lock>`_. +As part of the improvements, options available in previous series of the SDK (2.x and 3.x) are no longer needed. + +Notable options that are no longer utilized (or available) in the 4.x SDK: + +* ``lockmode`` ClusterOption has been deprecated as it is a no-op (i.e. has no functionality) and will be removed in a future version of the SDK +* ``unlock_gil`` option is no longer available + +Due to the underlying architecture of the 4.x SDK it is recommended to share a cluster instance across multiple threads. However, creating a +cluster instance per thread can work as well. As should be the standard for *all* applications, we recommend sufficient testing to determine +the path that best fits the needs of *your* application. + +See the `Python threading docs <https://docs.python.org/3/library/threading.html>`_ for details on thread-based parallelism. + +Process-based Parallelism +========================== + +Due to the underlying architecture of the 4.x SDK a cluster instance **cannot** be shared across across multiple processes; therefore, +a cluster instance must be created for each process used in a multiprocessing scenario. As should be the standard for *all* applications, +we recommend sufficient testing to determine the path that best fits the needs of *your* application. + +See the `Python multiprocessing docs <https://docs.python.org/3/library/multiprocessing.html>`_ for details on process-based parallelism. diff --git a/docs/couchbase_api/results.rst b/docs/couchbase_api/results.rst new file mode 100644 index 000000000..2b513cd2e --- /dev/null +++ b/docs/couchbase_api/results.rst @@ -0,0 +1,178 @@ +============== +Results +============== + +.. module:: couchbase.result + +AnalyticsResult +================= + +.. class:: AnalyticsResult + + .. automethod:: rows + .. automethod:: metadata + +ClusterInfoResult +================= + +.. class:: ClusterInfoResult + + .. autoproperty:: is_community + .. autoproperty:: is_enterprise + .. autoproperty:: server_version + .. autoproperty:: server_version_full + .. autoproperty:: server_version_short + +CounterResult +================= + +.. class:: CounterResult + + .. autoproperty:: cas + .. autoproperty:: content + .. autoproperty:: key + +DiagnosticsResult +================= + +.. class:: DiagnosticsResult + + .. autoproperty:: id + .. autoproperty:: endpoints + .. autoproperty:: state + .. autoproperty:: sdk + .. autoproperty:: version + .. automethod:: as_json + +ExistsResult +================= + +.. class:: ExistsResult + + .. autoproperty:: exists + +GetResult +================= + +.. class:: GetResult + + .. autoproperty:: cas + .. autoproperty:: content_as + .. autoproperty:: key + .. autoproperty:: expiry_time + + +LookupInResult +================= + +.. class:: LookupInResult + + .. autoproperty:: cas + .. autoproperty:: content_as + +MultiCounterResult +===================== + +.. class:: MultiCounterResult + + .. autoproperty:: all_ok + .. autoproperty:: exceptions + .. autoproperty:: results + +MultiGetResult +===================== + +.. class:: MultiGetResult + + .. autoproperty:: all_ok + .. autoproperty:: exceptions + .. autoproperty:: results + +MultiExistsResult +===================== + +.. class:: MultiExistsResult + + .. autoproperty:: all_ok + .. autoproperty:: exceptions + .. autoproperty:: results + +MultiMutationResult +===================== + +.. class:: MultiMutationResult + + .. autoproperty:: all_ok + .. autoproperty:: exceptions + .. autoproperty:: results + +MutateInResult +================= + +.. class:: MutateInResult + + .. autoproperty:: cas + .. autoproperty:: content_as + +MutationResult +================= + +.. class:: MutationResult + + .. autoproperty:: cas + .. automethod:: mutation_token + +PingResult +================= + +.. class:: PingResult + + .. autoproperty:: id + .. autoproperty:: endpoints + .. autoproperty:: sdk + .. autoproperty:: version + .. automethod:: as_json + + +QueryResult +================= + +.. class:: QueryResult + + .. automethod:: rows + .. automethod:: metadata + +SearchResult +================= + +.. class:: SearchResult + + .. automethod:: rows + .. automethod:: metadata + +ViewResult +================= + +.. class:: ViewResult + + .. automethod:: rows + .. automethod:: metadata + +ScanResult +================= + +.. class:: ScanResult + + .. autoproperty:: id + .. autoproperty:: ids_only + .. autoproperty:: cas + .. autoproperty:: content_as + .. autoproperty:: expiry_time + +ScanResultIterable +=================== + +.. class:: ScanResultIterable + + .. automethod:: rows + .. automethod:: cancel_scan diff --git a/docs/couchbase_api/subdocument.rst b/docs/couchbase_api/subdocument.rst new file mode 100644 index 000000000..b02ad48bd --- /dev/null +++ b/docs/couchbase_api/subdocument.rst @@ -0,0 +1,66 @@ +============== +Subdocument +============== + +.. contents:: + :local: + +.. module:: couchbase.subdocument + +Enumerations +====================== + +.. autoenum:: StoreSemantics + +Lookup Operations +====================== + +.. autofunction:: count +.. autofunction:: exists +.. autofunction:: get + +Mutation Operations +====================== + +.. autofunction:: array_addunique +.. autofunction:: array_append +.. autofunction:: array_insert +.. autofunction:: array_prepend +.. autofunction:: counter +.. autofunction:: decrement +.. autofunction:: increment +.. autofunction:: remove +.. autofunction:: replace +.. autofunction:: upsert + +Options +====================== + +.. module:: couchbase.options + :noindex: + +.. autoclass:: LookupInOptions + :noindex: + +.. autoclass:: MutateInOptions + :noindex: + +Results +====================== + +.. module:: couchbase.result + :noindex: + +.. class:: LookupInResult + :noindex: + + .. autoproperty:: cas + :noindex: + .. autoproperty:: content_as + :noindex: + +.. class:: MutateInResult + :noindex: + + .. autoproperty:: cas + :noindex: diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 000000000..a6bb26396 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,133 @@ +.. Couchbase Python Client Library documentation master file, created by + sphinx-quickstart on Thu Apr 14 13:34:44 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +=================================================== +Welcome to the Couchbase Python SDK documentation! +=================================================== + +Getting Started with the Python SDK +------------------------------------- + +:doc:`using_the_python_sdk` + Useful information for getting started and using the Python SDK. + +Synchronous API +--------------- + +:doc:`couchbase_api/couchbase_core` + API reference for Cluster, Bucket, Scope and Collection objects. + +:doc:`couchbase_api/couchbase_n1ql` + API reference for query (SQL++) operations. + +:doc:`couchbase_api/couchbase_analytics` + API reference for analytics operations. + +:doc:`couchbase_api/couchbase_search` + API reference for full text search (FTS) operations. + +:doc:`couchbase_api/couchbase_transactions` + API reference for Distributed ACID transactions with the Python SDK. + +:doc:`couchbase_api/couchbase_rangescan` + API reference for range scan operations. + +:doc:`couchbase_api/couchbase_diagnostics` + API reference for diagnostic operations. + +:doc:`couchbase_api/couchbase_binary_collection` + API reference for BinaryCollection operations. + +:doc:`couchbase_api/couchbase_datastructures` + API reference for datastructure operations. + +:doc:`couchbase_api/couchbase_management` + API reference for management operations. + +Global API +----------------- +:doc:`couchbase_api/authentication` + API reference for Authentication. + +:doc:`couchbase_api/exceptions` + API reference for Exceptions. + +:doc:`couchbase_api/management_options` + API reference for mangement operation options. + +:doc:`couchbase_api/options` + API reference for operation options. + +:doc:`couchbase_api/results` + API reference for operation results. + +:doc:`couchbase_api/subdocument` + API reference for subdocument operation Specs. + +:doc:`couchbase_api/parallelism` + API reference for using parallelism paradigms in Python. + +Asynchronous APIs +----------------- +:doc:`acouchbase_api/acouchbase` + API reference for the asyncio (acouchbase) API. + +:doc:`txcouchbase_api/txcouchbase` + API reference for the Twisted (txcouchbase) API. + +Indices and tables +------------------ + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + +.. Hidden TOCs + +.. toctree:: + :caption: Using the Couchbase Python SDK + :maxdepth: 2 + :hidden: + + using_the_python_sdk + +.. toctree:: + :caption: Synchronous API + :maxdepth: 2 + :hidden: + + couchbase_api/couchbase_analytics + couchbase_api/couchbase_binary_collection + couchbase_api/couchbase_core + couchbase_api/couchbase_datastructures + couchbase_api/couchbase_diagnostics + couchbase_api/couchbase_n1ql + couchbase_api/couchbase_management + couchbase_api/couchbase_search + couchbase_api/couchbase_rangescan + couchbase_api/couchbase_transactions + couchbase_api/couchbase_views + +.. toctree:: + :caption: Global API + :maxdepth: 2 + :hidden: + + couchbase_api/authentication + couchbase_api/exceptions + couchbase_api/management_options + couchbase_api/options + couchbase_api/results + couchbase_api/subdocument + couchbase_api/parallelism + + +.. toctree:: + :caption: Asynchronous APIs + :maxdepth: 2 + :hidden: + + acouchbase_api/acouchbase + txcouchbase_api/txcouchbase diff --git a/docs/make.bat b/docs/make.bat index 5c255ee4b..32bb24529 100644 --- a/docs/make.bat +++ b/docs/make.bat @@ -1,190 +1,35 @@ -@ECHO OFF - -REM Command file for Sphinx documentation - -if "%SPHINXBUILD%" == "" ( - set SPHINXBUILD=sphinx-build -) -set BUILDDIR=build -set ALLSPHINXOPTS=-d %BUILDDIR%/doctrees %SPHINXOPTS% source -set I18NSPHINXOPTS=%SPHINXOPTS% source -if NOT "%PAPER%" == "" ( - set ALLSPHINXOPTS=-D latex_paper_size=%PAPER% %ALLSPHINXOPTS% - set I18NSPHINXOPTS=-D latex_paper_size=%PAPER% %I18NSPHINXOPTS% -) - -if "%1" == "" goto help - -if "%1" == "help" ( - :help - echo.Please use `make ^<target^>` where ^<target^> is one of - echo. html to make standalone HTML files - echo. dirhtml to make HTML files named index.html in directories - echo. singlehtml to make a single large HTML file - echo. pickle to make pickle files - echo. json to make JSON files - echo. htmlhelp to make HTML files and a HTML help project - echo. qthelp to make HTML files and a qthelp project - echo. devhelp to make HTML files and a Devhelp project - echo. epub to make an epub - echo. latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter - echo. text to make text files - echo. man to make manual pages - echo. texinfo to make Texinfo files - echo. gettext to make PO message catalogs - echo. changes to make an overview over all changed/added/deprecated items - echo. linkcheck to check all external links for integrity - echo. doctest to run all doctests embedded in the documentation if enabled - goto end -) - -if "%1" == "clean" ( - for /d %%i in (%BUILDDIR%\*) do rmdir /q /s %%i - del /q /s %BUILDDIR%\* - goto end -) - -if "%1" == "html" ( - %SPHINXBUILD% -b html %ALLSPHINXOPTS% %BUILDDIR%/html - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/html. - goto end -) - -if "%1" == "dirhtml" ( - %SPHINXBUILD% -b dirhtml %ALLSPHINXOPTS% %BUILDDIR%/dirhtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/dirhtml. - goto end -) - -if "%1" == "singlehtml" ( - %SPHINXBUILD% -b singlehtml %ALLSPHINXOPTS% %BUILDDIR%/singlehtml - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The HTML pages are in %BUILDDIR%/singlehtml. - goto end -) - -if "%1" == "pickle" ( - %SPHINXBUILD% -b pickle %ALLSPHINXOPTS% %BUILDDIR%/pickle - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the pickle files. - goto end -) - -if "%1" == "json" ( - %SPHINXBUILD% -b json %ALLSPHINXOPTS% %BUILDDIR%/json - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can process the JSON files. - goto end -) - -if "%1" == "htmlhelp" ( - %SPHINXBUILD% -b htmlhelp %ALLSPHINXOPTS% %BUILDDIR%/htmlhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run HTML Help Workshop with the ^ -.hhp project file in %BUILDDIR%/htmlhelp. - goto end -) - -if "%1" == "qthelp" ( - %SPHINXBUILD% -b qthelp %ALLSPHINXOPTS% %BUILDDIR%/qthelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; now you can run "qcollectiongenerator" with the ^ -.qhcp project file in %BUILDDIR%/qthelp, like this: - echo.^> qcollectiongenerator %BUILDDIR%\qthelp\CouchbasePythonSDK.qhcp - echo.To view the help file: - echo.^> assistant -collectionFile %BUILDDIR%\qthelp\CouchbasePythonSDK.ghc - goto end -) - -if "%1" == "devhelp" ( - %SPHINXBUILD% -b devhelp %ALLSPHINXOPTS% %BUILDDIR%/devhelp - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. - goto end -) - -if "%1" == "epub" ( - %SPHINXBUILD% -b epub %ALLSPHINXOPTS% %BUILDDIR%/epub - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The epub file is in %BUILDDIR%/epub. - goto end -) - -if "%1" == "latex" ( - %SPHINXBUILD% -b latex %ALLSPHINXOPTS% %BUILDDIR%/latex - if errorlevel 1 exit /b 1 - echo. - echo.Build finished; the LaTeX files are in %BUILDDIR%/latex. - goto end -) - -if "%1" == "text" ( - %SPHINXBUILD% -b text %ALLSPHINXOPTS% %BUILDDIR%/text - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The text files are in %BUILDDIR%/text. - goto end -) - -if "%1" == "man" ( - %SPHINXBUILD% -b man %ALLSPHINXOPTS% %BUILDDIR%/man - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The manual pages are in %BUILDDIR%/man. - goto end -) - -if "%1" == "texinfo" ( - %SPHINXBUILD% -b texinfo %ALLSPHINXOPTS% %BUILDDIR%/texinfo - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The Texinfo files are in %BUILDDIR%/texinfo. - goto end -) - -if "%1" == "gettext" ( - %SPHINXBUILD% -b gettext %I18NSPHINXOPTS% %BUILDDIR%/locale - if errorlevel 1 exit /b 1 - echo. - echo.Build finished. The message catalogs are in %BUILDDIR%/locale. - goto end -) - -if "%1" == "changes" ( - %SPHINXBUILD% -b changes %ALLSPHINXOPTS% %BUILDDIR%/changes - if errorlevel 1 exit /b 1 - echo. - echo.The overview file is in %BUILDDIR%/changes. - goto end -) - -if "%1" == "linkcheck" ( - %SPHINXBUILD% -b linkcheck %ALLSPHINXOPTS% %BUILDDIR%/linkcheck - if errorlevel 1 exit /b 1 - echo. - echo.Link check complete; look for any errors in the above output ^ -or in %BUILDDIR%/linkcheck/output.txt. - goto end -) - -if "%1" == "doctest" ( - %SPHINXBUILD% -b doctest %ALLSPHINXOPTS% %BUILDDIR%/doctest - if errorlevel 1 exit /b 1 - echo. - echo.Testing of doctests in the sources finished, look at the ^ -results in %BUILDDIR%/doctest/output.txt. - goto end -) - -:end +@ECHO OFF + +pushd %~dp0 + +REM Command file for Sphinx documentation + +if "%SPHINXBUILD%" == "" ( + set SPHINXBUILD=sphinx-build +) +set SOURCEDIR=. +set BUILDDIR=_build + +%SPHINXBUILD% >NUL 2>NUL +if errorlevel 9009 ( + echo. + echo.The 'sphinx-build' command was not found. Make sure you have Sphinx + echo.installed, then set the SPHINXBUILD environment variable to point + echo.to the full path of the 'sphinx-build' executable. Alternatively you + echo.may add the Sphinx directory to PATH. + echo. + echo.If you don't have Sphinx installed, grab it from + echo.https://www.sphinx-doc.org/ + exit /b 1 +) + +if "%1" == "" goto help + +%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% +goto end + +:help +%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% + +:end +popd diff --git a/docs/source/api/admin.rst b/docs/source/api/admin.rst deleted file mode 100644 index d5ef07e9c..000000000 --- a/docs/source/api/admin.rst +++ /dev/null @@ -1,23 +0,0 @@ -========================= -Administrative Operations -========================= - -.. module:: couchbase.admin - -The :class:`~.Admin` provides several convenience methods -to perform common API requests. - -.. warning:: The interface here is provided as a convenience only - and its interface may change. - - -To create an administrative handle, simply instantiate a new -:class:`Admin` object. Note that unlike the :class:`~.Bucket`, -the :class:`Admin` constructor does not accept a connection string. This is -deliberate, as the administrative API communicates with a single node, on -a well defined port (whereas the :class:`Bucket` object communicates with -one or more nodes using a variety of different protocols). - - -.. autoclass:: Admin - :members: \ No newline at end of file diff --git a/docs/source/api/async.rst b/docs/source/api/async.rst deleted file mode 100644 index e87954e46..000000000 --- a/docs/source/api/async.rst +++ /dev/null @@ -1,159 +0,0 @@ -====================== -Asynchronous Interface -====================== - -.. module:: couchbase.async - - -.. _async_intro: - -The asynchronous interface to the SDK is a work in progress, and is currently -intended to be used by integrators into higher level async wrappers. See -the ``txcouchbase`` package for an example. - -This document largely explains the current internals of how the Couchbase -async module works in a lower level. For a higher level overview, see: -http://blog.couchbase.com/python-sdk-and-twisted - - -Key-Value Interface -=================== - -The Key-Value interface of the async subsystem functions as closely -as possible like its synchronous counterpart. The primary difference is that -where the synchronous interface would return an instance of a -:class:`~couchbase.result.Result` or a -:class:`~couchbase.result.MultiResult`, -the asynchronous interface returns an -:class:`~couchbase.result.AsyncResult` object. - -The :class:`~couchbase.result.AsyncResult` -object contains fields for two callbacks which are -invoked when the result is ready. One is the -:attr:`~couchbase.result.AsyncResult.callback` -field which is called with a ``Result`` or ``MultiResult`` upon success, -and the other is the -:attr:`~couchbase.result.AsyncResult.errback` field which is invoked -with an exception object upon error. - -The semantics of when an exception is passed follows the rules of the -``quiet`` parameter just like the synchronous API. - -.. currentmodule:: couchbase.async.bucket - -.. class:: couchbase.result.AsyncResult - - .. autoattribute:: callback - - .. autoattribute:: errback - - -.. autoclass:: AsyncBucket - :members: - :show-inheritance: - -Views Interface -=============== - -Different from the key-value interface, the synchronous view API -returns a :class:`~couchbase.views.iterator.View` object which -is itself an iterator which yields results. Because this is a synchronous -API, the iterator interface must be replaced with a class interface -which must be subclassed by a user. - -.. currentmodule:: couchbase.async.view - -.. class:: AsyncViewBase - - .. automethod:: __init__ - .. automethod:: on_rows - .. automethod:: on_error - .. automethod:: on_done - .. automethod:: start - -I/O Interface -============= - -The async API is divided into several sections. In order to have an async -client which interacts with other async libraries and frameworks, it is -necessary to make the Couchbase extension aware of that environment. To this -end, the ``IOPS`` interface is provided. The ``IOPS`` -API is entirely separate from the key-value API and should be treated as -belonging to a different library. It is simply the extension's I/O -abstraction. - -.. currentmodule:: couchbase.iops.base - -.. class:: Event - - This class represents an `Event`. This concept should be familiar - to the intended audience, who should be familiar with event loops - and their terminology. It represents a certain event in the future, - which shall be triggered either by something happening or by the - passage of time. - - When said event takes place, the object should be signalled via - the :meth:`ready` method. - - .. automethod:: ready - -.. class:: IOEvent - - A subclass of :class:`Event`, this represents a socket. Events applied - to this socket are triggered when the socket becomes available for reading - or writing. - - .. automethod:: ready_r - .. automethod:: ready_w - .. automethod:: ready_rw - .. automethod:: fileno - -.. class:: TimerEvent - - Subclass of :class:`Event` which represents a passage of time. - -.. class:: IOPS - - .. automethod:: __init__ - .. automethod:: update_event - .. automethod:: update_timer - .. automethod:: start_watching - .. automethod:: stop_watching - .. automethod:: io_event_factory - .. automethod:: timer_event_factory - -Action Constants ----------------- - -.. data:: PYCBC_EVACTION_WATCH - - Action indicating the specific event should be added to the event - loop's "watcher" list, and should be have its :meth:`~Event.ready` - method called when the IO implementation has detected the specific event - is ready - -.. data:: PYCBC_EVACTION_UNWATCH - - Action indicating that the specific object should not be notified - when the IO state changes. This is typically done by removing it from - the watcher list - -.. data:: PYCBC_EVACTION_CLEANUP - - Action to permanently erase any references to this event - - -IO Event Constants ------------------- -.. data:: LCB_READ_EVENT - - IO Flag indicating that this event should be notified on file readbility - -.. data:: LCB_WRITE_EVENT - - IO flag indicating that this event should be notified on file writeability - - -.. data:: LCB_RW_EVENT - - Equivalent to ``LCB_READ_EVENT|LCB_WRITE_EVENT`` diff --git a/docs/source/api/convertfuncs.rst b/docs/source/api/convertfuncs.rst deleted file mode 100644 index 5448d1fda..000000000 --- a/docs/source/api/convertfuncs.rst +++ /dev/null @@ -1,27 +0,0 @@ -==================== -Conversion Functions -==================== - -.. module:: couchbase - -By default, this library uses the default ``pickle`` and ``json`` -modules of the standard Python installation to perform conversion -to and from those formats. - -Sometimes there may be a wish to use a different implementation of -those functions (for example, ``cPickle`` or a faster JSON encoder/ -decoder). - -There are two functions available to change the default Pickle and JSON -converters. - -Note that these immediately affect *all* -:class:`~couchbase.bucket.Bucket` objects. If you wish to have -a finer grained control over which object uses which converters, you -may wish to consider writing your own -:class:`~couchbase.transcoder.Transcoder` -implementation instead. - -.. autofunction:: set_json_converters - -.. autofunction:: set_pickle_converters diff --git a/docs/source/api/couchbase.rst b/docs/source/api/couchbase.rst deleted file mode 100644 index 7d7df48e7..000000000 --- a/docs/source/api/couchbase.rst +++ /dev/null @@ -1,511 +0,0 @@ -================= -Bucket object -================= - -.. module:: couchbase.bucket -.. class:: Bucket - - .. automethod:: __init__ - - -.. _argtypes: - -Passing Arguments -================= - -.. currentmodule:: couchbase.bucket - -All keyword arguments passed to methods should be specified as keyword -arguments, and the user should not rely on their position within the keyword -specification - as this is subject to change. - -Thus, if a function is prototyped as:: - - def foo(self, key, foo=None, bar=1, baz=False) - -then arguments passed to ``foo()`` should *always* be in the form of :: - - obj.foo(key, foo=fooval, bar=barval, baz=bazval) - -and never like :: - - obj.foo(key, fooval, barval, bazval) - -Arguments To ``*_multi`` Methods --------------------------------- - -Arguments passed to ``*_multi`` methods involves passing an iterable of keys. -The iterable must have ``__len__`` and ``__iter__`` implemented. - -For operations which require values (i.e. the -:meth:`~couchbase.bucket.Bucket.upsert_multi` family), a ``dict`` must -be passed with the values set as the values which should be stored for the keys. - -Some of the multi methods accept keyword arguments; these arguments apply to -*all* the keys within the iterable passed. - - -You can also pass an -:class:`~couchbase.items.ItemCollection` as the ``keys`` or ``kv`` parameter. -The `Item` interfaces allows in-place modifications to an object across multiple -operations avoiding the need for copying the result into your own data structure. - -See the documentation for :class:`~couchbase.items.Item` for more information. - - -.. _format_info: - -Key and Value Format -==================== - -.. currentmodule:: couchbase - -By default, keys are encoded as UTF-8, while values are encoded as JSON; -which was selected to be the default for compatibility and ease-of-use -with views. - - -Format Options --------------- - -The following constants may be used as values to the `format` option -in methods where this is supported. This is also the value returned in the -:attr:`~couchbase.result.ValueResult.flags` attribute of the -:class:`~couchbase.result.ValueResult` object from a -:meth:`~couchbase.bucket.Bucket.get` operation. - -Each format specifier has specific rules about what data types it accepts. - -.. data:: FMT_JSON - - Indicates the value is to be converted to JSON. This accepts any plain - Python object and internally calls :meth:`json.dumps(value)`. See - the Python `json` documentation for more information. - It is recommended you use this format if you intend to examine the value - in a MapReduce view function - -.. data:: FMT_PICKLE - - Convert the value to Pickle. This is the most flexible format as it accepts - just about any Python object. This should not be used if operating in - environments where other Couchbase clients in other languages might be - operating (as Pickle is a Python-specific format) - -.. data:: FMT_BYTES - - Pass the value as a byte string. No conversion is performed, but the value - must already be of a `bytes` type. In Python 2.x `bytes` is a synonym - for `str`. In Python 3.x, `bytes` and `str` are distinct types. Use this - option to store "binary" data. - An exception will be thrown if a `unicode` object is passed, as `unicode` - objects do not have any specific encoding. You must first encode the object - to your preferred encoding and pass it along as the value. - - Note that values with `FMT_BYTES` are retrieved as `byte` objects. - - `FMT_BYTES` is the quickest conversion method. - -.. data:: FMT_UTF8 - - Pass the value as a UTF-8 encoded string. This accepts `unicode` objects. - It may also accept `str` objects if their content is encodable as UTF-8 - (otherwise a :exc:`~couchbase.exceptions.ValueFormatError` is - thrown). - - Values with `FMT_UTF8` are retrieved as `unicode` objects (for Python 3 - `unicode` objects are plain `str` objects). - -.. data:: FMT_AUTO - - Automatically determine the format of the input type. The value of this - constant is an opaque object. - - The rules are as follows: - - If the value is a ``str``, :const:`FMT_UTF8` is used. If it is a ``bytes`` - object then :const:`FMT_BYTES` is used. If it is a ``list``, ``tuple`` - or ``dict``, ``bool``, or ``None`` then :const:`FMT_JSON` is used. - For anything else :const:`FMT_PICKLE` is used. - - -Key Format ----------- - -The above format options are only valid for *values* being passed to one -of the storage methods (see :meth:`couchbase.bucket.Bucket.upsert`). - -For *keys*, the acceptable inputs are those for :const:`FMT_UTF8` - -Single-Key Data Methods -======================= - -These methods all return a :class:`~couchbase.result.Result` object containing -information about the operation (such as status and value). - -.. currentmodule:: couchbase.bucket - - -Storing Data ------------- - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - These methods set the contents of a key in Couchbase. If successful, - they replace the existing contents (if any) of the key. - - .. automethod:: upsert - - .. automethod:: insert - - .. automethod:: replace - - -Retrieving Data ---------------- - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: get - -Modifying Data --------------- - -These methods modify existing values in Couchbase - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - - .. automethod:: append - - .. automethod:: prepend - -Entry Operations ----------------- - -These methods affect an entry in Couchbase. They do not -directly modify the value, but may affect the entry's accessibility -or duration. - - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: remove - - .. automethod:: lock - - .. automethod:: unlock - - .. automethod:: touch - - -Counter Operations ------------------- - -These are atomic counter operations for Couchbase. They increment -or decrement a counter. A counter is a key whose value can be parsed -as an integer. Counter values may be retrieved (without modification) -using the :meth:`Bucket.get` method - -.. currentmodule:: couchbase.bucket - -.. class:: Bucket - - .. automethod:: counter - - -Multi-Key Data Methods -====================== - -These methods tend to be more efficient than their single-key -equivalents. They return a :class:`couchbase.result.MultiResult` object (which is -a dict subclass) which contains class:`couchbase.result.Result` objects as the -values for its keys - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: upsert_multi - - .. automethod:: get_multi - - .. automethod:: insert_multi - - .. automethod:: replace_multi - - .. automethod:: append_multi - - .. automethod:: prepend_multi - - .. automethod:: remove_multi - - .. automethod:: counter_multi - - .. automethod:: lock_multi - - .. automethod:: unlock_multi - - .. automethod:: touch_multi - -Batch Operation Pipeline -======================== - -In addition to the multi methods, you may also use the `Pipeline` context -manager to schedule multiple operations of different types - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: pipeline - -.. class:: Pipeline - - .. autoattribute:: results - - -MapReduce/View Methods -====================== - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: query - -Design Document Management -========================== - -.. currentmodule:: couchbase.bucketmanager - - -To perform design document management operations, you must first get -an instance of the :class:`BucketManager`. You can do this by invoking -the :meth:`~couchbase.bucket.Bucket.bucket_manager` method on the -:class:`~couchbase.bucket.Bucket` object. - -.. note:: - Design document management functions are async. This means that any - successful return value simply means that the operation was *scheduled* - successfuly on the server. It is possible that the view or design will - still not yet exist after creating, deleting, or publishing a design - document. Therefore it may be recommended to verify that the view exists - by "polling" until the view does not fail. This may be accomplished by - specifying the ``syncwait`` parameter to the various design methods which - accept them. - -.. note:: - The normal process for dealing with views and design docs is to first - create a `development` design document. Such design documents are - prefixed with the string ``dev_``. They operate on a small subset of - cluster data and as such are ideal for testing as they do not impact - load very much. - - Once you are satisfied with the behavior of the development design doc, - you can `publish` it into a production mode design doc. Such design - documents always operate on the full cluster dataset. - - The view and design functions accept a ``use_devmode`` parameter which - prefixes the design name with ``dev_`` if not already prefixed. - - -.. class:: BucketManager - - - .. automethod:: design_create - .. automethod:: design_get - .. automethod:: design_publish - .. automethod:: design_delete - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: bucket_manager - - -Flushing (clearing) the Bucket -============================== - -For some stages of development and/or deployment, it might be useful -to be able to clear the bucket of its contents. - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: flush - - -Informational Methods -===================== - -These methods do not operate on keys directly, but offer various -information about things - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: stats - - .. automethod:: lcb_version - - .. automethod:: observe - - .. automethod:: observe_multi - -Item API Methods -================ - -These methods are specifically for the :class:`~couchbase.items.Item` -API. Most of the `multi` methods will accept `Item` objects as well, -however there are some special methods for this interface - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: append_items - .. automethod:: prepend_items - -Durability Constraints -====================== - -Durability constraints ensure safer protection against data loss. - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. automethod:: endure - .. automethod:: endure_multi - .. automethod:: durability - -Attributes -========== - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - .. autoattribute:: quiet - - .. autoattribute:: transcoder - - .. autoattribute:: data_passthrough - - .. autoattribute:: unlock_gil - - .. autoattribute:: timeout - - .. autoattribute:: views_timeout - - .. autoattribute:: bucket - - .. autoattribute:: server_nodes - - .. attribute:: default_format - - Specify the default format (default: :const:`~couchbase.FMT_JSON`) - to encode your data before storing in Couchbase. It uses the - flags field to store the format. - - See :ref:`format_info` for the possible values - - On a :meth:`~couchbase.bucket.Bucket.get` the - original value will be returned. This means the JSON will be - decoded, respectively the object will be unpickled. - - .. seealso:: - - :ref:`format_info` and :attr:`data_passthrough` - - .. attribute:: quiet - - It controlls whether to raise an exception when the client - executes operations on non-existent keys (default: `False`). - If it is `False` it will raise - :exc:`couchbase.exceptions.NotFoundError` exceptions. When - set to `True` the operations will not raise an exception, but - still set an error inside the :class:`~couchbase.result.Result` object. - - .. autoattribute:: lockmode - - -Private APIs -============ - -.. currentmodule:: couchbase.bucket -.. class:: Bucket - - The following APIs are not supported because using them is inherently - dangerous. They are provided as workarounds for specific problems which - may be encountered by users, and for potential testing of certain states - and/or modifications which are not attainable with the public API. - - .. automethod:: _close - - .. automethod:: _cntl - - .. automethod:: _cntlstr - - .. automethod:: _vbmap - - -.. _connopts: - -Additional Connection Options -============================= - -.. currentmodule:: couchbase.bucket - -This section is intended to document some of the less common connection -options and formats of the connection string -(see :meth:`couchbase.bucket.Bucket.__init__`). - - -Using Custom Ports -------------------- - -If you require to connect to an alternate port for bootstrapping the client -(either because your administrator has configured the cluster to listen on -alternate ports, or because you are using the built-in ``cluster_run`` -script provided with the server source code), you may do so in the host list -itself. - -Simply provide the host in the format of ``host:port``. - -Note that the port is dependent on the *scheme* used. In this case, the scheme -dictates what specific service the port points to. - - -=============== ======== -Scheme Protocol -=============== ======== -``couchbase`` memcached port (default is ``11210``) -``couchbases`` SSL-encrypted memcached port (default is ``11207``) -``http`` REST API/Administrative port (default is ``8091``) -=============== ======== - - -Options in Connection String ----------------------------- - -Additional client options may be specified within the connection -string itself. These options are derived from the underlying -*libcouchbase* library and thus will accept any input accepted -by the library itself. The following are some influential options: - - -- ``config_total_timeout``. Number of seconds to wait for the client - bootstrap to complete. - -- ``config_node_timeout``. Maximum number of time to wait (in seconds) - to attempt to bootstrap from the current node. If the bootstrap times - out (and the ``config_total_timeout`` setting is not reached), the - bootstrap is then attempted from the next node (or an exception is - raised if no more nodes remain). - -- ``config_cache``. If set, this will refer to a file on the - filesystem where cached "bootstrap" information may be stored. This - path may be shared among multiple instance of the Couchbase client. - Using this option may reduce overhead when using many short-lived - instances of the client. - - If the file does not exist, it will be created. diff --git a/docs/source/api/exceptions.rst b/docs/source/api/exceptions.rst deleted file mode 100644 index 466a4b906..000000000 --- a/docs/source/api/exceptions.rst +++ /dev/null @@ -1,154 +0,0 @@ -================= -Exception Objects -================= - -.. module:: couchbase.exceptions - -------------------------------- -Exception Types and Classifiers -------------------------------- - -.. versionadded:: 1.2.1 - -Couchbase exceptions may be caught in two flavors. You can catch an exception -either by its explicit subclass or by its base class. - - -Normally you should catch exception classes for cases you don't have a specific -handler for, and error details for things that need special handling. Thus -for example: - -:: - - try: - cb.get("foo") - except NotFoundError: - print("Item does not exist") - except CouchbaseTransientError: - print("Transient error received. Handling and backing off") - -Where `NotFoundError` is a specific error detail code indicating the item has -not been found, and `CouchbaseTransientError` is an error category indicating -the specific cause is likely transient. - -As your application evolves you may wish to examine the specific error code -received as well as log the information to the screen. - -Employing the error classifier pattern will prove scalable when additional -error codes are added to the library. Sticking to catching error codes rather -than specific error categories will allow your application to deal gracefully -with future error codes so long as they match a specific category. - - -You may also employ a different use model, for example: - -:: - - try: - cb.get("foo") - except CouchbaseError as e: - if e.is_data and isinstance(e, NotFoundError): - # handle not found - pass - elif e.is_network: - print("Got network error") - elif e.is_data: - print("Got other data-related error") - else: - print("Got unhandled error code") - - ---------------------- -Base Exception Object ---------------------- - -This object is the base class for all exceptions thrown by Couchbase which -are specific to the library. Other standard Python exceptions may still be -thrown depending on the condition. - -.. autoexception:: CouchbaseError - :members: - --------------------- -Exception Categories --------------------- - -These categories form the base exception classes - -.. autoexception:: CouchbaseInternalError -.. autoexception:: CouchbaseNetworkError -.. autoexception:: CouchbaseInputError -.. autoexception:: CouchbaseFatalError -.. autoexception:: CouchbaseDataError -.. autoexception:: CouchbaseTransientError - - - ------------------ -Exception Details ------------------ - -The following codes are exception details. They all derive from -:exc:`CouchbaseError`. Many of them will have multiple error categories and thus -be inherited from multiple exception categories. - -.. autoexception:: ArgumentError - :show-inheritance: -.. autoexception:: ValueFormatError - :show-inheritance: -.. autoexception:: AuthError - :show-inheritance: -.. autoexception:: DeltaBadvalError - :show-inheritance: -.. autoexception:: TooBigError - :show-inheritance: -.. autoexception:: BusyError - :show-inheritance: -.. autoexception:: InternalError - :show-inheritance: -.. autoexception:: InvalidError - :show-inheritance: -.. autoexception:: NoMemoryError - :show-inheritance: -.. autoexception:: RangeError - :show-inheritance: -.. autoexception:: LibcouchbaseError - :show-inheritance: -.. autoexception:: TemporaryFailError - :show-inheritance: -.. autoexception:: KeyExistsError - :show-inheritance: -.. autoexception:: NotFoundError - :show-inheritance: -.. autoexception:: DlopenFailedError - :show-inheritance: -.. autoexception:: DlsymFailedError - :show-inheritance: -.. autoexception:: NetworkError - :show-inheritance: -.. autoexception:: NotMyVbucketError - :show-inheritance: -.. autoexception:: NotStoredError - :show-inheritance: -.. autoexception:: NotSupportedError - :show-inheritance: -.. autoexception:: UnknownCommandError - :show-inheritance: -.. autoexception:: UnknownHostError - :show-inheritance: -.. autoexception:: ProtocolError - :show-inheritance: -.. autoexception:: TimeoutError - :show-inheritance: -.. autoexception:: ConnectError - :show-inheritance: -.. autoexception:: BucketNotFoundError - :show-inheritance: -.. autoexception:: ClientNoMemoryError - :show-inheritance: -.. autoexception:: ClientTemporaryFailError - :show-inheritance: -.. autoexception:: BadHandleError - :show-inheritance: -.. autoexception:: HTTPError - :show-inheritance: diff --git a/docs/source/api/gcouchbase.rst b/docs/source/api/gcouchbase.rst deleted file mode 100644 index db774a443..000000000 --- a/docs/source/api/gcouchbase.rst +++ /dev/null @@ -1,38 +0,0 @@ -================== -`gevent` Interface -================== - - -.. module:: gcouchbase - -The ``gcouchbase`` module offers a complete API which is fully compatible -with the :class:`couchbase.bucket.Bucket` API, but is fully aware -and optimized for the gevent :class:`~gevent.hub.Hub`. - -Currently, this has been tested with `gevent` version 0.13 and 1.0.0. - - -As the `gcouchbase` implementation relies on `gevent` internal APIs -itself there may be incompatibilities between minor gevent releases, -although this is not expected. - -Example usage:: - - from gcouchbase import Bucket - cb = Bucket('couchbase://localhost/default') - - # Like the normal Bucket API - res = cb.upsert("foo", "bar") - res = cb.get("foo") - - - viewiter = cb.query("beer", "brewery_beers", limit=4) - for row in viewiter: - print("Have row {0}".format(row)) - - - -.. module:: gcouchbase.bucket - -.. autoclass:: Bucket - :show-inheritance: \ No newline at end of file diff --git a/docs/source/api/items.rst b/docs/source/api/items.rst deleted file mode 100644 index 6f24eadfc..000000000 --- a/docs/source/api/items.rst +++ /dev/null @@ -1,87 +0,0 @@ -============ -Item Objects -============ - -.. versionadded:: 1.1.0 - -.. module:: couchbase.items - -The :class:`~couchbase.items.Item` class is a subclass of -:class:`~couchbase.result.ValueResult`. It differs from its parent in that -it is instantiable by the user, and can also contain fields not originally -defined in the superclass (i.e. it has a ``__dict__`` field). - -These objects may be passed (via either the -:class:`couchbase.items.ItemOptionDict` or -:class:`couchbase.items.ItemSequence` containers) to any of the ``_multi`` -functions of the :class:`~couchbase.bucket.Bucket` objects. - -Since the `Item` structure is backwards-compatible (and therefore, -interchangeable) with any of the key-value subtypes of the -:class:`~couchbase.results.Result` object, a new `Result` object is not -created for each operation in the returned -:class:`~couchbase.result.MultiResult` dictionary. - -This approach allows you to maintain a persistent object representing your -data locally; periodically updating it from the Couchbase server. - -Using the `Item` collections also allows per-item options for any of the -``_multi`` methods. - - --------------- -Creating Items --------------- - -`Item` objects may be created simply by calling the zero-arg constructor:: - - from couchbase.items import Item - it = Item() - -Before an `Item` object can be passed to any of the `Bucket` methods, -it *must* have its key set. You can simply assign the key to the object's -`key` property:: - - it.key = "some_key" - -In order to store the actual item, you should assign it a value, and place -it inside one of the collections mentioned before. Here we'll use the -:class:`~couchbase.items.ItemOptionDict` which can also contain per-item -options:: - - from couchbase.items import ItemOptionDict - itmdict = ItemOptionDict() - - # Need to add the value: - it.value = "some string" - - itmdict.add(it, format=couchbase.FMT_UTF8) - -To actually store the item, you pass the *collection* to the -:meth:`~couchbase.bucket.Bucket.upsert_multi` method, and it will function as -normally:: - - mres = cb.set_multi(itmdict) - -``mres`` is a `MultiResult` object. The value for each key will now contain -the `Item` passed originally within the collection. The normal fields including -``cas``, ``flags``. - - ---------------- -Class Reference ---------------- - -.. versionadded:: 1.1.0 - -.. autoclass:: Item - :members: - -.. autoclass:: ItemCollection - -.. autoclass:: ItemOptionDict - :show-inheritance: - :members: - -.. autoclass:: ItemSequence - :show-inheritance: diff --git a/docs/source/api/logging.rst b/docs/source/api/logging.rst deleted file mode 100644 index df50b4377..000000000 --- a/docs/source/api/logging.rst +++ /dev/null @@ -1,27 +0,0 @@ -======= -Logging -======= - -.. currentmodule::couchbase - -.. versionadded:: 2.0.0 - -Logging may be enabled programmatically via :meth:`couchbase.enable_logging()` -or via the environment, setting the ``LCB_LOGLEVEL`` environment variable -to a value between `0` and `5`. - -"Programmatic" logging uses Python's standard ``logging`` module. If you -are not familiar with Python's logging module, see https://docs.python.org/2/howto/logging.html#logging-basic-tutorial -for a basic tutorial (note, the `logging` module is also available on -Python 3) - -Note the environment variable method is actually a variable interpreted -by the underlying C library (`libcouchbase`) and is available on all -C library versions starting from 2.4.0. - -Note that logging messages themselves are currently limited to output from the C -library - - - -.. autofunction:: couchbase.enable_logging \ No newline at end of file diff --git a/docs/source/api/n1ql.rst b/docs/source/api/n1ql.rst deleted file mode 100644 index bfe1832dd..000000000 --- a/docs/source/api/n1ql.rst +++ /dev/null @@ -1,31 +0,0 @@ -############ -N1QL Queries -############ - -.. warning:: - At the time of writing, N1QL is still an experimental feature. As such, - both the server-side API as well as the client API are subject to change. - - -.. currentmodule:: couchbase.n1ql - -.. class:: N1QLQuery - - .. automethod:: __init__ - .. automethod:: set_option - .. autoattribute:: consistency - .. autoattribute:: encoded - .. autoattribute:: adhoc - .. automethod:: consistent_with_ops - .. automethod:: consistent_with_all - -.. autodata:: CONSISTENCY_NONE -.. autodata:: CONSISTENCY_REQUEST - - -.. class:: N1QLRequest - - .. automethod:: __init__ - .. automethod:: __iter__ - .. automethod:: execute - .. automethod:: get_single_result \ No newline at end of file diff --git a/docs/source/api/results.rst b/docs/source/api/results.rst deleted file mode 100644 index a472c73ff..000000000 --- a/docs/source/api/results.rst +++ /dev/null @@ -1,122 +0,0 @@ -############## -Result Objects -############## - -.. currentmodule:: couchbase.result - -This is the base class for all result operations - -.. autoclass:: Result - - .. autoattribute:: rc - - .. autoattribute:: success - - .. autoattribute:: errstr - - .. autoattribute:: key - - -.. autoclass:: OperationResult - :show-inheritance: - :no-undoc-members: - - .. autoattribute cas - - -.. autoclass:: ValueResult - :show-inheritance: - :members: - - .. autoattribute:: cas - - -.. autoclass:: HttpResult - :show-inheritance: - :members: - :no-inherited-members: - - - - -.. class:: MultiResult - - - This class is intended to serve as a container for multiple results returned - from an operation. It is a subclass of `dict` and may be used as such. The - keys will be the keys on which the operations were performed and the values - will be the results of the operation (i.e. a :class:`OperationResult` object) - - The :attr:`all_ok` field can be used to quickly examine the object for errors - (in case something like ``quiet`` was passed to - :meth:`~couchbase.bucket.Bucket.get_multi`), e.g. - - Using the `all_ok` field:: - - results = cb.get_multi(("foo", "bar", "baz"), quiet=True) - if not results.all_ok: - # process error handling here - print "Some results did not complete successfully" - - - If an exception is propagated during the operation, the ``MultiResult`` class - will still contain valid contents, except than being a return value, it will - be available via the thrown exceptions' - :attr:`~couchbase.exceptions.CouchbaseError.all_results` field. From this field - you can inspect the non-failed operations and handle them as approrpiate, while - only invoking error handling for those items which explicitly contained an error - - Using the ``MultiResult`` class from an exception handler:: - - try: - cb.insert({"foo":"fooval","bar":"barval"}) - except CouchbaseDataError as e: - for key, result in e.all_results.items(): - if not result.success: - print "Could not add {0}. Got error code {1}".format(key, result.rc) - - - .. autoattribute:: all_ok - -.. _observe_info: - -=============== -Observe Results -=============== - ------------------ -Observe Constants ------------------ - -These constants are returned as values for :attr:`ObserveInfo.flags` -field. - -.. data:: couchbase.OBS_FOUND - - The key exists on the given node's cache, though it may not have been - stored to disk yet. - -.. data:: couchbase.OBS_PERSISTED - - The key is persisted to the given node's disk. - -.. data:: couchbase.OBS_NOTFOUND - - The key is not present in the node's cache. - -.. data:: couchbase.OBS_LOGICALLY_DELETED - - The key is not present in the node's cache, however it is still present - on the persistent store. If the node would crash at this moment, the key - would still be present when it starts up again. - - This is equivalent to ``OBS_NOTFOUND | OBS_PERSISTED`` - --------------------- -`ObserveInfo` Object --------------------- - -.. module:: couchbase.result - -.. autoclass:: couchbase.result.ObserveInfo - :members: diff --git a/docs/source/api/threads.rst b/docs/source/api/threads.rst deleted file mode 100644 index d03095996..000000000 --- a/docs/source/api/threads.rst +++ /dev/null @@ -1,91 +0,0 @@ -============================== -Using With and Without Threads -============================== - -.. module:: couchbase.bucket - -You can use a single :class:`~couchbase.bucket.Bucket` object in -a single thread, and attain reasonable performance by having one -`Bucket` object per thread. However, you **cannot** use the same object -from multiple threads concurrently (but see below) - -As `couchbase` is a C extension, it is helpful to know how Python -deals with threads in general, and how it handles C extensions in -this context. - -Python utilizes something called a *Global Interpreter Lock* (*GIL*) to -allow Python to simulate concurrency. The basic idea is that the interpreter -can only ever utilize a single CPU core. Threading is acheived by having the -interpreter switch to a different thread every *n* instructions (where *n*) -is typically `100` - but is set in :meth:`sys.getcheckinterval` and -:meth:`sys.setcheckinterval`. When C extension code is being executed -however, Python has no way of knowing how many 'instructions' have passed -and thus by default keeps the interpreter lock on for the duration of the -extension function. - -See http://docs.python.org/2/c-api/init.html#thread-state-and-the-global-interpreter-lock -for more details on how Python handles the GIL. - -Since `couchbase` does I/O operations, it is inefficient to keep the entire -interpreter locked during the wait for I/O responses; thus it has the ability -to unlock the *GIL* right before it begins an I/O operation, and lock it -back again when the operation is complete. - -When operating in a threaded environment (i.e. with *other* Python threads) -running, it is helpful to have the GIL handling enabled, as it may potentially -block other threads' execution. When using in non-threaded mode (i.e. where -no other Python threads operate within the entire application), it would be -good to disable the GIL handling. This reduces locking/unlocking overhead and -potentially makes your program faster. - -The default is to have GIL handling enabled, so as to not unexpectedly hang -other threads in the case where an I/O operation takes a prolonged amount -of time. - -This behavior itself can be controlled by the -:attr:`~couchbase.bucket.Bucket.unlock_gil` attribute - - -.. _multiple_threads: - -Using a :class:`Bucket` from multiple threads ---------------------------------------------------- - -Sometimes it may be necessary to use a :class:`Bucket` object from -multiple threads. This is normally not a good option as there is no concurrency -gained from multiple Python threads (as they do not run in parallel, as above) -it might be necessary to have a single object which is being used from -an already-existing framework using threads, where there is typically little -contention between them. - -You may utilize the ``lockmode`` constructor option to enforce a specific -behavior when the :class:`Bucket` object is accessed from multiple -threads - -.. data:: LOCKMODE_EXC - -This is the default lockmode. If it is detected that the object is being used -from multiple threads, an exception will be raised indicating such. - -Internally this uses the C equivalent of the ``threading.Lock`` object (i.e. -``PyThread_allocate_lock()``). Upon each entry to a function it will try -to acquire the lock (without waiting). If the acquisition fails, the -exception is raised. - -*This is the default lockmode* - -.. data:: LOCKMODE_WAIT - -In this mode, a lock is also used, only in this case an exception is not -raised. Rather, the current thread patiently waits until the other thread -has completed its operation and the lock is then acquired. It is released once -the current thread has finished performing the operation. - -Without this option, odd behavior may be exhibited (including some crashes). -If you are sure that the :class:`Bucket` object will never be used from -multiple threads, or if you have some other locking mechanism in place, then -you may use :const:`LOCKMODE_NONE` - -.. data:: LOCKMODE_NONE - -No thread safety checks diff --git a/docs/source/api/transcoder.rst b/docs/source/api/transcoder.rst deleted file mode 100644 index 102777d78..000000000 --- a/docs/source/api/transcoder.rst +++ /dev/null @@ -1,61 +0,0 @@ -==================== -Transcoder Interface -==================== - -.. module:: couchbase.transcoder - -The **Transcoder** interface may be used to provide custom value and -key formats. Among the uses of the transcoder class, one may: - -* Implement compatibility with other client libraries - - The ``format`` option in the operations correspond to a flag value - which is stored in the server as meta data along with the key. These - flags are utilized by client libraries to determine how to interpret - the value on the server (which is just a set of bytes) into a more - complex and user-friendly type in the native client language. Some - clients may have different ideas about which flags mean which value format, - and some clients may use formats which are specific to that platform - (for example, :const:`~couchbase.FMT_PICKLE` which - is typically only native to Python objects). One may implement a - custom transcoder class which understands a wider variety of types - and flags - -* Implement Compression - - If storing large values, it may be handy to compress them before - storing them on the server. This provides for lower network overhead - and storage space on the server, at the expense of the computational - overhead of compressing and decompressing objects. One may add extra - flags to indicate a value has been compressed, and with which format. - -* Automatic conversion of different value types into custom classes - - The built-in transcoder only uses native Python types. One may wish - to interpret values as belong to user-defined classes. - -The :class:`Transcoder` class is implemented in C for efficiency, but -a pure-python implementation is available as the :class:`~TranscoderPP` -class in ``couchbase/transcoder.py``. - -Typically one would subclass the :class:`Transcoder` class, and -implement the needed methods, allowing for high efficient built-in -methods to perform the un-needed operations. - -Note that the :class:`~couchbase.bucket.Bucket` does not -use a :class:`Transcoder` object by default (however it internally uses -the same routines that the C-implemented :class:`Transcoder` does). Thus, -if no custom transcoding is needed, it is more efficient to set the -:attr:`~couchbase.bucket.Bucket.transcoder` to ``None``, which -is the default. - -.. class:: Transcoder - - - .. automethod:: encode_key(key) - .. automethod:: encode_value(value, flags) - .. automethod:: decode_key(key) - .. automethod:: decode_value(value, flags) - .. automethod:: determine_format(value) - -.. autoclass:: TranscoderPP diff --git a/docs/source/api/txcouchbase.rst b/docs/source/api/txcouchbase.rst deleted file mode 100644 index 2eab73bc6..000000000 --- a/docs/source/api/txcouchbase.rst +++ /dev/null @@ -1,46 +0,0 @@ -=================== -`Twisted` Interface -=================== - -.. module:: txcouchbase - -The Twisted interface is for use with the Twisted Python event and networking -library which may be found at http://www.twistedmatrix.com. This documentation -contains the API reference for how to use the ``txcouchbase`` module with -Twisted. - -.. currentmodule:: txcouchbase.bucket - -For the most part, the ``txcouchbase`` API functions like its synchronous -counterpart, :class:`~couchbase.bucket.Bucket`, except for its -asynchronous nature. Where the synchronous API returns a -:class:`~couchbase.result.Result` object, the ``txcouchbase`` API returns -a :class:`Deferred` which will have its callback invoked with a result. - -As such, we will omit the mentions of the normal key value operations, which -function identially to their synchronous conterparts documented in the -:class:`~couchbase.bucket.Bucket` class. - -The :class:`Bucket` interface for Twisted is subclassed from the lower-level -:class:`RawBucket` which returns :class:`~couchbase.result.AsyncResult` -objects rather than `Deferred` objects. This is largely due to performance -reasons (Deferreds result in a 3x performance slowdown). - -.. class:: RawBucket - - .. automethod:: __init__ - .. automethod:: registerDeferred - .. automethod:: connect - .. automethod:: defer - .. autoattribute:: connected - -.. class:: Bucket - - .. automethod:: __init__ - .. automethod:: queryAll - .. automethod:: queryEx - -.. class:: BatchedView - - .. automethod:: __iter__ - .. automethod:: __init__ diff --git a/docs/source/api/views.rst b/docs/source/api/views.rst deleted file mode 100644 index 35cde642c..000000000 --- a/docs/source/api/views.rst +++ /dev/null @@ -1,796 +0,0 @@ -############## -Querying Views -############## - -=============== -``View`` Object -=============== - -.. module:: couchbase.views.iterator - -.. class:: View - - .. automethod:: __init__ - - .. automethod:: __iter__ - -^^^^^^^^^^ -Attributes -^^^^^^^^^^ - - .. attribute:: errors - - Errors returned from the view engine itself - - - .. attribute:: indexed_rows - - Number of total rows indexed by the view. This is the number of results - before any filters or limitations applied. - This is only valid once the iteration has started - - - .. attribute:: row_processor - - An object to handle a single page of the paginated results. This - object should be an instance of a class conforming to the - :class:`RowProcessor` interface. By default, it is an instance of - :class:`RowProcessor` itself. - - - .. attribute:: raw - - The actual :class:`couchbase.bucket.HttpResult` object. - Note that this is only the *last* result returned. If using paginated - views, the view comprises several such objects, and is cleared each - time a new page is fetched. - - - .. attribute:: design - - Name of the design document being used - - - .. attribute:: view - - Name of the view being queired - - - .. attribute:: include_docs - - Whether documents are fetched along with each row - - - .. attribute:: rows_returned - - How many actual rows were returned from the server. - - This is incremented each time a new request is made. Note this may - be different from the amount of rows yielded by iterator from - :meth:`RowProcessor.handle_rows` if a custom :attr:`row_processor` - is being used - - -^^^^^^^^^^^^^^ -Row Processing -^^^^^^^^^^^^^^ - -.. class:: RowProcessor - - .. automethod:: handle_rows - - -.. class:: ViewRow - - This is the default class returned by the :class:`RowProcessor` - - .. attribute:: key - - The key emitted by the view's ``map`` function (first argument to ``emit``) - - - .. attribute:: value - - The value emitted by the view's ``map`` function (second argument to - ``emit``). If the view was queried with ``reduce`` enabled, then this - contains the reduced value after being processed by the ``reduce`` - function. - - - .. attribute:: docid - - This is the document ID for the row. This is always ``None`` if - ``reduce`` was specified. Otherwise it may be passed to one of the - ``get`` or ``set`` method to retrieve or otherwise access the - underlying document. Note that if ``include_docs`` was specified, - the :attr:`doc` already contains the document - - - .. attribute:: doc - - If ``include_docs`` was specified, contains the actual - :class:`couchbase.bucket.Result` object for the document. - - - -================ -``Query`` Object -================ - -.. module:: couchbase.views.params - - -.. class:: Query - - .. automethod:: __init__ - - .. automethod:: update - - .. autoattribute:: encoded - - -.. _view_options: - -^^^^^^^^^^^^ -View Options -^^^^^^^^^^^^ - -This document explains the various view options, and how they are treated -by the Couchbase library. - - -Many of the view options correspond to those listed here -http://www.couchbase.com/docs/couchbase-manual-2.0/couchbase-views-querying-rest-api.html - - -Note that these explain the view options and their values as they are passed -along to the server. - -.. _param_listings: - -These attributes are available as properties (with get and set) -and can also be used as keys within a constructor. - - -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Result Range and Sorting Properties -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following properties allow you to - -* Define a range to limit your results (i.e. `between foo and bar`) -* Define a specific subset of keys for which results should be yielded -* Reverse the sort order - -.. class:: Query - - - .. attribute:: mapkey_range - - Specify the range based on the contents of the keys emitted by the - view's ``map`` function. - - :Server Option: Maps to both ``startkey`` and ``endkey`` - :Value Type: - :ref:`viewtype_range` of :ref:`viewtype_jsonvalue` elements - - The result output depends on the type of keys and ranges used. - - One may specify a "full" range (that is, an exact match of the first - and/or last key to use), or a partial range where the start and end - ranges specify a *subset* of the key to be used as the start and end. - In such a case, the results begin with the first key which matches - the partial start key, and ends with the first key that matches the - partial end key. - - Additionally, keys may be *compound* keys, i.e. complex data types - such as lists. - - You may use the :const:`STRING_RANGE_END` to specify a wildcard - for an end range. - - Match all keys that start with "a" through keys starting with "f":: - - q.mapkey_range = ["a", "f"+q.STRING_RANGE_END] - q.inclusive_end = True - - If you have a view function that looks something like this:: - - function(doc, meta) { - if (doc.city && doc.event) { - emit([doc.country, doc.state, doc.city], doc.event) - } - } - - Then you may query for all events in a specific state by using:: - - q.mapkey_range = [ - ["USA", "NV", ""] - ["USA", "NV", q.STRING_RANGE_END] - ] - - While the first two elements are an exact match (i.e. only keys which - have ``["USA","NV", ...]`` in them, the third element should accept - anything, and thus has its start value as the empty string (i.e. lowest - range) and the magic ``q.STRING_RANGE_END`` as its lowest value. - - As such, the results may look like:: - - ViewRow(key=[u'USA', u'NV', u'Reno'], value=u'Air Races', docid=u'air_races_rno', doc=None) - ViewRow(key=[u'USA', u'NV', u'Reno'], value=u'Reno Rodeo', docid=u'rodeo_rno', doc=None) - ViewRow(key=[u'USA', u'NV', u'Reno'], value=u'Street Vibrations', docid=u'street_vibrations_rno', doc=None) - - # etc. - - .. autoattribute:: STRING_RANGE_END - - .. attribute:: dockey_range - - :Server Option: Maps to both ``startkey_docid`` and ``endkey_docid`` - :Value Type: - :ref:`viewtype_range` of :ref:`viewtype_string` elements. - - Specify the range based on the contents of the keys as they are stored - by :meth:`~couchbase.bucket.Bucket.upsert`. These are - returned as the "Document IDs" in each view result. - - You *must* use this attribute in conjunction with - :attr:`mapkey_range` option. Additionally, this option only has - any effect if you are emitting duplicate keys for different - document IDs. An example of this follows: - - Documents:: - - c.upsert("id_1", { "type" : "dummy" }) - c.upsert("id_2", { "type" : "dummy" }) - # ... - c.upsert("id_9", { "type" : "dummy" }) - - - View:: - - // This will emit "dummy" for ids 1..9 - - function map(doc, meta) { - emit(doc.type); - } - - - - Only get information about ``"dummy"`` docs for IDs 3 through 6:: - - q = Query() - q.mapkey_range = ["dummy", "dummy" + Query.STRING_RANGE_END] - q.dockey_range = ["id_3", "id_6"] - q.inclusive_end = True - - .. warning:: - - Apparently, only the first element of this parameter has any - effect. Currently the example above will start returning rows - from ``id_3`` (as expected), but does not stop after reaching - ``id_6``. - - .. attribute:: key - - .. attribute:: mapkey_single - - :Server Option: ``key`` - :Value Type: :ref:`viewtype_jsonvalue` - - Limit the view results to those keys which match the value to this - option exactly. - - View:: - - function(doc, meta) { - if (doc.type == "brewery") { - emit([meta.id]); - } else { - emit([doc.brewery_id, meta.id]); - } - } - - Example:: - - q.mapkey_single = "abbaye_de_maredsous" - - - Note that as the ``map`` function can return more than one result with - the same key, you may still get more than one result back. - - - .. attribute:: keys - - .. attribute:: mapkey_multi - - :Server Option: ``keys`` - :Value Type: :ref:`viewtype_jsonarray` - - Like :attr:`mapkey_single`, but specify a sequence of keys. - Only rows whose emitted keys match any of the keys specified here - will be returned. - - Example:: - - q.mapkey_multi = [ - ["abbaye_de_maresdous"], - ["abbaye_de_maresdous", "abbaye_de_maresdous-8"], - ["abbaye_do_maresdous", "abbaye_de_maresdous-10"] - ] - - - .. attribute:: inclusive_end - - :Server Option: ``inclusive_end`` - :Value Type: :ref:`viewtype_boolean` - - Declare that the range parameters' (for e.g. :attr:`mapkey_range` and - :attr:`dockey_range`) end key should also be returned for rows that - match it. By default, the resultset is terminated once the first key - matching the end range is found. - - .. attribute:: descending - - :Server Option: ``descending`` - :Value Type: :ref:`viewtype_boolean` - - -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Reduce Function Parameters -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -These options are valid only for views which have a ``reduce`` function, -and for which the ``reduce`` value is enabled - -.. class:: Query - - .. attribute:: reduce - - :Server Option: ``reduce`` - :Value Type: :ref:`viewtype_boolean` - - Note that if the view specified in the query (to e.g. - :meth:`couchbase.bucket.Bucket.query`) does not have a - reduce function specified, an exception will be thrown once the query - begins. - - .. attribute:: group - - :Server Option: ``group`` - :Value Type: :ref:`viewtype_boolean` - - Specify this option to have the results contain a breakdown of the - ``reduce`` function based on keys produced by ``map``. By default, - only a single row is returned indicating the aggregate value from all - the ``reduce`` invocations. - - Specifying this option will show a breakdown of the aggregate ``reduce`` - value based on keys. Each unique key in the result set will have its - own value. - - Setting this property will also set :attr:`reduce` to ``True`` - - .. attribute:: group_level - - :Server Option: ``group_level`` - :Value Type: :ref:`viewtype_num` - - This is analoguous to ``group``, except that it places a constraint - on how many elements of the compound key produced by ``map`` should be - displayed in the summary. For example if this parameter is set to - ``1`` then the results are returned for each unique first element in - the mapped keys. - - Setting this property will also set :attr:`reduce` to ``True`` - -^^^^^^^^^^^^^^^^^^^^^^^ -Pagination and Sampling -^^^^^^^^^^^^^^^^^^^^^^^ - -These options limit or paginate through the results - -.. class:: Query - - .. attribute:: skip - - :Server Option: ``skip`` - :Value Type: :ref:`viewtype_num` - - .. warning:: - Consider using :attr:`mapkey_range` instead. Using this property - with high values is typically inefficient. - - .. attribute:: limit - - :Server Option: ``limit`` - :Value Type: :ref:`viewtype_num` - - Set an absolute limit on how many rows should be returned in this - query. The number of rows returned will always be less or equal to - this number. - -^^^^^^^^^^^^^^^ -Control Options -^^^^^^^^^^^^^^^ - -These do not particularly affect the actual query behavior, but may -control some other behavior which may indirectly impact performance or -indexing operations. - -.. class:: Query - - .. attribute:: stale - - :Server Option: ``stale`` - - Specify the (re)-indexing behavior for the view itself. Views return - results based on indexes - which are not updated for each query by - default. Updating the index for each query would cause much performance - issues. However it is sometimes desirable to ensure consistency of data - (as sometimes there may be a delay between recently-updated keys and - the view index). - - This option allows to specify indexing behavior. It accepts a string - which can have one of the following values: - - * ``ok`` - - Stale indexes are allowable. This is the default. The constant - :data:`STALE_OK` may be used instead. - - * ``false`` - - Stale indexes are not allowable. Re-generate the index before - returning the results. Note that if there are many results, this - may take a considerable amount of time (on the order of several - seconds, typically). - The constant :data:`STALE_UPDATE_BEFORE` may be used instead. - - * ``update_after`` - - Return stale indexes for this result (so that the query does not - take a long time), but re-generated the index immediately after - returning. - The constant :data:`STALE_UPDATE_AFTER` may be used instead. - - A :ref:`viewtype_boolean` may be used as well, in which case - ``True`` is converted to ``"ok"``, and ``False`` - is converted to ``"false"`` - - .. attribute:: on_error - - :Server Option: ``on_error`` - :Value Type: - A string of either ``"stop"`` or ``"continue"``. You may use - the symbolic constants :data:`ONERROR_STOP` or - :data:`ONERROR_CONTINUE` - - .. attribute:: connection_timeout - - This parameter is a server-side option indicating how long - a given node should wait for another node to respond. This does - *not* directly set the client-side timeout. - - :Server Option: ``connection_timeout`` - :Value Type: :ref:`viewtype_num` - - .. attribute:: debug - - :Server Option: ``debug`` - :Value Type: :ref:`viewtype_boolean` - - If enabled, various debug output will be dumped in the resultset. - - .. attribute:: full_set - - :Server Option: ``full_set`` - :Value Type: :ref:`viewtype_boolean` - - If enabled, development views will operate over the entire data within - the bucket (and not just a limited subset). - - - ----------------------- -Value Type For Options ----------------------- - -.. currentmodule:: couchbase.views.params - -Different options accept different types, which shall be enumerated here - - -.. _viewtype_boolean: - -^^^^^^^^^^^^ -Boolean Type -^^^^^^^^^^^^ - -.. currentmodule:: couchbase.views.params - -Options which accept booleans may accept the following Python types: - - * Standard python ``bool`` types, like ``True`` and ``False`` - * Numeric values which evaluate to booleans - * Strings containing either ``"true"`` or ``"false"`` - -Other options passed as booleans will raise an error, as it is assumed that -perhaps it was passed accidentally due to a bug in the application. - - -.. _viewtype_num: - -^^^^^^^^^^^^ -Numeric Type -^^^^^^^^^^^^ - -.. currentmodule:: couchbase.views.params - -Options which accept numeric values accept the following Python types: - - * ``int``, ``long`` and ``float`` objects - * Strings which contain values convertible to said native numeric types - -It is an error to pass a ``bool`` as a number, despite the fact that in Python, -``bool`` are actually a subclass of ``int``. - - -.. _viewtype_jsonvalue: - -^^^^^^^^^^ -JSON Value -^^^^^^^^^^ - -.. currentmodule:: couchbase.views.params - -Options which accept JSON values accept native Python types (and any user- -defined classes) which can successfully be passed through ``json.dumps``. - -Do *not* pass an already-encoded JSON string, and do not URI-escape the -string either - as this will be done by the option handling layer (but see -:ref:`passthrough_values` for a way to circumvent this) - -Note that it is perfectly acceptable to pass JSON primitives (such as numbers, -strings, and booleans). - - -.. _viewtype_jsonarray: - -^^^^^^^^^^ -JSON Array -^^^^^^^^^^ - -.. currentmodule:: couchbase.views.params - -Options which accept JSON array values should be pass a Python type which -can be converted to a JSON array. This typically means any ordered Python -sequence (such as ``list`` and ``tuple``). Like :ref:`viewtype_jsonvalue`, -the contents of the list should *not* be URI-escaped, as this will be done -at the option handling layer - - -.. _viewtype_string: - -^^^^^^ -String -^^^^^^ - -.. currentmodule:: couchbase.views.params - -Options which accept strings accept so-called "semantic strings", specifically; -the following Python types are acceptable: - - * ``str`` and ``unicode`` objects - * ``int`` and ``long`` objects - -Note that ``bool``, ``none`` and other objects are not accepted - this is to -ensure that random objects passed don't simply end up being ``repr()``'d -and causing confusion in your view results. - -If you have a custom object which has a ``__str__`` method and would like to -use it as a string, you must explicitly do so prior to passing it as an option. - -.. _viewtype_range: - -^^^^^^^^^^^ -Range Value -^^^^^^^^^^^ - -.. currentmodule:: couchbase.view.params - -Range specifiers take a sequence (list or tuple) of one or two elements. - -If the sequence contains two items, the first is taken to be the *start* -of the range, and the second is taken to be its (non-inclusive) *end* - -If the sequence contains only a single item, it is taken to be the -*start* of the range, and no *end* will be specified. - -To specify a range which has an *end* but not a start, pass a two-element -sequence with the first element being an :data:`UNSPEC` value. - - -The type of each element is parameter-specific. - - -.. _viewtype_unspec: - -^^^^^^^^^^^^^^^^^ -Unspecified Value -^^^^^^^^^^^^^^^^^ - -.. currentmodule:: couchbase.views.params - -Conventionally, it is common for APIs to treat the value ``None`` as being -a default parameter of some sort. Unfortunately since view queries deal with -JSON, and ``None`` maps to a JSON ``null``, it is not possible for the view -processing functions to ignore ``None``. - -As an alternative, a special constant is provided as -:data:`UNSPEC`. You may use this as a placeholder value for any -option. When the view processing code encounters this value, it will -discard the option-value pair. - -^^^^^^^^^^^^^^^^^^^^^ -Convenience Constants -^^^^^^^^^^^^^^^^^^^^^ - -.. currentmodule:: couchbase.views.params - -These are convenience *value* constants for some of the options - -.. autoattribute:: couchbase.views.params.ONERROR_CONTINUE -.. autoattribute:: couchbase.views.params.ONERROR_STOP -.. autoattribute:: couchbase.views.params.STALE_OK -.. autoattribute:: couchbase.views.params.STALE_UPDATE_BEFORE -.. autoattribute:: couchbase.views.params.STALE_UPDATE_AFTER -.. autoattribute:: couchbase.views.params.UNSPEC - - - - -.. _passthrough_values: - ------------------------------------ -Circumventing Parameter Constraints ------------------------------------ - -.. currentmodule:: couchbase.views.params - -Sometimes it may be necessary to circumvent existing constraints placed by -the client library regarding view option validation. - -For this, there are ``passthrough`` and ``allow_unrecognized`` options -which may be set in order to allow the client to be more lax in its conversion -process. - -These options are present under various names in the various view query -functions. - - -* Passthrough - - Passthrough removes any conversion functions applied. It simply assumes - values for all options are strings, and then encodes them rather simply - -* Allowing Unrecognized Options - - If a newer version of a server is released has added a new option, older - versions of this library will not know about it, and will raise an error - when it is being used. In this scenario, one can use the 'allow unrecognized' - mode to *add* extra options, with their values being treated as simple - strings. - - This has the benefit of providing normal behavior for known options. - -================ -Geospatial Views -================ - -.. warning:: - - Geospatial views are considered an experimental feature in current - versions of Couchbase Server (the latest version at the time of - writing being 3.0.2). As such, the feature exposed in the SDK itself - is too inherently experimental - -Geospatial views are views which can index and filter items based on one or -more independent axes or coordinates. This allows greater application at -query-time to filter based on more than a single attribute. - -Filtering at query time is done though _ranges_. These ranges contain the -start and end values for each key passed to the `emit()` in the `map()` -function. Unlike Map-Reduce views and compound keys for *startkey* and -*endkey*, each item in a spatial range is independent from any other, and is -not sorted or evaluated in any particular order. - - -See `GeoCouch`_<https://github.com/couchbase/geocouch/wiki/Spatial-Views-API> -for more information. - -^^^^^^^^^^^^^^^^^^^^^^^^^ -Creating Geospatial Views -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Creating a geospatial view may be done in a manner similar to creating -a normal view; except that the design document defines the spatial -view in the ``spatial`` field, rather than in the ``views`` field. - -.. code-block:: python - - ddoc = { - 'spatial': { - 'geoview': - ''' - if (doc.loc) { - emit({ - type: "Point", - geometry: doc.loc - }, doc.name); - } - ''' - } - } - cb.bucket_manager().design_create('geo', ddoc) - - -The above snippet will create a geospatial design doc (``geo``) with a single -view (called ``geoview``). - - -^^^^^^^^^^^^^^^^^^^^^^^^^ -Querying Geospatial Views -^^^^^^^^^^^^^^^^^^^^^^^^^ - -To query a geospatial view, you must pass an instance of :class:`.SpatialQuery` -as the ``query`` keyword argument to either the :class:`.View` constructor, or -the :meth:`.Bucket.query` method. - -.. code-block:: python - - from couchbase.views.params import SpatialQuery - q = SpatialQuery(start_range=[0, -90, None], end_range=[180, 90, None]) - for row in bkt.query(query=q): - print "Key:", row.key - print "Value:", row.value - print "Geometry", row.geometry - - -.. currentmodule:: couchbase.views.params - -.. class:: SpatialQuery - - .. automethod:: __init__ - - .. attribute:: start_range - - The starting range to query. If querying geometries, this should be - the lower bounds of the longitudes and latitudes to filter. Use - `None` to indicate that a given dimension should not be bounded. - - .. code-block:: python - - q.start_range=[0, -90] - - .. attribute:: end_range - - The upper limit for the range. This contains the upper bounds for - the ranges specified in ``start_range``. - - .. code-block:: python - - q.end_range[180, 90] - - .. attribute:: skip - - See :attr:`.Query.skip` - - .. attribute:: limit - - See :attr:`.Query.limit` - - .. attribute:: stale - - See :attr:`.Query.stale` - diff --git a/docs/source/conf.py b/docs/source/conf.py deleted file mode 100644 index f8cd562af..000000000 --- a/docs/source/conf.py +++ /dev/null @@ -1,252 +0,0 @@ -# -*- coding: utf-8 -*- -# -# Couchbase Python Client Library documentation build configuration file, created by -# sphinx-quickstart on Fri Apr 5 17:46:04 2013. -# -# This file is execfile()d with the current directory set to its containing dir. -# -# Note that not all possible configuration values are present in this -# autogenerated file. -# -# All configuration values have a default; values that are commented out -# serve to show the default. - -import os -import sys - -# If extensions (or modules to document with autodoc) are in another directory, -# add these directories to sys.path here. If the directory is relative to the -# documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) -sys.path.insert(0, os.path.abspath('../..')) -#sys.path.insert(0, os.path.abspath('../../couchbase')) -#sys.path.insert(0, os.path.abspath(os.path.join(os.pardir, os.pardir, 'couchbase'))) -import couchbase_version - -# -- General configuration ----------------------------------------------------- - -# If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' - -# Add any Sphinx extension module names here, as strings. They can be extensions -# coming with Sphinx (named 'sphinx.ext.*') or your custom ones. -extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'numpydoc', 'sphinx.ext.autosummary'] -#extensions = ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', 'numpydoc'] - -# Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] - -# The suffix of source filenames. -source_suffix = '.rst' - -# The encoding of source files. -#source_encoding = 'utf-8-sig' - -# The master toctree document. -master_doc = 'index' - -# General information about the project. -project = 'Couchbase Python Client Library' -copyright = '2013-2015, Couchbase, Inc.' - -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The full version, including alpha/beta/rc tags. -release = couchbase_version.get_version() - -# The short X.Y version. -version = release.split('.')[:2] - -# The language for content autogenerated by Sphinx. Refer to documentation -# for a list of supported languages. -#language = None - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -#today = '' -# Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' - -# List of patterns, relative to source directory, that match files and -# directories to ignore when looking for source files. -exclude_patterns = [] - -# The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None - -# If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True - -# If true, the current module name will be prepended to all description -# unit titles (such as .. function::). -#add_module_names = True - -# If true, sectionauthor and moduleauthor directives will be shown in the -# output. They are ignored by default. -#show_authors = False - -# The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' - -# A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] - - -# -- Options for HTML output --------------------------------------------------- - -# The theme to use for HTML and HTML Help pages. See the documentation for -# a list of builtin themes. -html_theme = 'classic' - -# Theme options are theme-specific and customize the look and feel of a theme -# further. For a list of options available for each theme, see the -# documentation. -#html_theme_options = {} - -# Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] - -# The name for this set of Sphinx documents. If None, it defaults to -# "<project> v<release> documentation". -#html_title = None - -# A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None - -# The name of an image file (relative to this directory) to place at the top -# of the sidebar. -#html_logo = None - -# The name of an image file (within the static path) to use as favicon of the -# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 -# pixels large. -#html_favicon = None - -# Add any paths that contain custom static files (such as style sheets) here, -# relative to this directory. They are copied after the builtin static files, -# so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' - -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -#html_use_smartypants = True - -# Custom sidebar templates, maps document names to template names. -#html_sidebars = {} - -# Additional templates that should be rendered to pages, maps page names to -# template names. -#html_additional_pages = {} - -# If false, no module index is generated. -#html_domain_indices = True - -# If false, no index is generated. -#html_use_index = True - -# If true, the index is split into individual pages for each letter. -#html_split_index = False - -# If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True - -# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True - -# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True - -# If true, an OpenSearch description file will be output, and all pages will -# contain a <link> tag referring to it. The value of this option must be the -# base URL from which the finished HTML is served. -#html_use_opensearch = '' - -# This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None - -# Output file base name for HTML help builder. -htmlhelp_basename = 'CouchbasePythonClientLibrarydoc' - - -# -- Options for LaTeX output -------------------------------------------------- - -latex_elements = { - # The paper size ('letterpaper' or 'a4paper'). - #'papersize': 'letterpaper', - - # The font size ('10pt', '11pt' or '12pt'). - #'pointsize': '10pt', - - # Additional stuff for the LaTeX preamble. - #'preamble': '', -} - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, documentclass [howto/manual]). -latex_documents = [ - ('index', 'CouchbasePythonClientLibrary.tex', 'Couchbase Python Client Library Documentation', - 'Couchbase, Inc.', 'manual'), -] - -# The name of an image file (relative to this directory) to place at the top of -# the title page. -#latex_logo = None - -# For "manual" documents, if this is true, then toplevel headings are parts, -# not chapters. -#latex_use_parts = False - -# If true, show page references after internal links. -#latex_show_pagerefs = False - -# If true, show URL addresses after external links. -#latex_show_urls = False - -# Documents to append as an appendix to all manuals. -#latex_appendices = [] - -# If false, no module index is generated. -#latex_domain_indices = True - - -# -- Options for manual page output -------------------------------------------- - -# One entry per manual page. List of tuples -# (source start file, name, description, authors, manual section). -man_pages = [ - ('index', 'couchbasepythonclientlibrary', 'Couchbase Python Client Library Documentation', - ['Couchbase, Inc.'], 1) -] - -# If true, show URL addresses after external links. -#man_show_urls = False - - -# -- Options for Texinfo output ------------------------------------------------ - -# Grouping the document tree into Texinfo files. List of tuples -# (source start file, target name, title, author, -# dir menu entry, description, category) -texinfo_documents = [ - ('index', 'CouchbasePythonClientLibrary', 'Couchbase Python Client Library Documentation', - 'Couchbase, Inc.', 'CouchbasePythonClientLibrary', 'A python client library to store, retrieve and query data from a Couchbase cluster.', - 'Miscellaneous'), -] - -# Documents to append as an appendix to all manuals. -#texinfo_appendices = [] - -# If false, no module index is generated. -#texinfo_domain_indices = True - -# How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' - -autoclass_content = 'both' -numpydoc_show_class_members = False diff --git a/docs/source/index.rst b/docs/source/index.rst deleted file mode 100644 index dcbe01498..000000000 --- a/docs/source/index.rst +++ /dev/null @@ -1,64 +0,0 @@ -.. Couchbase Python Client Library documentation master file, created by - sphinx-quickstart on Fri Apr 5 17:46:04 2013. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - -Couchbase Python Client Library Documentation -============================================= - -Contents: - -.. toctree:: - :maxdepth: 2 - - api/couchbase - api/views - api/n1ql - api/results - api/exceptions - api/transcoder - api/threads - api/convertfuncs - api/items - api/logging - -Asynchronous APIs -================= - -.. toctree:: - :maxdepth: 2 - - api/txcouchbase - api/gcouchbase - -Internal APIs -============= - -These are internal APIs whose interfaces are subject to change. They may -nevertheless be helpful for debugging or extending existing functionality. - -.. toctree:: - :maxdepth: 2 - - api/async - - -Administrative APIs -=================== - -These APIs allow simple manipulation of buckets themselves. They require -administrative privileges. Currently these APIs are only available for -the synchronous `couchbase` interface. - -.. toctree:: - :maxdepth: 2 - - api/admin - - -Indices and tables -================== - -* :ref:`genindex` -* :ref:`modindex` -* :ref:`search` diff --git a/docs/txcouchbase_api/txcouchbase.rst b/docs/txcouchbase_api/txcouchbase.rst new file mode 100644 index 000000000..638617781 --- /dev/null +++ b/docs/txcouchbase_api/txcouchbase.rst @@ -0,0 +1,18 @@ +=========== +txcouchbase +=========== + +.. note:: + Further updates to the acouchbase docs will come with future 4.x releases. In the meantime, + check out the provided examples in the :txcouchbase_examples:`Github repo <>`. + +.. note:: + The minimum required Twisted version is 21.7.0. + +.. warning:: + The 4.x SDK introduced a breaking change where the ``txcouchbase`` package must be imported *prior* to importing the reactor. This is so that the asyncio reactor can be installed. + +.. toctree:: + :maxdepth: 2 + + txcouchbase_core diff --git a/docs/txcouchbase_api/txcouchbase_core.rst b/docs/txcouchbase_api/txcouchbase_core.rst new file mode 100644 index 000000000..1d3681e19 --- /dev/null +++ b/docs/txcouchbase_api/txcouchbase_core.rst @@ -0,0 +1,19 @@ +============================== +Twisted (txcouchbase) Core API +============================== + +.. note:: + Further updates to the acouchbase docs will come with future 4.x releases. In the meantime, + check out the provided examples in the :txcouchbase_examples:`Github repo <>`. + +Cluster +============== + +Bucket +============== + +Scope +============== + +Collection +============== diff --git a/docs/using_the_python_sdk.rst b/docs/using_the_python_sdk.rst new file mode 100644 index 000000000..1c8a6e781 --- /dev/null +++ b/docs/using_the_python_sdk.rst @@ -0,0 +1,184 @@ +======================= +Using the Python SDK +======================= + +The Couchbase Python SDK library allows you to connect to a Couchbase cluster from Python. +The Python SDK uses the high-performance C++ library, Couchbase++, to handle communicating to the cluster over the Couchbase binary protocol. + +Useful Links +======================= + +* :python_sdk_github:`Source <>` +* :python_sdk_jira:`Bug Tracker <>` +* :python_sdk_docs:`Python docs on the Couchbase website <>` +* :python_sdk_release_notes:`Release Notes <>` +* :python_sdk_compatibility:`Compatibility Guide <>` +* :couchbase_dev_portal:`Couchbase Developer Portal <>` + +How to Engage +======================= + +* :couchbase_discord:`Join Discord and contribute <>`. + The Couchbase Discord server is a place where you can collaborate about all things Couchbase. + Connect with others from the community, learn tips and tricks, and ask questions. +* Ask and/or answer questions on the :python_sdk_forums:`Python SDK Forums <>`. + + +Installing the SDK +======================= + +.. note:: + Best practice is to use a Python virtual environment such as venv or pyenv. + Checkout: + + * Linux/MacOS: `pyenv <https://github.com/pyenv/>`_ + * Windows: `pyenv-win <https://github.com/pyenv-win/pyenv-win>`_ + + +.. note:: + The Couchbase Python SDK provides wheels for Windows, MacOS and Linux platforms (via manylinux) for supported versions of Python. + See :python_sdk_version_compat:`Python Version Compatibility <>` docs for details. + +Prereqs +++++++++++ + +If not on platform that has a binary wheel availble, the following is needed: + +* A supported Python version (see :python_sdk_version_compat:`Python Version Compatibility <>` for details) +* A C++ compiler supporting C++ 17 +* CMake (version >= 3.18) +* Git (if not on a platform that offers wheels) +* OpenSSL + + * SDK versions >= 4.1.9 provide wheels that **do not** require OpenSSL. + * SDK versions < 4.1.9 provide wheels that *require* OpenSSL 1.1.1. + * SDK versions >= 4.1 can be built against a desired version of OpenSSL. + +* If using the Twisted Framework and the txcouchbase API, Twisted >= 21.7.0 is required. + +.. warning:: + Some older linux platforms to not provide defaults (Python version, OpenSSL, C++ 17 support, etc.) that meet the Python SDK's minimum requirements. Be sure to update to the minimum requirements prior to installing the SDK. + See the `dockerfiles folder <https://github.com/couchbase/couchbase-python-client/tree/master/examples/dockerfiles>`_ in the Python SDK examples folder for references to working setups for various linux platforms. + +.. note:: + Starting with Python 3.11.5, macOS installers and Windows builders from python.org now use `OpenSSL 3.0 <https://docs.python.org/3/whatsnew/3.11.html#notable-changes-in-3-11-5>`_. + A potential side-effect of this change is an ``ImportError: DLL load failed while importing pycbc_core`` error when a version of the Python SDK prior to 4.1.9. As a work-around, + set the ``PYCBC_OPENSSL_DIR`` environment variable to the path where the OpenSSL 1.1 libraries can be found (``libssl-1_1.dll`` and ``libcrypto-1_1.dll`` for Windows; ``libssl.1.1.dylib`` and ``libcrypto.1.1.dylib`` for macOS). + Alternatively, the SDK can be built from source using a version of OpenSSL > 1.1.x. + +After the above have been installed, pip install ``setuptools`` and ``wheel`` (see command below). + +.. code-block:: console + + $ python3 -m pip install --upgrade pip setuptools wheel + +Install +++++++++++ + +.. code-block:: console + + $ python3 -m pip install couchbase + +Introduction +======================= + +Connecting to a Couchbase cluster is as simple as creating a new ``Cluster`` instance to represent the ``Cluster`` +you are using, and then using the ``bucket`` and ``collection`` commands against this to open a connection to open +your specific ``bucket`` and ``collection``. You are able to execute most operations immediately, and they will be +queued until the connection is successfully established. + +Here is a simple example of creating a ``Cluster`` instance, retrieving a document and using SQL++ (a.k.a. N1QL). + +.. code-block:: python + + # needed for any cluster connection + from couchbase.auth import PasswordAuthenticator + from couchbase.cluster import Cluster + # options for a cluster and SQL++ (N1QL) queries + from couchbase.options import ClusterOptions, QueryOptions + + # get a reference to our cluster + auth = PasswordAuthenticator('username', 'password') + cluster = Cluster.connect('couchbase://localhost', ClusterOptions(auth)) + + # get a reference to our bucket + cb = cluster.bucket('travel-sample') + + # get a reference to the default collection + cb_coll = cb.default_collection() + + # get a document + result = cb_coll.get('airline_10') + print(f'Document content: {result.content_as[dict]}') + + # using SQL++ (a.k.a N1QL) + call_sign = 'CBS' + sql_query = 'SELECT VALUE name FROM `travel-sample` WHERE type = "airline" AND callsign = $1' + query_res = cluster.query(sql_query, QueryOptions(positional_parameters=[call_sign])) + for row in query_res: + print(f'Found row: {row}') + +Source Control +======================= + +The source control is available on :python_sdk_github:`Github <>`. +Once you have cloned the repository, you may contribute changes through our gerrit server. +For more details see :python_sdk_contribute:`CONTRIBUTING.md <>`. + +Migrating from 3.x to 4.x +=========================== + +The Python SDK 4.x implements the :python_sdk_api_version:`SDK API 3 spec <>`, so all the steps outlined in the :python_sdk_api_version:`SDK 3 migration docs <>` apply to a migration from a Python SDK 2.x directly to Python SDK 4.x. + +Importantly, the Python SDK 4.x has been substantially reworked to use a new backend (Couchbase++ instead of libcouchbase.) +Though the API surfaces are intended to be compatible, any code that relies on undocumented or uncommitted internal details is not guaranteed to work. +Key areas that have been reworked: + +* The ``couchbase_core`` package has been removed. The 4.x SDK provides appropriate import paths within the ``couchbase`` package (or possibly the ``acouchbase``/``txcouchbase`` packages if using one of the async APIs) for anything that is needed with respect to the APIs provided by the SDK. +* As there is a new backend, the previous ``_libcouchbase`` c-extension has been removed +* Remnants of the 2.x API in previous Python 3.x SDK versions have been removed or deprecated + + * Key items that have been **removed**: + + * The ``ClassicAuthenticator`` class + * Key-value operations are no longer available with a ``bucket`` instance. Use a ``collection`` instance for key-value operations. + * A ``cluster`` and ``bucket`` instance do not inherit from the same base class + * The ``Client`` class has been removed + * ``Items`` API + * ``Admin`` cluster + + * Key items that have been **deprecated**: + + * Datastructure methods provided by the ``collection`` instance have been deprecated and replaced with their respective APIs (i.e. ``CouchbaseList``, ``CouchbaseMap``, ``CouchbaseQueue`` and ``CouchbaseSet``) + * ``OperationResult`` (deprecated, still available from ``couchbase.result``) + * ``ValueResult`` (deprecated, still available from ``couchbase.result``) + +* Import paths have been reorganized to follow consistent patterns. While the import paths that existed in 3.x SDK are mostly available (see previous points on removal of ``couchbase_core`` package), some paths are deprecated and will be removed in a future release. + + * All authenticators should be imported from ``couchbase.auth`` + * All constants should be imported from ``couchbase.constants`` + * All options should be imported from ``couchbase.options`` + * All management options should be imported from ``couchbase.management.options`` + * All results should be imported from ``couchbase.result`` + * All exceptions should be imported from ``couchbase.exceptions`` + * Enumerations and Classes related to operations should be imported from that operation's path. For example, ``QueryScanConsistency`` should be imported from ``couchbase.n1ql`` (i.e. ``from couchbase.n1ql import QueryScanConsistency``) + +* Changes to the async APIs (``acouchbase`` and ``txcouchbase``): + + * While multi-operations (``get_multi``, ``upsert_multi``, etc.) still exist for the ``couchbase`` API they have been removed from the async APIs (``acouchbase`` and ``txcouchbase``) as each of the async APIs are built with libraries that have mechanisms to handle multi/bulk operations (``asyncio`` has ``asyncio.gather(...)`` and ``Twisted`` has ``DeferredList(...)``). + * If using the ``txcouchbase`` API, the reactor that should be installed is the ``asyncioreactor``. Therefore, the ``txcouchbase`` package *needs* to be imported prior to importing the ``reactor``. See example import below. + + .. code-block:: python + + # this is new with Python SDK 4.x, it needs to be imported prior to + # importing the twisted reactor + import txcouchbase + + from twisted.internet import reactor + +License +======================= + +The Couchbase Python SDK is licensed under the Apache License 2.0. + +See :python_sdk_license:`LICENSE <>` for further details. diff --git a/examples/abench.py b/examples/abench.py deleted file mode 100755 index 47b244911..000000000 --- a/examples/abench.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -try: - import asyncio -except ImportError: - import trollius - asyncio = trollius - -import argparse -from time import time - - -from couchbase.user_constants import FMT_BYTES -from couchbase.experimental import enable; enable() -from acouchbase.bucket import Bucket - -ap = argparse.ArgumentParser() - -ap.add_argument('-t', '--threads', default=4, type=int, - help="Number of threads to spawn. 0 means no threads " - "but workload will still run in the main thread") - -ap.add_argument('-d', '--delay', default=0, type=float, - help="Number of seconds to wait between each op. " - "may be a fraction") - -ap.add_argument('-p', '--password', default=None, type=str) -ap.add_argument('-U', '--connstr', default='couchbase://localhost/default', type=str) -ap.add_argument('-D', '--duration', default=10, type=int, - help="Duration of run (in seconds)") - -ap.add_argument('--ksize', default=12, type=int, - help="Key size to use") - -ap.add_argument('--vsize', default=128, type=int, - help="Value size to use") -ap.add_argument('-g', '--global-instance', - help="Use global instance", default=False, - action='store_true') -ap.add_argument('--batch', '-N', type=int, help="Batch size", default=1) - -options = ap.parse_args() - -GLOBAL_INSTANCE = None -CONN_OPTIONS = {'connstr': options.connstr} - - -def make_instance(): - global GLOBAL_INSTANCE - if options.global_instance: - if not GLOBAL_INSTANCE: - GLOBAL_INSTANCE = Bucket(**CONN_OPTIONS) - return GLOBAL_INSTANCE - else: - return Bucket(**CONN_OPTIONS) - - -class Worker(object): - def __init__(self): - self.delay = options.delay - self.key = 'K' * options.ksize - self.value = b'V' * options.vsize - self.kv = {} - for x in range(options.batch): - self.kv[self.key + str(x)] = self.value - - self.wait_time = 0 - self.opcount = 0 - - try: - exec(''' -@asyncio.coroutine -def run(self): - self.end_time = time() + options.duration - cb = make_instance() - yield from cb.connect() - - while time() < self.end_time: - begin_time = time() - yield from cb.upsert_multi(self.kv, format=FMT_BYTES) - self.wait_time += time() - begin_time - self.opcount += options.batch - ''') - except SyntaxError: - @asyncio.coroutine - def run(self): - self.end_time = time() + options.duration - cb = make_instance() - yield trollius.From(cb.connect()) - - while time() < self.end_time: - begin_time = time() - yield trollius.From(cb.upsert_multi(self.kv, format=FMT_BYTES)) - self.wait_time += time() - begin_time - self.opcount += options.batch - -global_begin = None -tasks = [] -worker_threads = [] -loop = asyncio.get_event_loop() -for x in range(options.threads): - w = Worker() - worker_threads.append(w) - tasks.append(asyncio.async(w.run())) - -global_begin = time() -loop.run_until_complete(asyncio.wait(tasks)) -global_duration = time() - global_begin -total_ops = sum([w.opcount for w in worker_threads]) -total_time = 0 -for t in worker_threads: - total_time += t.wait_time - -print("Total run took an absolute time of %0.2f seconds" % (global_duration,)) -print("Did a total of %d operations" % (total_ops,)) -print("Total wait time of %0.2f seconds" % (total_time,)) -print("[WAIT] %0.2f ops/second" % (float(total_ops)/float(total_time),)) -print("[ABS] %0.2f ops/second" % (float(total_ops)/float(global_duration),)) diff --git a/examples/acouchbase/acouchbase_diagnostics_operations.py b/examples/acouchbase/acouchbase_diagnostics_operations.py new file mode 100644 index 000000000..93c85a42b --- /dev/null +++ b/examples/acouchbase/acouchbase_diagnostics_operations.py @@ -0,0 +1,71 @@ +from datetime import timedelta + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, use: from couchbase.options import PingOptions +from couchbase.bucket import PingOptions +from couchbase.diagnostics import PingState, ServiceType +from couchbase.exceptions import UnAmbiguousTimeoutException +from couchbase.options import WaitUntilReadyOptions + + +async def ok(cluster): + result = await cluster.ping() + for _, reports in result.endpoints.items(): + for report in reports: + if not report.state == PingState.OK: + return False + return True + + +async def main(): + cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) + await cluster.on_connect() + + cluster_ready = False + try: + await cluster.wait_until_ready(timedelta(seconds=3), + WaitUntilReadyOptions(service_types=[ServiceType.KeyValue, ServiceType.Query])) + cluster_ready = True + except UnAmbiguousTimeoutException as ex: + print('Cluster not ready in time: {}'.format(ex)) + + if cluster_ready is False: + quit() + + # For Server versions 6.5 or later you do not need to open a bucket here + bucket = cluster.bucket("beer-sample") + await bucket.on_connect() + collection = bucket.default_collection() + + ping_result = await cluster.ping() + + for endpoint, reports in ping_result.endpoints.items(): + for report in reports: + print( + "{0}: {1} took {2}".format( + endpoint.value, + report.remote, + report.latency)) + + ping_result = await cluster.ping() + print(ping_result.as_json()) + + print("Cluster is okay? {}".format(await ok(cluster))) + + ping_result = await cluster.ping(PingOptions(service_types=[ServiceType.Query])) + print(ping_result.as_json()) + + diag_result = await cluster.diagnostics() + print(diag_result.as_json()) + + print("Cluster state: {}".format(diag_result.state)) + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/acouchbase/acouchbase_kv_operations.py b/examples/acouchbase/acouchbase_kv_operations.py new file mode 100644 index 000000000..caa2ce0e5 --- /dev/null +++ b/examples/acouchbase/acouchbase_kv_operations.py @@ -0,0 +1,155 @@ +from datetime import timedelta + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, use: from couchbase.options import DeltaValue, SignedInt64 +# **DEPRECATED**, import ALL options from `couchbase.options` +from couchbase.collection import (DecrementOptions, + DeltaValue, + GetOptions, + IncrementOptions, + InsertOptions, + RemoveOptions, + ReplaceOptions, + SignedInt64, + UpsertOptions) +from couchbase.durability import (ClientDurability, + Durability, + PersistTo, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import CASMismatchException, CouchbaseException + + +async def main(): + cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) + await cluster.on_connect() + bucket = cluster.bucket("default") + await bucket.on_connect() + collection = bucket.default_collection() + + # setup + try: + result = await collection.remove("document-key") + result = await collection.remove("document-key-opts") + except CouchbaseException as ex: + pass # may not exist in this example + + # Insert document + document = {"foo": "bar", "bar": "foo"} + result = await collection.insert("document-key", document) + print("Result: {}; CAS: {}".format(result, result.cas)) + + # Insert document with options + document = {"foo": "bar", "bar": "foo"} + opts = InsertOptions(timeout=timedelta(seconds=5)) + result = await collection.insert("document-key-opts", + document, + opts, + expiry=timedelta(seconds=30)) + + try: + # Replace document with CAS + document = {"foo": "bar", "bar": "foo"} + result = await collection.replace( + "document-key", + document, + cas=12345, + timeout=timedelta( + minutes=1)) + except CASMismatchException as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print('Caught CAS mismatch: {}'.format(ex)) + + try: + # Replace document with CAS + result = await collection.get("document-key") + doc = result.content_as[dict] + doc["bar"] = "baz" + opts = ReplaceOptions(cas=result.cas) + result = await collection.replace("document-key", doc, opts) + except CouchbaseException as ex: + print('Caught Couchbase exception: {}'.format(ex)) + + try: + # Upsert with Durability (Couchbase Server >= 6.5) level Majority + document = dict(foo="bar", bar="foo") + opts = UpsertOptions(durability=ServerDurability(Durability.MAJORITY)) + result = await collection.upsert("document-key", document, opts) + except CouchbaseException as ex: + print('Caught Couchbase exception: {}'.format(ex)) + + # @TODO: couchbase++ doesn't implement observe based durability + # try: + # # Upsert with observe based durability (Couchbase Server < 6.5) + # document = {"foo": "bar", "bar": "foo"} + # opts = UpsertOptions( + # durability=ClientDurability( + # ReplicateTo.ONE, + # PersistTo.ONE)) + # result = await collection.upsert("document-key", document, opts) + # except CouchbaseException as ex: + # print(ex) + + result = await collection.get("document-key") + print(result.content_as[dict]) + + opts = GetOptions(timeout=timedelta(seconds=5)) + result = await collection.get("document-key", opts) + print(result.content_as[dict]) + + try: + # remove document with options + result = await collection.remove( + "document-key", + RemoveOptions( + cas=12345, + durability=ServerDurability( + Durability.MAJORITY))) + except CouchbaseException as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print('Caught Couchbase exception: {}'.format(ex)) + + result = await collection.touch("document-key", timedelta(seconds=10)) + + result = await collection.get("document-key", GetOptions(with_expiry=True)) + print("Expiry of result: {}".format(result.expiryTime)) + + result = await collection.get_and_touch("document-key", timedelta(seconds=10)) + + # Increment binary value by 1 + await collection.binary().increment( + "counter-key", + IncrementOptions( + delta=DeltaValue(1))) + + # Increment binary value by 5, if key doesn't exist, seed it at 1000 + await collection.binary().increment( + "counter-key", + IncrementOptions( + delta=DeltaValue(5), + initial=SignedInt64(1000))) + + # Decrement binary value by 1 + await collection.binary().decrement( + "counter-key", + DecrementOptions( + delta=DeltaValue(1))) + + # Decrement binary value by 2, if key doesn't exist, seed it at 1000 + await collection.binary().decrement( + "counter-key", + DecrementOptions( + delta=DeltaValue(2), + initial=SignedInt64(1000))) + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/acouchbase/acouchbase_query_operations.py b/examples/acouchbase/acouchbase_query_operations.py new file mode 100644 index 000000000..3a6af9f56 --- /dev/null +++ b/examples/acouchbase/acouchbase_query_operations.py @@ -0,0 +1,123 @@ +import uuid + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, import ALL options from `couchbase.options` +from couchbase.cluster import (ClusterOptions, + QueryOptions, + QueryScanConsistency) +from couchbase.exceptions import ParsingFailedException +from couchbase.mutation_state import MutationState + +# use this to not have deprecation warnings +# from couchbase.options import ClusterOptions, QueryOptions +# from couchbase.n1ql import QueryScanConsistency + + +async def main(): + cluster = await Cluster.connect('couchbase://localhost', + ClusterOptions(PasswordAuthenticator('Administrator', 'password'))) + bucket = cluster.bucket("travel-sample") + collection = bucket.default_collection() + + # basic query + try: + result = cluster.query( + "SELECT * FROM `travel-sample` LIMIT 10;", QueryOptions(metrics=True)) + + async for row in result.rows(): + print(f'Found row: {row}') + + metrics = result.metadata().metrics() + print(f'Query execution time: {metrics.execution_time()}') + + except ParsingFailedException as ex: + import traceback + traceback.print_exc() + + # positional params + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`=$1 LIMIT 10" + result = cluster.query(q_str, "hotel") + rows = [r async for r in result] + + # positional params via QueryOptions + result = cluster.query(q_str, QueryOptions(positional_parameters=["hotel"])) + rows = [r async for r in result] + + # named params + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`=$doc_type LIMIT 10" + result = cluster.query(q_str, doc_type='hotel') + rows = [r async for r in result] + + # name params via QueryOptions + result = cluster.query(q_str, QueryOptions(named_parameters={'doc_type': 'hotel'})) + rows = [r async for r in result] + + # iterate over result/rows + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10" + result = cluster.query(q_str) + + # iterate over rows + async for row in result: + # each row is an serialized JSON object + name = row["name"] + callsign = row["callsign"] + print(f'Airline name: {name}, callsign: {callsign}') + + # query metrics + result = cluster.query("SELECT 1=1", QueryOptions(metrics=True)) + await result.execute() + + print("Execution time: {}".format( + result.metadata().metrics().execution_time())) + + # print scan consistency + result = cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10", + QueryOptions(scan_consistency=QueryScanConsistency.REQUEST_PLUS)) + rows = [r async for r in result] + + # Read your own writes + new_airline = { + "callsign": None, + "country": "United States", + "iata": "TX", + "icao": "TX99", + "id": 123456789, + "name": "Howdy Airlines", + "type": "airline" + } + + res = await collection.upsert( + "airline_{}".format(new_airline["id"]), new_airline) + + ms = MutationState(res) + + result = cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10", + QueryOptions(consistent_with=ms)) + rows = [r async for r in result] + + # client context id + result = cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='hotel' LIMIT 10", + QueryOptions(client_context_id="user-44{}".format(uuid.uuid4()))) + rows = [r async for r in result] + + # read only + result = cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='hotel' LIMIT 10", + QueryOptions(read_only=True)) + rows = [r async for r in result] + + agent_scope = bucket.scope("inventory") + + result = agent_scope.query( + "SELECT a.* FROM `airline` a WHERE a.country=$country LIMIT 10", + country='France') + rows = [r async for r in result] + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/acouchbase/acouchbase_subdoc_operations.py b/examples/acouchbase/acouchbase_subdoc_operations.py new file mode 100644 index 000000000..0eba43851 --- /dev/null +++ b/examples/acouchbase/acouchbase_subdoc_operations.py @@ -0,0 +1,272 @@ + +# @TODO: change back to subdocument to match 3.x??? +import couchbase.subdocument as SD +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator + +# import couchbase.subdocument as SD +# **DEPRECATED**, use from couchbase.options import MutateInOptions +from couchbase.collection import MutateInOptions +from couchbase.durability import (ClientDurability, + Durability, + PersistTo, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import (CASMismatchException, + CouchbaseException, + DocumentExistsException, + DurabilityImpossibleException, + PathExistsException, + PathNotFoundException, + SubdocCantInsertValueException, + SubdocPathMismatchException) + +JSON_DOC = { + "name": "Douglas Reynholm", + "email": "douglas@reynholmindustries.com", + "addresses": { + "billing": { + "line1": "123 Any Street", + "line2": "Anytown", + "country": "United Kingdom" + }, + "delivery": { + "line1": "123 Any Street", + "line2": "Anytown", + "country": "United Kingdom" + } + }, + "purchases": { + "complete": [ + 339, 976, 442, 666 + ], + "abandoned": [ + 157, 42, 999 + ] + } +} + + +async def main(): + cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) + await cluster.on_connect() + bucket = cluster.bucket("default") + await bucket.on_connect() + collection = bucket.default_collection() + + try: + await collection.insert("customer123", JSON_DOC) + except DocumentExistsException: + await collection.remove("customer123") + await collection.insert("customer123", JSON_DOC) + + result = await collection.lookup_in("customer123", + [SD.get("addresses.delivery.country")]) + country = result.content_as[str](0) # "United Kingdom" + print(country) + + result = await collection.lookup_in( + "customer123", [ + SD.exists("purchases.pending[-1]")]) + print('Path exists: {}.'.format(result.exists(0))) + # Path exists: False. + + # NOTE: result.content_as[bool](0) raises a PathNotFoundException + # this is b/c when checking if a path exists + # no content is returned + + try: + print("Path exists {}.".format(result.content_as[bool](0))) + except PathNotFoundException: + print("Path does not exist") + + result = await collection.lookup_in( + "customer123", + [SD.get("addresses.delivery.country"), + SD.exists("purchases.complete[-1]")]) + + print("{0}".format(result.content_as[str](0))) + print("Path exists: {}.".format(result.exists(1))) + # path exists: True. + + await collection.mutate_in("customer123", [SD.upsert("fax", "311-555-0151")]) + + await collection.mutate_in( + "customer123", [SD.insert("purchases.pending", [42, True, "None"])]) + + try: + await collection.mutate_in( + "customer123", [ + SD.insert( + "purchases.complete", + [42, True, "None"])]) + except PathExistsException: + print("Path exists, cannot use insert.") + + await collection.mutate_in( + "customer123", + (SD.remove("addresses.billing"), + SD.replace( + "email", + "dougr96@hotmail.com"))) + + # NOTE: the mutate_in() operation expects a tuple or list + await collection.mutate_in( + "customer123", (SD.array_append( + "purchases.complete", 777),)) + # purchases.complete is now [339, 976, 442, 666, 777] + + await collection.mutate_in( + "customer123", [ + SD.array_prepend( + "purchases.abandoned", 18)]) + # purchases.abandoned is now [18, 157, 42, 999] + + await collection.upsert("my_array", []) + await collection.mutate_in("my_array", [SD.array_append("", "some element")]) + # the document my_array is now ["some element"] + + await collection.mutate_in( + "my_array", [ + SD.array_append( + "", "elem1", "elem2", "elem3")]) + + # the document my_array is now ["some_element", "elem1", "elem2", "elem3"] + + await collection.mutate_in( + "my_array", [ + SD.array_append( + "", ["elem4", "elem5", "elem6"])]) + + # the document my_array is now ["some_element", "elem1", "elem2", "elem3", + # ["elem4", "elem5", "elem6"]]] + + await collection.mutate_in( + "my_array", (SD.array_append("", "elem7"), + SD.array_append("", "elem8"), + SD.array_append("", "elem9"))) + + await collection.upsert("some_doc", {}) + await collection.mutate_in( + "some_doc", [ + SD.array_prepend( + "some.array", "Hello", "World", create_parents=True)]) + # the document some_doc is now {"some":{"array":["Hello", "World"]}} + try: + await collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", 95)]) + print('Success!') + except PathExistsException: + print('Path already exists.') + + try: + await collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", 95)]) + print('Success!') + except PathExistsException: + print('Path already exists.') + + # cannot add JSON obj w/ array_addunique + try: + await collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", {"new": "object"})]) + except SubdocCantInsertValueException as ex: + print("Cannot add JSON value w/ array_addunique.", ex) + + # cannot use array_addunique if array contains JSON obj + await collection.mutate_in( + "customer123", [ + SD.upsert( + "purchases.cancelled", [{"Date": "Some date"}])]) + + try: + await collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.cancelled", 89)]) + except SubdocPathMismatchException as ex: + print("Cannot use array_addunique if array contains JSON objs.", ex) + + await collection.upsert("array", []) + await collection.mutate_in("array", [SD.array_append("", "hello", "world")]) + await collection.mutate_in("array", [SD.array_insert("[1]", "cruel")]) + + # exception raised if attempt to insert in out of bounds position + try: + await collection.mutate_in("array", [SD.array_insert("[6]", "!")]) + except PathNotFoundException: + print("Cannot insert to out of bounds index.") + + # can insert into nested arrays as long as the path is appropriate + await collection.mutate_in("array", [SD.array_append("", ["another", "array"])]) + await collection.mutate_in("array", [SD.array_insert("[3][2]", "!")]) + + result = await collection.mutate_in("customer123", (SD.counter("logins", 1),)) + customer_result = await collection.get("customer123") + num_logins = customer_result.content_as[dict]["logins"] + print('Number of logins: {}.'.format(num_logins)) + # Number of logins: 1. + + await collection.upsert("player432", {"gold": 1000}) + + await collection.mutate_in("player432", (SD.counter("gold", -150),)) + result = await collection.lookup_in("player432", (SD.get("gold"),)) + print("{} has {} gold remaining.".format( + "player432", result.content_as[int](0))) + # player432 has 850 gold remaining. + + await collection.mutate_in("customer123", [SD.upsert("level_0.level_1.foo.bar.phone", + dict( + num="311-555-0101", + ext=16 + ), create_parents=True)]) + + await collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 998)]) + + await collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 999)]) + + try: + await collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 999)], + MutateInOptions(cas=1234)) + except (DocumentExistsException, CASMismatchException) as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print(ex) + + # @TODO: couchbase++ doesn't implement observe based durability + # try: + # await collection.mutate_in( + # "key", [SD.insert("username", "dreynholm")], + # MutateInOptions(durability=ClientDurability( + # ReplicateTo.ONE, + # PersistTo.ONE))) + # except CouchbaseException as ex: + # print('Need to have more than 1 node for durability') + # print(ex) + + try: + await collection.mutate_in( + "customer123", [SD.insert("username", "dreynholm")], + MutateInOptions(durability=ServerDurability( + Durability.MAJORITY))) + except DurabilityImpossibleException as ex: + print('Need to have more than 1 node for durability') + print(ex) + + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/acouchbase/management/acouchbase_bucketmgmt.py b/examples/acouchbase/management/acouchbase_bucketmgmt.py new file mode 100644 index 000000000..176266a71 --- /dev/null +++ b/examples/acouchbase/management/acouchbase_bucketmgmt.py @@ -0,0 +1,80 @@ +import asyncio + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import BucketAlreadyExistsException, BucketDoesNotExistException +from couchbase.management.buckets import (BucketSettings, + BucketType, + ConflictResolutionType, + CreateBucketSettings) + + +async def retry(func, *args, back_off=0.5, limit=5, **kwargs): + for i in range(limit): + try: + return await func(*args, **kwargs) + except Exception: + print("Retry in {} seconds...".format((i + 1) * back_off)) + await asyncio.sleep((i + 1) * back_off) + + raise Exception( + "Unable to successfully receive result from {}".format(func)) + + +async def main(): + cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) + await cluster.on_connect() + bucket = cluster.bucket("default") + await bucket.on_connect() + + bucket_manager = cluster.buckets() + + await bucket_manager.create_bucket( + CreateBucketSettings( + name="hello", + flush_enabled=False, + ram_quota_mb=100, + num_replicas=0, + bucket_type=BucketType.COUCHBASE, + conflict_resolution_type=ConflictResolutionType.SEQUENCE_NUMBER)) + + # Another approach to catch if bucket already exists + try: + await bucket_manager.create_bucket( + CreateBucketSettings( + name="hello", + flush_enabled=False, + ram_quota_mb=100, + num_replicas=0, + bucket_type=BucketType.COUCHBASE, + conflict_resolution_type=ConflictResolutionType.SEQUENCE_NUMBER)) + except BucketAlreadyExistsException: + print("{} bucket previously created.".format("hello")) + + # Creating a bucket can take some time depending on what + # the cluster is doing, good to use retries + bucket = await retry(bucket_manager.get_bucket, "hello") + + bucket = await bucket_manager.get_bucket("hello") + print("Found bucket: {}".format(bucket.name)) + + await bucket_manager.update_bucket(BucketSettings(name="hello", flush_enabled=True)) + + await bucket_manager.flush_bucket("hello") + + await bucket_manager.drop_bucket("hello") + + # verify bucket dropped + try: + await bucket_manager.get_bucket("hello") + except BucketDoesNotExistException: + print("{} bucket dropped.".format("hello")) + + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/acouchbase/management/acouchbase_collectionmgmt.py b/examples/acouchbase/management/acouchbase_collectionmgmt.py new file mode 100644 index 000000000..e6fb150fd --- /dev/null +++ b/examples/acouchbase/management/acouchbase_collectionmgmt.py @@ -0,0 +1,88 @@ +import asyncio + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import (CollectionAlreadyExistsException, + CollectionNotFoundException, + ScopeAlreadyExistsException, + ScopeNotFoundException) +from couchbase.management.collections import CollectionSpec + + +async def retry(func, *args, back_off=0.5, limit=5, **kwargs): + for i in range(limit): + try: + return await func(*args, **kwargs) + except Exception: + print("Retry in {} seconds...".format((i + 1) * back_off)) + await asyncio.sleep((i + 1) * back_off) + + raise Exception( + "Unable to successfully receive result from {}".format(func)) + + +async def get_scope(collection_mgr, scope_name): + scopes = await collection_mgr.get_all_scopes() + return next((s for s in scopes if s.name == scope_name), None) + + +async def get_collection(collection_mgr, scope_name, coll_name): + scope = await get_scope(collection_mgr, scope_name) + if scope: + return next( + (c for c in scope.collections if c.name == coll_name), + None) + + return None + + +async def main(): + cluster = Cluster( + "couchbase://ec2-54-213-122-253.us-west-2.compute.amazonaws.com", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) + + await cluster.on_connect() + bucket = cluster.bucket("default") + await bucket.on_connect() + coll_manager = bucket.collections() + + try: + await coll_manager.create_scope("example-scope") + except ScopeAlreadyExistsException as ex: + print(ex) + + scope = await retry(get_scope, coll_manager, "example-scope") + print("Found scope: {}".format(scope.name)) + + collection_spec = CollectionSpec( + "example-collection", + scope_name="example-scope") + + try: + collection = await coll_manager.create_collection(collection_spec) + except CollectionAlreadyExistsException as ex: + print(ex) + + collection = await retry( + get_collection, + coll_manager, + "example-scope", + "example-collection") + print("Found collection: {}".format(collection.name)) + + try: + await coll_manager.drop_collection(collection_spec) + except CollectionNotFoundException as ex: + print(ex) + + try: + await coll_manager.drop_scope("example-scope") + except ScopeNotFoundException as ex: + print(ex) + + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/acouchbase/management/acouchbase_querymgmt.py b/examples/acouchbase/management/acouchbase_querymgmt.py new file mode 100644 index 000000000..a437461ca --- /dev/null +++ b/examples/acouchbase/management/acouchbase_querymgmt.py @@ -0,0 +1,116 @@ +from datetime import timedelta + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import (QueryIndexAlreadyExistsException, + QueryIndexNotFoundException, + WatchQueryIndexTimeoutException) + +# **DEPRECATED**, import ALL management options from `couchbase.management.options` +# from couchbase.management.queries import (CreatePrimaryQueryIndexOptions, +# CreateQueryIndexOptions, +# DropQueryIndexOptions, +# WatchQueryIndexOptions) +from couchbase.management.options import (CreatePrimaryQueryIndexOptions, + CreateQueryIndexOptions, + DropQueryIndexOptions, + WatchQueryIndexOptions) + + +async def main(): + + cluster = Cluster( + 'couchbase://localhost', + authenticator=PasswordAuthenticator( + 'Administrator', + 'password')) + + await cluster.on_connect() + + bucket_name = 'beer-sample' + + ixm = cluster.query_indexes() + try: + # expect exception as the travel-sample bucket should already come w/ a primary index + await ixm.create_primary_index(bucket_name) + except QueryIndexAlreadyExistsException: + print('Primary index already exists') + + # can create an index, and ignore if it already exists + await ixm.create_primary_index(bucket_name, CreatePrimaryQueryIndexOptions(ignore_if_exists=True)) + + n1ql = ''' + SELECT brewery.name, + beer.beerCount, + beer.brewery_id + FROM `beer-sample` AS brewery + JOIN ( + SELECT brewery_id, + COUNT(1) AS beerCount + FROM `beer-sample` + WHERE `type` = 'beer' + GROUP BY brewery_id) AS beer ON beer.brewery_id = META(brewery).id + WHERE brewery.`type` = 'brewery' + LIMIT 10; + ''' + + # lets create some secondary indexes that let the above query run + # w/o needing to do a primary index scan + beer_sample_type_idx = 'beer_sample_type' + fields = ['type'] + await ixm.create_index(bucket_name, + beer_sample_type_idx, + fields, + CreateQueryIndexOptions(deferred=True, + timeout=timedelta(seconds=120))) + + beer_sample_brewery_id_idx = 'beer_sample_type_brewery_id' + await ixm.create_index(bucket_name, + beer_sample_brewery_id_idx, + ('type', 'brewery_id'), + CreateQueryIndexOptions(deferred=True, + timeout=timedelta(seconds=120))) + + # get all the indexes for a bucket + indexes = await ixm.get_all_indexes(bucket_name) + + # we only care about the indexes that are deferred + ix_names = list(map(lambda i: i.name, [idx for idx in indexes if idx.state == 'deferred'])) + + print(f"Building indexes: {', '.join(ix_names)}") + # build the deferred indexes + await ixm.build_deferred_indexes(bucket_name) + + # lets wait until the indexes are built + wait_seconds = 30 + for _ in range(3): + try: + await ixm.watch_indexes(bucket_name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=wait_seconds))) + print('Indexes {} built!') + except WatchQueryIndexTimeoutException: + print(f"Indexes not build within {wait_seconds} seconds...") + + # now lets do query! + query_res = cluster.query(n1ql) + async for row in query_res.rows(): + print('Query row: {}'.format(row)) + + # time to drop the indexes + await ixm.drop_index(bucket_name, beer_sample_type_idx) + await ixm.drop_index(bucket_name, beer_sample_brewery_id_idx) + + try: + # Should get an exception if the index has already been dropped + await ixm.drop_index(bucket_name, beer_sample_brewery_id_idx) + except QueryIndexNotFoundException: + print('Query index already dropped') + + # can drop an index, and ignore if it doesn't exists + await ixm.drop_index(bucket_name, beer_sample_brewery_id_idx, DropQueryIndexOptions(ignore_if_not_exists=True)) + + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/acouchbase/management/acouchbase_usermgmt.py b/examples/acouchbase/management/acouchbase_usermgmt.py new file mode 100644 index 000000000..390f11133 --- /dev/null +++ b/examples/acouchbase/management/acouchbase_usermgmt.py @@ -0,0 +1,90 @@ +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, import ALL management options from `couchbase.management.options` +# from couchbase.management.queries import CreatePrimaryQueryIndexOptions +from couchbase.management.options import CreatePrimaryQueryIndexOptions +from couchbase.management.users import Role, User + + +async def main(): + bucket_name = "travel-sample" + username = "test-user" + pw = "test-passw0rd!" + + adm_cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) + await adm_cluster.on_connect() + # For Server versions 6.5 or later you do not need to open a bucket here + adm_bucket = adm_cluster.bucket(bucket_name) + await adm_bucket.on_connect() + + user_manager = adm_cluster.users() + user = User(username=username, display_name="Test User", + roles=[ + # Roles required for reading data from bucket + Role(name="data_reader", bucket="*"), + Role(name="query_select", bucket="*"), + # Roles require for writing data to bucket + Role(name="data_writer", bucket=bucket_name), + Role(name="query_insert", bucket=bucket_name), + Role(name="query_delete", bucket=bucket_name), + # Role required for idx creation on bucket + Role(name="query_manage_index", bucket=bucket_name), + ], password=pw) + + await user_manager.upsert_user(user) + + users_metadata = await user_manager.get_all_users() + for u in users_metadata: + print("User's display name: {}".format(u.user.display_name)) + roles = u.user.roles + for r in roles: + print( + "\tUser has role {}, applicable to bucket {}".format( + r.name, r.bucket)) + + user_cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator(username, pw)) + + await user_cluster.on_connect() + + # For Server versions 6.5 or later you do not need to open a bucket here + user_bucket = user_cluster.bucket(bucket_name) + await user_bucket.on_connect() + collection = user_bucket.default_collection() + + # create primary idx for testing purposes + await user_cluster.query_indexes().create_primary_index( + bucket_name, CreatePrimaryQueryIndexOptions( + ignore_if_exists=True)) + + # test k/v operations + airline_10 = await collection.get("airline_10") + print("Airline 10: {}".format(airline_10.content_as[dict])) + + airline_11 = { + "callsign": "MILE-AIR", + "iata": "Q5", + "id": 11, + "name": "40-Mile Air", + "type": "airline", + } + + await collection.upsert("airline_11", airline_11) + + query_res = user_cluster.query("SELECT * FROM `travel-sample` LIMIT 5;") + async for row in query_res.rows(): + print("Query row: {}".format(row)) + + # drop the user + await user_manager.drop_user(user.username) + + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/acouchbase/simple_async_txns.py b/examples/acouchbase/simple_async_txns.py new file mode 100644 index 000000000..2d3c03eaf --- /dev/null +++ b/examples/acouchbase/simple_async_txns.py @@ -0,0 +1,76 @@ +import asyncio +from datetime import timedelta +from typing import TYPE_CHECKING +from uuid import uuid4 + +from acouchbase.cluster import Cluster +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import TransactionFailed +from couchbase.options import ClusterOptions, TransactionConfig + +if TYPE_CHECKING: + from acouchbase.transactions import AttemptContext + +opts = ClusterOptions(authenticator=PasswordAuthenticator("Administrator", "password"), + transaction_config=TransactionConfig(expiration_time=timedelta(seconds=2))) + + +async def wait_for_all(tasks): + results = await asyncio.gather(*tasks, return_exceptions=True) + for r in results: + if isinstance(r, Exception): + raise r + return results + + +async def run(): + my_cluster = Cluster("couchbase://localhost", opts) + await my_cluster.on_connect() + + bucket = my_cluster.bucket("default") + await bucket.on_connect() + + coll = bucket.default_collection() + + key1 = str(uuid4()) + key2 = str(uuid4()) + + tasks = [coll.insert(key1, {"value": 10}), coll.insert(key2, {"value": 0})] + results = await wait_for_all(tasks) + print(f'insert results: {results}') + + async def txn_logic(ctx # type: AttemptContext + ): + t = [ctx.get(coll, key1), ctx.get(coll, key2)] + res = await wait_for_all(t) + print(f'get results: {results}') + doc1 = res[0] + doc2 = res[1] + doc1_content = doc1.content_as[dict] + doc2_content = doc2.content_as[dict] + print(f'doc1:{doc1}, doc2: {doc2}') + print(f'doc1_content: {doc1_content}, doc2_content: {doc2_content}') + if doc1_content["value"] > 0: + doc1_content["value"] = doc1_content["value"] - 1 + doc2_content["value"] = doc2_content["value"] + 1 + t = [ctx.replace(doc1, doc1_content), ctx.replace(doc2, doc2_content)] + await wait_for_all(t) + else: + raise RuntimeError("doc1 is exhausted") + + ok = True + while ok: + try: + print(f'txn_result: {await my_cluster.transactions.run(txn_logic)}') + except TransactionFailed as e: + print(f'txn raised exception: {e}') + ok = False + + tasks = [coll.get(key1), coll.get(key2)] + results = await wait_for_all(tasks) + print(f'after txns, we have {results}') + + +loop = asyncio.get_event_loop() +loop.run_until_complete(run()) +print('done') diff --git a/examples/acouchbase/vector_search_async.py b/examples/acouchbase/vector_search_async.py new file mode 100644 index 000000000..3f66e8f75 --- /dev/null +++ b/examples/acouchbase/vector_search_async.py @@ -0,0 +1,163 @@ +import asyncio +import json +import os +import pathlib +from typing import List, Optional +from uuid import uuid4 + +import couchbase.search as search +from acouchbase import get_event_loop +from acouchbase.cluster import AsyncCluster +from acouchbase.collection import AsyncCollection +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import DocumentNotFoundException, QueryIndexAlreadyExistsException +from couchbase.management.search import SearchIndex +from couchbase.options import ClusterOptions, SearchOptions +from couchbase.vector_search import VectorQuery, VectorSearch + +# NOTE: These paths should be updated accordingly if going to use. The paths are currently setup for running w/in +# the couchbase-python-client root directory. +# + +TEST_INDEX_NAME = 'test-search-vector-index' +TEST_COLL_INDEX_NAME = 'test-search-vector-coll-index' +TEST_PATH = os.path.join(pathlib.Path(__file__).parent, + 'tests', + 'test_cases') +TEST_INDEX_PATH = os.path.join(TEST_PATH, + f'{TEST_INDEX_NAME}-params.json') +TEST_COLL_INDEX_PATH = os.path.join(TEST_PATH, + f'{TEST_COLL_INDEX_NAME}-params.json') +TEST_VECTOR_SEARCH_DOCS_PATH = os.path.join(TEST_PATH, + 'test-vector-search-docs.json') +TEST_SEARCH_VECTOR_PATH = os.path.join(TEST_PATH, + 'test-search-vector.json') + + +async def check_doc_count(sixm, + idx_name: str, + min_count: int, + retries: Optional[int] = 20, + delay: Optional[int] = 30 + ) -> bool: + + indexed_docs = 0 + no_docs_cutoff = 300 + for i in range(retries): + # if no docs after waiting for a period of time, exit + if indexed_docs == 0 and i * delay >= no_docs_cutoff: + return 0 + indexed_docs = await sixm.get_indexed_documents_count(idx_name) + if indexed_docs >= min_count: + break + print(f'Found {indexed_docs} indexed docs, waiting a bit...') + await asyncio.sleep(delay) + + return indexed_docs + + +async def load_search_idx(cluster: AsyncCluster) -> None: + sixm = cluster.search_indexes() + params_json = None + with open(TEST_INDEX_PATH) as params_file: + input = params_file.read() + params_json = json.loads(input) + + if params_json is None: + print('Unabled to read search index params') + return + + idx = SearchIndex(name=TEST_INDEX_NAME, + idx_type='fulltext-index', + source_name='default', + source_type='couchbase', + params=params_json) + await sixm.upsert_index(idx) + indexed_docs = await check_doc_count(sixm, TEST_INDEX_NAME, 20, retries=10, delay=3) + print(f'Have {indexed_docs} indexed documents.') + + +async def drop_search_idx(cluster: AsyncCluster) -> None: + sixm = cluster.search_indexes() + await sixm.drop_index(TEST_INDEX_NAME) + + +def load_test_vector(): + vector = None + with open(TEST_SEARCH_VECTOR_PATH) as vector_file: + vector = json.loads(vector_file.read()) + + return vector + + +def read_docs(): + with open(TEST_VECTOR_SEARCH_DOCS_PATH) as input: + while line := input.readline(): + yield json.loads(line) + + +async def load_docs(root: str, collection: AsyncCollection) -> List[str]: + idx = 0 + keys = [] + for doc in read_docs(): + key = f'{root}_{idx}' + # the search index expects a type field w/ vector as the value + doc['type'] = 'vector' + await collection.upsert(key, doc) + keys.append(key) + idx += 1 + print(f'loaded {len(keys)} docs.') + return keys + + +async def remove_docs(keys: List[str], collection: AsyncCollection) -> None: + for key in keys: + try: + await collection.remove(key) + except DocumentNotFoundException: + pass + + +async def setup(cluster: AsyncCluster, collection: AsyncCollection) -> List[str]: + root = str(uuid4())[:8] + keys = await load_docs(root, collection) + try: + await load_search_idx(cluster) + except QueryIndexAlreadyExistsException: + pass + + return keys + + +async def teardown(cluster: AsyncCluster, collection: AsyncCollection, keys: List[str]) -> None: + await remove_docs(keys, collection) + await drop_search_idx(cluster) + + +async def main(): + auth = PasswordAuthenticator('Administrator', 'password') + opts = ClusterOptions(auth) + cluster = await AsyncCluster.connect('couchbase://localhost', opts) + bucket = cluster.bucket('default') + await bucket.on_connect() + collection = bucket.default_collection() + + # NOTE: validate paths (see note on line 18) if going to use setup/teardown functionality + keys = await setup(cluster, collection) + vector = load_test_vector() + + search_req = search.SearchRequest.create(search.MatchAllQuery()).with_vector_search( + VectorSearch.from_vector_query(VectorQuery('vector_field', vector))) + search_iter = cluster.search(TEST_INDEX_NAME, search_req, SearchOptions(limit=2)) + async for row in search_iter.rows(): + print(f'row: {row}') + + print(f'Metatdata: {search_iter.metadata()}') + + # NOTE: only use in conjunction w/ setup() method + await teardown(cluster, collection, keys) + + +if __name__ == '__main__': + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/basic.py b/examples/basic.py deleted file mode 100755 index 4b951b54d..000000000 --- a/examples/basic.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.bucket import Bucket -from couchbase import FMT_PICKLE -from couchbase.exceptions import KeyExistsError - - -# Connect to the default bucket on local host -cb = Bucket('couchbase://127.0.0.1/default') - -# If you want to store the Python objects pickled and not as JSON -#cb.default_format = FMT_PICKLE - -# Store a document -rv = cb.upsert('first', {'hello': 'world'}) -cas = rv.cas -print(rv) - -# Get the document -item = cb.get('first') -print(item) - -# Overwrite the existing document only if the CAS value matched -try: - # An exception will be raised if the CAS doesn't match - wrong_cas = cas + 123 - cb.upsert('first', {'hello': 'world', 'additional': True}, cas=wrong_cas) -except KeyExistsError: - # Get the correct current CAS value - rv = cb.get('first') - item, flags, correct_cas = rv.value, rv.flags, rv.cas - # Set it again, this time with the correct CAS value - rv = cb.upsert('first', - {'hello': 'world', 'additional': True}, - cas=correct_cas) - print(rv) - -# Delete the document only if the CAS value matches (it would also -# work without a cas value) -cb.remove('first', cas=rv.cas) - -# Make sure the document really got deleted -assert cb.get('first', quiet=True).success is False diff --git a/examples/bench.py b/examples/bench.py deleted file mode 100755 index 4c41f07aa..000000000 --- a/examples/bench.py +++ /dev/null @@ -1,129 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -#!/usr/bin/env python -import argparse -from threading import Thread -from time import sleep, time -from couchbase.bucket import Bucket, FMT_BYTES -from couchbase.transcoder import Transcoder - -ap = argparse.ArgumentParser() - -ap.add_argument('-t', '--threads', default=4, type=int, - help="Number of threads to spawn. 0 means no threads " - "but workload will still run in the main thread") - -ap.add_argument('-d', '--delay', default=0, type=float, - help="Number of seconds to wait between each op. " - "may be a fraction") - -ap.add_argument('-U', '--connstr', default='couchbase://localhost/default', - help="Connection string") - -ap.add_argument('-p', '--password', default=None, type=str) - -ap.add_argument('-D', '--duration', default=10, type=int, - help="Duration of run (in seconds)") -ap.add_argument('-T', '--transcoder', default=False, - action='store_true', - help="Use the Transcoder object rather than built-in " - "conversion routines") - -ap.add_argument('--ksize', default=12, type=int, - help="Key size to use") - -ap.add_argument('--vsize', default=128, type=int, - help="Value size to use") -ap.add_argument('--iops', default=False, action='store_true', - help="Use Pure-Python IOPS plugin") - -ap.add_argument('--batch', '-N', default=1, type=int, - help="Number of commands to schedule per iteration") - -options = ap.parse_args() -DO_UNLOCK_GIL = options.threads > 0 -TC = Transcoder() - - -class Worker(Thread): - def __init__(self): - self.delay = options.delay - self.key = 'K' * options.ksize - self.value = b'V' * options.vsize - self.kv = {} - for x in range(options.batch): - self.kv[self.key + str(x)] = self.value - self.wait_time = 0 - self.opcount = 0 - connopts = { "connstr" : options.connstr, - "unlock_gil": DO_UNLOCK_GIL } - if options.iops: - connopts["experimental_gevent_support"] = True - - self.cb = Bucket(**connopts) - - if options.transcoder: - self.cb.transcoder = TC - self.end_time = time() + options.duration - super(Worker, self).__init__() - - def run(self, *args, **kwargs): - cb = self.cb - - while time() < self.end_time: - begin_time = time() - rv = cb.upsert_multi(self.kv, format=FMT_BYTES) - assert rv.all_ok, "Operation failed: " - self.wait_time += time() - begin_time - - if self.delay: - sleep(self.delay) - - self.opcount += options.batch - - -global_begin = None -worker_threads = [] -if not options.threads: - # No threding requested: - w = Worker() - worker_threads.append(w) - global_begin = time() - w.run() -else: - for x in range(options.threads): - worker_threads.append(Worker()) - - global_begin = time() - for t in worker_threads: - t.start() - - for t in worker_threads: - t.join() - -global_duration = time() - global_begin -total_ops = sum([w.opcount for w in worker_threads]) -total_time = 0 -for t in worker_threads: - total_time += t.wait_time - -print("Total run took an absolute time of %0.2f seconds" % (global_duration,)) -print("Did a total of %d operations" % (total_ops,)) -print("Total wait time of %0.2f seconds" % (total_time,)) -print("[WAIT] %0.2f ops/second" % (float(total_ops)/float(total_time),)) -print("[ABS] %0.2f ops/second" % (float(total_ops)/float(global_duration),)) diff --git a/examples/connection-pool.py b/examples/connection-pool.py deleted file mode 100755 index 493279fe0..000000000 --- a/examples/connection-pool.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -""" -This file shows how to make a simple connection pool using Couchbase. -""" -from couchbase.bucket import Bucket -from Queue import Queue, Empty -from threading import Lock, Thread -from time import time -from argparse import ArgumentParser - - -class ClientUnavailableError(Exception): - pass - - -class BucketWrapper(Bucket): - """ - This is a simple subclass which adds usage statistics to inspect later on - """ - def __init__(self, **kwargs): - super(BucketWrapper, self).__init__(**kwargs) - self.use_count = 0 - self.use_time = 0 - self.last_use_time = 0 - - def start_using(self): - self.last_use_time = time() - - def stop_using(self): - self.use_time += time() - self.last_use_time - self.use_count += 1 - - -class Pool(object): - def __init__(self, initial=4, max_clients=10, **connargs): - """ - Create a new pool - :param int initial: The initial number of client objects to create - :param int max_clients: The maximum amount of clients to create. These - clients will only be created on demand and will potentially be - destroyed once they have been returned via a call to - :meth:`release_client` - :param connargs: Extra arguments to pass to the Connection object's - constructor - """ - self._q = Queue() - self._l = [] - self._connargs = connargs - self._cur_clients = 0 - self._max_clients = max_clients - self._lock = Lock() - - for x in range(initial): - self._q.put(self._make_client()) - self._cur_clients += 1 - - def _make_client(self): - ret = BucketWrapper(**self._connargs) - self._l.append(ret) - return ret - - def get_client(self, initial_timeout=0.05, next_timeout=200): - """ - Wait until a client instance is available - :param float initial_timeout: - how long to wait initially for an existing client to complete - :param float next_timeout: - if the pool could not obtain a client during the initial timeout, - and we have allocated the maximum available number of clients, wait - this long until we can retrieve another one - - :return: A connection object - """ - try: - return self._q.get(True, initial_timeout) - except Empty: - try: - self._lock.acquire() - if self._cur_clients == self._max_clients: - raise ClientUnavailableError("Too many clients in use") - cb = self._make_client() - self._cur_clients += 1 - cb.start_using() - return cb - except ClientUnavailableError as ex: - try: - return self._q.get(True, next_timeout) - except Empty: - raise ex - finally: - self._lock.release() - - def release_client(self, cb): - """ - Return a Connection object to the pool - :param Connection cb: the client to release - """ - cb.stop_using() - self._q.put(cb, True) - - -class CbThread(Thread): - def __init__(self, pool, opcount=10, remaining=10000): - super(CbThread, self).__init__() - self.pool = pool - self.remaining = remaining - self.opcount = opcount - - def run(self): - while self.remaining: - cb = self.pool.get_client() - kv = dict( - ("Key_{0}".format(x), str(x)) for x in range(self.opcount) - ) - cb.upsert_multi(kv) - self.pool.release_client(cb) - self.remaining -= 1 - - -def main(): - - ap = ArgumentParser() - ap.add_argument('-U', '--connstr', help="Connection string", - default='couchbase://localhost/default') - ap.add_argument("-O", "--opcount", help="How many operations to perform " - "at once", type=int, - default=10) - ap.add_argument('--pool-min', - help="Minimum pool size", default=4, type=int) - ap.add_argument('--pool-max', - help="Maximum pool size", default=10, type=int) - ap.add_argument('-t', '--threads', type=int, default=10, - help="Number of threads to launch") - - options = ap.parse_args() - - pool = Pool(initial=options.pool_min, - max_clients=options.pool_max, - connstr=options.connstr) - - thrs = [ - CbThread(pool, opcount=options.opcount) for _ in range(options.threads) - ] - - map(lambda thr: thr.start(), thrs) - map(lambda thr: thr.join(), thrs) - - for c in pool._l: - print "Have client {0}".format(c) - print "\tTime In Use: {0}, use count: {1}".format(c.use_time, - c.use_count) - - -if __name__ == "__main__": - main() diff --git a/examples/couchbase/couchbase_query_operations.py b/examples/couchbase/couchbase_query_operations.py new file mode 100644 index 000000000..3a6af9f56 --- /dev/null +++ b/examples/couchbase/couchbase_query_operations.py @@ -0,0 +1,123 @@ +import uuid + +from acouchbase.cluster import Cluster, get_event_loop +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, import ALL options from `couchbase.options` +from couchbase.cluster import (ClusterOptions, + QueryOptions, + QueryScanConsistency) +from couchbase.exceptions import ParsingFailedException +from couchbase.mutation_state import MutationState + +# use this to not have deprecation warnings +# from couchbase.options import ClusterOptions, QueryOptions +# from couchbase.n1ql import QueryScanConsistency + + +async def main(): + cluster = await Cluster.connect('couchbase://localhost', + ClusterOptions(PasswordAuthenticator('Administrator', 'password'))) + bucket = cluster.bucket("travel-sample") + collection = bucket.default_collection() + + # basic query + try: + result = cluster.query( + "SELECT * FROM `travel-sample` LIMIT 10;", QueryOptions(metrics=True)) + + async for row in result.rows(): + print(f'Found row: {row}') + + metrics = result.metadata().metrics() + print(f'Query execution time: {metrics.execution_time()}') + + except ParsingFailedException as ex: + import traceback + traceback.print_exc() + + # positional params + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`=$1 LIMIT 10" + result = cluster.query(q_str, "hotel") + rows = [r async for r in result] + + # positional params via QueryOptions + result = cluster.query(q_str, QueryOptions(positional_parameters=["hotel"])) + rows = [r async for r in result] + + # named params + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`=$doc_type LIMIT 10" + result = cluster.query(q_str, doc_type='hotel') + rows = [r async for r in result] + + # name params via QueryOptions + result = cluster.query(q_str, QueryOptions(named_parameters={'doc_type': 'hotel'})) + rows = [r async for r in result] + + # iterate over result/rows + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10" + result = cluster.query(q_str) + + # iterate over rows + async for row in result: + # each row is an serialized JSON object + name = row["name"] + callsign = row["callsign"] + print(f'Airline name: {name}, callsign: {callsign}') + + # query metrics + result = cluster.query("SELECT 1=1", QueryOptions(metrics=True)) + await result.execute() + + print("Execution time: {}".format( + result.metadata().metrics().execution_time())) + + # print scan consistency + result = cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10", + QueryOptions(scan_consistency=QueryScanConsistency.REQUEST_PLUS)) + rows = [r async for r in result] + + # Read your own writes + new_airline = { + "callsign": None, + "country": "United States", + "iata": "TX", + "icao": "TX99", + "id": 123456789, + "name": "Howdy Airlines", + "type": "airline" + } + + res = await collection.upsert( + "airline_{}".format(new_airline["id"]), new_airline) + + ms = MutationState(res) + + result = cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10", + QueryOptions(consistent_with=ms)) + rows = [r async for r in result] + + # client context id + result = cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='hotel' LIMIT 10", + QueryOptions(client_context_id="user-44{}".format(uuid.uuid4()))) + rows = [r async for r in result] + + # read only + result = cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='hotel' LIMIT 10", + QueryOptions(read_only=True)) + rows = [r async for r in result] + + agent_scope = bucket.scope("inventory") + + result = agent_scope.query( + "SELECT a.* FROM `airline` a WHERE a.country=$country LIMIT 10", + country='France') + rows = [r async for r in result] + +if __name__ == "__main__": + loop = get_event_loop() + loop.run_until_complete(main()) diff --git a/examples/couchbase/diagnostics_operations.py b/examples/couchbase/diagnostics_operations.py new file mode 100644 index 000000000..2e0706423 --- /dev/null +++ b/examples/couchbase/diagnostics_operations.py @@ -0,0 +1,65 @@ +from datetime import timedelta + +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, use: from couchbase.options import PingOptions +from couchbase.bucket import PingOptions +from couchbase.cluster import Cluster +from couchbase.diagnostics import PingState, ServiceType +from couchbase.exceptions import UnAmbiguousTimeoutException +from couchbase.options import WaitUntilReadyOptions + + +def ok(cluster): + result = cluster.ping() + for _, reports in result.endpoints.items(): + for report in reports: + if not report.state == PingState.OK: + return False + return True + + +cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) + +cluster_ready = False +try: + cluster.wait_until_ready(timedelta(seconds=3), + WaitUntilReadyOptions(service_types=[ServiceType.KeyValue, ServiceType.Query])) + cluster_ready = True +except UnAmbiguousTimeoutException as ex: + print('Cluster not ready in time: {}'.format(ex)) + +if cluster_ready is False: + quit() + +# For Server versions 6.5 or later you do not need to open a bucket here +bucket = cluster.bucket("beer-sample") +collection = bucket.default_collection() + +ping_result = cluster.ping() + +for endpoint, reports in ping_result.endpoints.items(): + for report in reports: + print( + "{0}: {1} took {2}".format( + endpoint.value, + report.remote, + report.latency)) + +ping_result = cluster.ping() +print(ping_result.as_json()) + +print("Cluster is okay? {}".format(ok(cluster))) + +ping_result = cluster.ping(PingOptions(service_types=[ServiceType.Query])) +print(ping_result.as_json()) + + +diag_result = cluster.diagnostics() +print(diag_result.as_json()) + +print("Cluster state: {}".format(diag_result.state)) diff --git a/examples/couchbase/kv_operations.py b/examples/couchbase/kv_operations.py new file mode 100644 index 000000000..75dc597bd --- /dev/null +++ b/examples/couchbase/kv_operations.py @@ -0,0 +1,146 @@ +from datetime import timedelta + +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster + +# **DEPRECATED**, import ALL options from `couchbase.options` +from couchbase.collection import (DecrementOptions, + DeltaValue, + GetOptions, + IncrementOptions, + InsertOptions, + RemoveOptions, + ReplaceOptions, + SignedInt64, + UpsertOptions) +from couchbase.durability import (ClientDurability, + Durability, + PersistTo, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import CASMismatchException, CouchbaseException + +cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) +bucket = cluster.bucket("default") +collection = bucket.default_collection() + +# setup +for key in ['document-key', 'document-key-opts']: + try: + result = collection.remove(key) + except CouchbaseException as ex: + pass # may not exist in this example + +# Insert document +document = {"foo": "bar", "bar": "foo"} +result = collection.insert("document-key", document) +print("Result: {}; CAS: {}".format(result, result.cas)) + +# Insert document with options +document = {"foo": "bar", "bar": "foo"} +opts = InsertOptions(timeout=timedelta(seconds=5)) +result = collection.insert("document-key-opts", + document, + opts, + expiry=timedelta(seconds=30)) + +try: + # Replace document with CAS + document = {"foo": "bar", "bar": "foo"} + result = collection.replace( + "document-key", + document, + cas=12345, + timeout=timedelta( + minutes=1)) +except CASMismatchException as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print('Caught CAS mismatch: {}'.format(ex)) + +try: + # Replace document with CAS + result = collection.get("document-key") + doc = result.content_as[dict] + doc["bar"] = "baz" + opts = ReplaceOptions(cas=result.cas) + result = collection.replace("document-key", doc, opts) +except CouchbaseException as ex: + print('Caught Couchbase exception: {}'.format(ex)) + +try: + # Upsert with Durability (Couchbase Server >= 6.5) level Majority + document = dict(foo="bar", bar="foo") + opts = UpsertOptions(durability=ServerDurability(Durability.MAJORITY)) + result = collection.upsert("document-key", document, opts) +except CouchbaseException as ex: + print('Caught Couchbase exception: {}'.format(ex)) + +# @TODO: couchbase++ doesn't implement observe based durability +# try: +# # Upsert with observe based durability (Couchbase Server < 6.5) +# document = {"foo": "bar", "bar": "foo"} +# opts = UpsertOptions( +# durability=ClientDurability( +# ReplicateTo.ONE, +# PersistTo.ONE)) +# result = collection.upsert("document-key", document, opts) +# except CouchbaseException as ex: +# print(ex) + +result = collection.get("document-key") +print(result.content_as[dict]) + +opts = GetOptions(timeout=timedelta(seconds=5)) +result = collection.get("document-key", opts) +print(result.content_as[dict]) + +try: + # remove document with options + result = collection.remove( + "document-key", + RemoveOptions( + cas=12345, + durability=ServerDurability( + Durability.MAJORITY))) +except CouchbaseException as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print('Caught Couchbase exception: {}'.format(ex)) + +result = collection.touch("document-key", timedelta(seconds=10)) + +result = collection.get("document-key", GetOptions(with_expiry=True)) +print("Expiry of result: {}".format(result.expiryTime)) + +result = collection.get_and_touch("document-key", timedelta(seconds=10)) + +# Increment binary value by 1 +collection.binary().increment( + "counter-key", + IncrementOptions( + delta=DeltaValue(1))) + +# Increment binary value by 5, if key doesn't exist, seed it at 1000 +collection.binary().increment( + "counter-key", + IncrementOptions( + delta=DeltaValue(5), + initial=SignedInt64(1000))) + +# Decrement binary value by 1 +collection.binary().decrement( + "counter-key", + DecrementOptions( + delta=DeltaValue(1))) + +# Decrement binary value by 2, if key doesn't exist, seed it at 1000 +collection.binary().decrement( + "counter-key", + DecrementOptions( + delta=DeltaValue(2), + initial=SignedInt64(1000))) diff --git a/examples/couchbase/management/bucketmgmt.py b/examples/couchbase/management/bucketmgmt.py new file mode 100644 index 000000000..9b762b72c --- /dev/null +++ b/examples/couchbase/management/bucketmgmt.py @@ -0,0 +1,74 @@ +import time + +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.exceptions import BucketAlreadyExistsException, BucketDoesNotExistException +from couchbase.management.buckets import (BucketSettings, + BucketType, + ConflictResolutionType, + CreateBucketSettings) + + +def retry(func, *args, back_off=0.5, limit=5, **kwargs): + for i in range(limit): + try: + return func(*args, **kwargs) + except Exception: + print("Retry in {} seconds...".format((i + 1) * back_off)) + time.sleep((i + 1) * back_off) + + raise Exception( + "Unable to successfully receive result from {}".format(func)) + + +cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) +# For Server versions 6.5 or later you do not need to open a bucket here +bucket = cluster.bucket("default") +collection = bucket.default_collection() + +bucket_manager = cluster.buckets() + +bucket_manager.create_bucket( + CreateBucketSettings( + name="hello", + flush_enabled=False, + ram_quota_mb=100, + num_replicas=0, + bucket_type=BucketType.COUCHBASE, + conflict_resolution_type=ConflictResolutionType.SEQUENCE_NUMBER)) + +# Another approach to catch if bucket already exists +try: + bucket_manager.create_bucket( + CreateBucketSettings( + name="hello", + flush_enabled=False, + ram_quota_mb=100, + num_replicas=0, + bucket_type=BucketType.COUCHBASE, + conflict_resolution_type=ConflictResolutionType.SEQUENCE_NUMBER)) +except BucketAlreadyExistsException: + print("{} bucket previously created.".format("hello")) + +# Creating a bucket can take some time depending on what +# the cluster is doing, good to use retries +bucket = retry(bucket_manager.get_bucket, "hello") + +bucket = bucket_manager.get_bucket("hello") +print("Found bucket: {}".format(bucket.name)) + +bucket_manager.update_bucket(BucketSettings(name="hello", flush_enabled=True)) + +bucket_manager.flush_bucket("hello") + +bucket_manager.drop_bucket("hello") + +# verify bucket dropped +try: + bucket_manager.get_bucket("hello") +except BucketDoesNotExistException: + print("{} bucket dropped.".format("hello")) diff --git a/examples/couchbase/management/collectionmgmt.py b/examples/couchbase/management/collectionmgmt.py new file mode 100644 index 000000000..69505a740 --- /dev/null +++ b/examples/couchbase/management/collectionmgmt.py @@ -0,0 +1,80 @@ +import time + +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.exceptions import (CollectionAlreadyExistsException, + CollectionNotFoundException, + ScopeAlreadyExistsException, + ScopeNotFoundException) +from couchbase.management.collections import CollectionSpec + + +def retry(func, *args, back_off=0.5, limit=5, **kwargs): + for i in range(limit): + try: + return func(*args, **kwargs) + except Exception: + print("Retry in {} seconds...".format((i + 1) * back_off)) + time.sleep((i + 1) * back_off) + + raise Exception( + "Unable to successfully receive result from {}".format(func)) + + +def get_scope(collection_mgr, scope_name): + return next((s for s in collection_mgr.get_all_scopes() + if s.name == scope_name), None) + + +def get_collection(collection_mgr, scope_name, coll_name): + scope = get_scope(collection_mgr, scope_name) + if scope: + return next( + (c for c in scope.collections if c.name == coll_name), + None) + + return None + + +cluster = Cluster( + "couchbase://ec2-54-213-122-253.us-west-2.compute.amazonaws.com", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) + +bucket = cluster.bucket("default") +coll_manager = bucket.collections() + +try: + coll_manager.create_scope("example-scope") +except ScopeAlreadyExistsException as ex: + print(ex) + +scope = retry(get_scope, coll_manager, "example-scope") +print("Found scope: {}".format(scope.name)) + +collection_spec = CollectionSpec( + "example-collection", + scope_name="example-scope") + +try: + collection = coll_manager.create_collection(collection_spec) +except CollectionAlreadyExistsException as ex: + print(ex) + +collection = retry( + get_collection, + coll_manager, + "example-scope", + "example-collection") +print("Found collection: {}".format(collection.name)) + +try: + coll_manager.drop_collection(collection_spec) +except CollectionNotFoundException as ex: + print(ex) + +try: + coll_manager.drop_scope("example-scope") +except ScopeNotFoundException as ex: + print(ex) diff --git a/examples/couchbase/management/querymgmt.py b/examples/couchbase/management/querymgmt.py new file mode 100644 index 000000000..40ce0cb34 --- /dev/null +++ b/examples/couchbase/management/querymgmt.py @@ -0,0 +1,101 @@ +from datetime import timedelta + +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.exceptions import (QueryIndexAlreadyExistsException, + QueryIndexNotFoundException, + WatchQueryIndexTimeoutException) + +# **DEPRECATED**, import ALL management options from `couchbase.management.options` +# from couchbase.management.queries import (CreatePrimaryQueryIndexOptions, +# CreateQueryIndexOptions, +# DropQueryIndexOptions, +# WatchQueryIndexOptions) +from couchbase.management.options import (CreatePrimaryQueryIndexOptions, + CreateQueryIndexOptions, + DropQueryIndexOptions, + WatchQueryIndexOptions) + +cluster = Cluster( + 'couchbase://localhost', + authenticator=PasswordAuthenticator( + 'Administrator', + 'password')) + +bucket_name = 'beer-sample' + +ixm = cluster.query_indexes() +try: + # expect exception as the travel-sample bucket should already come w/ a primary index + ixm.create_primary_index(bucket_name) +except QueryIndexAlreadyExistsException: + print('Primary index already exists') + +# can create an index, and ignore if it already exists +ixm.create_primary_index(bucket_name, CreatePrimaryQueryIndexOptions(ignore_if_exists=True)) + + +n1ql = ''' +SELECT brewery.name, + beer.beerCount, + beer.brewery_id +FROM `beer-sample` AS brewery + JOIN ( + SELECT brewery_id, + COUNT(1) AS beerCount + FROM `beer-sample` + WHERE `type` = 'beer' + GROUP BY brewery_id) AS beer ON beer.brewery_id = META(brewery).id +WHERE brewery.`type` = 'brewery' +LIMIT 10; +''' + +# lets create some secondary indexes that let the above query run +# w/o needing to do a primary index scan +beer_sample_type_idx = 'beer_sample_type' +fields = ['type'] +ixm.create_index(bucket_name, beer_sample_type_idx, fields, + CreateQueryIndexOptions(deferred=True, timeout=timedelta(seconds=120))) + +beer_sample_brewery_id_idx = 'beer_sample_type_brewery_id' +ixm.create_index(bucket_name, beer_sample_brewery_id_idx, ('type', 'brewery_id'), + CreateQueryIndexOptions(deferred=True, timeout=timedelta(seconds=120))) + +# get all the indexes for a bucket +indexes = ixm.get_all_indexes(bucket_name) + +# we only care about the indexes that are deferred +ix_names = list(map(lambda i: i.name, [idx for idx in indexes if idx.state == 'deferred'])) + +print(f"Building indexes: {', '.join(ix_names)}") +# build the deferred indexes +ixm.build_deferred_indexes(bucket_name) + +# lets wait until the indexes are built +wait_seconds = 30 +for _ in range(3): + try: + ixm.watch_indexes(bucket_name, + ix_names, + WatchQueryIndexOptions(timeout=timedelta(seconds=wait_seconds))) + print('Indexes {} built!') + except WatchQueryIndexTimeoutException: + print(f"Indexes not build within {wait_seconds} seconds...") + +# now lets do query! +query_res = cluster.query(n1ql) +for row in query_res.rows(): + print('Query row: {}'.format(row)) + +# time to drop the indexes +ixm.drop_index(bucket_name, beer_sample_type_idx) +ixm.drop_index(bucket_name, beer_sample_brewery_id_idx) + +try: + # Should get an exception if the index has already been dropped + ixm.drop_index(bucket_name, beer_sample_brewery_id_idx) +except QueryIndexNotFoundException: + print('Query index already dropped') + +# can drop an index, and ignore if it doesn't exists +ixm.drop_index(bucket_name, beer_sample_brewery_id_idx, DropQueryIndexOptions(ignore_if_not_exists=True)) diff --git a/examples/couchbase/management/usermgmt.py b/examples/couchbase/management/usermgmt.py new file mode 100644 index 000000000..8ad7d881c --- /dev/null +++ b/examples/couchbase/management/usermgmt.py @@ -0,0 +1,78 @@ +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster + +# **DEPRECATED**, import ALL management options from `couchbase.management.options` +# from couchbase.management.queries import CreatePrimaryQueryIndexOptions +from couchbase.management.options import CreatePrimaryQueryIndexOptions +from couchbase.management.users import Role, User + +bucket_name = "travel-sample" +username = "test-user" +pw = "test-passw0rd!" + +adm_cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) +# For Server versions 6.5 or later you do not need to open a bucket here +adm_bucket = adm_cluster.bucket(bucket_name) + +user_manager = adm_cluster.users() +user = User(username=username, display_name="Test User", + roles=[ + # Roles required for reading data from bucket + Role(name="data_reader", bucket="*"), + Role(name="query_select", bucket="*"), + # Roles require for writing data to bucket + Role(name="data_writer", bucket=bucket_name), + Role(name="query_insert", bucket=bucket_name), + Role(name="query_delete", bucket=bucket_name), + # Role required for idx creation on bucket + Role(name="query_manage_index", bucket=bucket_name), + ], password=pw) + +user_manager.upsert_user(user) + +users_metadata = user_manager.get_all_users() +for u in users_metadata: + print("User's display name: {}".format(u.user.display_name)) + roles = u.user.roles + for r in roles: + print( + "\tUser has role {}, applicable to bucket {}".format( + r.name, r.bucket)) + +user_cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator(username, pw)) + +# For Server versions 6.5 or later you do not need to open a bucket here +user_bucket = user_cluster.bucket(bucket_name) +collection = user_bucket.default_collection() + +# create primary idx for testing purposes +user_cluster.query_indexes().create_primary_index( + bucket_name, CreatePrimaryQueryIndexOptions( + ignore_if_exists=True)) + +# test k/v operations +airline_10 = collection.get("airline_10") +print("Airline 10: {}".format(airline_10.content_as[dict])) + +airline_11 = { + "callsign": "MILE-AIR", + "iata": "Q5", + "id": 11, + "name": "40-Mile Air", + "type": "airline", +} + +collection.upsert("airline_11", airline_11) + +query_res = user_cluster.query("SELECT * FROM `travel-sample` LIMIT 5;") +for row in query_res.rows(): + print("Query row: {}".format(row)) + +# drop the user +user_manager.drop_user(user.username) diff --git a/examples/couchbase/simple_txns.py b/examples/couchbase/simple_txns.py new file mode 100644 index 000000000..6c5171a44 --- /dev/null +++ b/examples/couchbase/simple_txns.py @@ -0,0 +1,57 @@ +import logging +from datetime import timedelta +from sys import stdout +from typing import TYPE_CHECKING +from uuid import uuid4 + +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.exceptions import TransactionFailed +from couchbase.options import ClusterOptions, TransactionConfig + +if TYPE_CHECKING: + from couchbase.transactions import AttemptContext + +logging.basicConfig(stream=stdout, level=logging.DEBUG, format='%(asctime)s|%(levelname)s|%(name)s| %(message)s') + +opts = ClusterOptions(authenticator=PasswordAuthenticator("Administrator", "password"), + transaction_config=TransactionConfig(expiration_time=timedelta(seconds=2))) + +my_cluster = Cluster("couchbase://localhost", opts) + +coll = my_cluster.bucket("default").default_collection() + +key1 = str(uuid4()) +key2 = str(uuid4()) + +coll.insert(key1, {"value": 10}) +coll.insert(key2, {"value": 0}) + + +def txn_logic(ctx # type: AttemptContext + ): + doc1 = ctx.get(coll, key1) + doc2 = ctx.get(coll, key2) + doc1_content = doc1.content_as[dict] + doc2_content = doc2.content_as[dict] + if doc1_content["value"] > 0: + doc1_content["value"] = doc1_content["value"] - 1 + doc2_content["value"] = doc2_content["value"] + 1 + ctx.replace(doc1, doc1_content) + ctx.replace(doc2, doc2_content) + else: + raise RuntimeError("doc1 is exhausted") + + +ok = True +logging.info(f'doc 1 starts off as: {coll.get(key1).content_as[dict]}') +logging.info(f'doc 2 starts off as: {coll.get(key2).content_as[dict]}') +while ok: + try: + logging.info(f'txn_result: {my_cluster.transactions.run(txn_logic)}') + except TransactionFailed as e: + logging.info(f'got {e}') + ok = False + +logging.info(f'doc 1 is now: {coll.get(key1).content_as[dict]}') +logging.info(f'doc 2 is now: {coll.get(key2).content_as[dict]}') diff --git a/examples/couchbase/subdoc_operations.py b/examples/couchbase/subdoc_operations.py new file mode 100644 index 000000000..01d1d1d49 --- /dev/null +++ b/examples/couchbase/subdoc_operations.py @@ -0,0 +1,264 @@ +# @TODO: change back to subdocument to match 3.x??? +import couchbase.subdocument as SD +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster + +# import couchbase.subdocument as SD +# **DEPRECATED**, use from couchbase.options import MutateInOptions +from couchbase.collection import MutateInOptions +from couchbase.durability import (ClientDurability, + Durability, + PersistTo, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import (CASMismatchException, + CouchbaseException, + DocumentExistsException, + DurabilityImpossibleException, + PathExistsException, + PathNotFoundException, + SubdocCantInsertValueException, + SubdocPathMismatchException) + +cluster = Cluster( + "couchbase://localhost", + authenticator=PasswordAuthenticator( + "Administrator", + "password")) +bucket = cluster.bucket("default") +collection = bucket.default_collection() + +json_doc = { + "name": "Douglas Reynholm", + "email": "douglas@reynholmindustries.com", + "addresses": { + "billing": { + "line1": "123 Any Street", + "line2": "Anytown", + "country": "United Kingdom" + }, + "delivery": { + "line1": "123 Any Street", + "line2": "Anytown", + "country": "United Kingdom" + } + }, + "purchases": { + "complete": [ + 339, 976, 442, 666 + ], + "abandoned": [ + 157, 42, 999 + ] + } +} + +try: + collection.insert("customer123", json_doc) +except DocumentExistsException: + collection.remove("customer123") + collection.insert("customer123", json_doc) + +result = collection.lookup_in("customer123", + [SD.get("addresses.delivery.country")]) +country = result.content_as[str](0) # "United Kingdom" +print(country) + +result = collection.lookup_in( + "customer123", [ + SD.exists("purchases.pending[-1]")]) +print('Path exists: {}.'.format(result.exists(0))) +# Path exists: False. + +# NOTE: result.content_as[bool](0) raises a PathNotFoundException +# this is b/c when checking if a path exists +# no content is returned + +try: + print("Path exists {}.".format(result.content_as[bool](0))) +except PathNotFoundException: + print("Path does not exist") + +result = collection.lookup_in( + "customer123", + [SD.get("addresses.delivery.country"), + SD.exists("purchases.complete[-1]")]) + +print("{0}".format(result.content_as[str](0))) +print("Path exists: {}.".format(result.exists(1))) +# path exists: True. + +collection.mutate_in("customer123", [SD.upsert("fax", "311-555-0151")]) + +collection.mutate_in( + "customer123", [SD.insert("purchases.pending", [42, True, "None"])]) + +try: + collection.mutate_in( + "customer123", [ + SD.insert( + "purchases.complete", + [42, True, "None"])]) +except PathExistsException: + print("Path exists, cannot use insert.") + +collection.mutate_in( + "customer123", + (SD.remove("addresses.billing"), + SD.replace( + "email", + "dougr96@hotmail.com"))) + +# NOTE: the mutate_in() operation expects a tuple or list +collection.mutate_in( + "customer123", (SD.array_append( + "purchases.complete", 777),)) +# purchases.complete is now [339, 976, 442, 666, 777] + +collection.mutate_in( + "customer123", [ + SD.array_prepend( + "purchases.abandoned", 18)]) +# purchases.abandoned is now [18, 157, 42, 999] + +collection.upsert("my_array", []) +collection.mutate_in("my_array", [SD.array_append("", "some element")]) +# the document my_array is now ["some element"] + +collection.mutate_in( + "my_array", [ + SD.array_append( + "", "elem1", "elem2", "elem3")]) + +# the document my_array is now ["some_element", "elem1", "elem2", "elem3"] + +collection.mutate_in( + "my_array", [ + SD.array_append( + "", ["elem4", "elem5", "elem6"])]) + +# the document my_array is now ["some_element", "elem1", "elem2", "elem3", +# ["elem4", "elem5", "elem6"]]] + +collection.mutate_in( + "my_array", (SD.array_append("", "elem7"), + SD.array_append("", "elem8"), + SD.array_append("", "elem9"))) + + +collection.upsert("some_doc", {}) +collection.mutate_in( + "some_doc", [ + SD.array_prepend( + "some.array", "Hello", "World", create_parents=True)]) +# the document some_doc is now {"some":{"array":["Hello", "World"]}} +try: + collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", 95)]) + print('Success!') +except PathExistsException: + print('Path already exists.') + +try: + collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", 95)]) + print('Success!') +except PathExistsException: + print('Path already exists.') + +# cannot add JSON obj w/ array_addunique +try: + collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", {"new": "object"})]) +except SubdocCantInsertValueException as ex: + print("Cannot add JSON value w/ array_addunique.", ex) + + +# cannot use array_addunique if array contains JSON obj +collection.mutate_in( + "customer123", [ + SD.upsert( + "purchases.cancelled", [{"Date": "Some date"}])]) + +try: + collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.cancelled", 89)]) +except SubdocPathMismatchException as ex: + print("Cannot use array_addunique if array contains JSON objs.", ex) + +collection.upsert("array", []) +collection.mutate_in("array", [SD.array_append("", "hello", "world")]) +collection.mutate_in("array", [SD.array_insert("[1]", "cruel")]) + +# exception raised if attempt to insert in out of bounds position +try: + collection.mutate_in("array", [SD.array_insert("[6]", "!")]) +except PathNotFoundException: + print("Cannot insert to out of bounds index.") + +# # can insert into nested arrays as long as the path is appropriate +collection.mutate_in("array", [SD.array_append("", ["another", "array"])]) +collection.mutate_in("array", [SD.array_insert("[3][2]", "!")]) + + +result = collection.mutate_in("customer123", (SD.counter("logins", 1),)) +num_logins = collection.get("customer123").content_as[dict]["logins"] +print('Number of logins: {}.'.format(num_logins)) +# Number of logins: 1. + +collection.upsert("player432", {"gold": 1000}) + +collection.mutate_in("player432", (SD.counter("gold", -150),)) +result = collection.lookup_in("player432", (SD.get("gold"),)) +print("{} has {} gold remaining.".format( + "player432", result.content_as[int](0))) +# player432 has 850 gold remaining. + +collection.mutate_in("customer123", [SD.upsert("level_0.level_1.foo.bar.phone", + dict( + num="311-555-0101", + ext=16 + ), create_parents=True)]) + +collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 998)]) + +collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 999)]) + +try: + collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 999)], + MutateInOptions(cas=1234)) +except (DocumentExistsException, CASMismatchException) as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print(ex) + +# @TODO: couchbase++ doesn't implement observe based durability +# try: +# collection.mutate_in( +# "key", [SD.insert("username", "dreynholm")], +# MutateInOptions(durability=ClientDurability( +# ReplicateTo.ONE, +# PersistTo.ONE))) +# except CouchbaseException as ex: +# print('Need to have more than 1 node for durability') +# print(ex) + +try: + collection.mutate_in( + "customer123", [SD.insert("username", "dreynholm")], + MutateInOptions(durability=ServerDurability( + Durability.MAJORITY))) +except DurabilityImpossibleException as ex: + print('Need to have more than 1 node for durability') + print(ex) diff --git a/examples/couchbase/vector_search.py b/examples/couchbase/vector_search.py new file mode 100644 index 000000000..a78a7fd94 --- /dev/null +++ b/examples/couchbase/vector_search.py @@ -0,0 +1,160 @@ +import json +import os +import pathlib +from time import sleep +from typing import List, Optional +from uuid import uuid4 + +import couchbase.search as search +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.collection import Collection +from couchbase.exceptions import DocumentNotFoundException, QueryIndexAlreadyExistsException +from couchbase.management.search import SearchIndex +from couchbase.options import ClusterOptions, SearchOptions +from couchbase.vector_search import VectorQuery, VectorSearch + +# NOTE: These paths should be updated accordingly if going to use. The paths are currently setup for running w/in +# the couchbase-python-client root directory. +# + +TEST_INDEX_NAME = 'test-search-vector-index' +TEST_COLL_INDEX_NAME = 'test-search-vector-coll-index' +TEST_PATH = os.path.join(pathlib.Path(__file__).parent, + 'tests', + 'test_cases') +TEST_INDEX_PATH = os.path.join(TEST_PATH, + f'{TEST_INDEX_NAME}-params.json') +TEST_COLL_INDEX_PATH = os.path.join(TEST_PATH, + f'{TEST_COLL_INDEX_NAME}-params.json') +TEST_VECTOR_SEARCH_DOCS_PATH = os.path.join(TEST_PATH, + 'test-vector-search-docs.json') +TEST_SEARCH_VECTOR_PATH = os.path.join(TEST_PATH, + 'test-search-vector.json') + + +def check_doc_count(sixm, + idx_name: str, + min_count: int, + retries: Optional[int] = 20, + delay: Optional[int] = 30 + ) -> bool: + + indexed_docs = 0 + no_docs_cutoff = 300 + for i in range(retries): + # if no docs after waiting for a period of time, exit + if indexed_docs == 0 and i * delay >= no_docs_cutoff: + return 0 + indexed_docs = sixm.get_indexed_documents_count(idx_name) + if indexed_docs >= min_count: + break + print(f'Found {indexed_docs} indexed docs, waiting a bit...') + sleep(delay) + + return indexed_docs + + +def load_search_idx(cluster: Cluster) -> None: + sixm = cluster.search_indexes() + params_json = None + with open(TEST_INDEX_PATH) as params_file: + input = params_file.read() + params_json = json.loads(input) + + if params_json is None: + print('Unabled to read search index params') + return + + idx = SearchIndex(name=TEST_INDEX_NAME, + idx_type='fulltext-index', + source_name='default', + source_type='couchbase', + params=params_json) + sixm.upsert_index(idx) + indexed_docs = check_doc_count(sixm, TEST_INDEX_NAME, 20, retries=10, delay=3) + print(f'Have {indexed_docs} indexed documents.') + + +def drop_search_idx(cluster: Cluster) -> None: + sixm = cluster.search_indexes() + sixm.drop_index(TEST_INDEX_NAME) + + +def load_test_vector(): + vector = None + with open(TEST_SEARCH_VECTOR_PATH) as vector_file: + vector = json.loads(vector_file.read()) + + return vector + + +def read_docs(): + with open(TEST_VECTOR_SEARCH_DOCS_PATH) as input: + while line := input.readline(): + yield json.loads(line) + + +def load_docs(root: str, collection: Collection) -> List[str]: + idx = 0 + keys = [] + for doc in read_docs(): + key = f'{root}_{idx}' + # the search index expects a type field w/ vector as the value + doc['type'] = 'vector' + collection.upsert(key, doc) + keys.append(key) + idx += 1 + print(f'loaded {len(keys)} docs.') + return keys + + +def remove_docs(keys: List[str], collection: Collection) -> None: + for key in keys: + try: + collection.remove(key) + except DocumentNotFoundException: + pass + + +def setup(cluster: Cluster, collection: Collection) -> List[str]: + root = str(uuid4())[:8] + keys = load_docs(root, collection) + try: + load_search_idx(cluster) + except QueryIndexAlreadyExistsException: + pass + + return keys + + +def teardown(cluster: Cluster, collection: Collection, keys: List[str]) -> None: + remove_docs(keys, collection) + drop_search_idx(cluster) + + +def main(): + auth = PasswordAuthenticator('Administrator', 'password') + opts = ClusterOptions(auth) + cluster = Cluster.connect('couchbase://localhost', opts) + bucket = cluster.bucket('default') + collection = bucket.default_collection() + + # NOTE: validate paths (see note on line 17) if going to use setup/teardown functionality + keys = setup(cluster, collection) + vector = load_test_vector() + + search_req = search.SearchRequest.create(search.MatchAllQuery()).with_vector_search( + VectorSearch.from_vector_query(VectorQuery('vector_field', vector))) + search_iter = cluster.search(TEST_INDEX_NAME, search_req, SearchOptions(limit=2)) + for row in search_iter.rows(): + print(f'row: {row}') + + print(f'Metatdata: {search_iter.metadata()}') + + # NOTE: only use in conjunction w/ setup() method + teardown(cluster, collection, keys) + + +if __name__ == '__main__': + main() diff --git a/examples/dockerfiles/centos7/binary_install.Dockerfile b/examples/dockerfiles/centos7/binary_install.Dockerfile new file mode 100644 index 000000000..0600afad4 --- /dev/null +++ b/examples/dockerfiles/centos7/binary_install.Dockerfile @@ -0,0 +1,68 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 centos:centos7 + +# Update the following ARGs to desired specification + +# Always required: +# - Python >= 3.7 +# - OpenSSL >= 1.1.1 + +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN yum -y update && \ + yum install -y git-all python3-devel python3-pip \ + python3-setuptools gcc gcc-c++ openssl-devel make + +# OPTIONAL: useful tools +RUN yum install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN yum install -y lsof lshw sysstat net-tools tar + +# install new Python version +RUN yum install -y libffi-devel +RUN cd /tmp && \ + curl -L -o Python-$PYTHON_VERSION.tgz https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Update OpenSSL +RUN yum install -y pcre-devel zlib-devel gd-devel perl-ExtUtils-Embed libxslt-devel perl-Test-Simple +RUN cd /usr/src && \ + curl -L -o openssl-$OPENSSL_VERSION.tar.gz https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ + tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ + mv openssl-$OPENSSL_VERSION openssl && \ + cd openssl && \ + ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl --libdir=/lib64 shared zlib-dynamic && \ + make -j4 && \ + make install && \ + mv /usr/bin/openssl /usr/bin/openssl-backup && \ + ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK diff --git a/examples/dockerfiles/centos7/source_install.Dockerfile b/examples/dockerfiles/centos7/source_install.Dockerfile new file mode 100644 index 000000000..d3856025a --- /dev/null +++ b/examples/dockerfiles/centos7/source_install.Dockerfile @@ -0,0 +1,93 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 centos:centos7 + +# Update the following ARGs to desired specification + +# CMake must be >= 3.18 +ARG CMAKE_VERSION=3.19.8 +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN yum -y update && \ + yum install -y git-all python3-devel python3-pip \ + python3-setuptools gcc gcc-c++ openssl-devel make + +# OPTIONAL: useful tools +RUN yum install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN yum install -y lsof lshw sysstat net-tools tar + +# install devtoolset-9 to update gcc +RUN yum -y update && \ + yum install -y centos-release-scl +RUN yum install -y devtoolset-9 +RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc +SHELL ["/bin/bash", "--login", "-c"] + +# install/update CMake +RUN yum -y remove cmake +RUN cd /tmp && \ + curl -L -o cmake-$CMAKE_VERSION.tar.gz https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz && \ + tar xf cmake-$CMAKE_VERSION.tar.gz && \ + cd cmake-$CMAKE_VERSION && \ + ./bootstrap && \ + make -j4 && \ + make install + +# install new Python version +RUN yum install -y libffi-devel +RUN cd /tmp && \ + curl -L -o Python-$PYTHON_VERSION.tgz https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Update OpenSSL +RUN yum install -y pcre-devel zlib-devel gd-devel perl-ExtUtils-Embed libxslt-devel perl-Test-Simple +RUN cd /usr/src && \ + curl -L -o openssl-$OPENSSL_VERSION.tar.gz https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ + tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ + mv openssl-$OPENSSL_VERSION openssl && \ + cd openssl && \ + ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl --libdir=/lib64 shared zlib-dynamic && \ + make -j4 && \ + make install && \ + mv /usr/bin/openssl /usr/bin/openssl-backup && \ + ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +# To do a source install: +# - make sure the build system has been setup appropriately +# - use the --no-binary option to force an install from source +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase diff --git a/examples/dockerfiles/centos8/binary_install.Dockerfile b/examples/dockerfiles/centos8/binary_install.Dockerfile new file mode 100644 index 000000000..576fa343e --- /dev/null +++ b/examples/dockerfiles/centos8/binary_install.Dockerfile @@ -0,0 +1,67 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 centos:centos8 + +# Update the following ARGs to desired specification + +# Always required: +# - Python >= 3.7 +# - OpenSSL >= 1.1.1 + +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN yum install -y python3-devel python3-pip python3-setuptools gcc gcc-c++ openssl openssl-devel cmake make + +# OPTIONAL: useful tools +RUN yum install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN yum install -y lsof lshw sysstat net-tools tar + +# OPTIONAL: update OpenSSL +# - CentOS 8 *should* come with a compatible version of OpenSSL (>= v1.1.1) +# RUN yum install -y pcre-devel zlib-devel gd-devel perl-ExtUtils-Embed libxslt-devel perl-Test-Simple +# RUN cd /usr/src && \ +# curl -L -o openssl-$OPENSSL_VERSION.tar.gz https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ +# tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ +# mv openssl-$OPENSSL_VERSION openssl && \ +# cd openssl && \ +# ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl --libdir=/lib64 shared zlib-dynamic && \ +# make -j4 && \ +# make install && \ +# mv /usr/bin/openssl /usr/bin/openssl-backup && \ +# ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl + +# install new Python version +RUN yum install -y libffi-devel +RUN cd /tmp && \ + curl -L -o Python-$PYTHON_VERSION.tgz https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK diff --git a/examples/dockerfiles/centos8/source_install.Dockerfile b/examples/dockerfiles/centos8/source_install.Dockerfile new file mode 100644 index 000000000..4cb4b129c --- /dev/null +++ b/examples/dockerfiles/centos8/source_install.Dockerfile @@ -0,0 +1,93 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 centos:centos8 + +# Update the following ARGs to desired specification + +# CMake must be >= 3.18 +ARG CMAKE_VERSION=3.19.8 +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN yum install -y python3-devel python3-pip python3-setuptools gcc gcc-c++ openssl openssl-devel cmake make + +# OPTIONAL: useful tools +RUN yum install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN yum install -y lsof lshw sysstat net-tools tar + +# get the stdc++ static libs +RUN dnf install -y dnf-plugins-core && \ + dnf upgrade -y && \ + dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \ + dnf config-manager --set-enabled powertools && \ + yum install -y glibc-static && \ + yum install -y libstdc++-static + +# OPTIONAL: install/update CMake +# - CentOS 8 *should* offer a compatible version of CMake (>= v3.18) +# RUN cd /tmp && \ +# curl -L -o cmake-$CMAKE_VERSION.tar.gz https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz && \ +# tar xf cmake-$CMAKE_VERSION.tar.gz && \ +# cd cmake-$CMAKE_VERSION && \ +# ./bootstrap && \ +# make -j4 && \ +# make install + +# OPTIONAL: update OpenSSL +# - CentOS 8 *should* come with a compatible version of OpenSSL (>= v1.1.1) +# RUN yum install -y pcre-devel zlib-devel gd-devel perl-ExtUtils-Embed libxslt-devel perl-Test-Simple +# RUN cd /usr/src && \ +# curl -L -o openssl-$OPENSSL_VERSION.tar.gz https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ +# tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ +# mv openssl-$OPENSSL_VERSION openssl && \ +# cd openssl && \ +# ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl --libdir=/lib64 shared zlib-dynamic && \ +# make -j4 && \ +# make install && \ +# mv /usr/bin/openssl /usr/bin/openssl-backup && \ +# ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl + +# install new Python version +RUN yum install -y libffi-devel +RUN cd /tmp && \ + curl -L -o Python-$PYTHON_VERSION.tgz https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +# To do a source install: +# - make sure the build system has been setup appropriately +# - use the --no-binary option to force an install from source +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase diff --git a/examples/dockerfiles/debian10/binary_install.Dockerfile b/examples/dockerfiles/debian10/binary_install.Dockerfile new file mode 100644 index 000000000..6f0c9778c --- /dev/null +++ b/examples/dockerfiles/debian10/binary_install.Dockerfile @@ -0,0 +1,34 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 debian:buster + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make zlib1g-dev + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip + +# Install Couchbase Python SDK 4.x +ARG COUCHBASE_PYTHON_SDK=4.0.2 +RUN python3 -m pip install --upgrade pip setuptools wheel +RUN python3 -m pip install couchbase==$COUCHBASE_PYTHON_SDK + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/dockerfiles/debian10/source_install.Dockerfile b/examples/dockerfiles/debian10/source_install.Dockerfile new file mode 100644 index 000000000..38eb814c1 --- /dev/null +++ b/examples/dockerfiles/debian10/source_install.Dockerfile @@ -0,0 +1,44 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 debian:buster + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make zlib1g-dev + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip + +# Install Couchbase Python SDK 4.x +ARG COUCHBASE_PYTHON_SDK=4.0.2 +RUN python3 -m pip install --upgrade pip setuptools wheel +# To do a source install: +# - make sure the build system has been setup appropriately +# - use the --no-binary option to force an install from source +RUN python3 -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/dockerfiles/debian9/binary_install.Dockerfile b/examples/dockerfiles/debian9/binary_install.Dockerfile new file mode 100644 index 000000000..dd9047431 --- /dev/null +++ b/examples/dockerfiles/debian9/binary_install.Dockerfile @@ -0,0 +1,73 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 debian:stretch + +# Update the following ARGs to desired specification + +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make zlib1g-dev + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip + +# update OpenSSL to $OPENSSL_VERSION +ARG CORES=4 +RUN apt-get install -y build-essential checkinstall +RUN cd /usr/src && \ + wget https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ + tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ + mv openssl-$OPENSSL_VERSION openssl && \ + cd openssl && \ + ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl shared zlib && \ + make -j$CORES && \ + make install && \ + echo "/usr/local/openssl/lib" > /etc/ld.so.conf.d/openssl-$OPENSSL_VERSION.conf && \ + ldconfig -v && \ + mv /usr/bin/openssl /usr/bin/openssl-backup && \ + mv /usr/bin/c_rehash /usr/bin/c_rehash-backup && \ + cd /usr/src && \ + rm -rf /usr/src/openssl-$OPENSSL_VERSION /usr/src/openssl-$OPENSSL_VERSION.tar.gz + +# install new Python version +RUN apt-get install -y libffi-dev +RUN cd /usr/src && \ + wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall && \ + cd /usr/src && \ + rm -rf /usr/src/Python-$PYTHON_VERSION /usr/src/Python-$PYTHON_VERSION.tgz + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/dockerfiles/debian9/source_install.Dockerfile b/examples/dockerfiles/debian9/source_install.Dockerfile new file mode 100644 index 000000000..cacc74eb7 --- /dev/null +++ b/examples/dockerfiles/debian9/source_install.Dockerfile @@ -0,0 +1,126 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 debian:stretch + +# Update the following ARGs to desired specification + +# CMake must be >= 3.18 +ARG CMAKE_VERSION=3.19.8 +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make zlib1g-dev + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip + +# update GCC, use gcc >= 9 for c++ 17 +ARG ARCH=x86_64 +ARG CORES=4 +ARG GCC_VERSION=9.3.0 +ARG GCC_LIBS=lib64 +RUN cd /usr/src && \ + wget https://ftp.gnu.org/gnu/gcc/gcc-$GCC_VERSION/gcc-$GCC_VERSION.tar.gz && \ + tar -xvf gcc-$GCC_VERSION.tar.gz && \ + cd /usr/src/gcc-$GCC_VERSION && \ + sed -i 's\ftp://gcc.gnu.org\https://gcc.gnu.org\g' contrib/download_prerequisites && \ + contrib/download_prerequisites && \ + mkdir build && \ + cd build && \ + ../configure -v \ + --build=$ARCH-linux-gnu \ + --host=$ARCH-linux-gnu \ + --target=$ARCH-linux-gnu \ + --prefix=/usr/local/gcc-$GCC_VERSION \ + --enable-checking=release \ + --enable-languages=c,c++,fortran \ + --disable-multilib \ + --program-suffix=-$GCC_VERSION && \ + make -j$CORES && \ + make install-strip && \ + cd /usr/src && \ + rm -rf /usr/src/gcc-$GCC_VERSION /usr/src/gcc.tar.gz + +ENV CC=/usr/local/gcc-$GCC_VERSION/bin/gcc-$GCC_VERSION \ + CXX=/usr/local/gcc-$GCC_VERSION/bin/g++-$GCC_VERSION \ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/gcc-$GCC_VERSION/$GCC_LIBS + +# install/update CMake +RUN cd /usr/src && \ + wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz && \ + tar -xvf cmake-$CMAKE_VERSION.tar.gz && cd cmake-$CMAKE_VERSION && \ + ./bootstrap && \ + make -j$CORES && make install && \ + cd /usr/src && \ + rm -rf /usr/src/cmake-$CMAKE_VERSION /usr/src/cmake-$CMAKE_VERSION.tar.gz + +# update OpenSSL to $OPENSSL_VERSION +RUN apt-get install -y build-essential checkinstall + +RUN cd /usr/src && \ + wget https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ + tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ + mv openssl-$OPENSSL_VERSION openssl && \ + cd openssl && \ + ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl shared zlib && \ + make -j$CORES && \ + make install && \ + echo "/usr/local/openssl/lib" > /etc/ld.so.conf.d/openssl-$OPENSSL_VERSION.conf && \ + ldconfig -v && \ + mv /usr/bin/openssl /usr/bin/openssl-backup && \ + mv /usr/bin/c_rehash /usr/bin/c_rehash-backup && \ + cd /usr/src && \ + rm -rf /usr/src/openssl-$OPENSSL_VERSION /usr/src/openssl-$OPENSSL_VERSION.tar.gz + +# install new Python version +RUN apt-get install -y libffi-dev +RUN cd /usr/src && \ + wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall && \ + cd /usr/src && \ + rm -rf /usr/src/Python-$PYTHON_VERSION /usr/src/Python-$PYTHON_VERSION.tgz + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +# To do a source install: +# - make sure the build system has been setup appropriately +# - let the build system know where OpenSSL is installed by setting PYCBC_OPENSSL_DIR +# - use the --no-binary option to force an install from source +RUN PYCBC_OPENSSL_DIR=/usr/local/openssl $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/dockerfiles/rhel7/binary_install.Dockerfile b/examples/dockerfiles/rhel7/binary_install.Dockerfile new file mode 100644 index 000000000..2d591a05e --- /dev/null +++ b/examples/dockerfiles/rhel7/binary_install.Dockerfile @@ -0,0 +1,74 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 registry.access.redhat.com/ubi7/ubi:latest + +# Update to match RH subscription +ARG RH_USER=username +ARG RH_PW='password' + +# Update the following ARGs to desired specification + +# Always required: +# - Python >= 3.7 +# - OpenSSL >= 1.1.1 + +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +RUN subscription-manager register --username=$RH_USER --password=$RH_PW --auto-attach + +# basic setup +RUN yum install -y python3-devel python3-pip python3-setuptools gcc gcc-c++ openssl openssl-devel make + +# OPTIONAL: useful tools +RUN yum install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN yum install -y lsof lshw sysstat net-tools tar + +# install new Python version +RUN yum install -y libffi-devel +RUN cd /tmp && \ + curl -L -o Python-$PYTHON_VERSION.tgz https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Update OpenSSL +RUN yum install -y pcre-devel zlib-devel gd-devel perl-ExtUtils-Embed libxslt-devel perl-Test-Simple +RUN cd /usr/src && \ + curl -L -o openssl-$OPENSSL_VERSION.tar.gz https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ + tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ + mv openssl-$OPENSSL_VERSION openssl && \ + cd openssl && \ + ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl --libdir=/lib64 shared zlib-dynamic && \ + make -j4 && \ + make install && \ + mv /usr/bin/openssl /usr/bin/openssl-backup && \ + ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK + +RUN subscription-manager unregister diff --git a/examples/dockerfiles/rhel7/source_install.Dockerfile b/examples/dockerfiles/rhel7/source_install.Dockerfile new file mode 100644 index 000000000..8456b4490 --- /dev/null +++ b/examples/dockerfiles/rhel7/source_install.Dockerfile @@ -0,0 +1,99 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 registry.access.redhat.com/ubi7/ubi:latest + +# Update to match RH subscription +ARG RH_USER=username +ARG RH_PW='password' + +# Update the following ARGs to desired specification + +# CMake must be >= 3.18 +ARG CMAKE_VERSION=3.19.8 +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +RUN subscription-manager register --username=$RH_USER --password=$RH_PW --auto-attach + +# basic setup +RUN yum install -y python3-devel python3-pip python3-setuptools gcc gcc-c++ openssl openssl-devel make + +# OPTIONAL: useful tools +RUN yum install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN yum install -y lsof lshw sysstat net-tools tar + +# devtoolset-9 to update gcc +RUN yum install -y scl-utils scl-utils-build +RUN subscription-manager repos --enable rhel-7-server-optional-rpms \ + --enable rhel-server-rhscl-7-rpms +RUN yum install -y devtoolset-9 +RUN echo "source /opt/rh/devtoolset-9/enable" >> /etc/bashrc +SHELL ["/bin/bash", "--login", "-c"] + +# install/update CMake +RUN cd /tmp && \ + curl -L -o cmake-$CMAKE_VERSION.tar.gz https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz && \ + tar xf cmake-$CMAKE_VERSION.tar.gz && \ + cd cmake-$CMAKE_VERSION && \ + ./bootstrap && \ + make -j4 && \ + make install + +# install new Python version +RUN yum install -y libffi-devel +RUN cd /tmp && \ + curl -L -o Python-$PYTHON_VERSION.tgz https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Update OpenSSL +RUN yum install -y pcre-devel zlib-devel gd-devel perl-ExtUtils-Embed libxslt-devel perl-Test-Simple +RUN cd /usr/src && \ + curl -L -o openssl-$OPENSSL_VERSION.tar.gz https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ + tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ + mv openssl-$OPENSSL_VERSION openssl && \ + cd openssl && \ + ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl --libdir=/lib64 shared zlib-dynamic && \ + make -j4 && \ + make install && \ + mv /usr/bin/openssl /usr/bin/openssl-backup && \ + ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +# To do a source install: +# - make sure the build system has been setup appropriately +# - use the --no-binary option to force an install from source +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase + +RUN subscription-manager unregister diff --git a/examples/dockerfiles/rhel8/binary_install.Dockerfile b/examples/dockerfiles/rhel8/binary_install.Dockerfile new file mode 100644 index 000000000..32b95344b --- /dev/null +++ b/examples/dockerfiles/rhel8/binary_install.Dockerfile @@ -0,0 +1,75 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 registry.access.redhat.com/ubi8/ubi:latest + +# Update to match RH subscription +ARG RH_USER=username +ARG RH_PW='password' + +# Update the following ARGs to desired specification + +# Always required: +# - Python >= 3.7 +# - OpenSSL >= 1.1.1 + +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +RUN subscription-manager register --username=$RH_USER --password=$RH_PW --auto-attach + +# basic setup +RUN yum install -y python3-devel python3-pip python3-setuptools gcc gcc-c++ openssl openssl-devel make cmake + +# OPTIONAL: useful tools +RUN yum install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN yum install -y lsof lshw sysstat net-tools tar + +# OPTIONAL: update OpenSSL +# - RHEL8 *should* come with a compatible version of OpenSSL (>= v1.1.1) +# RUN yum install -y pcre-devel zlib-devel gd-devel perl-ExtUtils-Embed libxslt-devel perl-Test-Simple +# RUN cd /usr/src && \ +# curl -L -o openssl-$OPENSSL_VERSION.tar.gz https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ +# tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ +# mv openssl-$OPENSSL_VERSION openssl && \ +# cd openssl && \ +# ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl --libdir=/lib64 shared zlib-dynamic && \ +# make -j4 && \ +# make install && \ +# mv /usr/bin/openssl /usr/bin/openssl-backup && \ +# ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl + +# install new Python version +RUN yum install -y libffi-devel +RUN cd /tmp && \ + curl -L -o Python-$PYTHON_VERSION.tgz https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK + +RUN subscription-manager unregister diff --git a/examples/dockerfiles/rhel8/source_install.Dockerfile b/examples/dockerfiles/rhel8/source_install.Dockerfile new file mode 100644 index 000000000..9b1088058 --- /dev/null +++ b/examples/dockerfiles/rhel8/source_install.Dockerfile @@ -0,0 +1,95 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 registry.access.redhat.com/ubi8/ubi:latest + +# Update to match RH subscription +ARG RH_USER=username +ARG RH_PW='password' + +# Update the following ARGs to desired specification + +# CMake must be >= 3.18 +ARG CMAKE_VERSION=3.19.8 +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +RUN subscription-manager register --username=$RH_USER --password=$RH_PW --auto-attach + +# basic setup +RUN yum install -y python3-devel python3-pip python3-setuptools gcc gcc-c++ openssl openssl-devel make cmake + +# OPTIONAL: useful tools +RUN yum install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN yum install -y lsof lshw sysstat net-tools tar + +# OPTIONAL: install/update CMake +# - RHEL8 *should* offer a compatible version of CMake (>= v3.18) + +# RUN cd /tmp && \ +# curl -L -o cmake-$CMAKE_VERSION.tar.gz https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz && \ +# tar xf cmake-$CMAKE_VERSION.tar.gz && \ +# cd cmake-$CMAKE_VERSION && \ +# ./bootstrap && \ +# make -j4 && \ +# make install + +# OPTIONAL: update OpenSSL +# - RHEL8 *should* come with a compatible version of OpenSSL (>= v1.1.1) + +# RUN yum install -y pcre-devel zlib-devel gd-devel perl-ExtUtils-Embed libxslt-devel perl-Test-Simple +# RUN cd /usr/src && \ +# curl -L -o openssl-$OPENSSL_VERSION.tar.gz https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ +# tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ +# mv openssl-$OPENSSL_VERSION openssl && \ +# cd openssl && \ +# ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl --libdir=/lib64 shared zlib-dynamic && \ +# make -j4 && \ +# make install && \ +# mv /usr/bin/openssl /usr/bin/openssl-backup && \ +# ln -s /usr/local/openssl/bin/openssl /usr/bin/openssl + +# install new Python version +RUN yum install -y libffi-devel +RUN cd /tmp && \ + curl -L -o Python-$PYTHON_VERSION.tgz https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Install Couchbase Python SDK 4.x +# To do a source install: +# - make sure the build system has been setup appropriately +# - use the --no-binary option to force an install from source +# - need to dynamically link against stdc++ static libs on RHEL8, so set COUCHBASE_CXX_CLIENT_STATIC_STDLIB to OFF +RUN COUCHBASE_CXX_CLIENT_STATIC_STDLIB=OFF $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase + +RUN subscription-manager unregister diff --git a/examples/dockerfiles/ubuntu16/binary_install.Dockerfile b/examples/dockerfiles/ubuntu16/binary_install.Dockerfile new file mode 100644 index 000000000..c54e88413 --- /dev/null +++ b/examples/dockerfiles/ubuntu16/binary_install.Dockerfile @@ -0,0 +1,75 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 ubuntu:16.04 + +# Update the following ARGs to desired specification + +# Always required: +# - Python >= 3.7 +# - OpenSSL >= 1.1.1 + +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN apt-get install -y lsof lshw sysstat net-tools + +# install new Python version +RUN apt-get install -y libffi-dev +RUN cd /usr/src && \ + wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# update OpenSSL to $OPENSSL_VERSION +RUN apt-get install -y zlib1g-dev build-essential checkinstall + +RUN cd /usr/src && \ + wget https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ + tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ + mv openssl-$OPENSSL_VERSION openssl && \ + cd openssl && \ + ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl shared zlib && \ + make -j4 && \ + make install && \ + echo "/usr/local/openssl/lib" > /etc/ld.so.conf.d/openssl-$OPENSSL_VERSION.conf && \ + ldconfig -v && \ + mv /usr/bin/openssl /usr/bin/openssl-backup && \ + mv /usr/bin/c_rehash /usr/bin/c_rehash-backup + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/dockerfiles/ubuntu16/source_install.Dockerfile b/examples/dockerfiles/ubuntu16/source_install.Dockerfile new file mode 100644 index 000000000..84058fb72 --- /dev/null +++ b/examples/dockerfiles/ubuntu16/source_install.Dockerfile @@ -0,0 +1,101 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 ubuntu:16.04 + +# Update the following ARGs to desired specification + +# CMake must be >= 3.18 +ARG CMAKE_VERSION=3.19.8 +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make zlib1g-dev + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN apt-get install -y lsof lshw sysstat net-tools + +# update GCC +RUN apt-get update && \ + apt-get install -yq software-properties-common && \ + add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ + apt-get update && \ + apt-get install -yq gcc-9 g++-9 + +ENV CC=/usr/bin/gcc-9 \ + CXX=/usr/bin/g++-9 + +# install/update CMake +RUN cd /usr/src && \ + wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz && \ + tar -xvf cmake-$CMAKE_VERSION.tar.gz && cd cmake-$CMAKE_VERSION && \ + ./bootstrap && \ + make && make install + +# update OpenSSL to $OPENSSL_VERSION +RUN apt-get install -y build-essential checkinstall + +RUN cd /usr/src && \ + wget https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ + tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ + mv openssl-$OPENSSL_VERSION openssl && \ + cd openssl && \ + ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl shared zlib && \ + make -j4 && \ + make install && \ + echo "/usr/local/openssl/lib" > /etc/ld.so.conf.d/openssl-$OPENSSL_VERSION.conf && \ + ldconfig -v && \ + mv /usr/bin/openssl /usr/bin/openssl-backup && \ + mv /usr/bin/c_rehash /usr/bin/c_rehash-backup + +# install new Python version +RUN apt-get install -y libffi-dev +RUN cd /usr/src && \ + wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +# To do a source install: +# - make sure the build system has been setup appropriately +# - let the build system know where OpenSSL is installed by setting PYCBC_OPENSSL_DIR +# - use the --no-binary option to force an install from source +RUN PYCBC_OPENSSL_DIR=/usr/local/openssl $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/dockerfiles/ubuntu18/binary_install.Dockerfile b/examples/dockerfiles/ubuntu18/binary_install.Dockerfile new file mode 100644 index 000000000..9a8e07667 --- /dev/null +++ b/examples/dockerfiles/ubuntu18/binary_install.Dockerfile @@ -0,0 +1,72 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 ubuntu:18.04 + +# can update to a different timezone if desired +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Update the following ARGs to desired specification + +# Always required: +# - Python >= 3.7 +# - OpenSSL >= 1.1.1 + +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN apt-get install -y lsof lshw sysstat net-tools + +# OPTIONAL: update OpenSSL +# - Ubuntu 18.04 *should* come with a compatible version of OpenSSL (>= v1.1.1) + +# RUN apt-get install -y zlib1g-dev build-essential checkinstall +# RUN cd /usr/src && \ +# wget https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ +# tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ +# mv openssl-$OPENSSL_VERSION openssl && \ +# cd openssl && \ +# ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl shared zlib && \ +# make -j4 && \ +# make install && \ +# echo "/usr/local/openssl/lib" > /etc/ld.so.conf.d/openssl-$OPENSSL_VERSION.conf && \ +# ldconfig -v && \ +# mv /usr/bin/openssl /usr/bin/openssl-backup && \ +# mv /usr/bin/c_rehash /usr/bin/c_rehash-backup + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +# If installed/updated OpenSSL, might need to set PYCBC_OPENSSL_DIR +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/dockerfiles/ubuntu18/source_install.Dockerfile b/examples/dockerfiles/ubuntu18/source_install.Dockerfile new file mode 100644 index 000000000..6049ef440 --- /dev/null +++ b/examples/dockerfiles/ubuntu18/source_install.Dockerfile @@ -0,0 +1,106 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 ubuntu:18.04 + +# can update to a different timezone if desired +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Update the following ARGs to desired specification + +# CMake must be >= 3.18 +ARG CMAKE_VERSION=3.19.8 +# Python must be >= 3.7 +ARG PYTHON_VERSION=3.8.10 +# NOTE: the Python version chosen will impact what python executable to use when pip +# installing packages (see commands at bottom) +ARG PYTHON_EXE=python3.8 +# OpenSSL must be >= 1.1.1 +ARG OPENSSL_VERSION=1.1.1l +ARG COUCHBASE_PYTHON_SDK=4.0.2 + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make zlib1g-dev + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN apt-get install -y lsof lshw sysstat net-tools + +# update GCC +RUN apt-get update && \ + apt-get install -yq software-properties-common && \ + add-apt-repository ppa:ubuntu-toolchain-r/test -y && \ + apt-get update && \ + apt-get install -yq gcc-9 g++-9 + +ENV CC=/usr/bin/gcc-9 \ + CXX=/usr/bin/g++-9 + +# install/update CMake +RUN cd /usr/src && \ + wget https://github.com/Kitware/CMake/releases/download/v$CMAKE_VERSION/cmake-$CMAKE_VERSION.tar.gz && \ + tar -xvf cmake-$CMAKE_VERSION.tar.gz && cd cmake-$CMAKE_VERSION && \ + ./bootstrap && \ + make && make install + +# OPTIONAL: update OpenSSL +# - Ubuntu 18.04 *should* come with a compatible version of OpenSSL (>= v1.1.1) + +# RUN apt-get install -y build-essential checkinstall +# RUN cd /usr/src && \ +# wget https://www.openssl.org/source/old/1.1.1/openssl-$OPENSSL_VERSION.tar.gz && \ +# tar -xvf openssl-$OPENSSL_VERSION.tar.gz && \ +# mv openssl-$OPENSSL_VERSION openssl && \ +# cd openssl && \ +# ./config --prefix=/usr/local/openssl --openssldir=/usr/local/openssl shared zlib && \ +# make -j4 && \ +# make install && \ +# echo "/usr/local/openssl/lib" > /etc/ld.so.conf.d/openssl-$OPENSSL_VERSION.conf && \ +# ldconfig -v && \ +# mv /usr/bin/openssl /usr/bin/openssl-backup && \ +# mv /usr/bin/c_rehash /usr/bin/c_rehash-backup + +# # install new Python version +RUN apt-get install -y libffi-dev +RUN cd /usr/src && \ + wget https://www.python.org/ftp/python/$PYTHON_VERSION/Python-$PYTHON_VERSION.tgz && \ + tar -xf Python-$PYTHON_VERSION.tgz && \ + cd Python-$PYTHON_VERSION && \ + ./configure --enable-optimizations && \ + make altinstall + +# Install Couchbase Python SDK 4.x +RUN $PYTHON_EXE -m pip install --upgrade pip setuptools wheel +# To do a source install: +# - make sure the build system has been setup appropriately +# - if installed/updated OpenSSL, might need to set PYCBC_OPENSSL_DIR +# - use the --no-binary option to force an install from source +RUN $PYTHON_EXE -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/dockerfiles/ubuntu20/binary_install.Dockerfile b/examples/dockerfiles/ubuntu20/binary_install.Dockerfile new file mode 100644 index 000000000..2b0bdceb2 --- /dev/null +++ b/examples/dockerfiles/ubuntu20/binary_install.Dockerfile @@ -0,0 +1,36 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to install the Couchbase Python SDK >= 4.0.2. No optimization has been done. +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system. +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 ubuntu:20.04 + +# can update to a different timezone if desired +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make zlib1g-dev + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN apt-get install -y lsof lshw sysstat net-tools + +# Install Couchbase Python SDK 4.x +ARG COUCHBASE_PYTHON_SDK=4.0.2 +RUN python3 -m pip install --upgrade pip setuptools wheel +RUN python3 -m pip install couchbase==$COUCHBASE_PYTHON_SDK diff --git a/examples/dockerfiles/ubuntu20/source_install.Dockerfile b/examples/dockerfiles/ubuntu20/source_install.Dockerfile new file mode 100644 index 000000000..c75154425 --- /dev/null +++ b/examples/dockerfiles/ubuntu20/source_install.Dockerfile @@ -0,0 +1,50 @@ +# This is an *unsupported* and *unofficial* Dockerfile. The purpose of this Dockerfile is to demonstrate the steps +# required to have a working build system to build/install the Couchbase Python 4.x SDK. No optimization has been +# done. +# +# Build System checklist: +# - Compiler that supports C++ 17 +# - CMake >= 3.18 +# - OpenSSL >= 1.1.1 +# - Python >= 3.7 +# +# The 4.0.2 release of the Couchbase Python SDK provides manylinux wheels. A Python package wheel provides a pre-built +# binary that enables significantly faster install and users do not need to worry about setting up the appropriate +# build system (i.e. no need to install/update compiler and/or CMake). +# +# **NOTE:** All versions of the 4.x Python SDK, require OpenSSL >= 1.1.1 and Python >= 3.7 +# +# Example usage: +# build: +# docker build -t <name of image> -f <path including Dockerfile> <path to Dockerfile directory> +# run: +# docker run --rm --name <name of running container> -it <name of image> /bin/bash +# + +FROM --platform=linux/amd64 ubuntu:20.04 + +# can update to a different timezone if desired +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# basic setup +RUN apt-get update && \ + apt-get install -yq git-all python3-dev python3-pip \ + python3-setuptools build-essential libssl-dev make zlib1g-dev + +# OPTIONAL: useful tools +RUN apt-get install -y wget vim zip unzip +# OPTIONAL: more useful tools +# RUN apt-get install -y lsof lshw sysstat net-tools + +# Install Couchbase Python SDK 4.x +ARG COUCHBASE_PYTHON_SDK=4.0.2 +RUN python3 -m pip install --upgrade pip setuptools wheel +# To do a source install: +# - make sure the build system has been setup appropriately +# - use the --no-binary option to force an install from source +RUN python3 -m pip install couchbase==$COUCHBASE_PYTHON_SDK --no-binary couchbase + +# cleanup +RUN apt-get autoremove && apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/examples/docloader.py b/examples/docloader.py deleted file mode 100644 index d5df22ae6..000000000 --- a/examples/docloader.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This file attempts to mimic the basic behavior of cbdocloader -from __future__ import print_function -from zipfile import ZipFile -import logging - -from time import sleep -from itertools import izip_longest -from os.path import basename - -import requests - -# Uncomment to use the cffi module (useful with pypy) -# import couchbase_ffi - -from couchbase.user_constants import FMT_JSON, FMT_BYTES -from couchbase.transcoder import Transcoder -from couchbase.bucket import Bucket -from couchbase.exceptions import CouchbaseNetworkError, CouchbaseTransientError - - -class DocLoader(object): - def __init__(self, bucket, srcfile, username, password, quota, cluster, - create_bucket=False): - self.bucket = bucket - self.srcfile = srcfile - self.username = username - self.password = password - self.quota = quota - self.cluster = cluster - self.should_create = create_bucket - self.logger = logging.getLogger('docloader') - self._retries = {} - - self._zf = ZipFile(self.srcfile, 'r') - self._client = None - self._htsess = requests.Session() - self._htsess.auth = (self.username, self.password) - - if not self.bucket: - self.bucket = basename(srcfile).replace('.zip', '') - - self.logger.info('Using bucket %s', self.bucket) - - @property - def cluster_prefix(self): - return 'http://{0}/pools/default'.format(self.cluster) - - @property - def bucket_spec(self): - return 'http://{0}/{1}'.format(self.cluster, self.bucket) - - def prepare_bucket(self): - """ - Resets and creates the destination bucket ( - only called if --create is true). - :return: - """ - self.logger.info('Deleting old bucket first') - del_url = '{0}/buckets/{1}'.format(self.cluster_prefix, self.bucket) - r = self._htsess.delete(del_url) - - try: - r.raise_for_status() - except: - self.logger.exception("Couldn't delete bucket") - - cr_url = '{0}/buckets'.format(self.cluster_prefix) - data = { - 'name': self.bucket, - 'ramQuotaMB': '{0}'.format(self.quota), - 'bucketType': 'couchbase', - 'authType': 'sasl', - 'saslPassword': '', - 'replicaNumber': '0' - } - r = self._htsess.post(cr_url, data) - r.raise_for_status() - - def make_client(self): - cb = None - while not cb: - # Try to connect via HTTP - try: - cb = Bucket(self.bucket_spec) - cb.transcoder = AlwaysJsonTranscoder() - cb.stats() - cb.timeout = 7.5 - except CouchbaseNetworkError as e: - self.logger.exception( - 'Got error while connecting. Sleeping for a bit') - sleep(1) - - self._client = cb - - def process_one_batch(self, curnames): - ret = {} - for name in curnames: - if not name: - continue - - comps = name.split('/') - if len(comps) != 3: - continue - - pfx, dirname = comps[0:2] - if dirname != 'docs': - self.logger.warn('Skipping {0} (not a document)'.format(name)) - continue - else: - docname = comps[-1].replace('.json', '') - if not docname: - self.logger.warn('No document path for {0}'.format(name)) - continue - - fp = None - try: - fp = self._zf.open(name, mode='r') - ret[docname] = fp.read() - except: - self.logger.error("Couldn't load %s (%s)", name, docname) - raise - finally: - if fp: - fp.close() - return ret - - def run_batch(self, kvs): - if not kvs: - return - - try: - self._client.upsert_multi(kvs) - except CouchbaseTransientError as e: - self.logger.info('Items have failed. Placing into retry queue: %r', e) - for k, v in e.all_results.items(): - if not v.success: - self._retries[k] = kvs[k] - - def flush_retries(self): - while self._retries: - self.logger.info('Retrying %d items', len(self._retries)) - tmp = self._retries - self._retries = {} - self.run_batch(tmp) - - def start_load(self): - docnames = self._zf.namelist() - self.logger.info('Will load %d docs', len(docnames)) - - for curnames in grouper(docnames, 1000): - self.flush_retries() - curdocs = self.process_one_batch(curnames) - self.run_batch(curdocs) - - self.flush_retries() - - def run(self): - r = self._htsess.get( - '{0}/buckets/{1}'.format(self.cluster_prefix, self.bucket)) - - if self.should_create: - self.logger.info('Recreating bucket as requested') - self.prepare_bucket() - elif r.status_code == 404: - self.logger.info('Bucket does not exist. Creating') - self.prepare_bucket() - elif r.status_code == 200: - pass - else: - r.raise_for_status() - - self.make_client() - self.start_load() - - -# The data we want to load is actually bytes, but we want it to be JSON. -# therefore a custom transcoder is needed which blindly sets the appropriate -# flags -class AlwaysJsonTranscoder(Transcoder): - def encode_value(self, value, flags): - value, flags = super(AlwaysJsonTranscoder, - self).encode_value(value, FMT_BYTES) - flags = FMT_JSON - return value, flags - - -# Direct from Python.org -def grouper(iterable, n, fillvalue=None): - "Collect data into fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx - args = [iter(iterable)] * n - return izip_longest(fillvalue=fillvalue, *args) - - -if __name__ == '__main__': - import sys - import couchbase - from argparse import ArgumentParser - - ap = ArgumentParser() - ap.add_argument('-f', '--file', - help='Path to zip file', required=True) - ap.add_argument('--force-create', - help='Always recreate the bucket', action='store_true') - ap.add_argument('-n', '--node', help='Address for cluster', - default='127.0.0.1:8091') - ap.add_argument('-u', '--username', - help='Administrative username', default='Administrator') - ap.add_argument('-p', '--password', - help='Administrative password', default="123456") - ap.add_argument('-s', '--size', - help='RAM quota size for the bucket if created', - type=int, default=100) - ap.add_argument('-b', '--bucket', - help=('Name of destination bucket ' - '(determined from filename if not provided)')) - ap.add_argument('-v', '--verbose', - help='Verbosity of logging', action='count') - - options = ap.parse_args() - if options.verbose: - lvl = logging.DEBUG - else: - lvl = logging.INFO - - logging.basicConfig( - stream=sys.stderr, level=lvl, - format='%(created)f - %(name)s - %(levelname)s - %(message)s') - - couchbase.enable_logging() - loader = DocLoader( - bucket=options.bucket, srcfile=options.file, username=options.username, - password=options.password, quota=options.size, cluster=options.node, - create_bucket=options.force_create) - loader.run() diff --git a/examples/gbench.py b/examples/gbench.py deleted file mode 100755 index 6046f0be5..000000000 --- a/examples/gbench.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -from time import sleep, time - -import gevent - -from couchbase import FMT_BYTES -from gcouchbase.bucket import Bucket - -ap = argparse.ArgumentParser() - -ap.add_argument('-t', '--threads', default=4, type=int, - help="Number of threads to spawn. 0 means no threads " - "but workload will still run in the main thread") - -ap.add_argument('-d', '--delay', default=0, type=float, - help="Number of seconds to wait between each op. " - "may be a fraction") -ap.add_argument('-U', '--connstr', default='couchbase://localhost/default', - help="Connection string") -ap.add_argument('-p', '--password', default=None, type=str) -ap.add_argument('-D', '--duration', default=10, type=int, - help="Duration of run (in seconds)") - -ap.add_argument('--ksize', default=12, type=int, - help="Key size to use") - -ap.add_argument('--vsize', default=128, type=int, - help="Value size to use") -ap.add_argument('--iops', default=None, type=str, - help="Use Pure-Python IOPS plugin") -ap.add_argument('-g', '--global-instance', - help="Use global instance", default=False, - action='store_true') -ap.add_argument('--batch', '-N', type=int, help="Batch size", default=1) - -options = ap.parse_args() - -GLOBAL_INSTANCE = None -CONN_OPTIONS = { - 'connstr' : options.connstr, - 'password': options.password -} - -GLOBAL_INSTANCE = Bucket(**CONN_OPTIONS) - -def make_instance(): - if options.global_instance: - return GLOBAL_INSTANCE - else: - return Bucket(**CONN_OPTIONS) - -class Worker(object): - def __init__(self): - self.delay = options.delay - self.key = 'K' * options.ksize - self.value = b'V' * options.vsize - self.kv = {} - for x in range(options.batch): - self.kv[self.key + str(x)] = self.value - - self.wait_time = 0 - self.opcount = 0 - - def run(self, *args, **kwargs): - self.end_time = time() + options.duration - cb = make_instance() - - while time() < self.end_time: - begin_time = time() - rv = cb.upsert_multi(self.kv, format=FMT_BYTES) - assert rv.all_ok, "Operation failed: " - self.wait_time += time() - begin_time - self.opcount += options.batch - -global_begin = None -gthreads = [] -worker_threads = [] -for x in range(options.threads): - w = Worker() - worker_threads.append(w) - t = gevent.spawn(w.run) - gthreads.append(t) - -global_begin = time() -for t in gthreads: - t.join() - -global_duration = time() - global_begin -total_ops = sum([w.opcount for w in worker_threads]) -total_time = 0 -for t in worker_threads: - total_time += t.wait_time - -print("Total run took an absolute time of %0.2f seconds" % (global_duration,)) -print("Did a total of %d operations" % (total_ops,)) -print("Total wait time of %0.2f seconds" % (total_time,)) -print("[WAIT] %0.2f ops/second" % (float(total_ops)/float(total_time),)) -print("[ABS] %0.2f ops/second" % (float(total_ops)/float(global_duration),)) diff --git a/examples/iops_demo.py b/examples/iops_demo.py deleted file mode 100644 index 6c8b38e5f..000000000 --- a/examples/iops_demo.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python -import gevent -import gevent.monkey; gevent.monkey.patch_all() -import sys - -from couchbase.bucket import Bucket - -def test(x): - c = Bucket('couchbase://localhost/default', experimental_gevent_support=True) - c.upsert("tmp-" + str(x), 1) - sys.stdout.write(str(x) + " ") - sys.stdout.flush() - -print("Gevent starting..") -gevent.joinall([gevent.spawn(test, x) for x in xrange(100)]) -print("") diff --git a/examples/item.py b/examples/item.py deleted file mode 100644 index 1fcf3937f..000000000 --- a/examples/item.py +++ /dev/null @@ -1,114 +0,0 @@ -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from couchbase.items import Item, ItemSequence -from couchbase.bucket import Bucket -from pprint import pprint -from random import randint - -class Player(Item): - def __init__(self, name, create_structure=False): - super(Player, self).__init__(name) - if create_structure: - self.value = { - 'email': None, - 'score': 0, - 'games': [] - } - - @classmethod - def create(cls, name, email, cb): - """ - Create the basic structure of a player - """ - it = cls(name, create_structure=True) - it.value['email'] = email - - # In an actual application you'd probably want to use 'add', - # but since this app might be run multiple times, you don't - # want to get KeyExistsError - cb.upsert_multi(ItemSequence([it])) - return it - - @classmethod - def load(cls, name, cb): - it = Player(name) - cb.get_multi(ItemSequence([it])) - return it - - def save(self, cb): - cb.replace_multi(ItemSequence([self])) - - @property - def name(self): - return self.key - - @property - def score(self): - return self.value['score'] - - @score.setter - def score(self, value): - self.value['score'] = value - - @property - def games(self): - return self.value['games'] - - @property - def email(self): - return self.value['email'] - @email.setter - def email(self, value): - self.value['email'] = value - -cb = Bucket('couchbase://localhost/default') -single_player = Player.create("bob", "bob@bobnob.com", cb) -single_player.score += 100 -single_player.save(cb) - -# Let's try multiple players -players = ItemSequence([Player(x, create_structure=True) - for x in ("joe", "jim", "bill", "larry")]) - -# Save them all -cb.upsert_multi(players) - -# Give them all some points -for p, options in players: - p.score += randint(20, 2000) - # also set the email? - if not p.email: - p.email = "{0}@{0}.notspecified.com".format(p.name) - -cb.replace_multi(players) -all_players = ItemSequence([x[0] for x in players] + [single_player]) - -INDENT = " " * 3 -for player in all_players.sequence: - print "Name:", player.name - print INDENT , player - - lines = [] - lines.append("email: {0}".format(player.email)) - lines.append("score: {0}".format(player.score)) - - for line in lines: - print INDENT, line - -cb.remove_multi(all_players) -cb.endure_multi(all_players, check_removed=True, replicate_to=0, - persist_to=1) diff --git a/examples/reversed_keys.py b/examples/reversed_keys.py deleted file mode 100755 index 4074e0588..000000000 --- a/examples/reversed_keys.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python - -from couchbase.transcoder import Transcoder -from couchbase.bucket import Bucket - - -class ReverseTranscoder(Transcoder): - def encode_key(self, key): - return super(ReverseTranscoder, self).encode_key(key[::-1]) - - def decode_key(self, key): - key = super(ReverseTranscoder, self).decode_key(key) - return key[::-1] - - -c_reversed = Bucket('couchbase://localhost/default', - transcoder=ReverseTranscoder()) -c_plain = Bucket('couchbase://localhost/default') - -c_plain.remove_multi(('ABC', 'CBA', 'XYZ', 'ZYX'), quiet=True) - -c_reversed.upsert("ABC", "This is a reversed key") - -rv = c_plain.get("CBA") -print("Got value for reversed key '{0}'".format(rv.value)) - -rv = c_reversed.get("ABC") -print("Got value for reversed key '{0}' again".format(rv.value)) - -c_plain.upsert("ZYX", "This is really ZYX") - -rv = c_reversed.get("XYZ") -print("Got value for '{0}': '{1}'".format(rv.key, rv.value)) diff --git a/examples/search_keywords.py b/examples/search_keywords.py deleted file mode 100644 index 539c2a18c..000000000 --- a/examples/search_keywords.py +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env python - -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -# This file demonstrates some of the functionalities available with -# view queries. This creates a bunch of key-value pairs where the value is -# a number. It also creates a view to index the key-value pairs by the -# number itself, and finally queries the view to return the ten items with -# the highest values. - -from argparse import ArgumentParser -import random -import pprint - -from couchbase.bucket import Bucket - -ap = ArgumentParser() - -ap.add_argument('-D', '--create-design', default=False, - action='store_true', - help='whether to create the design') - -ap.add_argument('-n', '--number-of-terms', default=10, - type=int, help="How many terms to generate") - -options = ap.parse_args() - -c = Bucket('couchbase://localhost/default') - -DESIGN = { - '_id': '_design/search_keywords', - 'language': 'javascript', - 'views': { - 'top_keywords': { - 'map': - """ - function(doc) { - if (typeof doc === 'number') { - emit(doc, null); - } - } - """ - } - } -} - -if options.create_design: - bm = c.bucket_manager() - bm.design_create('search_keywords', DESIGN, use_devmode=False, syncwait=5) - -NOUNS = ['cow', 'cat', 'dog', 'computer', 'WMD'] -ADJECTIVES = ['happy', 'sad', 'thoughtful', 'extroverted'] - -kv = {} - -for x in range(options.number_of_terms): - n = random.choice(NOUNS) - a = random.choice(ADJECTIVES) - kv[" ".join([a, n])] = random.randint(1, 100000) - -c.upsert_multi(kv) - -vret = c.query('search_keywords', - 'top_keywords', - limit=10, - descending=True) - -for row in vret: - pprint.pprint(row, indent=4) - -# Sample output: -#[ { u'id': u'WMD sad', u'key': 92772, u'value': None}, -# { u'id': u'WMD thoughtful', u'key': 76222, u'value': None}, -# { u'id': u'cow happy', u'key': 71984, u'value': None}, -# { u'id': u'computer sad', u'key': 68849, u'value': None}, -# { u'id': u'cat thoughtful', u'key': 68417, u'value': None}, -# { u'id': u'computer thoughtful', u'key': 67518, u'value': None}, -# { u'id': u'dog thoughtful', u'key': 67350, u'value': None}, -# { u'id': u'computer extroverted', u'key': 63279, u'value': None}, -# { u'id': u'cow thoughtful', u'key': 60962, u'value': None}, -# { u'id': u'cow sad', u'key': 49510, u'value': None}] diff --git a/examples/twist-sample.py b/examples/twist-sample.py deleted file mode 100644 index c58db7e38..000000000 --- a/examples/twist-sample.py +++ /dev/null @@ -1,34 +0,0 @@ -from twisted.internet import reactor -from twisted.internet.defer import inlineCallbacks, Deferred - -from txcouchbase.connection import Connection - -class MyClient(object): - def __init__(self): - self.cb = Connection(bucket='default') - self.do_set() - - def on_op_error(self, msg): - print "Got operation error!" + str(msg) - - def do_set(self): - self.cb.set("foo", "bar").addCallback(self.on_set) - - def on_set(self, res): - print res - self.cb.get("foo").addCallback(self.on_get) - - def on_get(self, res): - print res - -@inlineCallbacks -def run_sync_example(): - cb = Connection(bucket='default') - rv_set = yield cb.set("foo", "bar") - print rv_set - rv_get = yield cb.get("foo") - print rv_get - -cb = MyClient() -run_sync_example() -reactor.run() diff --git a/examples/txbasic.py b/examples/txbasic.py deleted file mode 100644 index 468cb30aa..000000000 --- a/examples/txbasic.py +++ /dev/null @@ -1,15 +0,0 @@ -from twisted.internet import reactor - -from txcouchbase.connection import Connection as TxCouchbase - -cb = TxCouchbase(bucket='default') -def on_set(ret): - print("Set key. Result", ret) - -def on_get(ret): - print("Got key. Result", ret) - reactor.stop() - -cb.set("key", "value").addCallback(on_set) -cb.get("key").addCallback(on_get) -reactor.run() diff --git a/examples/txbench.py b/examples/txbench.py deleted file mode 100755 index afce65709..000000000 --- a/examples/txbench.py +++ /dev/null @@ -1,138 +0,0 @@ -#!/usr/bin/env python -# -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import argparse -from time import time - -from twisted.internet import reactor - -from txcouchbase.bucket import RawBucket, Bucket -from couchbase import FMT_BYTES -from couchbase.transcoder import Transcoder - -ap = argparse.ArgumentParser() - -ap.add_argument('-t', '--threads', default=4, type=int, - help="Number of threads to spawn. 0 means no threads " - "but workload will still run in the main thread") - -ap.add_argument('-d', '--delay', default=0, type=float, - help="Number of seconds to wait between each op. " - "may be a fraction") - -ap.add_argument('-C', '--clients', default=1, type=int, - help="Number of clients (nthreads are per-client)") - -ap.add_argument('--deferreds', action='store_true', default=False, - help="Whether to use Deferreds (or normal callbacks)") -ap.add_argument('-U', '--connstr', default='couchbase://localhost/default', - help="Connection string") -ap.add_argument('-p', '--password', default=None, type=str) -ap.add_argument('-D', '--duration', default=10, type=int, - help="Duration of run (in seconds)") - -ap.add_argument('-T', '--transcoder', default=False, - action='store_true', - help="Use the Transcoder object rather than built-in " - "conversion routines") - -ap.add_argument('--ksize', default=12, type=int, - help="Key size to use") - -ap.add_argument('--vsize', default=128, type=int, - help="Value size to use") - -ap.add_argument('--batch', '-N', type=int, default=1, help="Batch size to use") - -options = ap.parse_args() - -class Runner(object): - def __init__(self, cb): - self.cb = cb - self.delay = options.delay - self.key = 'K' * options.ksize - self.value = b'V' * options.vsize - self.kv = {} - for x in range(options.batch): - self.kv[self.key + str(x)] = self.value - self.wait_time = 0 - self.opcount = 0 - self.end_time = time() + options.duration - self._do_stop = False - self.start() - - def _schedule_raw(self, *args): - opres = self.cb.upsert_multi(self.kv, format=FMT_BYTES) - opres.callback = self._schedule_raw - self.opcount += 1 - - def _schedule_deferred(self, *args): - rv = self.cb.upsertMulti(self.kv, format=FMT_BYTES) - rv.addCallback(self._schedule_deferred) - self.opcount += options.batch - - def start(self): - if options.deferreds: - self._schedule_deferred() - else: - self.cb._async_raw = True - self._schedule_raw() - - def stop(self): - self._do_stop = True - -global_begin = time() -runners = [] -clients = [] -kwargs = { - 'connstr' : options.connstr, - 'password': options.password, - 'unlock_gil': False -} -if options.transcoder: - kwargs['transcoder'] = Transcoder() - -for _ in range(options.clients): - cls = Bucket if options.deferreds else RawBucket - cb = cls(**kwargs) - clients.append(cb) - d = cb.connect() - - def _on_connected(unused, client): - for _ in range(options.threads): - r = Runner(client) - runners.append(r) - d.addCallback(_on_connected, cb) - -def stop_all(): - [r.stop() for r in runners] - reactor.stop() - -reactor.callLater(options.duration, stop_all) -reactor.run() - - -global_duration = time() - global_begin -total_ops = sum([r.opcount for r in runners]) -total_time = 0 -for r in runners: - total_time += r.wait_time - -print("Total run took an absolute time of %0.2f seconds" % (global_duration,)) -print("Did a total of %d operations" % (total_ops,)) -print("[ABS] %0.2f ops/second" % (float(total_ops)/float(global_duration),)) diff --git a/examples/txcouchbase/txcouchbase_diagnostics_operations.py b/examples/txcouchbase/txcouchbase_diagnostics_operations.py new file mode 100644 index 000000000..40efcb3c7 --- /dev/null +++ b/examples/txcouchbase/txcouchbase_diagnostics_operations.py @@ -0,0 +1,82 @@ +from datetime import timedelta + +# this is new with Python SDK 4.0, it needs to be imported prior to +# importing the twisted reactor +import txcouchbase # nopep8 # isort:skip # noqa: E402, F401 + +from twisted.internet import defer, reactor + +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, use: from couchbase.options import PingOptions +from couchbase.bucket import PingOptions +from couchbase.diagnostics import PingState, ServiceType +from couchbase.exceptions import UnAmbiguousTimeoutException +from couchbase.options import WaitUntilReadyOptions +from txcouchbase.cluster import Cluster + + +@defer.inlineCallbacks +def ok(cluster): + result = yield cluster.ping() + for _, reports in result.endpoints.items(): + for report in reports: + if not report.state == PingState.OK: + return False + return True + + +@defer.inlineCallbacks +def main(): + # create a cluster object + cluster = Cluster('couchbase://localhost', + authenticator=PasswordAuthenticator('Administrator', 'password')) + + # @TODO: if connection fails, this will hang + yield cluster.on_connect() + + cluster_ready = False + try: + yield cluster.wait_until_ready(timedelta(seconds=3), + WaitUntilReadyOptions(service_types=[ServiceType.KeyValue, ServiceType.Query])) + cluster_ready = True + except UnAmbiguousTimeoutException as ex: + print('Cluster not ready in time: {}'.format(ex)) + + if cluster_ready is False: + quit() + + # For Server versions 6.5 or later you do not need to open a bucket here + bucket = cluster.bucket("beer-sample") + yield bucket.on_connect() + + ping_result = yield cluster.ping() + + for endpoint, reports in ping_result.endpoints.items(): + for report in reports: + print( + "{0}: {1} took {2}".format( + endpoint.value, + report.remote, + report.latency)) + + ping_result = yield cluster.ping() + print(ping_result.as_json()) + + cluster_ok = yield ok(cluster) + print("Cluster is okay? {}".format(cluster_ok)) + + ping_result = yield cluster.ping(PingOptions(service_types=[ServiceType.Query])) + print(ping_result.as_json()) + + diag_result = yield cluster.diagnostics() + print(diag_result.as_json()) + + print("Cluster state: {}".format(diag_result.state)) + + reactor.stop() + + +if __name__ == "__main__": + main() + reactor.run() diff --git a/examples/txcouchbase/txcouchbase_kv_operations.py b/examples/txcouchbase/txcouchbase_kv_operations.py new file mode 100644 index 000000000..00a91cdc1 --- /dev/null +++ b/examples/txcouchbase/txcouchbase_kv_operations.py @@ -0,0 +1,170 @@ +from datetime import timedelta + +# this is new with Python SDK 4.0, it needs to be imported prior to +# importing the twisted reactor +import txcouchbase # nopep8 # isort:skip # noqa: E402, F401 + +from twisted.internet import defer, reactor + +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, use: from couchbase.options import DeltaValue, SignedInt64 +# **DEPRECATED**, import ALL options from `couchbase.options` +from couchbase.collection import (DecrementOptions, + DeltaValue, + GetOptions, + IncrementOptions, + InsertOptions, + RemoveOptions, + ReplaceOptions, + SignedInt64, + UpsertOptions) +from couchbase.durability import (ClientDurability, + Durability, + PersistTo, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import CASMismatchException, CouchbaseException +from txcouchbase.cluster import Cluster + + +@defer.inlineCallbacks +def main(): + # create a cluster object + cluster = Cluster('couchbase://localhost', + authenticator=PasswordAuthenticator('Administrator', 'password')) + + # @TODO: if connection fails, this will hang + yield cluster.on_connect() + # create a bucket object + bucket = cluster.bucket('default') + yield bucket.on_connect() + + collection = bucket.default_collection() + + try: + yield collection.remove("document-key") + except CouchbaseException as ex: + pass # may not exist in this example + + try: + yield collection.remove("document-key-opts") + except CouchbaseException as ex: + pass # may not exist in this example + + # Insert document + document = {"foo": "bar", "bar": "foo"} + result = yield collection.insert("document-key", document) + print("Result: {}; CAS: {}".format(result, result.cas)) + + # Insert document with options + document = {"foo": "bar", "bar": "foo"} + opts = InsertOptions(timeout=timedelta(seconds=5)) + result = yield collection.insert("document-key-opts", + document, + opts, + expiry=timedelta(seconds=30)) + + try: + # Replace document with CAS + document = {"foo": "bar", "bar": "foo"} + result = yield collection.replace( + "document-key", + document, + cas=12345, + timeout=timedelta( + minutes=1)) + except CASMismatchException as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print('Caught CAS mismatch: {}'.format(ex)) + + try: + # Replace document with CAS + result = yield collection.get("document-key") + doc = result.content_as[dict] + doc["bar"] = "baz" + opts = ReplaceOptions(cas=result.cas) + result = yield collection.replace("document-key", doc, opts) + except CouchbaseException as ex: + print('Caught Couchbase exception: {}'.format(ex)) + + try: + # Upsert with Durability (Couchbase Server >= 6.5) level Majority + document = dict(foo="bar", bar="foo") + opts = UpsertOptions(durability=ServerDurability(Durability.MAJORITY)) + result = yield collection.upsert("document-key", document, opts) + except CouchbaseException as ex: + print('Caught Couchbase exception: {}'.format(ex)) + + # @TODO: couchbase++ doesn't implement observe based durability + # try: + # # Upsert with observe based durability (Couchbase Server < 6.5) + # document = {"foo": "bar", "bar": "foo"} + # opts = UpsertOptions( + # durability=ClientDurability( + # ReplicateTo.ONE, + # PersistTo.ONE)) + # result = yield collection.upsert("document-key", document, opts) + # except CouchbaseException as ex: + # print(ex) + + result = yield collection.get("document-key") + print(result.content_as[dict]) + + opts = GetOptions(timeout=timedelta(seconds=5)) + result = yield collection.get("document-key", opts) + print(result.content_as[dict]) + + try: + # remove document with options + result = yield collection.remove( + "document-key", + RemoveOptions( + cas=12345, + durability=ServerDurability( + Durability.MAJORITY))) + except CouchbaseException as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print('Caught Couchbase exception: {}'.format(ex)) + + result = yield collection.touch("document-key", timedelta(seconds=10)) + + result = yield collection.get("document-key", GetOptions(with_expiry=True)) + print("Expiry of result: {}".format(result.expiryTime)) + + result = yield collection.get_and_touch("document-key", timedelta(seconds=10)) + + # Increment binary value by 1 + yield collection.binary().increment( + "counter-key", + IncrementOptions( + delta=DeltaValue(1))) + + # Increment binary value by 5, if key doesn't exist, seed it at 1000 + yield collection.binary().increment( + "counter-key", + IncrementOptions( + delta=DeltaValue(5), + initial=SignedInt64(1000))) + + # Decrement binary value by 1 + yield collection.binary().decrement( + "counter-key", + DecrementOptions( + delta=DeltaValue(1))) + + # Decrement binary value by 2, if key doesn't exist, seed it at 1000 + yield collection.binary().decrement( + "counter-key", + DecrementOptions( + delta=DeltaValue(2), + initial=SignedInt64(1000))) + + reactor.stop() + + +if __name__ == "__main__": + main() + reactor.run() diff --git a/examples/txcouchbase/txcouchbase_query_operations.py b/examples/txcouchbase/txcouchbase_query_operations.py new file mode 100644 index 000000000..641d39ce2 --- /dev/null +++ b/examples/txcouchbase/txcouchbase_query_operations.py @@ -0,0 +1,134 @@ +import uuid + +# this is new with Python SDK 4.0, it needs to be imported prior to +# importing the twisted reactor +import txcouchbase # nopep8 # isort:skip # noqa: E402, F401 + +from twisted.internet import defer, reactor + +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, import ALL options from `couchbase.options` +from couchbase.cluster import (ClusterOptions, + QueryOptions, + QueryScanConsistency) +from couchbase.exceptions import ParsingFailedException +from couchbase.mutation_state import MutationState +from txcouchbase.cluster import Cluster + +# use this to not have deprecation warnings +# from couchbase.options import ClusterOptions, QueryOptions +# from couchbase.n1ql import QueryScanConsistency + + +@defer.inlineCallbacks +def main(): + + # create a cluster object + cluster = Cluster('couchbase://localhost', + ClusterOptions(PasswordAuthenticator('Administrator', 'password'))) + + # @TODO: if connection fails, this will hang + yield cluster.on_connect() + # create a bucket object + bucket = cluster.bucket('travel-sample') + yield bucket.on_connect() + + collection = bucket.default_collection() + + # basic query + try: + result = yield cluster.query( + "SELECT * FROM `travel-sample` LIMIT 10;", QueryOptions(metrics=True)) + + for row in result.rows(): + print(f'Found row: {row}') + + metrics = result.metadata().metrics() + print(f'Query execution time: {metrics.execution_time()}') + + except ParsingFailedException as ex: + import traceback + traceback.print_exc() + + # positional params + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`=$1 LIMIT 10" + result = yield cluster.query(q_str, "hotel") + rows = [r for r in result] + + # positional params via QueryOptions + result = yield cluster.query(q_str, QueryOptions(positional_parameters=["hotel"])) + rows = [r for r in result] + + # named params + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`=$doc_type LIMIT 10" + result = yield cluster.query(q_str, doc_type='hotel') + rows = [r for r in result] + + # name params via QueryOptions + result = yield cluster.query(q_str, QueryOptions(named_parameters={'doc_type': 'hotel'})) + rows = [r for r in result] + + # iterate over result/rows + q_str = "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10" + result = yield cluster.query(q_str) + + # iterate over rows + for row in result: + # each row is an serialized JSON object + name = row["name"] + callsign = row["callsign"] + print(f'Airline name: {name}, callsign: {callsign}') + + # query metrics + result = yield cluster.query("SELECT 1=1", QueryOptions(metrics=True)) + rows = [r for r in result] + + print("Execution time: {}".format( + result.metadata().metrics().execution_time())) + + # print scan consistency + result = yield cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10", + QueryOptions(scan_consistency=QueryScanConsistency.REQUEST_PLUS)) + rows = [r for r in result] + + # Read your own writes + new_airline = { + "callsign": None, + "country": "United States", + "iata": "TX", + "icao": "TX99", + "id": 123456789, + "name": "Howdy Airlines", + "type": "airline" + } + + res = yield collection.upsert( + "airline_{}".format(new_airline["id"]), new_airline) + + ms = MutationState(res) + + result = yield cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='airline' LIMIT 10", + QueryOptions(consistent_with=ms)) + rows = [r for r in result] + + # client context id + result = yield cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='hotel' LIMIT 10", + QueryOptions(client_context_id="user-44{}".format(uuid.uuid4()))) + rows = [r for r in result] + + # read only + result = yield cluster.query( + "SELECT ts.* FROM `travel-sample` ts WHERE ts.`type`='hotel' LIMIT 10", + QueryOptions(read_only=True)) + rows = [r for r in result] + + reactor.stop() + + +if __name__ == "__main__": + main() + reactor.run() diff --git a/examples/txcouchbase/txcouchbase_subdoc_operations.py b/examples/txcouchbase/txcouchbase_subdoc_operations.py new file mode 100644 index 000000000..c4489d804 --- /dev/null +++ b/examples/txcouchbase/txcouchbase_subdoc_operations.py @@ -0,0 +1,282 @@ +from datetime import timedelta + +# this is new with Python SDK 4.0, it needs to be imported prior to +# importing the twisted reactor +import txcouchbase # nopep8 # isort:skip # noqa: E402, F401 + +from twisted.internet import defer, reactor + +import couchbase.subdocument as SD +from couchbase.auth import PasswordAuthenticator + +# **DEPRECATED**, use from couchbase.options import MutateInOptions +from couchbase.collection import MutateInOptions +from couchbase.durability import (ClientDurability, + Durability, + PersistTo, + ReplicateTo, + ServerDurability) +from couchbase.exceptions import (CASMismatchException, + CouchbaseException, + DocumentExistsException, + PathExistsException, + PathNotFoundException, + SubdocCantInsertValueException, + SubdocPathMismatchException) +from txcouchbase.cluster import Cluster + + +JSON_DOC = { + "name": "Douglas Reynholm", + "email": "douglas@reynholmindustries.com", + "addresses": { + "billing": { + "line1": "123 Any Street", + "line2": "Anytown", + "country": "United Kingdom" + }, + "delivery": { + "line1": "123 Any Street", + "line2": "Anytown", + "country": "United Kingdom" + } + }, + "purchases": { + "complete": [ + 339, 976, 442, 666 + ], + "abandoned": [ + 157, 42, 999 + ] + } +} + + +@defer.inlineCallbacks +def main(): + # create a cluster object + cluster = Cluster('couchbase://localhost', + authenticator=PasswordAuthenticator('Administrator', 'password')) + + # @TODO: if connection fails, this will hang + yield cluster.on_connect() + # create a bucket object + bucket = cluster.bucket('default') + yield bucket.on_connect() + + collection = bucket.default_collection() + + try: + yield collection.insert("customer123", JSON_DOC) + except DocumentExistsException as ex: + yield collection.remove("customer123") + yield collection.insert("customer123", JSON_DOC) + + result = yield collection.lookup_in("customer123", + [SD.get("addresses.delivery.country")]) + country = result.content_as[str](0) # "United Kingdom" + print(country) + + result = yield collection.lookup_in( + "customer123", [ + SD.exists("purchases.pending[-1]")]) + print('Path exists: {}.'.format(result.exists(0))) + # Path exists: False. + + # NOTE: result.content_as[bool](0) raises a PathNotFoundException + # this is b/c when checking if a path exists + # no content is returned + + try: + print("Path exists {}.".format(result.content_as[bool](0))) + except PathNotFoundException: + print("Path does not exist") + + result = yield collection.lookup_in( + "customer123", + [SD.get("addresses.delivery.country"), + SD.exists("purchases.complete[-1]")]) + + print("{0}".format(result.content_as[str](0))) + print("Path exists: {}.".format(result.exists(1))) + # path exists: True. + + yield collection.mutate_in("customer123", [SD.upsert("fax", "311-555-0151")]) + + yield collection.mutate_in( + "customer123", [SD.insert("purchases.pending", [42, True, "None"])]) + + try: + yield collection.mutate_in( + "customer123", [ + SD.insert( + "purchases.complete", + [42, True, "None"])]) + except PathExistsException: + print("Path exists, cannot use insert.") + + yield collection.mutate_in( + "customer123", + (SD.remove("addresses.billing"), + SD.replace( + "email", + "dougr96@hotmail.com"))) + + # NOTE: the mutate_in() operation expects a tuple or list + yield collection.mutate_in( + "customer123", (SD.array_append( + "purchases.complete", 777),)) + # purchases.complete is now [339, 976, 442, 666, 777] + + yield collection.mutate_in( + "customer123", [ + SD.array_prepend( + "purchases.abandoned", 18)]) + # purchases.abandoned is now [18, 157, 42, 999] + + yield collection.upsert("my_array", []) + yield collection.mutate_in("my_array", [SD.array_append("", "some element")]) + # the document my_array is now ["some element"] + + yield collection.mutate_in( + "my_array", [ + SD.array_append( + "", "elem1", "elem2", "elem3")]) + + # the document my_array is now ["some_element", "elem1", "elem2", "elem3"] + + yield collection.mutate_in( + "my_array", [ + SD.array_append( + "", ["elem4", "elem5", "elem6"])]) + + # the document my_array is now ["some_element", "elem1", "elem2", "elem3", + # ["elem4", "elem5", "elem6"]]] + + yield collection.mutate_in( + "my_array", (SD.array_append("", "elem7"), + SD.array_append("", "elem8"), + SD.array_append("", "elem9"))) + + yield collection.upsert("some_doc", {}) + yield collection.mutate_in( + "some_doc", [ + SD.array_prepend( + "some.array", "Hello", "World", create_parents=True)]) + # the document some_doc is now {"some":{"array":["Hello", "World"]}} + try: + yield collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", 95)]) + print('Success!') + except PathExistsException: + print('Path already exists.') + + try: + yield collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", 95)]) + print('Success!') + except PathExistsException: + print('Path already exists.') + + # cannot add JSON obj w/ array_addunique + try: + yield collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.complete", {"new": "object"})]) + except SubdocCantInsertValueException as ex: + print("Cannot add JSON value w/ array_addunique.", ex) + + # cannot use array_addunique if array contains JSON obj + yield collection.mutate_in( + "customer123", [ + SD.upsert( + "purchases.cancelled", [{"Date": "Some date"}])]) + + try: + yield collection.mutate_in( + "customer123", [ + SD.array_addunique( + "purchases.cancelled", 89)]) + except SubdocPathMismatchException as ex: + print("Cannot use array_addunique if array contains JSON objs.", ex) + + yield collection.upsert("array", []) + yield collection.mutate_in("array", [SD.array_append("", "hello", "world")]) + yield collection.mutate_in("array", [SD.array_insert("[1]", "cruel")]) + + # exception raised if attempt to insert in out of bounds position + try: + yield collection.mutate_in("array", [SD.array_insert("[6]", "!")]) + except PathNotFoundException: + print("Cannot insert to out of bounds index.") + + # can insert into nested arrays as long as the path is appropriate + yield collection.mutate_in("array", [SD.array_append("", ["another", "array"])]) + yield collection.mutate_in("array", [SD.array_insert("[3][2]", "!")]) + + result = yield collection.mutate_in("customer123", (SD.counter("logins", 1),)) + customer_result = yield collection.get("customer123") + num_logins = customer_result.content_as[dict]["logins"] + print('Number of logins: {}.'.format(num_logins)) + # Number of logins: 1. + + yield collection.upsert("player432", {"gold": 1000}) + + yield collection.mutate_in("player432", (SD.counter("gold", -150),)) + result = yield collection.lookup_in("player432", (SD.get("gold"),)) + print("{} has {} gold remaining.".format( + "player432", result.content_as[int](0))) + # player432 has 850 gold remaining. + + yield collection.mutate_in("customer123", [SD.upsert("level_0.level_1.foo.bar.phone", + dict( + num="311-555-0101", + ext=16 + ), create_parents=True)]) + + yield collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 998)]) + + yield collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 999)]) + + try: + yield collection.mutate_in( + "customer123", [SD.array_append("purchases.complete", 999)], + MutateInOptions(cas=1234)) + except (DocumentExistsException, CASMismatchException) as ex: + # we expect an exception here as the CAS value is chosen + # for example purposes + print(ex) + + # @TODO: couchbase++ doesn't implement observe based durability + # try: + # yield collection.mutate_in( + # "key", [SD.insert("username", "dreynholm")], + # MutateInOptions(durability=ClientDurability( + # ReplicateTo.ONE, + # PersistTo.ONE))) + # except CouchbaseException as ex: + # print('Need to have more than 1 node for durability') + # print(ex) + + try: + yield collection.mutate_in( + "customer123", [SD.insert("username", "dreynholm")], + MutateInOptions(durability=ServerDurability( + Durability.MAJORITY))) + except CouchbaseException as ex: + print('Need to have more than 1 node for durability') + print(ex) + + reactor.stop() + + +if __name__ == "__main__": + main() + reactor.run() diff --git a/examples/txcouchbase/vector_search_tx_inline.py b/examples/txcouchbase/vector_search_tx_inline.py new file mode 100644 index 000000000..c338b3be2 --- /dev/null +++ b/examples/txcouchbase/vector_search_tx_inline.py @@ -0,0 +1,177 @@ +# IMPORTANT -- the txcouchbase import must occur PRIOR to importing the reactor +import txcouchbase + +import json +import os +import pathlib +from time import sleep +from typing import List, Optional +from uuid import uuid4 + +from twisted.internet import reactor, defer, task + +import couchbase.search as search +from couchbase.auth import PasswordAuthenticator +from txcouchbase.cluster import TxCluster +from txcouchbase.collection import TxCollection +from couchbase.exceptions import DocumentNotFoundException, QueryIndexAlreadyExistsException, CouchbaseException +from couchbase.management.search import SearchIndex +from couchbase.options import ClusterOptions, SearchOptions +from couchbase.vector_search import VectorQuery, VectorSearch + +# NOTE: These paths should be updated accordingly if going to use. The paths are currently setup for running w/in +# the couchbase-python-client root directory. +# + +TEST_INDEX_NAME = 'test-search-vector-index' +TEST_COLL_INDEX_NAME = 'test-search-vector-coll-index' +TEST_PATH = os.path.join(pathlib.Path(__file__).parent, + 'tests', + 'test_cases') +TEST_INDEX_PATH = os.path.join(TEST_PATH, + f'{TEST_INDEX_NAME}-params.json') +TEST_COLL_INDEX_PATH = os.path.join(TEST_PATH, + f'{TEST_COLL_INDEX_NAME}-params.json') +TEST_VECTOR_SEARCH_DOCS_PATH = os.path.join(TEST_PATH, + 'test-vector-search-docs.json') +TEST_SEARCH_VECTOR_PATH = os.path.join(TEST_PATH, + 'test-search-vector.json') + + +@defer.inlineCallbacks +def check_doc_count(sixm, + idx_name: str, + min_count: int, + retries: Optional[int] = 20, + delay: Optional[int] = 30 + ) -> bool: + + indexed_docs = 0 + no_docs_cutoff = 300 + for i in range(retries): + # if no docs after waiting for a period of time, exit + if indexed_docs == 0 and i * delay >= no_docs_cutoff: + return 0 + indexed_docs = yield sixm.get_indexed_documents_count(idx_name) + if indexed_docs >= min_count: + break + print(f'Found {indexed_docs} indexed docs, waiting a bit...') + sleep(delay) + + return indexed_docs + + +@defer.inlineCallbacks +def load_search_idx(cluster: TxCluster) -> None: + sixm = cluster.search_indexes() + params_json = None + with open(TEST_INDEX_PATH) as params_file: + input = params_file.read() + params_json = json.loads(input) + + if params_json is None: + print('Unabled to read search index params') + return + + idx = SearchIndex(name=TEST_INDEX_NAME, + idx_type='fulltext-index', + source_name='default', + source_type='couchbase', + params=params_json) + yield sixm.upsert_index(idx) + indexed_docs = yield check_doc_count(sixm, TEST_INDEX_NAME, 20, retries=10, delay=3) + print(f'Have {indexed_docs} indexed documents.') + + +def load_test_vector(): + vector = None + with open(TEST_SEARCH_VECTOR_PATH) as vector_file: + vector = json.loads(vector_file.read()) + + return vector + + +def read_docs(): + with open(TEST_VECTOR_SEARCH_DOCS_PATH) as input: + while line := input.readline(): + yield json.loads(line) + + +@defer.inlineCallbacks +def load_docs(root: str, collection: TxCollection) -> List[str]: + idx = 0 + keys = [] + for doc in read_docs(): + key = f'{root}_{idx}' + # the search index expects a type field w/ vector as the value + doc['type'] = 'vector' + yield collection.upsert(key, doc) + keys.append(key) + idx += 1 + print(f'loaded {len(keys)} docs.') + return keys + + +@defer.inlineCallbacks +def setup(cluster: TxCluster, collection: TxCollection) -> List[str]: + root = str(uuid4())[:8] + keys = yield load_docs(root, collection) + try: + yield load_search_idx(cluster) + except QueryIndexAlreadyExistsException: + pass + + return keys + + +@defer.inlineCallbacks +def remove_docs(keys: List[str], collection: TxCollection) -> None: + for key in keys: + try: + yield collection.remove(key) + except DocumentNotFoundException: + pass + + +@defer.inlineCallbacks +def drop_search_idx(cluster: TxCluster) -> None: + sixm = cluster.search_indexes() + yield sixm.drop_index(TEST_INDEX_NAME) + + +@defer.inlineCallbacks +def teardown(cluster: TxCluster, collection: TxCollection, keys: List[str]) -> None: + yield remove_docs(keys, collection) + yield drop_search_idx(cluster) + + +@defer.inlineCallbacks +def main(): + auth = PasswordAuthenticator('Administrator', 'password') + opts = ClusterOptions(auth) + cluster = TxCluster('couchbase://localhost', opts) + yield cluster.on_connect() + bucket = cluster.bucket('default') + yield bucket.on_connect() + collection = bucket.default_collection() + + # NOTE: validate paths (see note on line 22) if going to use setup/teardown functionality + keys = yield setup(cluster, collection) + vector = load_test_vector() + + search_req = search.SearchRequest.create(search.MatchAllQuery()).with_vector_search( + VectorSearch.from_vector_query(VectorQuery('vector_field', vector))) + search_iter = yield cluster.search(TEST_INDEX_NAME, search_req, SearchOptions(limit=2)) + for row in search_iter.rows(): + print(f'row: {row}') + + print(f'Metatdata: {search_iter.metadata()}') + + # NOTE: only use in conjunction w/ setup() method + yield teardown(cluster, collection, keys) + reactor.stop() + + +if __name__ == "__main__": + main() + reactor.run() diff --git a/examples/txview.py b/examples/txview.py deleted file mode 100644 index e944689d9..000000000 --- a/examples/txview.py +++ /dev/null @@ -1,12 +0,0 @@ -from twisted.internet import reactor - -from txcouchbase.connection import Connection - -def on_view_rows(res): - for row in res: - print "Got row", row.key - -cb = Connection(bucket='beer-sample') -d = cb.queryAll("beer", "brewery_beers", limit=20) -d.addCallback(on_view_rows) -reactor.run() diff --git a/gcouchbase/__init__.py b/gcouchbase/__init__.py deleted file mode 100644 index b794fd409..000000000 --- a/gcouchbase/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '0.1.0' diff --git a/gcouchbase/bucket.py b/gcouchbase/bucket.py deleted file mode 100644 index 5b482fc6e..000000000 --- a/gcouchbase/bucket.py +++ /dev/null @@ -1,136 +0,0 @@ -from gevent.event import AsyncResult, Event -from gevent.hub import get_hub, getcurrent, Waiter - -from couchbase.async.bucket import AsyncBucket -from couchbase.async.view import AsyncViewBase -from couchbase.async.n1ql import AsyncN1QLRequest -from couchbase.views.iterator import AlreadyQueriedError -try: - from gcouchbase.iops_gevent0x import IOPS -except ImportError: - from gcouchbase.iops_gevent10 import IOPS - - -class GRowsHandler(object): - def __init__(self): - """ - Subclass of :class:`~.AsyncViewBase` - This doesn't expose an API different from the normal - synchronous view API. It's just implemented differently - """ - - # We use __double_underscore to mangle names. This is because - # the views class has quite a bit of data attached to it. - self.__waiter = Waiter() - self.__raw_rows = [] - self.__done_called = False - self.start() - self.raw.rows_per_call = 100000 - - def _callback(self, *args): - # This method overridden from the parent. Rather than do the processing - # on demand, we must defer it for later. This is done by copying the - # rows to a list. In the typical case we shouldn't accumulate all - # the rows in the buffer (since .switch() will typically have something - # waiting for us). However if the view is destroyed prematurely, - # or if the user is not actively iterating over us, or if something - # else happens (such as more rows arriving during a get request with - # include_docs), we simply accumulate the rows here. - self.__raw_rows.append(self.raw.rows) - if self.raw.done: - self._clear() - self.__waiter.switch() - - def _errback(self, mres, *args): - self._clear() - self.__waiter.throw(*args) - - def __iter__(self): - if not self._do_iter: - raise AlreadyQueriedError.pyexc("Already queried") - - while self._do_iter and not self.__done_called: - self.__waiter.get() - - rowset_list = self.__raw_rows - self.__raw_rows = [] - for rowset in rowset_list: - for row in self._process_payload(rowset): - yield row - - self._do_iter = False - - -class GView(GRowsHandler, AsyncViewBase): - def __init__(self, *args, **kwargs): - AsyncViewBase.__init__(self, *args, **kwargs) - GRowsHandler.__init__(self) - - -class GN1QLRequest(GRowsHandler, AsyncN1QLRequest): - def __init__(self, *args, **kwargs): - AsyncN1QLRequest.__init__(self, *args, **kwargs) - GRowsHandler.__init__(self) - -class Bucket(AsyncBucket): - def __init__(self, *args, **kwargs): - """ - This class is a 'GEvent'-optimized subclass of libcouchbase - which utilizes the underlying IOPS structures and the gevent - event primitives to efficiently utilize couroutine switching. - """ - super(Bucket, self).__init__(IOPS(), *args, **kwargs) - - def _do_ctor_connect(self): - if self.connected: - return - - self._connect() - self._evconn = AsyncResult() - self._conncb = self._on_connected - self._evconn.get() - self._evconn = None - - def _on_connected(self, err): - if err: - self._evconn.set_exception(err) - else: - self._evconn.set(None) - - def _waitwrap(self, cbasync): - cur_thread = getcurrent() - cbasync.callback = cur_thread.switch - cbasync.errback = lambda r, x, y, z: cur_thread.throw(x, y, z) - - return get_hub().switch() - - def _meth_factory(meth, name): - def ret(self, *args, **kwargs): - return self._waitwrap(meth(self, *args, **kwargs)) - return ret - - def _http_request(self, **kwargs): - res = super(Bucket, self)._http_request(**kwargs) - - w = Waiter() - res.callback = lambda x: w.switch(x) - res.errback = lambda x, c, o, b: w.throw(c, o, b) - return w.get() - - def query(self, *args, **kwargs): - kwargs['itercls'] = GView - return super(Bucket, self).query(*args, **kwargs) - - def n1ql_query(self, query, *args, **kwargs): - kwargs['itercls'] = GN1QLRequest - return super(Bucket, self).n1ql_query(query, *args, **kwargs) - - def _get_close_future(self): - ev = Event() - def _dtor_cb(*args): - ev.set() - self._dtorcb = _dtor_cb - return ev - - - locals().update(AsyncBucket._gen_memd_wrappers(_meth_factory)) diff --git a/gcouchbase/connection.py b/gcouchbase/connection.py deleted file mode 100644 index 0f89ed7c2..000000000 --- a/gcouchbase/connection.py +++ /dev/null @@ -1,11 +0,0 @@ -from gcouchbase.bucket import Bucket -from couchbase.bucket import _depr -from couchbase.connstr import convert_1x_args - -class GConnection(Bucket): - def __init__(self, bucket, **kwargs): - _depr('gcouchbase.connection.GConnection', - 'gcouchbase.bucket.Bucket') - - kwargs = convert_1x_args(bucket, **kwargs) - super(GConnection, self).__init__(**kwargs) diff --git a/gcouchbase/gevent_bucket.py b/gcouchbase/gevent_bucket.py deleted file mode 120000 index e7b4468f8..000000000 --- a/gcouchbase/gevent_bucket.py +++ /dev/null @@ -1 +0,0 @@ -bucket.py \ No newline at end of file diff --git a/gcouchbase/iops_gevent0x.py b/gcouchbase/iops_gevent0x.py deleted file mode 100644 index ea7f36b69..000000000 --- a/gcouchbase/iops_gevent0x.py +++ /dev/null @@ -1,87 +0,0 @@ -from gevent.core import event as LibeventEvent -from gevent.core import timer as LibeventTimer -from gevent.core import EV_READ, EV_WRITE - -from couchbase.iops.base import ( - IOEvent, TimerEvent, - LCB_READ_EVENT, LCB_WRITE_EVENT, LCB_RW_EVENT, - PYCBC_EVACTION_WATCH, PYCBC_EVACTION_UNWATCH -) - -EVENTMAP = { - LCB_READ_EVENT: EV_READ, - LCB_WRITE_EVENT: EV_WRITE, - LCB_RW_EVENT: EV_READ|EV_WRITE -} - -REVERSERMAP = { - EV_READ: LCB_READ_EVENT, - EV_WRITE: LCB_WRITE_EVENT, - EV_READ|EV_WRITE: LCB_RW_EVENT -} - -class GeventIOEvent(IOEvent): - def __init__(self): - super(GeventIOEvent, self).__init__() - self.ev = None - self._last_events = -1 - - def _ready_pre(self, unused, flags): - self.update(self.flags) - lcbflags = REVERSERMAP[flags] - self.ready(lcbflags) - - def update(self, flags): - if not self.ev: - self.ev = LibeventEvent(flags, self.fd, self._ready_pre) - - if self._last_events != self.ev.events: - self.ev.cancel() - # DANGER: this relies on the implementation details of the - # cython-level class. - LibeventEvent.__init__(self.ev, flags, self.fd, self._ready_pre) - - self.ev.add() - - def cancel(self): - if not self.ev: - return - self.ev.cancel() - -class GeventTimer(TimerEvent): - def __init__(self): - super(GeventTimer, self).__init__() - self._tmev = LibeventTimer(0, lambda: self.ready(0)) - self._tmev.cancel() - - def reset(self, usecs): - self._tmev.add(usecs / 1000000.0) - - def cancel(self): - self._tmev.cancel() - -class IOPS(object): - def update_event(self, event, action, flags): - if action == PYCBC_EVACTION_WATCH: - event.update(EVENTMAP[flags]) - - elif action == PYCBC_EVACTION_UNWATCH: - event.cancel() - - def update_timer(self, event, action, usecs): - if action == PYCBC_EVACTION_WATCH: - event.reset(usecs) - else: - event.cancel() - - def start_watching(self): - pass - - def stop_watching(self): - pass - - def io_event_factory(self): - return GeventIOEvent() - - def timer_event_factory(self): - return GeventTimer() diff --git a/gcouchbase/iops_gevent10.py b/gcouchbase/iops_gevent10.py deleted file mode 100644 index 7e958a60f..000000000 --- a/gcouchbase/iops_gevent10.py +++ /dev/null @@ -1,86 +0,0 @@ -from gevent.hub import get_hub -from gevent.core import timer as _PyxTimer -from time import time - -from couchbase.iops.base import ( - IOEvent, TimerEvent, - LCB_READ_EVENT, LCB_WRITE_EVENT, LCB_RW_EVENT, - PYCBC_EVACTION_WATCH, PYCBC_EVACTION_UNWATCH -) - -from couchbase.iops.base import ( - IOEvent, TimerEvent, - LCB_READ_EVENT, LCB_WRITE_EVENT, LCB_RW_EVENT, - PYCBC_EVACTION_WATCH, PYCBC_EVACTION_UNWATCH -) - -EVENTMAP = { - LCB_READ_EVENT: 1, - LCB_WRITE_EVENT: 2, - LCB_RW_EVENT: 3 -} - -REVERSERMAP = { - 1: LCB_READ_EVENT, - 2: LCB_WRITE_EVENT, - 3: LCB_RW_EVENT -} - -class GEventIOEvent(IOEvent): - def __init__(self): - self.ev = get_hub().loop.io(0,0) - - def ready_proxy(self, event): - self.ready(REVERSERMAP[event]) - - def watch(self, events): - self.ev.stop() - self.ev.fd = self.fd - self.ev.events = events - - self.ev.start(self.ready_proxy, pass_events=True) - -class GEventTimer(TimerEvent): - def __init__(self): - self.ev = get_hub().loop.timer(0) - - def ready_proxy(self, *args): - self.ready(0) - - def schedule(self, usecs): - seconds = usecs / 1000000.0 - # This isn't the "clean" way, but it's much quicker.. and - # since we're already using undocumented APIs, why not.. - _PyxTimer.__init__(self.ev, get_hub().loop, seconds) - self.ev.start(self.ready_proxy, 0) - - -class IOPS(object): - def update_event(self, event, action, flags): - if action == PYCBC_EVACTION_UNWATCH: - event.ev.stop() - return - - elif action == PYCBC_EVACTION_WATCH: - ev_event = EVENTMAP[flags] - event.watch(ev_event) - - def update_timer(self, event, action, usecs): - if action == PYCBC_EVACTION_UNWATCH: - event.ev.stop() - return - - elif action == PYCBC_EVACTION_WATCH: - event.schedule(usecs) - - def start_watching(self): - pass - - def stop_watching(self): - pass - - def io_event_factory(self): - return GEventIOEvent() - - def timer_event_factory(self): - return GEventTimer() diff --git a/gcouchbase/tests/__init__.py b/gcouchbase/tests/__init__.py deleted file mode 100644 index dab902367..000000000 --- a/gcouchbase/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from couchbase.tests import * diff --git a/gcouchbase/tests/test_api.py b/gcouchbase/tests/test_api.py deleted file mode 100644 index 09c4685d1..000000000 --- a/gcouchbase/tests/test_api.py +++ /dev/null @@ -1,31 +0,0 @@ -from couchbase.tests.base import ApiImplementationMixin, SkipTest -try: - import gevent -except ImportError as e: - raise SkipTest(e) - -from gcouchbase.bucket import Bucket, GView -from couchbase.tests.importer import get_configured_classes - - -class GEventImplMixin(ApiImplementationMixin): - factory = Bucket - viewfactory = GView - should_check_refcount = True - - def _implDtorHook(self): - import gc - if not self.cb.closed: - waiter = self.cb._get_close_future() - del self.cb - gc.collect() - if not waiter.wait(7): - raise Exception("Not properly cleaned up!") - - -skiplist = ('IopsTest', 'LockmodeTest', 'PipelineTest') - -configured_classes = get_configured_classes(GEventImplMixin, - skiplist=skiplist) - -globals().update(configured_classes) diff --git a/pycbc_build_setup.py b/pycbc_build_setup.py new file mode 100644 index 000000000..20cd33ed1 --- /dev/null +++ b/pycbc_build_setup.py @@ -0,0 +1,354 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import os +import platform +import shutil +import subprocess # nosec +import sys +from dataclasses import dataclass, field +from sysconfig import get_config_var +from typing import (Dict, + List, + Optional) + +from setuptools import Command, Extension + +# need at least setuptools v62.3.0 +from setuptools.command.build import build +from setuptools.command.build_ext import build_ext +from setuptools.errors import OptionError, SetupError + +CMAKE_EXE = os.environ.get('CMAKE_EXE', shutil.which('cmake')) +PYCBC_ROOT = os.path.dirname(__file__) +# PYCBC_CXXCBC_CACHE_DIR should only need to be used on Windows when setting the CPM cache (PYCBC_SET_CPM_CACHE=ON). +# It helps prevent issues w/ path lengths. +# NOTE: Setting the CPM cache on a Windows machine should be a _rare_ occasion. When doing so and setting +# PYCBC_CXXCBC_CACHE_DIR, be sure to copy the cache to <root source dir>\deps\couchbase-cxx-cache if building a sdist. +CXXCBC_CACHE_DIR = os.environ.get('PYCBC_CXXCBC_CACHE_DIR', os.path.join(PYCBC_ROOT, 'deps', 'couchbase-cxx-cache')) +ENV_TRUE = ['true', '1', 'y', 'yes', 'on'] + + +def check_for_cmake(): + if not CMAKE_EXE: + print('cmake executable not found. ' + 'Set CMAKE_EXE environment or update your path') + sys.exit(1) + + +def process_build_env_vars(): # noqa: C901 + # Set debug or release + build_type = os.getenv('PYCBC_BUILD_TYPE', 'Release') + if build_type == 'Debug': + # @TODO: extra Windows debug args? + if platform.system() != "Windows": + debug_flags = ' '.join(['-O0', '-g3']) + c_flags = os.getenv('CFLAGS', '') + cxx_flags = os.getenv('CXXFLAGS', '') + os.environ['CFLAGS'] = f'{c_flags} {debug_flags}' + os.environ['CXXFLAGS'] = f'{cxx_flags} {debug_flags}' + os.environ['PYCBC_BUILD_TYPE'] = build_type + cmake_extra_args = [] + + # Allows us to set the location of OpenSSL for the build. + ssl_dir = os.getenv('PYCBC_OPENSSL_DIR', None) + if ssl_dir is not None: + cmake_extra_args += [f'-DOPENSSL_ROOT_DIR={ssl_dir}'] + + # We use OpenSSL by default if building the SDK; however, starting with v4.1.9 we build our wheels using BoringSSL. + pycbc_use_openssl = os.getenv('PYCBC_USE_OPENSSL', 'true').lower() in ENV_TRUE + if pycbc_use_openssl is True: + cmake_extra_args += ['-DUSE_STATIC_BORINGSSL:BOOL=OFF'] + ssl_version = os.getenv('PYCBC_OPENSSL_VERSION', None) + if not ssl_version: + ssl_version = '1.1.1w' + cmake_extra_args += [f'-DOPENSSL_VERSION={ssl_version}'] + else: + cmake_extra_args += ['-DUSE_STATIC_BORINGSSL:BOOL=ON'] + + # v4.1.9: building with static stdlibc++ must be opted-in by user + use_static_stdlib = os.getenv('PYCBC_USE_STATIC_STDLIB', 'false').lower() in ENV_TRUE + if use_static_stdlib is True: + cmake_extra_args += ['-DUSE_STATIC_STDLIB:BOOL=ON'] + else: + cmake_extra_args += ['-DUSE_STATIC_STDLIB:BOOL=OFF'] + + # v4.3.4: Allow user to specify if the C++ core will download Mozilla CA bundle during build. + # Defaults to ON unless the CPM Cache is being used then we use the certs from the cache + download_mozilla_ca_bundle = os.getenv('PYCBC_DOWNLOAD_MOZILLA_CA_BUNDLE', None) + if download_mozilla_ca_bundle is not None: + if download_mozilla_ca_bundle.lower() in ENV_TRUE: + cmake_extra_args += ['-DDOWNLOAD_MOZILLA_CA_BUNDLE:BOOL=ON'] + else: + cmake_extra_args += ['-DDOWNLOAD_MOZILLA_CA_BUNDLE:BOOL=OFF'] + + sanitizers = os.getenv('PYCBC_SANITIZERS', None) + if sanitizers: + for x in sanitizers.split(','): + cmake_extra_args += [f'-DENABLE_SANITIZER_{x.upper()}=ON'] + + if os.getenv('PYCBC_VERBOSE_MAKEFILE', None): + cmake_extra_args += ['-DCMAKE_VERBOSE_MAKEFILE:BOOL=ON'] + + pycbc_cmake_system_version = os.getenv('PYCBC_CMAKE_SYSTEM_VERSION', None) + if pycbc_cmake_system_version is not None: + cmake_extra_args += [f'-DCMAKE_SYSTEM_VERSION={pycbc_cmake_system_version}'] + + pycbc_tls_key_log_file = os.getenv('PYCBC_TLS_KEY_LOG_FILE', None) + if pycbc_tls_key_log_file is not None: + cmake_extra_args += [f'-DCOUCHBASE_CXX_CLIENT_TLS_KEY_LOG_FILE={pycbc_tls_key_log_file}'] + + # now pop these in CMAKE_COMMON_VARIABLES, and they will be used by cmake... + os.environ['CMAKE_COMMON_VARIABLES'] = ' '.join(cmake_extra_args) + + +@dataclass +class CMakeConfig: + build_type: str + num_threads: int + set_cpm_cache: bool + env: Dict[str, str] = field(default_factory=dict) + config_args: List[str] = field(default_factory=list) + + @classmethod + def create_cmake_config(cls, # noqa: C901 + output_dir: str, + source_dir: str, + set_cpm_cache: Optional[bool] = None + ) -> CMakeConfig: + env = os.environ.copy() + num_threads = env.pop('PYCBC_CMAKE_PARALLEL_THREADS', '4') + build_type = env.pop('PYCBC_BUILD_TYPE') + cmake_generator = env.pop('PYCBC_CMAKE_SET_GENERATOR', None) + cmake_arch = env.pop('PYCBC_CMAKE_SET_ARCH', None) + cmake_config_args = [CMAKE_EXE, + source_dir, + f'-DCMAKE_BUILD_TYPE={build_type}', + f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY={output_dir}', + f'-DCMAKE_LIBRARY_OUTPUT_DIRECTORY_{build_type.upper()}={output_dir}'] + + cmake_config_args.extend( + [x for x in + os.environ.get('CMAKE_COMMON_VARIABLES', '').split(' ') + if x]) + + python3_find_strategy = env.get('PYCBC_PYTHON3_FIND_STRATEGY', 'LOCATION') + cmake_config_args += [f'-DPython3_FIND_STRATEGY={python3_find_strategy}'] + + python3_rootdir = env.get('PYCBC_PYTHON3_ROOT_DIR', None) + if python3_rootdir: + cmake_config_args += [f'-DPython3_ROOT_DIR={python3_rootdir}'] + + python3_executable = env.get('PYCBC_PYTHON3_EXECUTABLE', None) + if python3_executable is None and sys.executable: + # if sys.executable determines the path we want to use that to determine the + # Python version in our get_python_version CMake function. + python3_executable = sys.executable + env['PYCBC_PYTHON3_EXECUTABLE'] = python3_executable + if python3_executable: + cmake_config_args += [f'-DPython3_EXECUTABLE={python3_executable}'] + + python3_include = env.get('PYCBC_PYTHON3_INCLUDE_DIR', None) + if python3_include: + cmake_config_args += [f'-DPython3_INCLUDE_DIR={python3_include}'] + + if set_cpm_cache is None: + set_cpm_cache = env.pop('PYCBC_SET_CPM_CACHE', 'false').lower() in ENV_TRUE + use_cpm_cache = env.pop('PYCBC_USE_CPM_CACHE', 'true').lower() in ENV_TRUE + + if set_cpm_cache is True: + # if we are setting the cache, we don't want to attempt a build (it will fail). + use_cpm_cache = False + if os.path.exists(CXXCBC_CACHE_DIR): + shutil.rmtree(CXXCBC_CACHE_DIR) + cmake_config_args += [f'-DCOUCHBASE_CXX_CPM_CACHE_DIR={CXXCBC_CACHE_DIR}', + '-DCPM_DOWNLOAD_ALL=ON', + '-DCPM_USE_NAMED_CACHE_DIRECTORIES=ON', + '-DCPM_USE_LOCAL_PACKAGES=OFF'] + + if use_cpm_cache is True: + if not os.path.exists(CXXCBC_CACHE_DIR): + raise OptionError(f'Cannot use cached dependencies, path={CXXCBC_CACHE_DIR} does not exist.') + cmake_config_args += ['-DCPM_DOWNLOAD_ALL=OFF', + '-DCPM_USE_NAMED_CACHE_DIRECTORIES=ON', + '-DCPM_USE_LOCAL_PACKAGES=OFF', + f'-DCPM_SOURCE_CACHE={CXXCBC_CACHE_DIR}'] + # v4.3.4: If the user has not specifically provided what they want for downloading the Mozilla CA bundle, + # we turn this off to use the bundle from the CPM Cache. If the user wants the bundle downloaded, + # we make sure to not set the CA_BUNDLE_ROOT path. + user_defined_download_mozilla = next( + (arg for arg in cmake_config_args if '-DDOWNLOAD_MOZILLA_CA_BUNDLE' in arg), None) + if user_defined_download_mozilla is None: + cmake_config_args += [f'-DCOUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE_ROOT={CXXCBC_CACHE_DIR}', + '-DDOWNLOAD_MOZILLA_CA_BUNDLE:BOOL=OFF'] + elif user_defined_download_mozilla == '-DDOWNLOAD_MOZILLA_CA_BUNDLE:BOOL=OFF': + cmake_config_args.append(f'-DCOUCHBASE_CXX_CLIENT_EMBED_MOZILLA_CA_BUNDLE_ROOT={CXXCBC_CACHE_DIR}') + + if platform.system() == "Windows": + cmake_config_args += [f'-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_{build_type.upper()}={output_dir}'] + + if cmake_generator: + if cmake_generator.upper() == 'TRUE': + cmake_config_args += ['-G', 'Visual Studio 16 2019'] + else: + cmake_config_args += ['-G', f'{cmake_generator}'] + + if cmake_arch: + if cmake_arch.upper() == 'TRUE': + if sys.maxsize > 2 ** 32: + cmake_config_args += ['-A', 'x64'] + else: + cmake_config_args += ['-A', f'{cmake_arch}'] + # maybe?? + # '-DCMAKE_WINDOWS_EXPORT_ALL_SYMBOLS=TRUE', + + return CMakeConfig(build_type, + num_threads, + set_cpm_cache, + env, + cmake_config_args) + + +class CMakeExtension(Extension): + def __init__(self, name, sourcedir=''): + Extension.__init__(self, name, sources=[]) + self.sourcedir = os.path.abspath(sourcedir) + + +class CMakeConfigureExt(Command): + description = 'Configure Python Operational SDK C Extension' + user_options = [] + + def initialize_options(self) -> None: + return + + def finalize_options(self) -> None: + return + + def run(self) -> None: + check_for_cmake() + process_build_env_vars() + build_ext = self.get_finalized_command('build_ext') + if len(self.distribution.ext_modules) != 1: + raise SetupError('Should have only the Python SDK extension module.') + ext = self.distribution.ext_modules[0] + output_dir = os.path.abspath(os.path.dirname(build_ext.get_ext_fullpath(ext.name))) + set_cpm_cache = os.environ.get('PYCBC_SET_CPM_CACHE', 'true').lower() in ENV_TRUE + cmake_config = CMakeConfig.create_cmake_config(output_dir, ext.sourcedir, set_cpm_cache=set_cpm_cache) + if not os.path.exists(build_ext.build_temp): + os.makedirs(build_ext.build_temp) + print(f'cmake config args: {cmake_config.config_args}') + # configure (i.e. cmake ..) + subprocess.check_call(cmake_config.config_args, # nosec + cwd=build_ext.build_temp, + env=cmake_config.env) + + self._clean_cache_cpm_dependencies() + + def _clean_cache_cpm_dependencies(self): + import re + from fileinput import FileInput + from pathlib import Path + + cxx_cache_path = Path(CXXCBC_CACHE_DIR) + cmake_cpm = next((p for p in cxx_cache_path.glob('cpm/*') if f'{p}'.endswith('.cmake')), None) + if cmake_cpm is not None: + with FileInput(files=[cmake_cpm], inplace=True) as cpm_cmake: + for line in cpm_cmake: + # used so that we don't have a dependency on git w/in environment + if 'find_package(Git REQUIRED)' in line: + line = re.sub(r'Git REQUIRED', 'Git', line) + # remove ending whitespace to avoid double spaced output + print(line.rstrip()) + + +class CMakeBuildExt(build_ext): + + def get_ext_filename(self, ext_name): + ext_path = ext_name.split('.') + ext_suffix = get_config_var('EXT_SUFFIX') + ext_suffix = "." + ext_suffix.split('.')[-1] + return os.path.join(*ext_path) + ext_suffix + + def build_extension(self, ext): # noqa: C901 + check_for_cmake() + process_build_env_vars() + if isinstance(ext, CMakeExtension): + output_dir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))) + cmake_config = CMakeConfig.create_cmake_config(output_dir, ext.sourcedir) + + cmake_build_args = [CMAKE_EXE, + '--build', + '.', + '--config', + f'{cmake_config.build_type}', + '--parallel', + f'{cmake_config.num_threads}'] + + if not os.path.exists(self.build_temp): + os.makedirs(self.build_temp) + print(f'cmake config args: {cmake_config.config_args}') + # configure (i.e. cmake ..) + subprocess.check_call(cmake_config.config_args, # nosec + cwd=self.build_temp, + env=cmake_config.env) + print(f'cmake build args: {cmake_build_args}') + # build (i.e. cmake --build .) + subprocess.check_call(cmake_build_args, # nosec + cwd=self.build_temp, + env=cmake_config.env) + + else: + super().build_extension(ext) + + def _clean_cache_cpm_dependencies(self): + import re + from fileinput import FileInput + from pathlib import Path + + cxx_cache_path = Path(CXXCBC_CACHE_DIR) + cmake_cpm = next((p for p in cxx_cache_path.glob('cpm/*') if f'{p}'.endswith('.cmake')), None) + if cmake_cpm is not None: + with FileInput(files=[cmake_cpm], inplace=True) as cpm_cmake: + for line in cpm_cmake: + # used so that we don't have a dependency on git w/in environment + if 'find_package(Git REQUIRED)' in line: + line = re.sub(r'Git REQUIRED', 'Git', line) + # remove ending whitespace to avoid double spaced output + print(line.rstrip()) + + +class BuildCommand(build): + def finalize_options(self): + # Setting the build_base to an absolute path will make sure that build (i.e. temp) and lib dirs are in sync + # and that our binary is copied appropriately after the build is complete. Particularly useful to avoid Windows + # complaining about long paths. + # NOTE: if setting the build_temp and/or build_lib, the paths should include the build_base path. + # EX: PYCBC_BUILD_BASE=C:\Users\Admin\build + # PYCBC_BUILD_TEMP=C:\Users\Admin\build\tmp + # PYCBC_BUILD_LIB=C:\Users\Admin\build\lib + env = os.environ.copy() + pycbc_build_base = env.pop('PYCBC_BUILD_BASE', None) + if pycbc_build_base: + self.build_base = pycbc_build_base + pycbc_build_temp = env.pop('PYCBC_BUILD_TEMP', None) + if pycbc_build_temp: + self.build_temp = pycbc_build_temp + pycbc_build_lib = env.pop('PYCBC_BUILD_LIB', None) + if pycbc_build_lib: + self.build_lib = pycbc_build_lib + super().finalize_options() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 000000000..2b0fa82ab --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,47 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel", +] +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +minversion = "6.0" +#addopts = "-ra -q" +testpaths = [ + "tests", + "acouchbase/tests", + "couchbase/tests", + "txcouchbase/tests", +] +python_classes = [ + "*Tests" +] +python_files = [ + "*_t.py" +] +markers = [ + "pycbc_couchbase: marks a test for the couchbase API (deselect with '-m \"not pycbc_couchbase\"')", + "pycbc_acouchbase: marks a test for the acouchbase API (deselect with '-m \"not pycbc_acouchbase\"')", + "pycbc_txcouchbase: marks a test for the txcouchbase API (deselect with '-m \"not pycbc_txcouchbase\"')", + "pycbc_diag: marks a test as a diagnostic API test", + "pycbc_kv: marks a test as a Key-Value API test", + "pycbc_streaming: marks a test as a streaming (query, search, analytics, or views) API test", + "pycbc_mgmt: marks a test as a management API test", + "pycbc_misc: marks a test as a miscellaneous (connect, rate_limit) API test", + "pycbc_txn: marks a test as a transactions API test", + "pycbc_slow_mgmt: marks a test as a management API test that is slow", +] + +[tool.autopep8] +max_line_length = 120 +in-place = true +recursive = true + +[tool.isort] +multi_line_output = 1 +force_grid_wrap = 3 +use_parentheses = true +ensure_newline_before_comments = true +line_length = 120 +order_by_type = true diff --git a/setup.py b/setup.py index 4ef3c4422..285e1e282 100644 --- a/setup.py +++ b/setup.py @@ -1,141 +1,83 @@ -#!/usr/bin/env python -import sys -import os.path +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import platform -import warnings -import couchbase_version - -try: - if os.environ.get('PYCBC_NO_DISTRIBUTE'): - raise ImportError() +import sys - from setuptools import setup, Extension -except ImportError: - from distutils.core import setup, Extension +from setuptools import find_packages, setup -extoptions = {} -pkgdata = {} -pkgversion = None +sys.path.append('.') +import couchbase_version # nopep8 # isort:skip # noqa: E402 +from pycbc_build_setup import (BuildCommand, # nopep8 # isort:skip # noqa: E402 + CMakeBuildExt, + CMakeConfigureExt, + CMakeExtension, + CMAKE_EXE) try: couchbase_version.gen_version() except couchbase_version.CantInvokeGit: pass -pkgversion = couchbase_version.get_version() - - -LCB_NAME = None -if sys.platform != 'win32': - extoptions['libraries'] = ['couchbase'] - if sys.platform == 'darwin': - warnings.warn('Adding /usr/local to search path for OS X') - extoptions['library_dirs'] = ['/usr/local/lib'] - extoptions['include_dirs'] = ['/usr/local/include'] -else: - warnings.warn("I'm detecting you're running windows." - "You might want to modify " - "the 'setup.py' script to use appropriate paths") - - - # The layout i have here is an ..\lcb-winbuild, in which there are subdirs - # called 'x86' and 'x64', for x86 and x64 architectures. The default - # 'nmake install' on libcouchbase will install them to 'deps' - bit_type = platform.architecture()[0] - lcb_root = os.path.join(os.path.pardir, 'lcb-winbuild') - - if bit_type.startswith('32'): - lcb_root = os.path.join(lcb_root, 'x86') - else: - lcb_root = os.path.join(lcb_root, 'x64') - - lcb_root = os.path.join(lcb_root, 'deps') - - extoptions['libraries'] = ['libcouchbase'] - ## Enable these lines for debug builds - #extoptions['extra_compile_args'] = ['/Zi'] - #extoptions['extra_link_args'] = ['/DEBUG'] - extoptions['library_dirs'] = [os.path.join(lcb_root, 'lib')] - extoptions['include_dirs'] = [os.path.join(lcb_root, 'include')] - extoptions['define_macros'] = [('_CRT_SECURE_NO_WARNINGS', 1)] - pkgdata['couchbase'] = ['libcouchbase.dll'] - - -SOURCEMODS = [ - 'exceptions', - 'ext', - 'result', - 'opresult', - 'callbacks', - 'cntl', - 'convert', - 'bucket', - 'store', - 'constants', - 'multiresult', - 'miscops', - 'typeutil', - 'oputil', - 'get', - 'counter', - 'http', - 'htresult', - 'ctranscoder', - 'observe', - 'iops', - 'connevents', - 'pipeline', - 'views', - 'n1ql', - ] - -if platform.python_implementation() != 'PyPy': - extoptions['sources'] = [ os.path.join("src", m + ".c") for m in SOURCEMODS ] - module = Extension('couchbase._libcouchbase', **extoptions) - setup_kw = {'ext_modules': [module]} -else: - warnings.warn('The C extension libary does not work on PyPy. ' - 'You should install the couchbase_ffi module. Installation of this ' - 'module will continue but will be unusable without couchbase_ffi') - setup_kw = {} - -setup( - name = 'couchbase', - version = pkgversion, - url="https://github.com/couchbase/couchbase-python-client", - author="Couchbase, Inc.", - author_email="mark.nunberg@couchbase.com", - license="Apache License 2.0", - description="Python Client for Couchbase", - long_description=open("README.rst", "r").read(), - keywords=["couchbase", "nosql", "pycouchbase", "libcouchbase"], - - classifiers=[ - "Development Status :: 5 - Production/Stable", - "License :: OSI Approved :: Apache Software License", - "Intended Audience :: Developers", - "Operating System :: OS Independent", - "Programming Language :: Python", - "Programming Language :: Python :: 2", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: Implementation :: CPython", - "Topic :: Database", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules"], - - packages = [ - 'couchbase', - 'couchbase.views', - 'couchbase.iops', - 'couchbase.async', - 'couchbase.tests', - 'couchbase.tests.cases', - 'gcouchbase', - 'txcouchbase' - ], - package_data = pkgdata, - tests_require = [ 'nose', 'testresources>=0.2.7' ], - test_suite = 'couchbase.tests.test_sync', - **setup_kw -) +PYCBC_README = os.path.join(os.path.dirname(__file__), 'README.md') +PYCBC_VERSION = couchbase_version.get_version() + + +package_data = {} +# some more Windows tomfoolery... +if platform.system() == 'Windows': + package_data = {'couchbase': ['pycbc_core.pyd']} + +# request installing cmake from PyPI if no CMake executable was found. +# otherwise, we want to use the system executable. +setup_requires = [] +if not CMAKE_EXE: + setup_requires += ["cmake>=3.19.0,<4.0.0"] + +print(f'Python SDK version: {PYCBC_VERSION}') + +setup(name='couchbase', + version=PYCBC_VERSION, + ext_modules=[CMakeExtension('couchbase.pycbc_core')], + cmdclass={'build': BuildCommand, + 'build_ext': CMakeBuildExt, + 'configure_ext': CMakeConfigureExt}, + python_requires='>=3.7', + setup_requires=setup_requires, + packages=find_packages( + include=['acouchbase', 'couchbase', 'txcouchbase', 'couchbase.*', 'acouchbase.*', 'txcouchbase.*'], + exclude=['acouchbase.tests', 'couchbase.tests', 'txcouchbase.tests']), + package_data=package_data, + url="https://github.com/couchbase/couchbase-python-client", + author="Couchbase, Inc.", + author_email="PythonPackage@couchbase.com", + license="Apache License 2.0", + description="Python Client for Couchbase", + long_description=open(PYCBC_README, "r").read(), + long_description_content_type='text/markdown', + keywords=["couchbase", "nosql", "pycouchbase", "libcouchbase"], + classifiers=[ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Developers", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: Implementation :: CPython", + "Topic :: Database", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules"], + ) diff --git a/sphinx_requirements.txt b/sphinx_requirements.txt new file mode 100644 index 000000000..53a6f6594 --- /dev/null +++ b/sphinx_requirements.txt @@ -0,0 +1,5 @@ +Sphinx~=7.2 +sphinx-rtd-theme~=2.0 +sphinx-copybutton~=0.5 +enum-tools~=0.9 +sphinx-toolbox~=3.5 diff --git a/src/analytics.cxx b/src/analytics.cxx new file mode 100644 index 000000000..6622b1a35 --- /dev/null +++ b/src/analytics.cxx @@ -0,0 +1,568 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "analytics.hxx" + +#include <core/analytics_scan_consistency.hxx> +#include <core/operations/document_analytics.hxx> + +#include "exceptions.hxx" +#include "n1ql.hxx" +#include "result.hxx" +#include "tracing.hxx" + +couchbase::core::analytics_scan_consistency +str_to_scan_consistency_type(std::string consistency) +{ + if (consistency.compare("not_bounded") == 0) { + return couchbase::core::analytics_scan_consistency::not_bounded; + } + if (consistency.compare("request_plus") == 0) { + return couchbase::core::analytics_scan_consistency::request_plus; + } + + // TODO: better exception + PyErr_SetString(PyExc_ValueError, "Invalid Analytics Scan Consistency type."); + return {}; +} + +PyObject* +analytics_status_to_string(couchbase::core::operations::analytics_response::analytics_status status) +{ + std::string status_str; + switch (status) { + case couchbase::core::operations::analytics_response::analytics_status::running: { + status_str = "running"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::success: { + status_str = "success"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::errors: { + status_str = "errors"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::completed: { + status_str = "completed"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::stopped: { + status_str = "stopped"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::timedout: { + status_str = "timeout"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::closed: { + status_str = "closed"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::fatal: { + status_str = "fatal"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::aborted: { + status_str = "aborted"; + break; + } + case couchbase::core::operations::analytics_response::analytics_status::unknown: + default: { + status_str = "unknown"; + break; + } + } + // should not be able to reach here, since this is an enum class + return PyUnicode_FromString(status_str.c_str()); +} + +PyObject* +get_result_metrics(couchbase::core::operations::analytics_response::analytics_metrics metrics) +{ + PyObject* pyObj_metrics = PyDict_New(); + std::chrono::duration<unsigned long long, std::nano> int_nsec = metrics.elapsed_time; + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(int_nsec.count()); + if (-1 == PyDict_SetItemString(pyObj_metrics, "elapsed_time", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_nsec = metrics.execution_time; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_nsec.count()); + if (-1 == PyDict_SetItemString(pyObj_metrics, "execution_time", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.result_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "result_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.result_size); + if (-1 == PyDict_SetItemString(pyObj_metrics, "result_size", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.error_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "error_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.processed_objects); + if (-1 == PyDict_SetItemString(pyObj_metrics, "processed_objects", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.warning_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "warning_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + return pyObj_metrics; +} + +PyObject* +get_result_metadata(couchbase::core::operations::analytics_response::analytics_meta_data metadata, + bool include_metrics) +{ + PyObject* pyObj_metadata = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(metadata.request_id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "request_id", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(metadata.client_context_id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "client_context_id", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = analytics_status_to_string(metadata.status); + if (-1 == PyDict_SetItemString(pyObj_metadata, "status", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (metadata.signature.has_value()) { + pyObj_tmp = PyUnicode_FromString(metadata.signature.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "signature", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + } + + PyObject* pyObj_warnings = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& warning : metadata.warnings) { + PyObject* pyObj_warning = PyDict_New(); + + pyObj_tmp = PyLong_FromLong(warning.code); + if (-1 == PyDict_SetItemString(pyObj_warning, "code", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(warning.message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_warning, "message", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == PyList_Append(pyObj_warnings, pyObj_warning)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_warning); + } + + if (-1 == PyDict_SetItemString(pyObj_metadata, "warnings", pyObj_warnings)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_warnings); + + PyObject* pyObj_errors = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& error : metadata.errors) { + PyObject* pyObj_error = PyDict_New(); + + pyObj_tmp = PyLong_FromLong(error.code); + if (-1 == PyDict_SetItemString(pyObj_error, "code", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(error.message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error, "message", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == PyList_Append(pyObj_errors, pyObj_error)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_error); + } + + if (-1 == PyDict_SetItemString(pyObj_metadata, "errors", pyObj_errors)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_errors); + + if (include_metrics) { + PyObject* pyObject_metrics = get_result_metrics(metadata.metrics); + + if (-1 == PyDict_SetItemString(pyObj_metadata, "metrics", pyObject_metrics)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObject_metrics); + } + + return pyObj_metadata; +} + +result* +create_result_from_analytics_response(couchbase::core::operations::analytics_response resp, + bool include_metrics) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_payload = PyDict_New(); + + PyObject* pyObject_metadata = get_result_metadata(resp.meta, include_metrics); + if (-1 == PyDict_SetItemString(pyObj_payload, "metadata", pyObject_metadata)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObject_metadata); + + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_payload)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_payload); + + return res; +} + +void +create_analytics_result(couchbase::core::operations::analytics_response resp, + bool include_metrics, + std::shared_ptr<rows_queue<PyObject*>> rows, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + auto set_exception = false; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_args = NULL; + PyObject* pyObj_func = NULL; + PyObject* pyObj_callback_res = nullptr; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing analytics operation."); + // lets clear any errors + PyErr_Clear(); + rows->put(pyObj_exc); + } else { + for (auto const& row : resp.rows) { + PyObject* pyObj_row = PyBytes_FromStringAndSize(row.c_str(), row.length()); + rows->put(pyObj_row); + } + + auto res = create_result_from_analytics_response(resp, include_metrics); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + // None indicates done (i.e. raise StopIteration) + Py_INCREF(Py_None); + rows->put(Py_None); + rows->put(reinterpret_cast<PyObject*>(res)); + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Analytics operation error."); + rows->put(pyObj_exc); + } + + // This is for txcouchbase -- let it knows we're done w/ the analytics request + if (pyObj_callback != nullptr) { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, PyBool_FromLong(static_cast<long>(1))); + } + + if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_CallObject(pyObj_func, pyObj_args); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + pycbc_set_python_exception( + PycbcError::InternalSDKError, __FILE__, __LINE__, "Analytics complete callback failed."); + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +streamed_result* +handle_analytics_query([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + char* statement = nullptr; + + char* scan_consistency = nullptr; + char* bucket_name = nullptr; + char* scope_name = nullptr; + char* scope_qualifier = nullptr; + char* client_context_id = nullptr; + + std::uint64_t timeout = 0; + std::uint64_t streaming_timeout_us = 0; + // booleans, but use int to read from kwargs + int metrics = 0; + int readonly = 0; + int priority = 0; + + PyObject* pyObj_raw = nullptr; + PyObject* pyObj_named_parameters = nullptr; + PyObject* pyObj_positional_parameters = nullptr; + PyObject* pyObj_serializer = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_row_callback = nullptr; + PyObject* pyObj_span = nullptr; + + static const char* kw_list[] = { "conn", + "statement", + "bucket_name", + "scope_name", + "scope_qualifier", + "client_context_id", + "scan_consistency", + "timeout", + "streaming_timeout", + "metrics", + "readonly", + "priority", + "named_parameters", + "positional_parameters", + "raw", + "serializer", + "callback", + "errback", + "row_callback", + "span", + nullptr }; + + const char* kw_format = "O!s|sssssKKiiiOOOOOOOO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &statement, + &bucket_name, + &scope_name, + &scope_qualifier, + &client_context_id, + &scan_consistency, + &timeout, + &streaming_timeout_us, + &metrics, + &readonly, + &priority, + &pyObj_named_parameters, + &pyObj_positional_parameters, + &pyObj_raw, + &pyObj_serializer, + &pyObj_callback, + &pyObj_errback, + &pyObj_row_callback, + &pyObj_span); + if (!ret) { + PyErr_SetString(PyExc_ValueError, "Unable to parse arguments"); + return nullptr; + } + + connection* conn = nullptr; + + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + PyErr_SetString(PyExc_ValueError, "passed null connection"); + return nullptr; + } + PyErr_Clear(); + + couchbase::core::operations::analytics_request req{ statement }; + // positional parameters + std::vector<couchbase::core::json_string> positional_parameters{}; + if (pyObj_positional_parameters && PyList_Check(pyObj_positional_parameters)) { + size_t nargs = static_cast<size_t>(PyList_Size(pyObj_positional_parameters)); + size_t ii; + for (ii = 0; ii < nargs; ++ii) { + PyObject* pyOb_param = PyList_GetItem(pyObj_positional_parameters, ii); + if (!pyOb_param) { + // TODO: handle this better + PyErr_SetString(PyExc_ValueError, "Unable to parse positional argument."); + return nullptr; + } + // PyList_GetItem returns borrowed ref, inc while using, decr after done + Py_INCREF(pyOb_param); + if (PyUnicode_Check(pyOb_param)) { + auto res = std::string(PyUnicode_AsUTF8(pyOb_param)); + positional_parameters.push_back(couchbase::core::json_string{ std::move(res) }); + } + //@TODO: exception if this check fails?? + Py_DECREF(pyOb_param); + pyOb_param = nullptr; + } + } + if (positional_parameters.size() > 0) { + req.positional_parameters = positional_parameters; + } + + // named parameters + std::map<std::string, couchbase::core::json_string> named_parameters{}; + if (pyObj_named_parameters && PyDict_Check(pyObj_named_parameters)) { + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_named_parameters, &pos, &pyObj_key, &pyObj_value)) { + std::string k; + if (PyUnicode_Check(pyObj_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_key)); + } + if (PyUnicode_Check(pyObj_value) && !k.empty()) { + auto res = std::string(PyUnicode_AsUTF8(pyObj_value)); + named_parameters.emplace(k, couchbase::core::json_string{ std::move(res) }); + } + } + } + if (named_parameters.size() > 0) { + req.named_parameters = named_parameters; + } + + if (0 < timeout) { + req.timeout = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + } + + // req.metrics = metrics == 1; + req.readonly = readonly == 1; + req.priority = priority == 1; + + if (scan_consistency) { + req.scan_consistency = + str_to_scan_consistency_type<couchbase::core::analytics_scan_consistency>(scan_consistency); + } + + if (scope_qualifier != nullptr) { + req.scope_qualifier = std::string(scope_qualifier); + } + + // raw options + std::map<std::string, couchbase::core::json_string> raw_options{}; + if (pyObj_raw && PyDict_Check(pyObj_raw)) { + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_raw, &pos, &pyObj_key, &pyObj_value)) { + std::string k; + if (PyUnicode_Check(pyObj_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_key)); + } + if (PyUnicode_Check(pyObj_value) && !k.empty()) { + auto res = std::string(PyUnicode_AsUTF8(pyObj_value)); + raw_options.emplace(k, couchbase::core::json_string{ std::move(res) }); + } + } + } + if (raw_options.size() > 0) { + req.raw = raw_options; + } + if (nullptr != pyObj_span) { + req.parent_span = std::make_shared<pycbc::request_span>(pyObj_span); + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + Py_XINCREF(pyObj_errback); + Py_XINCREF(pyObj_callback); + + // timeout is always set either to default, or timeout provided in options + auto streaming_timeout = couchbase::core::timeout_defaults::analytics_timeout; + if (streaming_timeout_us > 0) { + streaming_timeout = std::chrono::milliseconds(streaming_timeout_us / 1000ULL); + } + streamed_result* streamed_res = create_streamed_result_obj(streaming_timeout); + + // TODO: let the couchbase++ streaming stabilize a bit more... + // req.row_callback = [rows = streamed_res->rows](std::string&& row) { + // PyGILState_STATE state = PyGILState_Ensure(); + // PyObject* pyObj_row = PyBytes_FromStringAndSize(row.c_str(), row.length()); + // rows->put(pyObj_row); + // PyGILState_Release(state); + // return couchbase::core::utils::json::stream_control::next_row; + // }; + + { + Py_BEGIN_ALLOW_THREADS conn->cluster_.execute( + req, + [rows = streamed_res->rows, include_metrics = metrics, pyObj_callback, pyObj_errback]( + couchbase::core::operations::analytics_response resp) { + create_analytics_result(resp, include_metrics, rows, pyObj_callback, pyObj_errback); + }); + Py_END_ALLOW_THREADS + } + return streamed_res; +} diff --git a/src/analytics.hxx b/src/analytics.hxx new file mode 100644 index 000000000..0095655ed --- /dev/null +++ b/src/analytics.hxx @@ -0,0 +1,24 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" +#include "result.hxx" + +streamed_result* +handle_analytics_query(PyObject* self, PyObject* args, PyObject* kwargs); diff --git a/src/binary_ops.cxx b/src/binary_ops.cxx new file mode 100644 index 000000000..94330a34a --- /dev/null +++ b/src/binary_ops.cxx @@ -0,0 +1,734 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "binary_ops.hxx" +#include "exceptions.hxx" +#include "result.hxx" +#include "utils.hxx" + +template<typename T> +result* +add_extras_to_result([[maybe_unused]] const T& t, result* res) +{ + return res; +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::increment_response>( + const couchbase::core::operations::increment_response& resp, + result* res) +{ + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(resp.content); + if (-1 == PyDict_SetItemString(res->dict, "content", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + return res; +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::decrement_response>( + const couchbase::core::operations::decrement_response& resp, + result* res) +{ + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(resp.content); + if (-1 == PyDict_SetItemString(res->dict, "content", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + return res; +} + +template<typename T> +result* +create_base_result_from_binary_op_response(const char* key, const T& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(resp.cas.value()); + if (-1 == PyDict_SetItemString(res->dict, RESULT_CAS, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_mutation_token = create_mutation_token_obj(resp.token); + if (-1 == PyDict_SetItemString(res->dict, RESULT_MUTATION_TOKEN, pyObj_mutation_token)) { + Py_XDECREF(pyObj_mutation_token); + return nullptr; + } + Py_DECREF(pyObj_mutation_token); + + return res; +} + +template<typename Response> +void +create_result_from_binary_op_response(const char* key, + const Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* pyObj_args = NULL; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + if (resp.ctx.ec()) { + pyObj_exc = + build_exception_from_context(resp.ctx, __FILE__, __LINE__, "Binary operation error."); + if (pyObj_errback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_False); + barrier->set_value(Py_False); + if (-1 == PyDict_SetItemString(multi_result->dict, key, pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + // won't fall into logic path where pyObj_exc is decremented later + Py_DECREF(pyObj_exc); + } else { + barrier->set_value(pyObj_exc); + } + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_base_result_from_binary_op_response(key, resp); + if (res != nullptr) { + res = add_extras_to_result(resp, res); + } + + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_True); + barrier->set_value(Py_True); + if (-1 == + PyDict_SetItemString(multi_result->dict, key, reinterpret_cast<PyObject*>(res))) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(reinterpret_cast<PyObject*>(res)); + } else { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Binary operation error."); + if (pyObj_errback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_False); + barrier->set_value(Py_False); + if (-1 == PyDict_SetItemString(multi_result->dict, key, pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + // won't fall into logic path where pyObj_exc is decremented later + Py_DECREF(pyObj_exc); + } else { + barrier->set_value(pyObj_exc); + } + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<typename Request> +void +do_binary_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, + [key = req.id.key(), pyObj_callback, pyObj_errback, barrier, multi_result](response_type resp) { + create_result_from_binary_op_response( + key.c_str(), resp, pyObj_callback, pyObj_errback, barrier, multi_result); + }); + Py_END_ALLOW_THREADS +} + +PyObject* +prepare_and_execute_counter_op(struct counter_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + if (options->op_type == Operations::INCREMENT) { + auto req = couchbase::core::operations::increment_request{ options->id }; + req.timeout = options->timeout_ms; + req.delta = options->delta; + req.initial_value = options->initial_value; + if (options->expiry > 0) { + req.expiry = options->expiry; + } + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::increment_request_with_legacy_durability{ + req, options->persist_to, options->replicate_to + }; + do_binary_op(*(options->conn), + req_legacy_durability, + pyObj_callback, + pyObj_errback, + barrier, + multi_result); + Py_RETURN_NONE; + } + req.durability_level = options->durability_level; + do_binary_op(*(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + } else { + auto req = couchbase::core::operations::decrement_request{ options->id }; + req.timeout = options->timeout_ms; + req.delta = options->delta; + req.initial_value = options->initial_value; + if (options->expiry > 0) { + req.expiry = options->expiry; + } + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::decrement_request_with_legacy_durability{ + req, options->persist_to, options->replicate_to + }; + do_binary_op(*(options->conn), + req_legacy_durability, + pyObj_callback, + pyObj_errback, + barrier, + multi_result); + Py_RETURN_NONE; + } + req.durability_level = options->durability_level; + do_binary_op(*(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + } + Py_RETURN_NONE; +} + +PyObject* +prepare_and_execute_binary_mutation_op(struct binary_mutation_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + if (!PyBytes_Check(options->pyObj_value)) { + if (multi_result != nullptr) { + PyObject* pyObj_exc = pycbc_build_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Value should be bytes object."); + if (-1 == PyDict_SetItemString(multi_result->dict, options->id.key().c_str(), pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_exc); + Py_INCREF(Py_False); + barrier->set_value(Py_False); + Py_RETURN_NONE; + } + barrier->set_value(nullptr); + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Value should be bytes object."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + couchbase::core::utils::binary value; + try { + value = PyObject_to_binary(options->pyObj_value); + } catch (const std::exception& e) { + if (multi_result != nullptr) { + PyObject* pyObj_exc = + pycbc_build_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, e.what()); + if (-1 == PyDict_SetItemString(multi_result->dict, options->id.key().c_str(), pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_exc); + Py_INCREF(Py_False); + barrier->set_value(Py_False); + Py_RETURN_NONE; + } + barrier->set_value(nullptr); + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, e.what()); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + if (options->op_type == Operations::APPEND) { + auto req = couchbase::core::operations::append_request{ options->id }; + req.timeout = options->timeout_ms; + req.cas = options->cas; + req.value = value; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::append_request_with_legacy_durability{ req, + options->persist_to, + options->replicate_to }; + do_binary_op(*(options->conn), + req_legacy_durability, + pyObj_callback, + pyObj_errback, + barrier, + multi_result); + Py_RETURN_NONE; + } + req.durability_level = options->durability_level; + do_binary_op(*(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + } else { + auto req = couchbase::core::operations::prepend_request{ options->id }; + req.timeout = options->timeout_ms; + req.cas = options->cas; + req.value = value; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + // need to branch if using legacy durability + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::prepend_request_with_legacy_durability{ + req, options->persist_to, options->replicate_to + }; + do_binary_op(*(options->conn), + req_legacy_durability, + pyObj_callback, + pyObj_errback, + barrier, + multi_result); + Py_RETURN_NONE; + } + req.durability_level = options->durability_level; + do_binary_op(*(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + } + Py_RETURN_NONE; +} + +struct counter_options +get_counter_options(PyObject* op_args) +{ + struct counter_options opts; + + PyObject* pyObj_delta = PyDict_GetItemString(op_args, "delta"); + if (pyObj_delta != nullptr) { + auto delta = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_delta)); + opts.delta = delta; + } + + PyObject* pyObj_initial = PyDict_GetItemString(op_args, "initial"); + if (pyObj_initial != nullptr) { + auto initial = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_initial)); + opts.initial_value = initial; + } + + PyObject* pyObj_span = PyDict_GetItemString(op_args, "span"); + if (pyObj_span != nullptr) { + opts.span = pyObj_span; + } + + PyObject* pyObj_expiry = PyDict_GetItemString(op_args, "expiry"); + if (pyObj_expiry != nullptr) { + opts.expiry = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_expiry)); + } + + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (pyObj_timeout != nullptr) { + auto timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_timeout)); + auto timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + if (0 < timeout) { + opts.timeout_ms = timeout_ms; + } + } + + PyObject* pyObj_durability = PyDict_GetItemString(op_args, "durability"); + if (pyObj_durability) { + if (PyDict_Check(pyObj_durability)) { + auto durability = PyObject_to_durability(pyObj_durability); + opts.use_legacy_durability = true; + opts.persist_to = durability.first; + opts.replicate_to = durability.second; + } else if (PyLong_Check(pyObj_durability)) { + opts.durability_level = PyObject_to_durability_level(pyObj_durability); + } + } + + return opts; +} + +struct binary_mutation_options +get_binary_mutation_options(PyObject* op_args) +{ + struct binary_mutation_options opts; + + PyObject* pyObj_span = PyDict_GetItemString(op_args, "span"); + if (pyObj_span != nullptr) { + opts.span = pyObj_span; + } + + PyObject* pyObj_cas = PyDict_GetItemString(op_args, "cas"); + couchbase::cas cas = couchbase::cas{ 0 }; + if (pyObj_cas != nullptr) { + auto cas_int = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_cas)); + if (cas_int != 0) { + cas = couchbase::cas{ cas_int }; + } + } + opts.cas = cas; + + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (pyObj_timeout != nullptr) { + auto timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_timeout)); + auto timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + if (0 < timeout) { + opts.timeout_ms = timeout_ms; + } + } + + PyObject* pyObj_durability = PyDict_GetItemString(op_args, "durability"); + if (pyObj_durability) { + if (PyDict_Check(pyObj_durability)) { + auto durability = PyObject_to_durability(pyObj_durability); + opts.use_legacy_durability = true; + opts.persist_to = durability.first; + opts.replicate_to = durability.second; + } else if (PyLong_Check(pyObj_durability)) { + opts.durability_level = PyObject_to_durability_level(pyObj_durability); + } + } + + return opts; +} + +PyObject* +handle_binary_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + char* bucket = nullptr; + char* scope = nullptr; + char* collection = nullptr; + char* key = nullptr; + Operations::OperationType op_type = Operations::UNKNOWN; + PyObject* pyObj_value = nullptr; + PyObject* pyObj_op_args = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + + static const char* kw_list[] = { "conn", "bucket", "scope", "collection_name", "key", "op_type", + "value", "op_args", nullptr }; + + const char* kw_format = "O!ssssI|OO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &bucket, + &scope, + &collection, + &key, + &op_type, + &pyObj_value, + &pyObj_op_args); + + if (!ret) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform binary operation. Unable to parse args/kwargs."); + return nullptr; + } + + connection* conn = nullptr; + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + pyObj_callback = PyDict_GetItemString(pyObj_op_args, "callback"); + pyObj_errback = PyDict_GetItemString(pyObj_op_args, "errback"); + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + + PyObject* pyObj_op_response = nullptr; + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + + switch (op_type) { + case Operations::APPEND: + case Operations::PREPEND: { + auto opts = get_binary_mutation_options(pyObj_op_args); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, key }; + opts.op_type = op_type; + if (pyObj_value != nullptr) { + opts.pyObj_value = pyObj_value; + } + + try { + pyObj_op_response = + prepare_and_execute_binary_mutation_op(&opts, pyObj_callback, pyObj_errback, barrier); + } catch (const std::system_error& e) { + barrier->set_value(nullptr); + pycbc_set_python_exception(e.code(), __FILE__, __LINE__, e.what()); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + pyObj_op_response = nullptr; + } + break; + } + case Operations::INCREMENT: + case Operations::DECREMENT: { + auto opts = get_counter_options(pyObj_op_args); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, key }; + opts.op_type = op_type; + + try { + pyObj_op_response = + prepare_and_execute_counter_op(&opts, pyObj_callback, pyObj_errback, barrier); + } catch (const std::system_error& e) { + barrier->set_value(nullptr); + pycbc_set_python_exception(e.code(), __FILE__, __LINE__, e.what()); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + pyObj_op_response = nullptr; + } + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized binary operation passed in."); + barrier->set_value(nullptr); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + + return pyObj_op_response; +} + +PyObject* +handle_binary_multi_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_conn = nullptr; + char* bucket = nullptr; + char* scope = nullptr; + char* collection = nullptr; + Operations::OperationType op_type = Operations::UNKNOWN; + PyObject* pyObj_op_args = nullptr; + + static const char* kw_list[] = { "conn", "bucket", "scope", "collection_name", + "op_type", "op_args", nullptr }; + + const char* kw_format = "O!sssIO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &bucket, + &scope, + &collection, + &op_type, + &pyObj_op_args); + if (!ret) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform binary operation. Unable to parse args/kwargs."); + return nullptr; + } + + connection* conn = nullptr; + + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + std::vector<std::future<PyObject*>> op_results{}; + + PyObject* pyObj_multi_result = create_result_obj(); + result* multi_result = reinterpret_cast<result*>(pyObj_multi_result); + + if (pyObj_op_args && PyDict_Check(pyObj_op_args)) { + PyObject *pyObj_doc_key, *pyObj_op_dict; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_op_args, &pos, &pyObj_doc_key, &pyObj_op_dict)) { + std::string k; + bool do_op = false; + PyObject* pyObj_op_response = nullptr; + if (PyUnicode_Check(pyObj_doc_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_doc_key)); + } + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + if (PyDict_Check(pyObj_op_dict) && !k.empty()) { + PyObject* pyObj_value = PyDict_GetItemString(pyObj_op_dict, "value"); + switch (op_type) { + case Operations::APPEND: + case Operations::PREPEND: { + auto opts = get_binary_mutation_options(pyObj_op_dict); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, k }; + opts.op_type = op_type; + if (pyObj_value != nullptr) { + opts.pyObj_value = pyObj_value; + } + + try { + pyObj_op_response = prepare_and_execute_binary_mutation_op( + &opts, nullptr, nullptr, barrier, multi_result); + } catch (const std::system_error& e) { + PyObject* pyObj_exc = pycbc_build_exception(e.code(), __FILE__, __LINE__, e.what()); + barrier->set_value(pyObj_exc); + } + break; + } + case Operations::INCREMENT: + case Operations::DECREMENT: { + auto opts = get_counter_options(pyObj_op_dict); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, k }; + opts.op_type = op_type; + + try { + pyObj_op_response = + prepare_and_execute_counter_op(&opts, nullptr, nullptr, barrier, multi_result); + } catch (const std::system_error& e) { + PyObject* pyObj_exc = pycbc_build_exception(e.code(), __FILE__, __LINE__, e.what()); + barrier->set_value(pyObj_exc); + } + break; + } + default: { + PyObject* pyObj_exc = pycbc_build_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized binary operation passed in."); + barrier->set_value(pyObj_exc); + break; + } + }; + } + + Py_XDECREF(pyObj_op_response); + op_results.emplace_back(std::move(f)); + } + } + + auto all_okay = true; + for (auto i = 0; i < op_results.size(); i++) { + PyObject* res = nullptr; + Py_BEGIN_ALLOW_THREADS res = op_results[i].get(); + Py_END_ALLOW_THREADS if (res == Py_False) + { + all_okay = false; + } + Py_XDECREF(res); + } + + if (all_okay) { + PyDict_SetItemString(multi_result->dict, "all_okay", Py_True); + } else { + PyDict_SetItemString(multi_result->dict, "all_okay", Py_False); + } + + return reinterpret_cast<PyObject*>(multi_result); +} diff --git a/src/binary_ops.hxx b/src/binary_ops.hxx new file mode 100644 index 000000000..b31fb9616 --- /dev/null +++ b/src/binary_ops.hxx @@ -0,0 +1,66 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" +#include "tracing.hxx" +#include <core/document_id.hxx> +#include <couchbase/cas.hxx> +#include <couchbase/persist_to.hxx> +#include <couchbase/replicate_to.hxx> + +struct counter_options { + // required + connection* conn; + couchbase::core::document_id id; + Operations::OperationType op_type{ Operations::UNKNOWN }; + uint64_t delta{ 1 }; + + // optional + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + uint32_t expiry{ 0 }; + couchbase::durability_level durability_level{ couchbase::durability_level::none }; + bool use_legacy_durability{ false }; + couchbase::replicate_to replicate_to{ couchbase::replicate_to::none }; + couchbase::persist_to persist_to{ couchbase::persist_to::none }; + std::optional<uint64_t> initial_value{}; + PyObject* span = nullptr; +}; + +struct binary_mutation_options { + // required + connection* conn; + couchbase::core::document_id id; + Operations::OperationType op_type{ Operations::UNKNOWN }; + PyObject* pyObj_value = nullptr; + + // optional + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + couchbase::durability_level durability_level{ couchbase::durability_level::none }; + bool use_legacy_durability{ false }; + couchbase::replicate_to replicate_to{ couchbase::replicate_to::none }; + couchbase::persist_to persist_to{ couchbase::persist_to::none }; + couchbase::cas cas; + PyObject* span = nullptr; +}; + +PyObject* +handle_binary_op(PyObject* self, PyObject* args, PyObject* kwargs); + +PyObject* +handle_binary_multi_op(PyObject* self, PyObject* args, PyObject* kwargs); diff --git a/src/bucket.c b/src/bucket.c deleted file mode 100644 index 4a2cb56db..000000000 --- a/src/bucket.c +++ /dev/null @@ -1,826 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "structmember.h" -#include "oputil.h" -#include "iops.h" -#include <libcouchbase/vbucket.h> - -PyObject *pycbc_DummyTuple; -PyObject *pycbc_DummyKeywords; - -static int -Bucket__init__(pycbc_Bucket *self, - PyObject *args, - PyObject *kwargs); - -static PyObject* -Bucket__connect(pycbc_Bucket *self); - -static void -Bucket_dtor(pycbc_Bucket *self); - -static PyTypeObject BucketType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static PyObject * -Bucket_get_format(pycbc_Bucket *self, void * unused) -{ - if (self->dfl_fmt) { - Py_INCREF(self->dfl_fmt); - return self->dfl_fmt; - } - - (void)unused; - Py_RETURN_NONE; -} - -static int -Bucket_set_format(pycbc_Bucket *self, PyObject *value, void *unused) -{ - if (value != pycbc_helpers.fmt_auto) { - if (!PyNumber_Check(value)) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, - "Format must be a number"); - return -1; - } - - if (Py_TYPE(value) == &PyBool_Type) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, - "Format must not be a boolean"); - return -1; - } - } - - Py_XDECREF(self->dfl_fmt); - Py_INCREF(value); - self->dfl_fmt = value; - - (void)unused; - return 0; -} - -static int -Bucket_set_transcoder(pycbc_Bucket *self, PyObject *value, void *unused) -{ - Py_XDECREF(self->tc); - if (PyObject_IsTrue(value)) { - self->tc = value; - Py_INCREF(self->tc); - } else { - self->tc = NULL; - } - (void)unused; - return 0; -} - -static PyObject* -Bucket_get_transcoder(pycbc_Bucket *self, void *unused) -{ - if (self->tc) { - Py_INCREF(self->tc); - return self->tc; - } - Py_INCREF(Py_None); - - (void)unused; - return Py_None; -} - -static PyObject * -Bucket_server_nodes(pycbc_Bucket *self, void *unused) -{ - const char * const *cnodes; - const char **curnode; - PyObject *ret_list; - cnodes = lcb_get_server_list(self->instance); - - if (!cnodes) { - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "Can't get server nodes"); - return NULL; - } - - ret_list = PyList_New(0); - if (!ret_list) { - return NULL; - } - - for (curnode = (const char**)cnodes; *curnode; curnode++) { - PyObject *tmpstr = pycbc_SimpleStringZ(*curnode); - PyList_Append(ret_list, tmpstr); - Py_DECREF(tmpstr); - } - - (void)unused; - - return ret_list; -} - -static PyObject * -Bucket_get_configured_replica_count(pycbc_Bucket *self, void *unused) -{ - PyObject *iret = pycbc_IntFromUL(lcb_get_num_replicas(self->instance)); - - (void)unused; - return iret; -} - -static PyObject * -Bucket_connected(pycbc_Bucket *self, void *unused) -{ - PyObject* ret = self->flags & PYCBC_CONN_F_CONNECTED ? Py_True : Py_False; - - if (ret == Py_False) { - void *handle = NULL; - lcb_error_t err; - err = lcb_cntl(self->instance, LCB_CNTL_GET, LCB_CNTL_VBCONFIG, &handle); - if (err == LCB_SUCCESS && handle != NULL) { - self->flags |= PYCBC_CONN_F_CONNECTED; - ret = Py_True; - } - } - - Py_INCREF(ret); - - (void)unused; - - return ret; -} - -static PyObject * -Bucket__instance_pointer(pycbc_Bucket *self, void *unused) -{ - PyObject *ret; - Py_uintptr_t ptri = (Py_uintptr_t)self->instance; - ret = pycbc_IntFromULL(ptri); - (void)unused; - return ret; -} - -static PyObject * -Bucket__thr_lockop(pycbc_Bucket *self, PyObject *arg) -{ - int rv; - int is_unlock = 0; - rv = PyArg_ParseTuple(arg, "i:is_unlock", &is_unlock); - - if (!rv) { - return NULL; - } - - if (!self->lockmode) { - PYCBC_EXC_WRAP(PYCBC_EXC_THREADING, 0, "lockmode is LOCKMODE_NONE"); - return NULL; - } - - if (is_unlock) { - PyThread_release_lock(self->lock); - } else { - if (!PyThread_acquire_lock(self->lock, WAIT_LOCK)) { - PYCBC_EXC_WRAP(PYCBC_EXC_THREADING, 0, "Couldn't lock"); - return NULL; - } - } - - Py_RETURN_NONE; -} - -static PyObject * -Bucket__close(pycbc_Bucket *self) -{ - lcb_error_t err; - - if (self->flags & PYCBC_CONN_F_CLOSED) { - Py_RETURN_NONE; - } - - self->flags |= PYCBC_CONN_F_CLOSED; - - lcb_destroy(self->instance); - - if (self->iopswrap) { - Py_XDECREF(self->iopswrap); - self->iopswrap = NULL; - } - - err = lcb_create(&self->instance, NULL); - pycbc_assert(err == LCB_SUCCESS); - if (err != LCB_SUCCESS) { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, - err, - "Internal error while closing object"); - return NULL; - } - - Py_RETURN_NONE; -} - -static void -timings_callback(lcb_t instance, - const void *cookie, - lcb_timeunit_t timeunit, - lcb_uint32_t min, - lcb_uint32_t max, - lcb_uint32_t total, - lcb_uint32_t maxtotal) -{ - PyObject *arr = (PyObject *)cookie; - PyObject *dict; - double divisor = 1.0; - double d_min, d_max; - - if (timeunit == LCB_TIMEUNIT_NSEC) { - divisor = 1000000; - } else if (timeunit == LCB_TIMEUNIT_USEC) { - divisor = 1000; - } else if (timeunit == LCB_TIMEUNIT_MSEC) { - divisor = 1; - } else if (timeunit == LCB_TIMEUNIT_SEC) { - divisor = 0.001; - } - - d_min = (double)min / divisor; - d_max = (double)max / divisor; - - dict = PyDict_New(); - PyList_Append(arr, dict); - PyDict_SetItemString(dict, "min", PyFloat_FromDouble(d_min)); - PyDict_SetItemString(dict, "max", PyFloat_FromDouble(d_max)); - PyDict_SetItemString(dict, "count", pycbc_IntFromUL(total)); - - (void)maxtotal; - (void)instance; -} - -static PyObject * -Bucket__start_timings(pycbc_Bucket *self) -{ - lcb_disable_timings(self->instance); - lcb_enable_timings(self->instance); - Py_RETURN_NONE; -} - -static PyObject * -Bucket__clear_timings(pycbc_Bucket *self) -{ - lcb_disable_timings(self->instance); - Py_RETURN_NONE; -} - -static PyObject * -Bucket__get_timings(pycbc_Bucket *self) -{ - PyObject *ll = PyList_New(0); - lcb_get_timings(self->instance, ll, timings_callback); - return ll; -} - -static PyObject * -Bucket__mutinfo(pycbc_Bucket *self) -{ - PyObject *ll = PyList_New(0); - size_t ii, vbmax; - lcbvb_CONFIG *cfg = NULL; - lcb_error_t rc; - - rc = lcb_cntl(self->instance, LCB_CNTL_GET, LCB_CNTL_VBCONFIG, &cfg); - if (rc != LCB_SUCCESS) { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, rc, "Couldn't get vBucket config"); - return NULL; - } - - vbmax = vbucket_config_get_num_vbuckets(cfg); - for (ii = 0; ii < vbmax; ++ii) { - lcb_KEYBUF kb = { 0 }; - const lcb_MUTATION_TOKEN *mt; - lcb_error_t rc = LCB_SUCCESS; - PyObject *cur; - - kb.type = LCB_KV_VBID; - kb.contig.nbytes = (size_t)ii; - - mt = lcb_get_mutation_token(self->instance, &kb, &rc); - if (mt == NULL) { - continue; - } - cur = Py_BuildValue("HKK", LCB_MUTATION_TOKEN_VB(mt), - LCB_MUTATION_TOKEN_ID(mt), LCB_MUTATION_TOKEN_SEQ(mt)); - PyList_Append(ll, cur); - Py_DECREF(cur); - } - - return ll; -} - - -static PyGetSetDef Bucket_TABLE_getset[] = { - { "default_format", - (getter)Bucket_get_format, - (setter)Bucket_set_format, - PyDoc_STR("The default format to use for encoding values " - "(passed to transcoder)") - }, - { "server_nodes", - (getter)Bucket_server_nodes, - NULL, - PyDoc_STR("Get a list of the current nodes in the cluster") - }, - { "configured_replica_count", - (getter)Bucket_get_configured_replica_count, - NULL, - PyDoc_STR("Get the number of configured replicas for the bucket") - }, - - { "transcoder", - (getter)Bucket_get_transcoder, - (setter)Bucket_set_transcoder, - PyDoc_STR("The :class:`~couchbase.transcoder.Transcoder` " - "object being used.\n\n" - "" - "This is normally ``None`` unless a custom " - ":class:`couchbase.transcoder.Transcoder` " - "is being used\n") - }, - - { "connected", - (getter)Bucket_connected, - NULL, - PyDoc_STR("Boolean read only property indicating whether\n" - "this instance has been connected.\n" - "\n" - "Note that this will still return true even if\n" - "it is subsequently closed via :meth:`_close`\n") - }, - - { "_instance_pointer", - (getter)Bucket__instance_pointer, - NULL, - PyDoc_STR("Gets the C level pointer for the underlying C " - "handle") - }, - - { NULL } -}; - -static struct PyMemberDef Bucket_TABLE_members[] = { - { "quiet", T_UINT, offsetof(pycbc_Bucket, quiet), - 0, - PyDoc_STR("Whether to suppress errors when keys are not found " - "(in :meth:`get` and :meth:`delete` operations).\n" - "\n" - "An error is still returned within the :class:`Result`\n" - "object") - }, - - { "data_passthrough", T_UINT, offsetof(pycbc_Bucket, data_passthrough), - 0, - PyDoc_STR("When this flag is set, values are always returned " - "as raw bytes\n") - }, - - { "unlock_gil", T_UINT, offsetof(pycbc_Bucket, unlock_gil), - READONLY, - PyDoc_STR("Whether GIL manipulation is enabeld for " - "this connection object.\n" - "\n" - "This attribute can only be set from the constructor.\n") - }, - - { "bucket", T_OBJECT_EX, offsetof(pycbc_Bucket, bucket), - READONLY, - PyDoc_STR("Name of the bucket this object is connected to") - }, - - { "lockmode", T_INT, offsetof(pycbc_Bucket, lockmode), - READONLY, - PyDoc_STR("How access from multiple threads is handled.\n" - "See :ref:`multiple_threads` for more information\n") - }, - - { "_privflags", T_UINT, offsetof(pycbc_Bucket, flags), - 0, - PyDoc_STR("Internal flags.") - }, - - { "_conncb", T_OBJECT_EX, offsetof(pycbc_Bucket, conncb), - 0, - PyDoc_STR("Internal connection callback.") - }, - - { "_dtorcb", T_OBJECT_EX, offsetof(pycbc_Bucket, dtorcb), - 0, - PyDoc_STR("Internal destruction callback") - }, - - { "_dur_persist_to", T_BYTE, - offsetof(pycbc_Bucket, dur_global.persist_to), - 0, - PyDoc_STR("Internal default persistence settings") - }, - - { "_dur_replicate_to", T_BYTE, - offsetof(pycbc_Bucket, dur_global.replicate_to), - 0, - PyDoc_STR("Internal default replication settings") - }, - - { "_dur_timeout", T_ULONG, - offsetof(pycbc_Bucket, dur_timeout), - 0, - PyDoc_STR("Internal ") - }, - - { "_dur_testhook", T_OBJECT_EX, - offsetof(pycbc_Bucket, dur_testhook), - 0, - PyDoc_STR("Internal hook for durability tests") - }, - - { NULL } -}; - -static PyMethodDef Bucket_TABLE_methods[] = { - -#define OPFUNC(name, doc) \ -{ #name, (PyCFunction)pycbc_Bucket_##name, METH_VARARGS|METH_KEYWORDS, \ - PyDoc_STR(doc) } - - /** Basic Operations */ - OPFUNC(upsert, "Unconditionally store a key in Couchbase"), - OPFUNC(insert, "Add a key in Couchbase if it does not already exist"), - OPFUNC(replace, "Replace an existing key in Couchbase"), - OPFUNC(append, "Append to an existing value in Couchbase"), - OPFUNC(prepend, "Prepend to an existing value in Couchbase"), - OPFUNC(upsert_multi, NULL), - OPFUNC(insert_multi, NULL), - OPFUNC(replace_multi, NULL), - OPFUNC(append_multi, NULL), - OPFUNC(prepend_multi, NULL), - - OPFUNC(get, "Get a key from Couchbase"), - OPFUNC(touch, "Update the expiration time of a key in Couchbase"), - OPFUNC(lock, "Lock a key in Couchbase"), - OPFUNC(get_multi, NULL), - OPFUNC(touch_multi, NULL), - OPFUNC(lock_multi, NULL), - OPFUNC(_rget, NULL), - OPFUNC(_rgetix, NULL), - - OPFUNC(remove, "Delete a key in Couchbase"), - OPFUNC(unlock, "Unlock a previously-locked key in Couchbase"), - OPFUNC(remove_multi, "Multi-key variant of delete"), - OPFUNC(unlock_multi, "Multi-key variant of unlock"), - - OPFUNC(counter, "Modify a counter in Couchbase"), - OPFUNC(counter_multi, "Multi-key variant of counter"), - OPFUNC(_stats, "Get various server statistics"), - - OPFUNC(_http_request, "Internal routine for HTTP requests"), - OPFUNC(_view_request, "Internal routine for view requests"), - OPFUNC(_n1ql_query, "Internal routine for N1QL queries"), - - OPFUNC(observe, "Get replication/persistence status for keys"), - OPFUNC(observe_multi, "multi-key variant of observe"), - - OPFUNC(endure_multi, "Check durability requirements"), - - -#undef OPFUNC - - - { "_thr_lockop", - (PyCFunction)Bucket__thr_lockop, - METH_VARARGS, - PyDoc_STR("Unconditionally lock/unlock the connection object " - "if 'lockmode' has been set. For testing uses only") - }, - - { "_close", - (PyCFunction)Bucket__close, - METH_NOARGS, - PyDoc_STR( - "Close the instance's underlying socket resources\n" - "\n" - "Note that operations pending on the connection may\n" - "fail.\n" - "\n") - }, - - { "_connect", - (PyCFunction)Bucket__connect, - METH_NOARGS, - PyDoc_STR( - "Connect this instance. This is typically called by one of\n" - "the wrapping constructors\n") - }, - - { "_pipeline_begin", - (PyCFunction)pycbc_Bucket__start_pipeline, - METH_NOARGS, - PyDoc_STR("Enter pipeline mode. Internal use") - }, - - { "_pipeline_end", - (PyCFunction)pycbc_Bucket__end_pipeline, - METH_NOARGS, - PyDoc_STR( - "End pipeline mode and wait for operations to complete") - }, - - { "_start_timings", - (PyCFunction)Bucket__start_timings, - METH_NOARGS, - PyDoc_STR("Start recording timings") - }, - - { "_get_timings", - (PyCFunction)Bucket__get_timings, - METH_NOARGS, - PyDoc_STR("Get all timings since the last call to start_timings") - }, - - { "_stop_timings", - (PyCFunction)Bucket__clear_timings, - METH_NOARGS, - PyDoc_STR("Clear and disable timings") - }, - - { "_cntl", - (PyCFunction)pycbc_Bucket__cntl, - METH_VARARGS|METH_KEYWORDS, - NULL - }, - - { "_cntlstr", (PyCFunction)pycbc_Bucket__cntlstr, - METH_VARARGS|METH_KEYWORDS, - NULL - }, - - { "_vbmap", - (PyCFunction)pycbc_Bucket__vbmap, - METH_VARARGS, - PyDoc_STR("Returns a tuple of (vbucket, server index) for a key") - }, - - { "_mutinfo", - (PyCFunction)Bucket__mutinfo, - METH_NOARGS, - PyDoc_STR("Gets known mutation information") - }, - - { NULL, NULL, 0, NULL } -}; - -static int -Bucket__init__(pycbc_Bucket *self, - PyObject *args, PyObject *kwargs) -{ - int rv; - int conntype = LCB_TYPE_BUCKET; - - lcb_error_t err; - PyObject *unlock_gil_O = NULL; - PyObject *iops_O = NULL; - PyObject *dfl_fmt = NULL; - PyObject *tc = NULL; - - struct lcb_create_st create_opts = { 0 }; - - - /** - * This xmacro enumerates the constructor keywords, targets, and types. - * This was converted into an xmacro to ease the process of adding or - * removing various parameters. - */ -#define XCTOR_ARGS(X) \ - X("connection_string", &create_opts.v.v3.connstr, "z") \ - X("connstr", &create_opts.v.v3.connstr, "z") \ - X("username", &create_opts.v.v3.username, "z") \ - X("password", &create_opts.v.v3.passwd, "z") \ - X("quiet", &self->quiet, "I") \ - X("unlock_gil", &unlock_gil_O, "O") \ - X("transcoder", &tc, "O") \ - X("default_format", &dfl_fmt, "O") \ - X("lockmode", &self->lockmode, "i") \ - X("_flags", &self->flags, "I") \ - X("_conntype", &conntype, "i") \ - X("_iops", &iops_O, "O") - - static char *kwlist[] = { - #define X(s, target, type) s, - XCTOR_ARGS(X) - #undef X - NULL - }; - - #define X(s, target, type) type - static char *argspec = "|" XCTOR_ARGS(X); - #undef X - - if (self->init_called) { - PyErr_SetString(PyExc_RuntimeError, "__init__ was already called"); - return -1; - } - - self->init_called = 1; - self->flags = 0; - self->unlock_gil = 1; - self->lockmode = PYCBC_LOCKMODE_EXC; - - #define X(s, target, type) target, - rv = PyArg_ParseTupleAndKeywords(args, kwargs, argspec, kwlist, - XCTOR_ARGS(X) NULL); - #undef X - - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return -1; - } - - if (unlock_gil_O && PyObject_IsTrue(unlock_gil_O) == 0) { - self->unlock_gil = 0; - } - - create_opts.version = 3; - create_opts.v.v3.type = conntype; - - if (iops_O && iops_O != Py_None) { - self->iopswrap = pycbc_iowrap_new(self, iops_O); - create_opts.v.v3.io = pycbc_iowrap_getiops(self->iopswrap); - self->unlock_gil = 0; - } - - if (dfl_fmt == Py_None || dfl_fmt == NULL) { - /** Set to 0 if None or NULL */ - dfl_fmt = pycbc_IntFromL(PYCBC_FMT_JSON); - - } else { - Py_INCREF(dfl_fmt); /* later decref */ - } - - rv = Bucket_set_format(self, dfl_fmt, NULL); - Py_XDECREF(dfl_fmt); - if (rv == -1) { - return rv; - } - - /** Set the transcoder */ - if (tc && Bucket_set_transcoder(self, tc, NULL) == -1) { - return -1; - } - -#if defined(WITH_THREAD) - if (!self->unlock_gil) { - self->lockmode = PYCBC_LOCKMODE_NONE; - } - - if (self->lockmode != PYCBC_LOCKMODE_NONE) { - self->lock = PyThread_allocate_lock(); - } -#else - self->unlock_gil = 0; - self->lockmode = PYCBC_LOCKMODE_NONE; -#endif - - err = lcb_create(&self->instance, &create_opts); - if (err != LCB_SUCCESS) { - self->instance = NULL; - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, err, - "Couldn't create instance. Either bad " - "credentials/hosts/bucket names were " - "passed, or there was an internal error in creating the " - "object"); - return -1; - } - - if (pycbc_log_handler) { - err = lcb_cntl(self->instance, LCB_CNTL_SET, LCB_CNTL_LOGGER, - &pycbc_lcb_logprocs); - if (err != LCB_SUCCESS) { - self->instance = NULL; - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, err, "Couldn't create log handler"); - return -1; - } - } - - pycbc_callbacks_init(self->instance); - lcb_set_cookie(self->instance, self); - { - const char *bucketstr; - err = lcb_cntl(self->instance, LCB_CNTL_GET, LCB_CNTL_BUCKETNAME, &bucketstr); - if (err == LCB_SUCCESS && bucketstr != NULL) { - self->bucket = pycbc_SimpleStringZ(bucketstr); - } - } - return 0; -} - -static PyObject* -Bucket__connect(pycbc_Bucket *self) -{ - lcb_error_t err; - - if (self->flags & PYCBC_CONN_F_CONNECTED) { - Py_RETURN_NONE; - } - - err = lcb_connect(self->instance); - - if (err != LCB_SUCCESS) { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, err, - "Couldn't schedule connection. This might be a result of " - "an invalid hostname."); - return NULL; - } - - pycbc_oputil_wait_common(self); - if ((self->flags & PYCBC_CONN_F_ASYNC) == 0) { - err = lcb_get_bootstrap_status(self->instance); - if (err != LCB_SUCCESS) { - PYCBC_EXCTHROW_WAIT(err); - return NULL; - } - } - Py_RETURN_NONE; -} - -static void -Bucket_dtor(pycbc_Bucket *self) -{ - if (self->flags & PYCBC_CONN_F_CLOSED) { - lcb_destroy(self->instance); - self->instance = NULL; - } - - if (self->instance) { - lcb_set_cookie(self->instance, NULL); - pycbc_schedule_dtor_event(self); - } - - Py_XDECREF(self->dtorcb); - Py_XDECREF(self->dfl_fmt); - Py_XDECREF(self->tc); - Py_XDECREF(self->bucket); - Py_XDECREF(self->conncb); - Py_XDECREF(self->dur_testhook); - Py_XDECREF(self->iopswrap); - - if (self->instance) { - lcb_destroy(self->instance); - } - -#ifdef WITH_THREAD - if (self->lock) { - PyThread_free_lock(self->lock); - self->lock = NULL; - } -#endif - - Py_TYPE(self)->tp_free((PyObject*)self); -} - -int -pycbc_BucketType_init(PyObject **ptr) -{ - PyTypeObject *p = &BucketType; - *ptr = (PyObject*)p; - - if (p->tp_name) { - return 0; - } - - p->tp_name = "Bucket"; - p->tp_new = PyType_GenericNew; - p->tp_init = (initproc)Bucket__init__; - p->tp_dealloc = (destructor)Bucket_dtor; - - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_doc = PyDoc_STR("The connection object"); - - p->tp_basicsize = sizeof(pycbc_Bucket); - - p->tp_methods = Bucket_TABLE_methods; - p->tp_members = Bucket_TABLE_members; - p->tp_getset = Bucket_TABLE_getset; - - pycbc_DummyTuple = PyTuple_New(0); - pycbc_DummyKeywords = PyDict_New(); - - return PyType_Ready(p); -} diff --git a/src/callbacks.c b/src/callbacks.c deleted file mode 100644 index 8797ade61..000000000 --- a/src/callbacks.c +++ /dev/null @@ -1,550 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" - -#define CB_THREADS - -#ifdef CB_THREADS - -static void -cb_thr_end(pycbc_Bucket *self) -{ - PYCBC_CONN_THR_END(self); - Py_INCREF((PyObject *)self); -} - -static void -cb_thr_begin(pycbc_Bucket *self) -{ - if (Py_REFCNT(self) > 1) { - Py_DECREF(self); - PYCBC_CONN_THR_BEGIN(self); - return; - } - - pycbc_assert(self->unlock_gil == 0); - Py_DECREF(self); -} - -#define CB_THR_END cb_thr_end -#define CB_THR_BEGIN cb_thr_begin -#else -#define CB_THR_END(x) -#define CB_THR_BEGIN(x) -#endif - - -enum { - RESTYPE_BASE = 1 << 0, - RESTYPE_VALUE = 1 << 1, - RESTYPE_OPERATION = 1 << 2, - - /* Extra flag indicating it's ok if it already exists */ - RESTYPE_EXISTS_OK = 1 << 3, - - /* Don't modify "remaining" count */ - RESTYPE_VARCOUNT = 1 << 4 -}; - -static void -maybe_push_operr(pycbc_MultiResult *mres, pycbc_Result *res, lcb_error_t err, - int check_enoent) -{ - if (err == LCB_SUCCESS || mres->errop) { - return; - } - - if (check_enoent && - (mres->mropts & PYCBC_MRES_F_QUIET) && - err == LCB_KEY_ENOENT) { - return; - } - - mres->errop = (PyObject*)res; - Py_INCREF(mres->errop); -} - - -static void -operation_completed(pycbc_Bucket *self, pycbc_MultiResult *mres) -{ - pycbc_assert(self->nremaining); - --self->nremaining; - - if ((self->flags & PYCBC_CONN_F_ASYNC) == 0) { - if (!self->nremaining) { - lcb_breakout(self->instance); - } - return; - } - - if (mres) { - pycbc_AsyncResult *ares; - ares = (pycbc_AsyncResult *)mres; - if (--ares->nops) { - return; - } - pycbc_asyncresult_invoke(ares); - } -} - -static int -get_common_objects(const lcb_RESPBASE *resp, pycbc_Bucket **conn, - pycbc_Result **res, int restype, pycbc_MultiResult **mres) - -{ - PyObject *hkey; - PyObject *mrdict; - int rv; - - pycbc_assert(pycbc_multiresult_check(resp->cookie)); - *mres = (pycbc_MultiResult*)resp->cookie; - *conn = (*mres)->parent; - - CB_THR_END(*conn); - - rv = pycbc_tc_decode_key(*conn, resp->key, resp->nkey, &hkey); - - if (rv < 0) { - pycbc_multiresult_adderr(*mres); - return -1; - } - - mrdict = pycbc_multiresult_dict(*mres); - - *res = (pycbc_Result*)PyDict_GetItem(mrdict, hkey); - - if (*res) { - int exists_ok = (restype & RESTYPE_EXISTS_OK) || - ( (*mres)->mropts & PYCBC_MRES_F_UALLOCED); - - if (!exists_ok) { - if ((*conn)->flags & PYCBC_CONN_F_WARNEXPLICIT) { - PyErr_WarnExplicit(PyExc_RuntimeWarning, - "Found duplicate key", - __FILE__, __LINE__, - "couchbase._libcouchbase", - NULL); - - } else { - PyErr_WarnEx(PyExc_RuntimeWarning, - "Found duplicate key", - 1); - } - /** - * We need to destroy the existing object and re-create it. - */ - PyDict_DelItem(mrdict, hkey); - *res = NULL; - - } else { - Py_XDECREF(hkey); - } - - } - - if (*res == NULL) { - /** - * Now, get/set the result object - */ - if ( (*mres)->mropts & PYCBC_MRES_F_ITEMS) { - *res = (pycbc_Result*)pycbc_item_new(*conn); - - } else if (restype & RESTYPE_BASE) { - *res = (pycbc_Result*)pycbc_result_new(*conn); - - } else if (restype & RESTYPE_OPERATION) { - *res = (pycbc_Result*)pycbc_opresult_new(*conn); - - } else if (restype & RESTYPE_VALUE) { - *res = (pycbc_Result*)pycbc_valresult_new(*conn); - - } else { - abort(); - } - - PyDict_SetItem(mrdict, hkey, (PyObject*)*res); - - (*res)->key = hkey; - Py_DECREF(*res); - } - - if (resp->rc) { - (*res)->rc = resp->rc; - } - - if (resp->rc != LCB_SUCCESS) { - (*mres)->all_ok = 0; - } - - return 0; -} - -static void -invoke_endure_test_notification(pycbc_Bucket *self, pycbc_Result *resp) -{ - PyObject *ret; - PyObject *argtuple = Py_BuildValue("(O)", resp); - ret = PyObject_CallObject(self->dur_testhook, argtuple); - pycbc_assert(ret); - - Py_XDECREF(ret); - Py_XDECREF(argtuple); -} - -/** - * Common handler for durability - */ -static void -durability_chain_common(lcb_t instance, int cbtype, const lcb_RESPBASE *resp) -{ - pycbc_Bucket *conn; - pycbc_OperationResult *res = NULL; - pycbc_MultiResult *mres; - lcb_durability_opts_t dopts = { 0 }; - lcb_CMDENDURE cmd = { 0 }; - lcb_MULTICMD_CTX *mctx = NULL; - int rv; - lcb_error_t err; - int is_delete = cbtype == LCB_CALLBACK_REMOVE; - - rv = get_common_objects(resp, &conn, (pycbc_Result**)&res, - RESTYPE_OPERATION|RESTYPE_VARCOUNT, &mres); - - if (rv == -1) { - operation_completed(conn, mres); - CB_THR_BEGIN(conn); - return; - } - - res->rc = resp->rc; - if (resp->rc == LCB_SUCCESS) { - const lcb_MUTATION_TOKEN *mutinfo = NULL; - - Py_XDECREF(res->mutinfo); - - mutinfo = lcb_resp_get_mutation_token(cbtype, resp); - if (mutinfo && LCB_MUTATION_TOKEN_ISVALID(mutinfo)) { - /* Create the mutation token tuple: (vb,uuid,seqno) */ - res->mutinfo = Py_BuildValue("HKK", - LCB_MUTATION_TOKEN_VB(mutinfo), - LCB_MUTATION_TOKEN_ID(mutinfo), - LCB_MUTATION_TOKEN_SEQ(mutinfo)); - } else { - Py_INCREF(Py_None); - res->mutinfo = Py_None; - } - res->cas = resp->cas; - } - - /** For remove, we check quiet */ - maybe_push_operr(mres, (pycbc_Result*)res, resp->rc, is_delete ? 1 : 0); - - if ((mres->mropts & PYCBC_MRES_F_DURABILITY) == 0 || resp->rc != LCB_SUCCESS) { - operation_completed(conn, mres); - CB_THR_BEGIN(conn); - return; - } - - if (conn->dur_testhook && conn->dur_testhook != Py_None) { - invoke_endure_test_notification(conn, (pycbc_Result *)res); - } - - /** Setup global options */ - dopts.v.v0.persist_to = mres->dur.persist_to; - dopts.v.v0.replicate_to = mres->dur.replicate_to; - dopts.v.v0.timeout = conn->dur_timeout; - dopts.v.v0.check_delete = is_delete; - if (mres->dur.persist_to < 0 || mres->dur.replicate_to < 0) { - dopts.v.v0.cap_max = 1; - } - - lcb_sched_enter(conn->instance); - mctx = lcb_endure3_ctxnew(conn->instance, &dopts, &err); - if (mctx == NULL) { - goto GT_DONE; - } - - cmd.cas = resp->cas; - LCB_CMD_SET_KEY(&cmd, resp->key, resp->nkey); - err = mctx->addcmd(mctx, (lcb_CMDBASE*)&cmd); - if (err != LCB_SUCCESS) { - goto GT_DONE; - } - - err = mctx->done(mctx, mres); - if (err == LCB_SUCCESS) { - mctx = NULL; - lcb_sched_leave(conn->instance); - } - - GT_DONE: - if (mctx) { - mctx->fail(mctx); - } - if (err != LCB_SUCCESS) { - res->rc = err; - maybe_push_operr(mres, (pycbc_Result*)res, err, 0); - operation_completed(conn, mres); - - } - - CB_THR_BEGIN(conn); -} - -static void -value_callback(lcb_t instance, int cbtype, const lcb_RESPBASE *resp) -{ - int rv; - pycbc_Bucket *conn = NULL; - pycbc_ValueResult *res = NULL; - pycbc_MultiResult *mres = NULL; - - rv = get_common_objects(resp, &conn, (pycbc_Result**)&res, RESTYPE_VALUE, - &mres); - - if (rv < 0) { - goto GT_DONE; - } - - if (resp->rc == LCB_SUCCESS) { - res->cas = resp->cas; - } else { - maybe_push_operr(mres, (pycbc_Result*)res, resp->rc, - cbtype != LCB_CALLBACK_COUNTER); - goto GT_DONE; - } - - if (cbtype == LCB_CALLBACK_GET || cbtype == LCB_CALLBACK_GETREPLICA) { - const lcb_RESPGET *gresp = (const lcb_RESPGET *)resp; - lcb_U32 eflags; - - res->flags = gresp->itmflags; - if (mres->mropts & PYCBC_MRES_F_FORCEBYTES) { - eflags = PYCBC_FMT_BYTES; - } else { - eflags = gresp->itmflags; - } - - rv = pycbc_tc_decode_value(mres->parent, gresp->value, gresp->nvalue, - eflags, &res->value); - if (rv < 0) { - pycbc_multiresult_adderr(mres); - } - } else if (cbtype == LCB_CALLBACK_COUNTER) { - const lcb_RESPCOUNTER *cresp = (const lcb_RESPCOUNTER *)resp; - res->value = pycbc_IntFromULL(cresp->value); - } - - GT_DONE: - operation_completed(conn, mres); - CB_THR_BEGIN(conn); - (void)instance; -} - -static void -keyop_simple_callback(lcb_t instance, int cbtype, const lcb_RESPBASE *resp) -{ - int rv; - int optflags = RESTYPE_OPERATION; - pycbc_Bucket *conn = NULL; - pycbc_OperationResult *res = NULL; - pycbc_MultiResult *mres = NULL; - - if (cbtype == LCB_CALLBACK_ENDURE) { - optflags |= RESTYPE_EXISTS_OK; - } - - rv = get_common_objects(resp, &conn, (pycbc_Result**)&res, optflags, &mres); - - if (rv == 0) { - res->rc = resp->rc; - maybe_push_operr(mres, (pycbc_Result*)res, resp->rc, 0); - } - if (resp->cas) { - res->cas = resp->cas; - } - - operation_completed(conn, mres); - CB_THR_BEGIN(conn); - (void)instance; - -} - -static void -stats_callback(lcb_t instance, int cbtype, const lcb_RESPBASE *resp_base) -{ - pycbc_MultiResult *mres; - PyObject *value; - PyObject *skey, *knodes; - PyObject *mrdict; - const lcb_RESPSTATS *resp = (const lcb_RESPSTATS *)resp_base; - - - mres = (pycbc_MultiResult*)resp->cookie; - CB_THR_END(mres->parent); - - if (!resp->server) { - pycbc_Bucket *parent = mres->parent; - operation_completed(mres->parent, mres); - CB_THR_BEGIN(parent); - return; - } - - if (resp->rc != LCB_SUCCESS) { - if (mres->errop == NULL) { - pycbc_Result *res = - (pycbc_Result*)pycbc_result_new(mres->parent); - res->rc = resp->rc; - res->key = Py_None; Py_INCREF(res->key); - maybe_push_operr(mres, res, resp->rc, 0); - } - CB_THR_BEGIN(mres->parent); - return; - } - - skey = pycbc_SimpleStringN(resp->key, resp->nkey); - value = pycbc_SimpleStringN(resp->value, resp->nvalue); - { - PyObject *intval = pycbc_maybe_convert_to_int(value); - if (intval) { - Py_DECREF(value); - value = intval; - - } else { - PyErr_Clear(); - } - } - - mrdict = pycbc_multiresult_dict(mres); - knodes = PyDict_GetItem(mrdict, skey); - if (!knodes) { - knodes = PyDict_New(); - PyDict_SetItem(mrdict, skey, knodes); - } - - PyDict_SetItemString(knodes, resp->server, value); - - Py_DECREF(skey); - Py_DECREF(value); - - CB_THR_BEGIN(mres->parent); - (void)instance; -} - - - -static void -observe_callback(lcb_t instance, int cbtype, const lcb_RESPBASE *resp_base) -{ - int rv; - pycbc_ObserveInfo *oi; - pycbc_Bucket *conn; - pycbc_ValueResult *vres; - pycbc_MultiResult *mres; - const lcb_RESPOBSERVE *oresp = (const lcb_RESPOBSERVE *)resp_base; - - if (resp_base->rflags & LCB_RESP_F_FINAL) { - mres = (pycbc_MultiResult*)resp_base->cookie; - operation_completed(mres->parent, mres); - return; - } - - rv = get_common_objects(resp_base, &conn, (pycbc_Result**)&vres, - RESTYPE_VALUE|RESTYPE_EXISTS_OK|RESTYPE_VARCOUNT, &mres); - if (rv < 0) { - goto GT_DONE; - } - - if (resp_base->rc != LCB_SUCCESS) { - maybe_push_operr(mres, (pycbc_Result*)vres, resp_base->rc, 0); - goto GT_DONE; - } - - if (!vres->value) { - vres->value = PyList_New(0); - } - - oi = pycbc_observeinfo_new(conn); - if (oi == NULL) { - pycbc_multiresult_adderr(mres); - goto GT_DONE; - } - - oi->from_master = oresp->ismaster; - oi->flags = oresp->status; - oi->cas = oresp->cas; - PyList_Append(vres->value, (PyObject*)oi); - Py_DECREF(oi); - - GT_DONE: - CB_THR_BEGIN(conn); - (void)instance; (void)cbtype; -} - -static int -start_global_callback(lcb_t instance, pycbc_Bucket **selfptr) -{ - *selfptr = (pycbc_Bucket *)lcb_get_cookie(instance); - if (!*selfptr) { - return 0; - } - CB_THR_END(*selfptr); - Py_INCREF((PyObject *)*selfptr); - return 1; -} - -static void -end_global_callback(lcb_t instance, pycbc_Bucket *self) -{ - Py_DECREF((PyObject *)(self)); - - self = (pycbc_Bucket *)lcb_get_cookie(instance); - if (self) { - CB_THR_BEGIN(self); - } -} - -static void -bootstrap_callback(lcb_t instance, lcb_error_t err) -{ - pycbc_Bucket *self; - - if (!start_global_callback(instance, &self)) { - return; - } - pycbc_invoke_connected_event(self, err); - end_global_callback(instance, self); -} - -void -pycbc_callbacks_init(lcb_t instance) -{ - lcb_install_callback3(instance, LCB_CALLBACK_STORE, durability_chain_common); - lcb_install_callback3(instance, LCB_CALLBACK_REMOVE, durability_chain_common); - lcb_install_callback3(instance, LCB_CALLBACK_UNLOCK, keyop_simple_callback); - lcb_install_callback3(instance, LCB_CALLBACK_TOUCH, keyop_simple_callback); - lcb_install_callback3(instance, LCB_CALLBACK_ENDURE, keyop_simple_callback); - lcb_install_callback3(instance, LCB_CALLBACK_GET, value_callback); - lcb_install_callback3(instance, LCB_CALLBACK_GETREPLICA, value_callback); - lcb_install_callback3(instance, LCB_CALLBACK_COUNTER, value_callback); - lcb_install_callback3(instance, LCB_CALLBACK_OBSERVE, observe_callback); - lcb_install_callback3(instance, LCB_CALLBACK_STATS, stats_callback); - lcb_set_bootstrap_callback(instance, bootstrap_callback); - - pycbc_http_callbacks_init(instance); -} diff --git a/src/client.cxx b/src/client.cxx new file mode 100644 index 000000000..7122dd743 --- /dev/null +++ b/src/client.cxx @@ -0,0 +1,477 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "client.hxx" +#include <cstdlib> +#include <structmember.h> + +#include <core/meta/version.hxx> + +#include "analytics.hxx" +#include "binary_ops.hxx" +#include "connection.hxx" +#include "diagnostics.hxx" +#include "exceptions.hxx" +#include "kv_ops.hxx" +#include "kv_range_scan.hxx" +#include "logger.hxx" +#include "n1ql.hxx" +#include "result.hxx" +#include "search.hxx" +#include "subdoc_ops.hxx" +#include "views.hxx" + +#include "management/analytics_management.hxx" +#include "management/bucket_management.hxx" +#include "management/collection_management.hxx" +#include "management/eventing_function_management.hxx" +#include "management/management.hxx" +#include "management/query_index_management.hxx" +#include "management/search_index_management.hxx" +#include "management/user_management.hxx" +#include "management/view_index_management.hxx" + +#include "transactions/transactions.hxx" + +void +add_ops_enum(PyObject* pyObj_module) +{ + PyObject* pyObj_enum_module = PyImport_ImportModule("enum"); + if (!pyObj_enum_module) { + return; + } + PyObject* pyObj_enum_class = PyObject_GetAttrString(pyObj_enum_module, "Enum"); + + PyObject* pyObj_enum_values = PyUnicode_FromString(Operations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("Operations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "operations", pyObj_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_operations); + return; + } + + add_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_cluster_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_bucket_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_collection_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_user_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_query_index_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_analytics_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_search_index_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_view_index_mgmt_ops_enum(pyObj_module, pyObj_enum_class); + add_eventing_function_mgmt_ops_enum(pyObj_module, pyObj_enum_class); +} + +void +add_constants(PyObject* module) +{ + if (PyModule_AddIntConstant(module, "FMT_JSON", PYCBC_FMT_JSON) < 0) { + Py_XDECREF(module); + return; + } + if (PyModule_AddIntConstant(module, "FMT_BYTES", PYCBC_FMT_BYTES) < 0) { + Py_XDECREF(module); + return; + } + if (PyModule_AddIntConstant(module, "FMT_UTF8", PYCBC_FMT_UTF8) < 0) { + Py_XDECREF(module); + return; + } + if (PyModule_AddIntConstant(module, "FMT_PICKLE", PYCBC_FMT_PICKLE) < 0) { + Py_XDECREF(module); + return; + } + if (PyModule_AddIntConstant(module, "FMT_LEGACY_MASK", PYCBC_FMT_LEGACY_MASK) < 0) { + Py_XDECREF(module); + return; + } + if (PyModule_AddIntConstant(module, "FMT_COMMON_MASK", PYCBC_FMT_COMMON_MASK) < 0) { + Py_XDECREF(module); + return; + } + auto cxxcbc_metadata = couchbase::core::meta::sdk_build_info_json(); + if (PyModule_AddStringConstant(module, "CXXCBC_METADATA", cxxcbc_metadata.c_str())) { + Py_XDECREF(module); + return; + } +} + +static PyObject* +binary_operation(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_binary_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, __FILE__, __LINE__, "Unable to perform binary operation."); + } + return res; +} + +static PyObject* +binary_multi_operation(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_binary_multi_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to perform binary multi operation."); + } + return res; +} + +static PyObject* +kv_operation(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_kv_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, __FILE__, __LINE__, "Unable to perform KV operation."); + } + return res; +} + +static PyObject* +kv_multi_operation(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_kv_multi_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to perform KV multi operation."); + } + return res; +} + +static PyObject* +kv_range_scan_operation(PyObject* self, PyObject* args, PyObject* kwargs) +{ + scan_iterator* res = handle_kv_range_scan_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to perform KV range scan operation."); + } + return reinterpret_cast<PyObject*>(res); +} + +static PyObject* +subdoc_operation(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_subdoc_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to perform subdocument operation."); + } + return res; +} + +static PyObject* +diagnostics_operation(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_diagnostics_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to perform diagnostics operation."); + } + return res; +} + +static PyObject* +n1ql_query(PyObject* self, PyObject* args, PyObject* kwargs) +{ + streamed_result* res = handle_n1ql_query(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, __FILE__, __LINE__, "Unable to perform N1QL query."); + } + return reinterpret_cast<PyObject*>(res); +} + +static PyObject* +analytics_query(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyGILState_STATE state = PyGILState_Ensure(); + streamed_result* res = handle_analytics_query(self, args, kwargs); + if (!res) { + PyErr_SetString(PyExc_Exception, "Unable to perform analytics query."); + } + PyGILState_Release(state); + return reinterpret_cast<PyObject*>(res); +} + +static PyObject* +search_query(PyObject* self, PyObject* args, PyObject* kwargs) +{ + streamed_result* res = handle_search_query(self, args, kwargs); + if (!res) { + PyErr_SetString(PyExc_Exception, "Unable to perform search query."); + } + return reinterpret_cast<PyObject*>(res); +} + +static PyObject* +view_query(PyObject* self, PyObject* args, PyObject* kwargs) +{ + streamed_result* res = handle_view_query(self, args, kwargs); + if (!res) { + PyErr_SetString(PyExc_Exception, "Unable to perform view query."); + } + return reinterpret_cast<PyObject*>(res); +} + +static PyObject* +cluster_info(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_mgmt_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to perform cluster info operation."); + } + return res; +} + +static PyObject* +management_operation(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_mgmt_op(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to perform management operation."); + } + return res; +} + +static PyObject* +open_or_close_bucket(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_open_or_close_bucket(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, __FILE__, __LINE__, "Unable to open/close bucket."); + } + return res; +} + +static PyObject* +create_connection(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_create_connection(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, __FILE__, __LINE__, "Unable to create connection."); + } + return res; +} + +static PyObject* +get_connection_information(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = get_connection_info(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() == nullptr) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to get connection information."); + } + return res; +} + +static PyObject* +close_connection(PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* res = handle_close_connection(self, args, kwargs); + if (res == nullptr && PyErr_Occurred() != nullptr) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, __FILE__, __LINE__, "Unable to close connection."); + } + return res; +} + +static PyObject* +shutdown_logger(PyObject* self, PyObject* Py_UNUSED(ignored)) +{ + Py_BEGIN_ALLOW_THREADS couchbase::core::logger::shutdown(); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +static struct PyMethodDef methods[] = { + { "create_connection", + (PyCFunction)create_connection, + METH_VARARGS | METH_KEYWORDS, + "Create connection object" }, + { "get_connection_info", + (PyCFunction)get_connection_information, + METH_VARARGS | METH_KEYWORDS, + "Get connection options" }, + { "open_or_close_bucket", + (PyCFunction)open_or_close_bucket, + METH_VARARGS | METH_KEYWORDS, + "Open or close a bucket" }, + { "close_connection", + (PyCFunction)close_connection, + METH_VARARGS | METH_KEYWORDS, + "Close a connection" }, + { "kv_operation", + (PyCFunction)kv_operation, + METH_VARARGS | METH_KEYWORDS, + "Handle all key/value operations" }, + { "kv_multi_operation", + (PyCFunction)kv_multi_operation, + METH_VARARGS | METH_KEYWORDS, + "Handle all key/value multi operations" }, + { "kv_range_scan_operation", + (PyCFunction)kv_range_scan_operation, + METH_VARARGS | METH_KEYWORDS, + "Handle all key/value range scan operations" }, + { "subdoc_operation", + (PyCFunction)subdoc_operation, + METH_VARARGS | METH_KEYWORDS, + "Handle all subdoc operations" }, + { "binary_operation", + (PyCFunction)binary_operation, + METH_VARARGS | METH_KEYWORDS, + "Handle all binary operations" }, + { "binary_multi_operation", + (PyCFunction)binary_multi_operation, + METH_VARARGS | METH_KEYWORDS, + "Handle all binary multi operations" }, + { "diagnostics_operation", + (PyCFunction)diagnostics_operation, + METH_VARARGS | METH_KEYWORDS, + "Handle all diagnostics operations" }, + { "n1ql_query", (PyCFunction)n1ql_query, METH_VARARGS | METH_KEYWORDS, "Execute N1QL Query" }, + { "analytics_query", + (PyCFunction)analytics_query, + METH_VARARGS | METH_KEYWORDS, + "Execute analytics Query" }, + { "search_query", + (PyCFunction)search_query, + METH_VARARGS | METH_KEYWORDS, + "Execute search Query" }, + { "view_query", + (PyCFunction)view_query, + METH_VARARGS | METH_KEYWORDS, + "Execute map reduce views Query" }, + { "cluster_info", + (PyCFunction)cluster_info, + METH_VARARGS | METH_KEYWORDS, + "Get information on the connected cluster" }, + { "management_operation", + (PyCFunction)management_operation, + METH_VARARGS | METH_KEYWORDS, + "Handle all management operations" }, + { "create_transactions", + (PyCFunction)pycbc_txns::create_transactions, + METH_VARARGS | METH_KEYWORDS, + "Create a transactions object" }, + { "create_transaction_context", + (PyCFunction)pycbc_txns::create_transaction_context, + METH_VARARGS | METH_KEYWORDS, + "Create a transaction context object" }, + { "create_new_attempt_context", + (PyCFunction)pycbc_txns::create_new_attempt_context, + METH_VARARGS | METH_KEYWORDS, + "Create a new attempt context object" }, + { "transaction_op", + (PyCFunction)pycbc_txns::transaction_op, + METH_VARARGS | METH_KEYWORDS, + "perform a transaction kv operation" }, + { "transaction_query_op", + (PyCFunction)pycbc_txns::transaction_query_op, + METH_VARARGS | METH_KEYWORDS, + "perform a transactional query" }, + { "transaction_commit", + (PyCFunction)pycbc_txns::transaction_commit, + METH_VARARGS | METH_KEYWORDS, + "Commit a transaction" }, + { "transaction_rollback", + (PyCFunction)pycbc_txns::transaction_rollback, + METH_VARARGS | METH_KEYWORDS, + "Rollback a transaction" }, + { "destroy_transactions", + (PyCFunction)pycbc_txns::destroy_transactions, + METH_VARARGS | METH_KEYWORDS, + "shut down transactions object" }, + { "shutdown_logger", (PyCFunction)shutdown_logger, METH_NOARGS, "shut down C++ logger" }, + { nullptr, nullptr, 0, nullptr } +}; + +static PyModuleDef +init_pycbc_core_module() +{ + PyModuleDef mod = {}; + mod.m_base = PyModuleDef_HEAD_INIT; + mod.m_name = "pycbc_core"; + mod.m_doc = "Python interface to couchbase-client-cxx"; + mod.m_size = -1; + mod.m_methods = methods; + return mod; +} + +static PyModuleDef pycbc_core_module = init_pycbc_core_module(); + +PyMODINIT_FUNC +PyInit_pycbc_core(void) +{ + Py_Initialize(); + PyObject* m = PyModule_Create(&pycbc_core_module); + if (m == nullptr) { + return nullptr; + } + + if (add_result_objects(m) == nullptr) { + Py_DECREF(m); + return nullptr; + } + + if (add_exception_objects(m) == nullptr) { + Py_DECREF(m); + return nullptr; + } + + if (add_logger_objects(m) == nullptr) { + Py_DECREF(m); + return nullptr; + } + + add_ops_enum(m); + add_constants(m); + return pycbc_txns::add_transaction_objects(m); +} diff --git a/src/client.hxx b/src/client.hxx new file mode 100644 index 000000000..335427a94 --- /dev/null +++ b/src/client.hxx @@ -0,0 +1,281 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +// NOLINTNEXTLINE +#include "Python.h" // NOLINT +#include "structmember.h" + +#include <future> +#include <list> +#include <thread> + +#include <core/cluster.hxx> +#include <core/logger/logger.hxx> +#include <core/operations.hxx> + +#define PY_SSIZE_T_CLEAN + +class Operations +{ +public: + enum OperationType { + UNKNOWN, + GET, + GET_PROJECTED, + GET_AND_LOCK, + GET_AND_TOUCH, + GET_ANY_REPLICA, + GET_ALL_REPLICAS, + EXISTS, + TOUCH, + UNLOCK, + INSERT, + UPSERT, + REPLACE, + REMOVE, + MUTATE_IN, + LOOKUP_IN, + LOOKUP_IN_ALL_REPLICAS, + LOOKUP_IN_ANY_REPLICA, + DIAGNOSTICS, + PING, + INCREMENT, + DECREMENT, + APPEND, + PREPEND, + N1QL_QUERY, + CLUSTER_MGMT_CLUSTER_INFO, + KV_RANGE_SCAN, + KV_PREFIX_SCAN, + KV_SAMPLING_SCAN + }; + + Operations() + : Operations{ UNKNOWN } + { + } + constexpr Operations(OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(Operations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(Operations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "GET " + "GET_PROJECTED " + "GET_AND_LOCK " + "GET_AND_TOUCH " + "GET_ANY_REPLICA " + "GET_ALL_REPLICAS " + "EXISTS " + "TOUCH " + "UNLOCK " + "INSERT " + "UPSERT " + "REPLACE " + "REMOVE " + "MUTATE_IN " + "LOOKUP_IN " + "LOOKUP_IN_ALL_REPLICAS " + "LOOKUP_IN_ANY_REPLICA " + "DIAGNOSTICS " + "PING " + "INCREMENT " + "DECREMENT " + "APPEND " + "PREPEND " + "N1QL_QUERY " + "CLUSTER_MGMT_CLUSTER_INFO " + "KV_RANGE_SCAN " + "KV_PREFIX_SCAN " + "KV_SAMPLING_SCAN"; + + return ops; + } + +private: + OperationType operation; +}; + +enum { + PYCBC_FMT_LEGACY_JSON = 0x00, + PYCBC_FMT_LEGACY_PICKLE = 0x01, + PYCBC_FMT_LEGACY_BYTES = 0x02, + PYCBC_FMT_LEGACY_UTF8 = 0x04, + PYCBC_FMT_LEGACY_MASK = 0x07, + + PYCBC_FMT_COMMON_PICKLE = (0x01U << 24), + PYCBC_FMT_COMMON_JSON = (0x02U << 24), + PYCBC_FMT_COMMON_BYTES = (0x03U << 24), + PYCBC_FMT_COMMON_UTF8 = (0x04U << 24), + PYCBC_FMT_COMMON_MASK = (0xFFU << 24), + + PYCBC_FMT_JSON = PYCBC_FMT_LEGACY_JSON | PYCBC_FMT_COMMON_JSON, + PYCBC_FMT_PICKLE = PYCBC_FMT_LEGACY_PICKLE | PYCBC_FMT_COMMON_PICKLE, + PYCBC_FMT_BYTES = PYCBC_FMT_LEGACY_BYTES | PYCBC_FMT_COMMON_BYTES, + PYCBC_FMT_UTF8 = PYCBC_FMT_LEGACY_UTF8 | PYCBC_FMT_COMMON_UTF8 +}; + +struct callback_context { +private: + PyObject* callback; + PyObject* errback; + PyObject* transcoder; + PyObject* row_callback; + +public: + callback_context(PyObject* callback, + PyObject* errback, + PyObject* transcoder, + PyObject* row_callback) + : callback{ callback } + , errback{ errback } + , transcoder{ transcoder } + , row_callback{ row_callback } + { + PyGILState_STATE state = PyGILState_Ensure(); + Py_XINCREF(callback); + Py_XINCREF(errback); + Py_XINCREF(transcoder); + Py_XINCREF(row_callback); + PyGILState_Release(state); + } + + callback_context() + : callback_context{ nullptr, nullptr, nullptr, nullptr } + { + } + callback_context(PyObject* callback, PyObject* errback) + : callback_context{ callback, errback, nullptr, nullptr } + { + } + callback_context(PyObject* callback, PyObject* errback, PyObject* transcoder) + : callback_context{ callback, errback, transcoder, nullptr } + { + } + + callback_context(const callback_context& ctx) + : callback_context{ ctx.callback, ctx.errback, ctx.transcoder, ctx.row_callback } + { + } + + callback_context(callback_context&& ctx) + : callback{ ctx.callback } + , errback{ ctx.errback } + , transcoder{ ctx.transcoder } + , row_callback{ ctx.row_callback } + { + ctx.callback = nullptr; + ctx.errback = nullptr; + ctx.transcoder = nullptr; + ctx.row_callback = nullptr; + } + + ~callback_context() + { + PyGILState_STATE state = PyGILState_Ensure(); + Py_XDECREF(callback); + Py_XDECREF(errback); + Py_XDECREF(transcoder); + Py_XDECREF(row_callback); + PyGILState_Release(state); + } + + // maybe use [[nodiscard]] to force assignment (or at least gen a compiler warning)? + // const = no mucking w/ ref counts + PyObject* get_callback() const + { + return callback; + } + + PyObject* get_errback() const + { + return errback; + } + + PyObject* get_transcoder() const + { + return transcoder; + } + + PyObject* get_row_callback() const + { + return row_callback; + } +}; + +#define RESULT_VALUE "value" +#define RESULT_CAS "cas" +#define RESULT_FLAGS "flags" +#define RESULT_EXPIRY "expiry" +#define RESULT_KEY "key" +#define RESULT_MUTATION_TOKEN "mutation_token" +#define RESULT_EXISTS "exists" + +struct connection { + asio::io_context io_; + couchbase::core::cluster cluster_; + std::list<std::thread> io_threads_; + + connection() + : connection{ 1 } + { + } + + connection(int num_io_threads) + : cluster_(couchbase::core::cluster(io_)) + { + for (int i = 0; i < num_io_threads; i++) { + // TODO: consider maybe catching exceptions and running run() again? For now, lets + // log the exception and rethrow (which will lead to a crash) + io_threads_.emplace_back([&] { + try { + io_.run(); + } catch (const std::exception& e) { + CB_LOG_ERROR(e.what()); + throw; + } catch (...) { + CB_LOG_ERROR("Unknown exception"); + throw; + } + }); + } + } +}; + +void +add_ops_enum(PyObject* module); + +void +add_constants(PyObject* module); diff --git a/src/cntl.c b/src/cntl.c deleted file mode 100644 index 8ad57ed85..000000000 --- a/src/cntl.c +++ /dev/null @@ -1,413 +0,0 @@ -/** - * Copyright 2014 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" - -/** - * We're only using a subset of commands we actually care about. We're not - * including the header directly because we might be using constants not - * defined in older headers (which would result in a runtime NOT_SUPPORTED - * error rather than a compilation failure). - */ -#define CNTL_SET 0x01 -#define CNTL_GET 0x00 -#define CNTL_OP_TIMEOUT 0x00 -#define CNTL_VIEW_TIMEOUT 0x01 -#define CNTL_RBUFSIZE 0x02 -#define CNTL_WBUFSIZE 0x03 -#define CNTL_VBMAP 0x07 -#define CNTL_CONFERRTHRESH 0x0c -#define CNTL_DURABILITY_TIMEOUT 0x0d -#define CNTL_DURABILITY_INTERVAL 0x0e -#define CNTL_HTTP_TIMEOUT 0x0f -#define CNTL_CONFIGURATION_TIMEOUT 0x12 -#define CNTL_SKIP_CONFIGURATION_ERRORS_ON_CONNECT 0x13 -#define CNTL_RANDOMIZE_BOOTSTRAP_HOSTS 0x14 -#define CNTL_CONFIG_CACHE_LOADED 0x15 -#define CNTL_MAX_REDIRECTS 0x17 - - -struct vbinfo_st { - int version; - - union { - struct { - /** Input parameters */ - const void *key; - lcb_size_t nkey; - /** Output */ - int vbucket; - int server_index; - } v0; - } v; -}; - -static PyObject * -handle_float_tmo(lcb_t instance, - int cmd, int mode, PyObject *val, lcb_error_t *err) -{ - lcb_uint32_t cval; - - if (val != NULL) { - if (PyFloat_Check(val)) { - double dv = PyFloat_AsDouble(val); - if (dv < 0) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, - "Timeout cannot be < 0"); - return NULL; - } - cval = dv * 1000000; - - } else { - cval = pycbc_IntAsL(val); - if (cval == (lcb_uint32_t)-1 && PyErr_Occurred()) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - } - } - if ( (*err = lcb_cntl(instance, mode, cmd, &cval)) != LCB_SUCCESS) { - return NULL; - } - - return pycbc_IntFromUL(cval); -} - -static PyObject * -handle_boolean(lcb_t instance, - int cmd, int mode, PyObject *val, lcb_error_t *err) -{ - int cval; - PyObject *ret; - - if (val != NULL) { - cval = PyObject_IsTrue(val); - } - - if ( (*err = lcb_cntl(instance, mode, cmd, &cval)) != LCB_SUCCESS) { - return NULL; - } - - if (cval) { - ret = Py_True; - } else { - ret = Py_False; - } - Py_INCREF(ret); - return ret; -} - -static PyObject * -handle_intval(lcb_t instance, - int cmd, int mode, PyObject *val, lcb_error_t *err) -{ - int cval; - - if (val != NULL) { - cval = pycbc_IntAsL(val); - if (cval == -1 && PyErr_Occurred()) { - PYCBC_EXCTHROW_ARGS(); - } - } - - if ((*err = lcb_cntl(instance, mode, cmd, &cval)) != LCB_SUCCESS) { - return NULL; - } - - return pycbc_IntFromL(cval); - -} - -typedef union { - float f; - int i; - unsigned u; - lcb_uint32_t u32; - lcb_size_t sz; - const char *str; -} uCNTL; - -typedef enum { - CTL_TYPE_INVALID, - CTL_TYPE_STRING, - CTL_TYPE_INT, - CTL_TYPE_SIZET, - CTL_TYPE_U32, - CTL_TYPE_FLOAT, - CTL_TYPE_UNSIGNED, - CTL_TYPE_TIMEOUT, - CTL_TYPE_COMPAT -} CTLTYPE; - -static CTLTYPE -get_ctltype(const char *s) -{ - if (!strcmp(s, "str") || !strcmp(s, "string")) { - return CTL_TYPE_STRING; - } - if (!strcmp(s, "int")) { - return CTL_TYPE_INT; - } - if (!strcmp(s, "uint") || !strcmp(s, "unsigned")) { - return CTL_TYPE_UNSIGNED; - } - if (!strcmp(s, "size_t") || !strcmp(s, "lcb_size_t")) { - return CTL_TYPE_SIZET; - } - if (!strcmp(s, "float")) { - return CTL_TYPE_FLOAT; - } - if (!strcmp(s, "uint32_t") || !strcmp(s, "lcb_uint32_t")) { - return CTL_TYPE_U32; - } - if (!strcmp(s, "timeout") || !strcmp(s, "interval")) { - return CTL_TYPE_TIMEOUT; - } - return CTL_TYPE_INVALID; -} -/** - * Convert an input object to a proper pointer target based on the value - * @param type The type string - * @param input The input PyObject - * @param output Target for value - * @return 1 on success, 0 on failure - */ -static int -convert_object_input(CTLTYPE t, PyObject* input, uCNTL *output) -{ - int rv = 1; - PyObject *tuple = PyTuple_New(1); - PyTuple_SET_ITEM(tuple, 0, input); - Py_INCREF(input); - - if (t == CTL_TYPE_STRING) { - rv = PyArg_ParseTuple(tuple, "s", &output->str); - - } else if (t == CTL_TYPE_INT) { - rv = PyArg_ParseTuple(tuple, "i", &output->i); - - } else if (t == CTL_TYPE_UNSIGNED) { - rv = PyArg_ParseTuple(tuple, "I", &output->u); - - } else if (t == CTL_TYPE_U32) { - unsigned long tmp = 0; - rv = PyArg_ParseTuple(tuple, "k", &tmp); - if (rv) { - output->u32 = tmp; - } - - } else if (t == CTL_TYPE_TIMEOUT) { - double d; - rv = PyArg_ParseTuple(tuple, "d", &d); - if (rv) { - if (d <= 0) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, - "Cannot set timeout of value <= 0. Use uint32 for that"); - return 0; - } - output->u32 = d * 1000000; - } - } else if (t == CTL_TYPE_FLOAT) { - rv = PyArg_ParseTuple(tuple, "f", &output->f); - - } else if (t == CTL_TYPE_SIZET) { - unsigned long tmp = 0; - rv = PyArg_ParseTuple(tuple, "k", &tmp); - if (rv) { - output->sz = tmp; - } - } else { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Bad format for value"); - rv = 0; - } - - Py_DECREF(tuple); - return rv; -} - -static PyObject * -convert_object_output(CTLTYPE t, void *retval) -{ - if (t == CTL_TYPE_STRING) { - return PyUnicode_FromString(*(char**)retval); - } else if (t == CTL_TYPE_UNSIGNED) { - return pycbc_IntFromUL(*(unsigned*)retval); - } else if (t == CTL_TYPE_U32) { - return pycbc_IntFromUL(*(lcb_uint32_t*)retval); - } else if (t == CTL_TYPE_INT) { - return pycbc_IntFromL(*(int*)retval); - } else if (t == CTL_TYPE_TIMEOUT) { - double d = *(lcb_uint32_t*)retval; - d /= 1000000; - return PyFloat_FromDouble(d); - } else if (t == CTL_TYPE_SIZET) { - return pycbc_IntFromULL(*(lcb_size_t*)retval); - } else if (t == CTL_TYPE_FLOAT) { - return PyFloat_FromDouble(*(float*)retval); - } else { - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "oops"); - return NULL; - } -} - -static PyObject * -handle_old_ctl(pycbc_Bucket *self, int cmd, PyObject *val) -{ - PyObject *ret = NULL; - int mode = (val == NULL) ? CNTL_GET : CNTL_SET; - lcb_error_t err = LCB_SUCCESS; - - switch (cmd) { - /** Timeout parameters */ - case CNTL_OP_TIMEOUT: - case CNTL_VIEW_TIMEOUT: - case CNTL_HTTP_TIMEOUT: - case CNTL_DURABILITY_INTERVAL: - case CNTL_DURABILITY_TIMEOUT: - case CNTL_CONFIGURATION_TIMEOUT: { - ret = handle_float_tmo(self->instance, cmd, mode, val, &err); - if (ret) { - return ret; - } - break; - } - - /** Boolean values */ - case CNTL_SKIP_CONFIGURATION_ERRORS_ON_CONNECT: - case CNTL_RANDOMIZE_BOOTSTRAP_HOSTS: - case CNTL_CONFIG_CACHE_LOADED: { - ret = handle_boolean(self->instance, cmd, mode, val, &err); - break; - } - - /** int values */ - case CNTL_MAX_REDIRECTS: { - ret = handle_intval(self->instance, cmd, mode, val, &err); - break; - } - - default: - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Couldn't determine type for cntl"); - break; - } - - if (ret) { - return ret; - } - - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, err, "lcb_cntl failed"); - return NULL; -} - -PyObject * -pycbc_Bucket__cntl(pycbc_Bucket *self, PyObject *args, PyObject *kwargs) -{ - int cmd = 0; - CTLTYPE type = CTL_TYPE_COMPAT; - const char *argt = NULL; - PyObject *val = NULL; - lcb_error_t err = LCB_SUCCESS; - uCNTL input; - - char *kwnames[] = { "op", "value", "value_type", NULL }; - if (!PyArg_ParseTupleAndKeywords( - args, kwargs, "i|Os", kwnames, &cmd, &val, &argt)) { - - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - if (argt) { - type = get_ctltype(argt); - if (type == CTL_TYPE_INVALID) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Invalid type string"); - return NULL; - } - } - if (type == CTL_TYPE_COMPAT) { - return handle_old_ctl(self, cmd, val); - } - - if (val) { - int rv; - rv = convert_object_input(type, val, &input); - if (!rv) { - return NULL; /* error raised */ - } - err = lcb_cntl(self->instance, LCB_CNTL_SET, cmd, &input); - if (err == LCB_SUCCESS) { - Py_RETURN_TRUE; - } else { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, err, "lcb_cntl: Problem setting value"); - return NULL; - } - } else { - err = lcb_cntl(self->instance, LCB_CNTL_GET, cmd, &input); - if (err != LCB_SUCCESS) { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, err, "lcb_cntl: problem retrieving value"); - return NULL; - } - return convert_object_output(type, &input); - } -} - -PyObject * -pycbc_Bucket__cntlstr(pycbc_Bucket *conn, PyObject *args, PyObject *kw) -{ - const char *key, *value; - lcb_error_t err; - char *kwlist[] = { "key", "value", NULL }; - - if (!PyArg_ParseTupleAndKeywords(args, kw, "ss", kwlist, &key, &value)) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - err = lcb_cntl_string(conn->instance, key, value); - if (err != LCB_SUCCESS) { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, err, "Couldn't modify setting"); - return NULL; - } - Py_RETURN_NONE; -} - -PyObject * -pycbc_Bucket__vbmap(pycbc_Bucket *conn, PyObject *args) -{ - pycbc_strlen_t slen = 0; - const char *s = NULL; - PyObject *rtuple; - struct vbinfo_st info; - lcb_error_t err; - - if (!PyArg_ParseTuple(args, "s#", &s, &slen)) { - PYCBC_EXCTHROW_ARGS(); - } - - memset(&info, 0, sizeof(info)); - info.v.v0.key = s; - info.v.v0.nkey = slen; - err = lcb_cntl(conn->instance, CNTL_GET, CNTL_VBMAP, &info); - if (err != LCB_SUCCESS) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "lcb_cntl failed"); - return NULL; - } - - rtuple = PyTuple_New(2); - PyTuple_SET_ITEM(rtuple, 0, pycbc_IntFromL(info.v.v0.vbucket)); - PyTuple_SET_ITEM(rtuple, 1, pycbc_IntFromL(info.v.v0.server_index)); - return rtuple; -} diff --git a/src/connection.cxx b/src/connection.cxx new file mode 100644 index 000000000..b5dd4532b --- /dev/null +++ b/src/connection.cxx @@ -0,0 +1,1520 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "connection.hxx" + +#include <core/io/ip_protocol.hxx> +#include <core/utils/connection_string.hxx> + +#include "exceptions.hxx" +#include "metrics.hxx" +#include "tracing.hxx" + +couchbase::core::io::ip_protocol +pyObj_to_ip_protocol(std::string ip_protocol) +{ + if (ip_protocol.compare("force_ipv4") == 0) { + return couchbase::core::io::ip_protocol::force_ipv4; + } else if (ip_protocol.compare("force_ipv6") == 0) { + return couchbase::core::io::ip_protocol::force_ipv6; + } else { + return couchbase::core::io::ip_protocol::any; + } +} + +PyObject* +ip_protocol_to_pyObj(couchbase::core::io::ip_protocol ip_protocol) +{ + if (ip_protocol == couchbase::core::io::ip_protocol::force_ipv4) { + return PyUnicode_FromString("force_ipv4"); + } else if (ip_protocol == couchbase::core::io::ip_protocol::force_ipv6) { + return PyUnicode_FromString("force_ipv6"); + } else { + return PyUnicode_FromString("any"); + } +} + +couchbase::core::tls_verify_mode +pyObj_to_tls_verify_mode(std::string tls_verify_mode) +{ + if (tls_verify_mode.compare("none") == 0) { + return couchbase::core::tls_verify_mode::none; + } else if (tls_verify_mode.compare("peer") == 0) { + return couchbase::core::tls_verify_mode::peer; + } else { + return couchbase::core::tls_verify_mode::none; + } +} + +PyObject* +tls_verify_mode_to_pyObj(couchbase::core::tls_verify_mode tls_verify_mode) +{ + if (tls_verify_mode == couchbase::core::tls_verify_mode::none) { + return PyUnicode_FromString("none"); + } else if (tls_verify_mode == couchbase::core::tls_verify_mode::peer) { + return PyUnicode_FromString("peer"); + } else { + return PyUnicode_FromString("none"); + } +} + +static void +dealloc_conn(PyObject* obj) +{ + auto conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(obj, "conn_")); + if (conn) { + auto barrier = std::make_shared<std::promise<void>>(); + auto f = barrier->get_future(); + conn->cluster_.close([barrier]() { + barrier->set_value(); + }); + f.get(); + conn->io_.stop(); + for (auto& t : conn->io_threads_) { + if (t.joinable()) { + t.join(); + } + } + } + CB_LOG_DEBUG("{}: dealloc_conn completed", "PYCBC"); + delete conn; +} + +void +bucket_op_callback(std::error_code ec, + bool open, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + + PyGILState_STATE state = PyGILState_Ensure(); + + if (ec.value()) { + std::string msg = "Error trying to "; + msg.append((open ? "open" : "close") + std::string(" bucket.")); + pyObj_exc = pycbc_build_exception(ec, __FILE__, __LINE__, msg); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(PyBool_FromLong(static_cast<long>(1))); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, PyBool_FromLong(static_cast<long>(1))); + } + } + + if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_CallObject(pyObj_func, pyObj_args); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + std::string msg; + msg.append((open ? "Open" : "Close") + std::string(" bucket callback failed")); + pycbc_set_python_exception(PycbcError::InternalSDKError, __FILE__, __LINE__, msg.c_str()); + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + CB_LOG_DEBUG("{}: open/close bucket callback completed", "PYCBC"); + PyGILState_Release(state); +} + +void +close_connection_callback(PyObject* pyObj_conn, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = NULL; + PyObject* pyObj_func = NULL; + PyObject* pyObj_callback_res = nullptr; + + PyGILState_STATE state = PyGILState_Ensure(); + + if (pyObj_callback == nullptr) { + barrier->set_value(PyBool_FromLong(static_cast<long>(1))); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, PyBool_FromLong(static_cast<long>(1))); + } + + if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_CallObject(pyObj_func, pyObj_args); + CB_LOG_DEBUG("{}: return from close conn callback.", "PYCBC"); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + pycbc_set_python_exception( + PycbcError::InternalSDKError, __FILE__, __LINE__, "Close connection callback failed."); + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + CB_LOG_DEBUG("{}: close conn callback completed", "PYCBC"); + auto conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + conn->io_.stop(); + // the pyObj_conn was incref'd before being passed into this callback, decref it here + Py_DECREF(pyObj_conn); + PyGILState_Release(state); +} + +void +create_connection_callback(PyObject* pyObj_conn, + std::error_code ec, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = NULL; + PyObject* pyObj_func = NULL; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + + PyGILState_STATE state = PyGILState_Ensure(); + + if (ec.value()) { + pyObj_exc = pycbc_build_exception(ec, __FILE__, __LINE__, "Error creating a connection."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(pyObj_conn); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_conn); + } + } + + if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_CallObject(pyObj_func, pyObj_args); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + pycbc_set_python_exception( + PycbcError::InternalSDKError, __FILE__, __LINE__, "Create connection callback failed."); + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + Py_DECREF(pyObj_conn); + CB_LOG_DEBUG("{}: create conn callback completed", "PYCBC"); + PyGILState_Release(state); +} + +couchbase::core::cluster_credentials +get_cluster_credentials(PyObject* pyObj_auth) +{ + couchbase::core::cluster_credentials auth{}; + PyObject* pyObj_username = PyDict_GetItemString(pyObj_auth, "username"); + if (pyObj_username != nullptr) { + auto username = std::string(PyUnicode_AsUTF8(pyObj_username)); + auth.username = username; + } + + PyObject* pyObj_password = PyDict_GetItemString(pyObj_auth, "password"); + if (pyObj_password != nullptr) { + auto pw = std::string(PyUnicode_AsUTF8(pyObj_password)); + auth.password = pw; + } + + PyObject* pyObj_cert_path = PyDict_GetItemString(pyObj_auth, "cert_path"); + if (pyObj_cert_path != nullptr) { + auto cert_path = std::string(PyUnicode_AsUTF8(pyObj_cert_path)); + auth.certificate_path = cert_path; + } + + PyObject* pyObj_key_path = PyDict_GetItemString(pyObj_auth, "key_path"); + if (pyObj_key_path != nullptr) { + auto key_path = std::string(PyUnicode_AsUTF8(pyObj_key_path)); + auth.key_path = key_path; + } + + PyObject* pyObj_allowed_sasl_mechanisms = + PyDict_GetItemString(pyObj_auth, "allowed_sasl_mechanisms"); + if (pyObj_allowed_sasl_mechanisms != nullptr && PyList_Check(pyObj_allowed_sasl_mechanisms)) { + if (auth.allowed_sasl_mechanisms.has_value()) { + auth.allowed_sasl_mechanisms.value().clear(); + } + size_t nargs = static_cast<size_t>(PyList_Size(pyObj_allowed_sasl_mechanisms)); + size_t ii; + std::vector<std::string> allowed_sasl_mechanisms{}; + for (ii = 0; ii < nargs; ++ii) { + PyObject* pyOb_mechanism = PyList_GetItem(pyObj_allowed_sasl_mechanisms, ii); + auto mechanism = std::string(PyUnicode_AsUTF8(pyOb_mechanism)); + allowed_sasl_mechanisms.emplace_back(mechanism); + } + auth.allowed_sasl_mechanisms = allowed_sasl_mechanisms; + } + + return auth; +} + +PyObject* +get_metrics_options(const couchbase::core::metrics::logging_meter_options& logging_options) +{ + PyObject* pyObj_opts = PyDict_New(); + std::chrono::duration<unsigned long long, std::milli> int_msec = logging_options.emit_interval; + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "emit_interval", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + return pyObj_opts; +} + +void +update_cluster_logging_meter_options(couchbase::core::cluster_options& options, + PyObject* pyObj_emit_interval) +{ + couchbase::core::metrics::logging_meter_options logging_options{}; + bool has_logging_meter_options = false; + + if (pyObj_emit_interval != nullptr) { + auto emit_interval = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_emit_interval)); + auto emit_interval_ms = std::chrono::milliseconds(std::max(0ULL, emit_interval / 1000ULL)); + logging_options.emit_interval = emit_interval_ms; + has_logging_meter_options = true; + } + + if (has_logging_meter_options) { + options.metrics_options = logging_options; + } +} + +PyObject* +get_tracing_options(const couchbase::core::tracing::threshold_logging_options& tracing_options) +{ + PyObject* pyObj_opts = PyDict_New(); + std::chrono::duration<unsigned long long, std::milli> int_msec = + tracing_options.orphaned_emit_interval; + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "orphaned_emit_interval", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromSize_t(tracing_options.orphaned_sample_size); + if (-1 == PyDict_SetItemString(pyObj_opts, "orphaned_sample_size", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = tracing_options.threshold_emit_interval; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "threshold_emit_interval", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromSize_t(tracing_options.threshold_sample_size); + if (-1 == PyDict_SetItemString(pyObj_opts, "threshold_sample_size", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = tracing_options.key_value_threshold; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "key_value_threshold", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = tracing_options.query_threshold; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "query_threshold", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = tracing_options.view_threshold; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "view_threshold", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = tracing_options.search_threshold; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "search_threshold", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = tracing_options.analytics_threshold; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "analytics_threshold", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = tracing_options.management_threshold; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "management_threshold", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = tracing_options.eventing_threshold; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "eventing_threshold", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + return pyObj_opts; +} + +void +update_cluster_tracing_options(couchbase::core::cluster_options& options, + PyObject* pyObj_tracing_opts) +{ + couchbase::core::tracing::threshold_logging_options tracing_options{}; + bool has_tracing_options = false; + + PyObject* pyObj_kv_threshold = PyDict_GetItemString(pyObj_tracing_opts, "key_value_threshold"); + if (pyObj_kv_threshold != nullptr) { + auto kv_threshold = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_kv_threshold)); + auto kv_threshold_ms = std::chrono::milliseconds(std::max(0ULL, kv_threshold / 1000ULL)); + tracing_options.key_value_threshold = kv_threshold_ms; + has_tracing_options = true; + } + + PyObject* pyObj_view_threshold = PyDict_GetItemString(pyObj_tracing_opts, "view_threshold"); + if (pyObj_view_threshold != nullptr) { + auto view_threshold = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_view_threshold)); + auto view_threshold_ms = std::chrono::milliseconds(std::max(0ULL, view_threshold / 1000ULL)); + tracing_options.view_threshold = view_threshold_ms; + has_tracing_options = true; + } + + PyObject* pyObj_query_threshold = PyDict_GetItemString(pyObj_tracing_opts, "query_threshold"); + if (pyObj_query_threshold != nullptr) { + auto query_threshold = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_query_threshold)); + auto query_threshold_ms = std::chrono::milliseconds(std::max(0ULL, query_threshold / 1000ULL)); + tracing_options.query_threshold = query_threshold_ms; + has_tracing_options = true; + } + + PyObject* pyObj_search_threshold = PyDict_GetItemString(pyObj_tracing_opts, "search_threshold"); + if (pyObj_search_threshold != nullptr) { + auto search_threshold = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_search_threshold)); + auto search_threshold_ms = + std::chrono::milliseconds(std::max(0ULL, search_threshold / 1000ULL)); + tracing_options.search_threshold = search_threshold_ms; + has_tracing_options = true; + } + + PyObject* pyObj_analytics_threshold = + PyDict_GetItemString(pyObj_tracing_opts, "analytics_threshold"); + if (pyObj_analytics_threshold != nullptr) { + auto analytics_threshold = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_analytics_threshold)); + auto analytics_threshold_ms = + std::chrono::milliseconds(std::max(0ULL, analytics_threshold / 1000ULL)); + tracing_options.analytics_threshold = analytics_threshold_ms; + has_tracing_options = true; + } + + PyObject* pyObj_eventing_threshold = + PyDict_GetItemString(pyObj_tracing_opts, "eventing_threshold"); + if (pyObj_eventing_threshold != nullptr) { + auto eventing_threshold = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_eventing_threshold)); + auto eventing_threshold_ms = + std::chrono::milliseconds(std::max(0ULL, eventing_threshold / 1000ULL)); + tracing_options.eventing_threshold = eventing_threshold_ms; + has_tracing_options = true; + } + + PyObject* pyObj_management_threshold = + PyDict_GetItemString(pyObj_tracing_opts, "management_threshold"); + if (pyObj_management_threshold != nullptr) { + auto management_threshold = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_management_threshold)); + auto management_threshold_ms = + std::chrono::milliseconds(std::max(0ULL, management_threshold / 1000ULL)); + tracing_options.management_threshold = management_threshold_ms; + has_tracing_options = true; + } + + PyObject* pyObj_threshold_sample_size = + PyDict_GetItemString(pyObj_tracing_opts, "threshold_sample_size"); + if (pyObj_threshold_sample_size != nullptr) { + auto threshold_sample_size = + static_cast<size_t>(PyLong_AsUnsignedLong(pyObj_threshold_sample_size)); + tracing_options.threshold_sample_size = threshold_sample_size; + has_tracing_options = true; + } + + PyObject* pyObj_threshold_emit_interval = + PyDict_GetItemString(pyObj_tracing_opts, "threshold_emit_interval"); + if (pyObj_threshold_emit_interval != nullptr) { + auto threshold_emit_interval = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_threshold_emit_interval)); + auto threshold_emit_interval_ms = + std::chrono::milliseconds(std::max(0ULL, threshold_emit_interval / 1000ULL)); + tracing_options.threshold_emit_interval = threshold_emit_interval_ms; + has_tracing_options = true; + } + + PyObject* pyObj_orphaned_emit_interval = + PyDict_GetItemString(pyObj_tracing_opts, "orphaned_emit_interval"); + if (pyObj_orphaned_emit_interval != nullptr) { + auto orphaned_emit_interval = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_orphaned_emit_interval)); + auto orphaned_emit_interval_ms = + std::chrono::milliseconds(std::max(0ULL, orphaned_emit_interval / 1000ULL)); + tracing_options.orphaned_emit_interval = orphaned_emit_interval_ms; + has_tracing_options = true; + } + + PyObject* pyObj_orphaned_sample_size = + PyDict_GetItemString(pyObj_tracing_opts, "orphaned_sample_size"); + if (pyObj_orphaned_sample_size != nullptr) { + auto orphaned_sample_size = + static_cast<size_t>(PyLong_AsUnsignedLong(pyObj_orphaned_sample_size)); + tracing_options.orphaned_sample_size = orphaned_sample_size; + has_tracing_options = true; + } + + if (has_tracing_options) { + options.tracing_options = tracing_options; + } +} + +void +update_cluster_timeout_options(couchbase::core::cluster_options& options, + PyObject* pyObj_timeout_opts) +{ + PyObject* pyObj_bootstrap_timeout = PyDict_GetItemString(pyObj_timeout_opts, "bootstrap_timeout"); + if (pyObj_bootstrap_timeout != nullptr) { + auto bootstrap_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_bootstrap_timeout)); + auto bootstrap_timeout_ms = + std::chrono::milliseconds(std::max(0ULL, bootstrap_timeout / 1000ULL)); + options.bootstrap_timeout = bootstrap_timeout_ms; + } + + PyObject* pyObj_resolve_timeout = PyDict_GetItemString(pyObj_timeout_opts, "resolve_timeout"); + if (pyObj_resolve_timeout != nullptr) { + auto resolve_timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_resolve_timeout)); + auto resolve_timeout_ms = std::chrono::milliseconds(std::max(0ULL, resolve_timeout / 1000ULL)); + options.resolve_timeout = resolve_timeout_ms; + } + + PyObject* pyObj_connect_timeout = PyDict_GetItemString(pyObj_timeout_opts, "connect_timeout"); + if (pyObj_connect_timeout != nullptr) { + auto connect_timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_connect_timeout)); + auto connect_timeout_ms = std::chrono::milliseconds(std::max(0ULL, connect_timeout / 1000ULL)); + options.connect_timeout = connect_timeout_ms; + } + + PyObject* pyObj_key_value_timeout = PyDict_GetItemString(pyObj_timeout_opts, "key_value_timeout"); + if (pyObj_key_value_timeout != nullptr) { + auto key_value_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_key_value_timeout)); + auto key_value_timeout_ms = + std::chrono::milliseconds(std::max(0ULL, key_value_timeout / 1000ULL)); + options.key_value_timeout = key_value_timeout_ms; + } + + PyObject* pyObj_key_value_durable_timeout = + PyDict_GetItemString(pyObj_timeout_opts, "key_value_durable_timeout"); + if (pyObj_key_value_durable_timeout != nullptr) { + auto key_value_durable_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_key_value_durable_timeout)); + auto key_value_durable_timeout_ms = + std::chrono::milliseconds(std::max(0ULL, key_value_durable_timeout / 1000ULL)); + options.key_value_durable_timeout = key_value_durable_timeout_ms; + } + + PyObject* pyObj_view_timeout = PyDict_GetItemString(pyObj_timeout_opts, "view_timeout"); + if (pyObj_view_timeout != nullptr) { + auto view_timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_view_timeout)); + auto view_timeout_ms = std::chrono::milliseconds(std::max(0ULL, view_timeout / 1000ULL)); + options.view_timeout = view_timeout_ms; + } + + PyObject* pyObj_query_timeout = PyDict_GetItemString(pyObj_timeout_opts, "query_timeout"); + if (pyObj_query_timeout != nullptr) { + auto query_timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_query_timeout)); + auto query_timeout_ms = std::chrono::milliseconds(std::max(0ULL, query_timeout / 1000ULL)); + options.query_timeout = query_timeout_ms; + } + + PyObject* pyObj_analytics_timeout = PyDict_GetItemString(pyObj_timeout_opts, "analytics_timeout"); + if (pyObj_analytics_timeout != nullptr) { + auto analytics_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_analytics_timeout)); + auto analytics_timeout_ms = + std::chrono::milliseconds(std::max(0ULL, analytics_timeout / 1000ULL)); + options.analytics_timeout = analytics_timeout_ms; + } + + PyObject* pyObj_search_timeout = PyDict_GetItemString(pyObj_timeout_opts, "search_timeout"); + if (pyObj_search_timeout != nullptr) { + auto search_timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_search_timeout)); + auto search_timeout_ms = std::chrono::milliseconds(std::max(0ULL, search_timeout / 1000ULL)); + options.search_timeout = search_timeout_ms; + } + + PyObject* pyObj_management_timeout = + PyDict_GetItemString(pyObj_timeout_opts, "management_timeout"); + if (pyObj_management_timeout != nullptr) { + auto management_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_management_timeout)); + auto management_timeout_ms = + std::chrono::milliseconds(std::max(0ULL, management_timeout / 1000ULL)); + options.management_timeout = management_timeout_ms; + } + + PyObject* pyObj_idle_http_connection_timeout = + PyDict_GetItemString(pyObj_timeout_opts, "idle_http_connection_timeout"); + if (pyObj_idle_http_connection_timeout != nullptr) { + auto idle_http_connection_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_idle_http_connection_timeout)); + auto idle_http_connection_timeout_ms = + std::chrono::milliseconds(std::max(0ULL, idle_http_connection_timeout / 1000ULL)); + options.idle_http_connection_timeout = idle_http_connection_timeout_ms; + } + + PyObject* pyObj_config_idle_redial_timeout = + PyDict_GetItemString(pyObj_timeout_opts, "config_idle_redial_timeout"); + if (pyObj_config_idle_redial_timeout != nullptr) { + auto config_idle_redial_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_config_idle_redial_timeout)); + auto config_idle_redial_timeout_ms = + std::chrono::milliseconds(std::max(0ULL, config_idle_redial_timeout / 1000ULL)); + options.config_idle_redial_timeout = config_idle_redial_timeout_ms; + } +} + +void +update_cluster_options(couchbase::core::cluster_options& options, + PyObject* pyObj_options, + PyObject* pyObj_auth) +{ + PyObject* pyObj_timeout_opts = PyDict_GetItemString(pyObj_options, "timeout_options"); + if (pyObj_timeout_opts != nullptr) { + update_cluster_timeout_options(options, pyObj_timeout_opts); + } + + PyObject* pyObj_tracing_opts = PyDict_GetItemString(pyObj_options, "tracing_options"); + if (pyObj_tracing_opts != nullptr) { + update_cluster_tracing_options(options, pyObj_tracing_opts); + } + + PyObject* pyObj_emit_interval = PyDict_GetItemString(pyObj_options, "emit_interval"); + if (pyObj_emit_interval != nullptr) { + update_cluster_logging_meter_options(options, pyObj_emit_interval); + } + + PyObject* pyObj_enable_tls = PyDict_GetItemString(pyObj_options, "enable_tls"); + if (pyObj_enable_tls != nullptr && pyObj_enable_tls == Py_True) { + options.enable_tls = true; + } + + PyObject* pyObj_trust_store_path = PyDict_GetItemString(pyObj_auth, "trust_store_path"); + if (pyObj_trust_store_path != nullptr) { + auto trust_store_path = std::string(PyUnicode_AsUTF8(pyObj_trust_store_path)); + options.trust_certificate = trust_store_path; + } else { + pyObj_trust_store_path = PyDict_GetItemString(pyObj_options, "trust_store_path"); + if (pyObj_trust_store_path != nullptr) { + auto trust_store_path = std::string(PyUnicode_AsUTF8(pyObj_trust_store_path)); + options.trust_certificate = trust_store_path; + } + } + + PyObject* pyObj_disable_mozilla_ca_certificates = + PyDict_GetItemString(pyObj_options, "disable_mozilla_ca_certificates"); + if (pyObj_disable_mozilla_ca_certificates != nullptr && + pyObj_disable_mozilla_ca_certificates == Py_True) { + options.disable_mozilla_ca_certificates = true; + } + + PyObject* pyObj_enable_mut_tokens = PyDict_GetItemString(pyObj_options, "enable_mutation_tokens"); + if (pyObj_enable_mut_tokens != nullptr && pyObj_enable_mut_tokens == Py_False) { + options.enable_mutation_tokens = false; + } + + PyObject* pyObj_enable_tcp_keep_alive = + PyDict_GetItemString(pyObj_options, "enable_tcp_keep_alive"); + if (pyObj_enable_tcp_keep_alive != nullptr && pyObj_enable_tcp_keep_alive == Py_False) { + options.enable_tcp_keep_alive = false; + } + + PyObject* pyObj_use_ip_protocol = PyDict_GetItemString(pyObj_options, "use_ip_protocol"); + if (pyObj_use_ip_protocol != nullptr) { + options.use_ip_protocol = + pyObj_to_ip_protocol(std::string(PyUnicode_AsUTF8(pyObj_use_ip_protocol))); + } + + PyObject* pyObj_enable_dns_srv = PyDict_GetItemString(pyObj_options, "enable_dns_srv"); + if (pyObj_enable_dns_srv != nullptr && pyObj_enable_dns_srv == Py_False) { + options.enable_dns_srv = false; + } + + PyObject* pyObj_show_queries = PyDict_GetItemString(pyObj_options, "show_queries"); + if (pyObj_show_queries != nullptr && pyObj_show_queries == Py_True) { + options.show_queries = true; + } + + PyObject* pyObj_enable_unordered_execution = + PyDict_GetItemString(pyObj_options, "enable_unordered_execution"); + if (pyObj_enable_unordered_execution != nullptr && pyObj_enable_unordered_execution == Py_False) { + options.enable_unordered_execution = false; + } + + PyObject* pyObj_enable_clustermap_notification = + PyDict_GetItemString(pyObj_options, "enable_clustermap_notification"); + if (pyObj_enable_clustermap_notification != nullptr && + pyObj_enable_clustermap_notification == Py_False) { + options.enable_clustermap_notification = false; + } + + PyObject* pyObj_enable_compression = PyDict_GetItemString(pyObj_options, "enable_compression"); + if (pyObj_enable_compression != nullptr && pyObj_enable_compression == Py_False) { + options.enable_compression = false; + } + + PyObject* pyObj_enable_tracing = PyDict_GetItemString(pyObj_options, "enable_tracing"); + if (pyObj_enable_tracing != nullptr && pyObj_enable_tracing == Py_False) { + options.enable_tracing = false; + } + + PyObject* pyObj_enable_metrics = PyDict_GetItemString(pyObj_options, "enable_metrics"); + if (pyObj_enable_metrics != nullptr && pyObj_enable_metrics == Py_False) { + options.enable_metrics = false; + } + + PyObject* pyObj_network = PyDict_GetItemString(pyObj_options, "network"); + if (pyObj_network != nullptr) { + auto network = std::string(PyUnicode_AsUTF8(pyObj_network)); + options.network = network; + } + + PyObject* pyObj_tls_verify = PyDict_GetItemString(pyObj_options, "tls_verify"); + if (pyObj_tls_verify != nullptr) { + options.tls_verify = pyObj_to_tls_verify_mode(std::string(PyUnicode_AsUTF8(pyObj_tls_verify))); + } + + PyObject* pyObj_tcp_keep_alive_interval = + PyDict_GetItemString(pyObj_options, "tcp_keep_alive_interval"); + if (pyObj_tcp_keep_alive_interval != nullptr) { + auto tcp_keep_alive_interval = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_tcp_keep_alive_interval)); + auto tcp_keep_alive_interval_ms = + std::chrono::milliseconds(std::max(0ULL, tcp_keep_alive_interval / 1000ULL)); + options.tcp_keep_alive_interval = tcp_keep_alive_interval_ms; + } + + PyObject* pyObj_config_poll_interval = + PyDict_GetItemString(pyObj_options, "config_poll_interval"); + if (pyObj_config_poll_interval != nullptr) { + auto config_poll_interval = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_config_poll_interval)); + auto config_poll_interval_ms = + std::chrono::milliseconds(std::max(0ULL, config_poll_interval / 1000ULL)); + options.config_poll_interval = config_poll_interval_ms; + } + + PyObject* pyObj_config_poll_floor = PyDict_GetItemString(pyObj_options, "config_poll_floor"); + if (pyObj_config_poll_floor != nullptr) { + auto config_poll_floor = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_config_poll_floor)); + auto config_poll_floor_ms = + std::chrono::milliseconds(std::max(0ULL, config_poll_floor / 1000ULL)); + options.config_poll_floor = config_poll_floor_ms; + } + + PyObject* pyObj_user_agent_extra = PyDict_GetItemString(pyObj_options, "user_agent_extra"); + if (pyObj_user_agent_extra != nullptr) { + auto user_agent_extra = std::string(PyUnicode_AsUTF8(pyObj_user_agent_extra)); + options.user_agent_extra = user_agent_extra; + } + + PyObject* pyObj_max_http_connections = + PyDict_GetItemString(pyObj_options, "max_http_connections"); + if (pyObj_max_http_connections != nullptr) { + auto max_http_connections = + static_cast<size_t>(PyLong_AsUnsignedLong(pyObj_max_http_connections)); + options.max_http_connections = max_http_connections; + } + + PyObject* pyObj_tracer = PyDict_GetItemString(pyObj_options, "tracer"); + if (pyObj_tracer != nullptr) { + options.tracer = std::make_shared<pycbc::request_tracer>(pyObj_tracer); + } + + PyObject* pyObj_meter = PyDict_GetItemString(pyObj_options, "meter"); + if (pyObj_meter != nullptr) { + options.meter = std::make_shared<pycbc::meter>(pyObj_meter); + } + + PyObject* pyObj_dns_nameserver = PyDict_GetItemString(pyObj_options, "dns_nameserver"); + PyObject* pyObj_dns_port = PyDict_GetItemString(pyObj_options, "dns_port"); + PyObject* pyObj_dns_srv_timeout = nullptr; + if (pyObj_timeout_opts != nullptr) { + pyObj_dns_srv_timeout = PyDict_GetItemString(pyObj_timeout_opts, "dns_srv_timeout"); + } + if (pyObj_dns_srv_timeout != nullptr || pyObj_dns_nameserver != nullptr || + pyObj_dns_port != nullptr) { + auto nameserver = pyObj_dns_nameserver != nullptr + ? std::string(PyUnicode_AsUTF8(pyObj_dns_nameserver)) + : options.dns_config.nameserver(); + auto port = pyObj_dns_port != nullptr + ? static_cast<uint16_t>(PyLong_AsUnsignedLong(pyObj_dns_port)) + : options.dns_config.port(); + auto dns_srv_timeout_ms = couchbase::core::timeout_defaults::dns_srv_timeout; + if (pyObj_dns_srv_timeout != nullptr) { + auto dns_srv_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_dns_srv_timeout)); + dns_srv_timeout_ms = std::chrono::milliseconds(std::max(0ULL, dns_srv_timeout / 1000ULL)); + } + options.dns_config = couchbase::core::io::dns::dns_config(nameserver, port, dns_srv_timeout_ms); + } + + PyObject* pyObj_dump_configuration = PyDict_GetItemString(pyObj_options, "dump_configuration"); + if (pyObj_dump_configuration != nullptr && pyObj_dump_configuration == Py_True) { + options.dump_configuration = true; + } + + PyObject* pyObj_preferred_server_group = + PyDict_GetItemString(pyObj_options, "preferred_server_group"); + if (pyObj_preferred_server_group != nullptr) { + options.server_group = std::string(PyUnicode_AsUTF8(pyObj_preferred_server_group)); + } + + PyObject* pyObj_enable_app_telemetry = + PyDict_GetItemString(pyObj_options, "enable_app_telemetry"); + if (pyObj_enable_app_telemetry != nullptr && pyObj_enable_app_telemetry == Py_False) { + options.enable_app_telemetry = false; + } + + PyObject* pyObj_app_telemetry_endpoint = + PyDict_GetItemString(pyObj_options, "app_telemetry_endpoint"); + if (pyObj_app_telemetry_endpoint != nullptr) { + auto app_telemetry_endpoint = std::string(PyUnicode_AsUTF8(pyObj_app_telemetry_endpoint)); + options.app_telemetry_endpoint = app_telemetry_endpoint; + } + + PyObject* pyObj_app_telemetry_backoff = + PyDict_GetItemString(pyObj_options, "app_telemetry_backoff"); + if (pyObj_app_telemetry_backoff != nullptr) { + auto app_telemetry_backoff = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_app_telemetry_backoff)); + auto app_telemetry_backoff_ms = + std::chrono::milliseconds(std::max(0ULL, app_telemetry_backoff / 1000ULL)); + options.app_telemetry_backoff_interval = app_telemetry_backoff_ms; + } + + PyObject* pyObj_app_telemetry_ping_interval = + PyDict_GetItemString(pyObj_options, "app_telemetry_ping_interval"); + if (pyObj_app_telemetry_ping_interval != nullptr) { + auto app_telemetry_ping_interval = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_app_telemetry_ping_interval)); + auto app_telemetry_ping_interval_ms = + std::chrono::milliseconds(std::max(0ULL, app_telemetry_ping_interval / 1000ULL)); + options.app_telemetry_ping_interval = app_telemetry_ping_interval_ms; + } + + PyObject* pyObj_app_telemetry_ping_timeout = + PyDict_GetItemString(pyObj_options, "app_telemetry_ping_timeout"); + if (pyObj_app_telemetry_ping_timeout != nullptr) { + auto app_telemetry_ping_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_app_telemetry_ping_timeout)); + auto app_telemetry_ping_timeout_ms = + std::chrono::milliseconds(std::max(0ULL, app_telemetry_ping_timeout / 1000ULL)); + options.app_telemetry_ping_timeout = app_telemetry_ping_timeout_ms; + } +} + +PyObject* +handle_create_connection([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + char* conn_str = nullptr; + PyObject* pyObj_auth = nullptr; + PyObject* pyObj_options = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_result = nullptr; + + static const char* kw_list[] = { "", "auth", "options", "callback", "errback", nullptr }; + + const char* kw_format = "s|OOOO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &conn_str, + &pyObj_auth, + &pyObj_options, + &pyObj_callback, + &pyObj_errback); + + if (!ret) { + std::string msg = "Cannot create connection. Unable to parse args/kwargs."; + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, msg.c_str()); + return nullptr; + } + + couchbase::core::utils::connection_string connection_str = + couchbase::core::utils::parse_connection_string(conn_str); + couchbase::core::cluster_credentials auth = get_cluster_credentials(pyObj_auth); + try { + update_cluster_options(connection_str.options, pyObj_options, pyObj_auth); + } catch (const std::invalid_argument& e) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, e.what()); + return nullptr; + } catch (const std::exception& e) { + PyErr_SetString(PyExc_Exception, e.what()); + return nullptr; + } + + PyObject* pyObj_num_io_threads = PyDict_GetItemString(pyObj_options, "num_io_threads"); + int num_io_threads = 1; + if (pyObj_num_io_threads != nullptr) { + num_io_threads = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_num_io_threads)); + } + + connection* const conn = new connection(num_io_threads); + PyObject* pyObj_conn = PyCapsule_New(conn, "conn_", dealloc_conn); + + if (pyObj_conn == nullptr) { + pycbc_set_python_exception(PycbcError::InternalSDKError, + __FILE__, + __LINE__, + "Cannot create connection. Unable to create PyCapsule."); + return nullptr; + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + + Py_XINCREF(pyObj_conn); + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + { + int callback_count = 0; + Py_BEGIN_ALLOW_THREADS conn->cluster_.open( + couchbase::core::origin(auth, connection_str), + [pyObj_conn, pyObj_callback, pyObj_errback, callback_count, barrier]( + std::error_code ec) mutable { + if (callback_count == 0) { + create_connection_callback(pyObj_conn, ec, pyObj_callback, pyObj_errback, barrier); + } + callback_count++; + }); + Py_END_ALLOW_THREADS + } + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} + +PyObject* +get_connection_info([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_conn = nullptr; + static const char* kw_list[] = { "", nullptr }; + + const char* kw_format = "O!"; + int ret = PyArg_ParseTupleAndKeywords( + args, kwargs, kw_format, const_cast<char**>(kw_list), &PyCapsule_Type, &pyObj_conn); + + if (!ret) { + std::string msg = "Cannot get connection options. Unable to parse args/kwargs."; + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, msg.c_str()); + return nullptr; + } + + connection* conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + auto cluster_info = conn->cluster_.origin(); + if (cluster_info.first) { + Py_RETURN_NONE; + } + auto opts = cluster_info.second.options(); + PyObject* pyObj_opts = PyDict_New(); + std::chrono::duration<unsigned long long, std::milli> int_msec = opts.bootstrap_timeout; + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "bootstrap_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.resolve_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "resolve_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.connect_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "connect_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.key_value_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "key_value_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.key_value_durable_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "key_value_durable_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.view_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "view_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.query_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "query_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.analytics_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "analytics_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.search_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "search_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.management_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "management_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.dns_config.timeout(); + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "dns_srv_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString(pyObj_opts, "enable_tls", opts.enable_tls ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + pyObj_tmp = PyUnicode_FromString(opts.trust_certificate.c_str()); + if (-1 == PyDict_SetItemString(pyObj_opts, "trust_certificate", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString(pyObj_opts, + "disable_mozilla_ca_certificates", + opts.disable_mozilla_ca_certificates ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == PyDict_SetItemString(pyObj_opts, + "enable_mutation_tokens", + opts.enable_mutation_tokens ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == PyDict_SetItemString(pyObj_opts, + "enable_tcp_keep_alive", + opts.enable_tcp_keep_alive ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + pyObj_tmp = ip_protocol_to_pyObj(opts.use_ip_protocol); + if (-1 == PyDict_SetItemString(pyObj_opts, "ip_protocol", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString( + pyObj_opts, "enable_dns_srv", opts.enable_dns_srv ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == + PyDict_SetItemString(pyObj_opts, "show_queries", opts.show_queries ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == PyDict_SetItemString(pyObj_opts, + "enable_unordered_execution", + opts.enable_unordered_execution ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == PyDict_SetItemString(pyObj_opts, + "enable_clustermap_notification", + opts.enable_clustermap_notification ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == PyDict_SetItemString( + pyObj_opts, "enable_compression", opts.enable_compression ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == PyDict_SetItemString( + pyObj_opts, "enable_tracing", opts.enable_tracing ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == PyDict_SetItemString( + pyObj_opts, "enable_metrics", opts.enable_metrics ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + pyObj_tmp = PyUnicode_FromString(opts.network.c_str()); + if (-1 == PyDict_SetItemString(pyObj_opts, "network", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = get_tracing_options(opts.tracing_options); + if (-1 == PyDict_SetItemString(pyObj_opts, "tracing_options", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = get_metrics_options(opts.metrics_options); + if (-1 == PyDict_SetItemString(pyObj_opts, "metrics_options", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = tls_verify_mode_to_pyObj(opts.tls_verify); + if (-1 == PyDict_SetItemString(pyObj_opts, "tls_verify", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == + PyDict_SetItemString(pyObj_opts, "has_tracer", opts.tracer != nullptr ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + if (-1 == + PyDict_SetItemString(pyObj_opts, "has_meter", opts.meter != nullptr ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + int_msec = opts.tcp_keep_alive_interval; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "tcp_keep_alive_interval", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.config_poll_interval; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "config_poll_interval", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.config_poll_floor; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "config_poll_floor", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.config_idle_redial_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "config_idle_redial_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromSize_t(opts.max_http_connections); + if (-1 == PyDict_SetItemString(pyObj_opts, "max_http_connections", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.idle_http_connection_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "idle_http_connection_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(opts.user_agent_extra.c_str()); + if (-1 == PyDict_SetItemString(pyObj_opts, "user_agent_extra", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + auto credentials = cluster_info.second.credentials(); + PyObject* pyObj_creds = PyDict_New(); + + pyObj_tmp = PyUnicode_FromString(credentials.username.c_str()); + if (-1 == PyDict_SetItemString(pyObj_creds, "username", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(credentials.password.c_str()); + if (-1 == PyDict_SetItemString(pyObj_creds, "password", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(credentials.certificate_path.c_str()); + if (-1 == PyDict_SetItemString(pyObj_creds, "certificate_path", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(credentials.key_path.c_str()); + if (-1 == PyDict_SetItemString(pyObj_creds, "key_path", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + PyObject* pyObj_allowed_sasl_mechanisms = PyList_New(static_cast<Py_ssize_t>(0)); + if (credentials.allowed_sasl_mechanisms.has_value()) { + for (auto const& mech : credentials.allowed_sasl_mechanisms.value()) { + pyObj_tmp = PyUnicode_FromString(mech.c_str()); + if (-1 == PyList_Append(pyObj_allowed_sasl_mechanisms, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + } + } + + if (-1 == + PyDict_SetItemString(pyObj_creds, "allowed_sasl_mechanisms", pyObj_allowed_sasl_mechanisms)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_allowed_sasl_mechanisms); + + if (-1 == PyDict_SetItemString(pyObj_opts, "credentials", pyObj_creds)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_creds); + + if (-1 == PyDict_SetItemString( + pyObj_opts, "dump_configuration", opts.dump_configuration ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + pyObj_tmp = PyUnicode_FromString(opts.server_group.c_str()); + if (-1 == PyDict_SetItemString(pyObj_opts, "preferred_server_group", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString( + pyObj_opts, "enable_app_telemetry", opts.enable_app_telemetry ? Py_True : Py_False)) { + PyErr_Print(); + PyErr_Clear(); + } + + pyObj_tmp = PyUnicode_FromString(opts.app_telemetry_endpoint.c_str()); + if (-1 == PyDict_SetItemString(pyObj_opts, "app_telemetry_endpoint", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.app_telemetry_backoff_interval; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "app_telemetry_backoff", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.app_telemetry_ping_interval; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "app_telemetry_ping_interval", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_msec = opts.app_telemetry_ping_timeout; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_opts, "app_telemetry_ping_timeout", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + return pyObj_opts; +} + +PyObject* +handle_close_connection([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_conn = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_result = nullptr; + + static const char* kw_list[] = { "", "callback", "errback", nullptr }; + + const char* kw_format = "O!|OO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &pyObj_callback, + &pyObj_errback); + + if (!ret) { + std::string msg = "Cannot close connection. Unable to parse args/kwargs."; + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, msg.c_str()); + return nullptr; + } + + connection* conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + Py_XINCREF(pyObj_conn); + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + { + int callback_count = 0; + Py_BEGIN_ALLOW_THREADS conn->cluster_.close( + [pyObj_conn, pyObj_callback, pyObj_errback, callback_count, barrier]() mutable { + if (callback_count == 0) { + close_connection_callback(pyObj_conn, pyObj_callback, pyObj_errback, barrier); + } else { + CB_LOG_DEBUG("close callback called {} times already!", callback_count); + callback_count++; + } + }); + Py_END_ALLOW_THREADS + } + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} + +PyObject* +handle_open_or_close_bucket([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + char* bucket_name = nullptr; + PyObject* pyObj_conn = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_result = nullptr; + + int open = 1; + + static const char* kw_list[] = { "", "", "callback", "errback", "open_bucket", nullptr }; + + // TODO: something about passing in a boolean corrupts the param before it + // "O!s|OOp" would cause the errback PyObject to be corrupted -- no idea why??? + // don't seem to have this issue in the KV ops... + const char* kw_format = "O!s|OOi"; + + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &bucket_name, + &pyObj_callback, + &pyObj_errback, + &open); + + if (!ret) { + std::string msg = "Cannot "; + msg.append(open == 1 ? "open" : "close"); + msg.append(" bucket. Unable to parse args/kwargs."); + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, msg.c_str()); + return nullptr; + } + + connection* conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + { + int callback_count = 0; + Py_BEGIN_ALLOW_THREADS if (open) + { + conn->cluster_.open_bucket( + bucket_name, + [pyObj_callback, pyObj_errback, callback_count, open, barrier](std::error_code ec) mutable { + // @TODO: should the c++ client execute this lambda more than once? + if (callback_count == 0) { + bucket_op_callback(ec, open, pyObj_callback, pyObj_errback, barrier); + } + callback_count++; + }); + } + else + { + conn->cluster_.close_bucket( + bucket_name, + [pyObj_callback, pyObj_errback, callback_count, open, barrier](std::error_code ec) mutable { + if (callback_count == 0) { + bucket_op_callback(ec, open, pyObj_callback, pyObj_errback, barrier); + } + callback_count++; + }); + } + Py_END_ALLOW_THREADS + } + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} diff --git a/src/connection.hxx b/src/connection.hxx new file mode 100644 index 000000000..48054b802 --- /dev/null +++ b/src/connection.hxx @@ -0,0 +1,32 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" + +PyObject* +handle_create_connection(PyObject* self, PyObject* args, PyObject* kwargs); + +PyObject* +get_connection_info(PyObject* self, PyObject* args, PyObject* kwargs); + +PyObject* +handle_close_connection(PyObject* self, PyObject* args, PyObject* kwargs); + +PyObject* +handle_open_or_close_bucket(PyObject* self, PyObject* args, PyObject* kwargs); diff --git a/src/connevents.c b/src/connevents.c deleted file mode 100644 index b8ce8a0c1..000000000 --- a/src/connevents.c +++ /dev/null @@ -1,130 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -/** - * This file contains connection events - */ -#include "pycbc.h" -#include "iops.h" - -void -pycbc_invoke_connected_event(pycbc_Bucket *conn, lcb_error_t err) -{ - PyObject *argtuple; - PyObject *arg; - PyObject *ret; - - if (conn->flags & PYCBC_CONN_F_CONNECTED) { - return; - } - - conn->flags |= PYCBC_CONN_F_CONNECTED; - - if (conn->conncb == NULL || PyObject_IsTrue(conn->conncb) == 0) { - return; - } - - if (err == LCB_SUCCESS) { - arg = Py_None; - Py_INCREF(Py_None); - } else { - arg = pycbc_exc_message(PYCBC_EXC_LCBERR, err, - "Error getting initial connection " - "to cluster"); - } - - argtuple = PyTuple_New(1); - PyTuple_SET_ITEM(argtuple, 0, arg); - - ret = PyObject_CallObject(conn->conncb, argtuple); - Py_XDECREF(ret); - Py_XDECREF(conn->conncb); - conn->conncb = NULL; - Py_DECREF(argtuple); -} - - -struct dtor_info_st { - PyObject *iowrap; - PyObject *dtorcb; - PyObject *conncb; -}; - -static void -dtor_callback(const void *arg) -{ - struct dtor_info_st *dti = (void*)arg; - - if (dti->conncb) { - PyObject *ret; - PyObject *exc; - PyObject *args = PyTuple_New(1); - - exc = pycbc_exc_message(PYCBC_EXC_DESTROYED, 0, - "Connection object was garbage collected"); - assert(exc); - - PyTuple_SET_ITEM(args, 0, exc); - ret = PyObject_CallObject(dti->conncb, args); - - Py_XDECREF(ret); - Py_DECREF(args); - Py_DECREF(dti->conncb); - dti->conncb = NULL; - } - - if (dti->dtorcb) { - PyObject *ret = PyObject_CallObject(dti->dtorcb, NULL); - Py_XDECREF(ret); - Py_DECREF(dti->dtorcb); - dti->dtorcb = NULL; - } - if (dti->iowrap) { - Py_DECREF(dti->iowrap); - } - free(dti); -} - -void -pycbc_schedule_dtor_event(pycbc_Bucket *self) -{ - struct dtor_info_st *dti; - - if ((self->flags & PYCBC_CONN_F_ASYNC_DTOR) == 0) { - return; - } - - pycbc_assert(self->instance); - dti = malloc(sizeof(*dti)); - if (!dti) { - fprintf(stderr, - "[PYCBC] Couldn't allocate memory for libcouchbase async " - "destruction. Instance will leak\n"); - - } else { - dti->iowrap = self->iopswrap; - dti->dtorcb = self->dtorcb; - dti->conncb = self->conncb; - } - - lcb_set_destroy_callback(self->instance, dtor_callback); - lcb_destroy_async(self->instance, dti); - - self->instance = NULL; - self->iopswrap = NULL; - self->dtorcb = NULL; - self->conncb = NULL; -} diff --git a/src/constants.c b/src/constants.c deleted file mode 100644 index 5256790da..000000000 --- a/src/constants.c +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "iops.h" - -/** - * Very simple file that simply adds LCB constants to the module - */ - -#define XERR(X) \ - X(SUCCESS) \ - X(AUTH_CONTINUE) \ - X(AUTH_ERROR) \ - X(DELTA_BADVAL) \ - X(E2BIG) \ - X(EBUSY) \ - X(ENOMEM) \ - X(ERANGE) \ - X(ERROR) \ - X(ETMPFAIL) \ - X(EINVAL) \ - X(CLIENT_ETMPFAIL) \ - X(KEY_EEXISTS) \ - X(KEY_ENOENT) \ - X(DLOPEN_FAILED) \ - X(DLSYM_FAILED) \ - X(NETWORK_ERROR) \ - X(NOT_MY_VBUCKET) \ - X(NOT_STORED) \ - X(NOT_SUPPORTED) \ - X(UNKNOWN_HOST) \ - X(PROTOCOL_ERROR) \ - X(ETIMEDOUT) \ - X(BUCKET_ENOENT) \ - X(CONNECT_ERROR) \ - X(EBADHANDLE) \ - X(SERVER_BUG) \ - X(PLUGIN_VERSION_MISMATCH) \ - X(INVALID_HOST_FORMAT) \ - X(INVALID_CHAR) \ - X(DURABILITY_ETOOMANY) \ - X(DUPLICATE_COMMANDS) \ - X(HTTP_ERROR) - -#define XHTTP(X) \ - X(HTTP_METHOD_GET) \ - X(HTTP_METHOD_POST) \ - X(HTTP_METHOD_PUT) \ - X(HTTP_METHOD_DELETE) - - - -#define XSTORAGE(X) \ - X(ADD) \ - X(REPLACE) \ - X(SET) \ - X(APPEND) \ - X(PREPEND) - -static void -do_all_constants(PyObject *module, void (*handler)(PyObject*, const char*, long)) -{ - #define ADD_MACRO(sym) handler(module, #sym, sym) - #define ADD_CONSTANT(name, val) handler(module, name, val) - - #define X(b) ADD_MACRO(LCB_##b); - XERR(X); - XSTORAGE(X); - XHTTP(X); - #undef X - - ADD_MACRO(PYCBC_CMD_GET); - ADD_MACRO(PYCBC_CMD_LOCK); - ADD_MACRO(PYCBC_CMD_TOUCH); - ADD_MACRO(PYCBC_CMD_GAT); - - ADD_MACRO(PYCBC_EXC_ARGUMENTS); - ADD_MACRO(PYCBC_EXC_ENCODING); - ADD_MACRO(PYCBC_EXC_LCBERR); - ADD_MACRO(PYCBC_EXC_INTERNAL); - ADD_MACRO(PYCBC_EXC_HTTP); - ADD_MACRO(PYCBC_EXC_THREADING); - ADD_MACRO(PYCBC_EXC_DESTROYED); - ADD_MACRO(PYCBC_EXC_PIPELINE); - - ADD_MACRO(LCB_TYPE_BUCKET); - ADD_MACRO(LCB_TYPE_CLUSTER); - ADD_MACRO(LCB_HTTP_TYPE_VIEW); - ADD_MACRO(LCB_HTTP_TYPE_MANAGEMENT); - - ADD_MACRO(PYCBC_RESFLD_CAS); - ADD_MACRO(PYCBC_RESFLD_FLAGS); - ADD_MACRO(PYCBC_RESFLD_KEY); - ADD_MACRO(PYCBC_RESFLD_VALUE); - ADD_MACRO(PYCBC_RESFLD_RC); - ADD_MACRO(PYCBC_RESFLD_HTCODE); - ADD_MACRO(PYCBC_RESFLD_URL); - - ADD_CONSTANT("FMT_JSON", PYCBC_FMT_JSON); - ADD_CONSTANT("FMT_BYTES", PYCBC_FMT_BYTES); - ADD_CONSTANT("FMT_UTF8", PYCBC_FMT_UTF8); - ADD_CONSTANT("FMT_PICKLE", PYCBC_FMT_PICKLE); - ADD_CONSTANT("FMT_LEGACY_MASK", PYCBC_FMT_LEGACY_MASK); - ADD_CONSTANT("FMT_COMMON_MASK", PYCBC_FMT_COMMON_MASK); - - ADD_CONSTANT("OBS_PERSISTED", LCB_OBSERVE_PERSISTED); - ADD_CONSTANT("OBS_FOUND", LCB_OBSERVE_FOUND); - ADD_CONSTANT("OBS_NOTFOUND", LCB_OBSERVE_NOT_FOUND); - ADD_CONSTANT("OBS_LOGICALLY_DELETED", - LCB_OBSERVE_PERSISTED| LCB_OBSERVE_NOT_FOUND); - - ADD_CONSTANT("OBS_MASK", - LCB_OBSERVE_PERSISTED|LCB_OBSERVE_FOUND| LCB_OBSERVE_NOT_FOUND); - - ADD_CONSTANT("LOCKMODE_WAIT", PYCBC_LOCKMODE_WAIT); - ADD_CONSTANT("LOCKMODE_EXC", PYCBC_LOCKMODE_EXC); - ADD_CONSTANT("LOCKMODE_NONE", PYCBC_LOCKMODE_NONE); - - ADD_MACRO(PYCBC_CONN_F_WARNEXPLICIT); - ADD_MACRO(PYCBC_CONN_F_CLOSED); - ADD_MACRO(PYCBC_CONN_F_ASYNC); - ADD_MACRO(PYCBC_CONN_F_ASYNC_DTOR); - - ADD_MACRO(PYCBC_EVACTION_WATCH); - ADD_MACRO(PYCBC_EVACTION_UNWATCH); - ADD_MACRO(PYCBC_EVACTION_SUSPEND); - ADD_MACRO(PYCBC_EVACTION_RESUME); - ADD_MACRO(PYCBC_EVACTION_CLEANUP); - ADD_MACRO(PYCBC_EVSTATE_INITIALIZED); - ADD_MACRO(PYCBC_EVSTATE_ACTIVE); - ADD_MACRO(PYCBC_EVSTATE_SUSPENDED); - ADD_MACRO(PYCBC_EVTYPE_IO); - ADD_MACRO(PYCBC_EVTYPE_TIMER); - ADD_MACRO(LCB_READ_EVENT); - ADD_MACRO(LCB_WRITE_EVENT); - ADD_MACRO(LCB_RW_EVENT); - - ADD_MACRO(LCB_ERRTYPE_DATAOP); - ADD_MACRO(LCB_ERRTYPE_FATAL); - ADD_MACRO(LCB_ERRTYPE_INTERNAL); - ADD_MACRO(LCB_ERRTYPE_NETWORK); - ADD_MACRO(LCB_ERRTYPE_TRANSIENT); - ADD_MACRO(LCB_ERRTYPE_INPUT); - - /* For CNTL constants */ - ADD_MACRO(LCB_CNTL_OP_TIMEOUT); - ADD_MACRO(LCB_CNTL_VIEW_TIMEOUT); - - /* View options */ - ADD_MACRO(LCB_CMDVIEWQUERY_F_INCLUDE_DOCS); - ADD_MACRO(LCB_CMDVIEWQUERY_F_SPATIAL); -} - -static void -do_constmod(PyObject *module, const char *name, long value) { - PyModule_AddIntConstant(module, name, value); -} - -void -pycbc_init_pyconstants(PyObject *module) -{ - do_all_constants(module, do_constmod); - /* We support built-in include_docs now! */ - PyModule_AddIntConstant(module, "_IMPL_INCLUDE_DOCS", 1); -} - - -PyObject * -pycbc_lcb_errstr(lcb_t instance, lcb_error_t err) -{ -#if PY_MAJOR_VERSION == 3 - - return PyUnicode_InternFromString(lcb_strerror(instance, err)); -#else - return PyString_InternFromString(lcb_strerror(instance, err)); -#endif -} - -static void -do_printmod(PyObject *module, const char *name, long value) -{ - printf("%s = %ld\n", name, value); -} - -PyObject * -pycbc_print_constants(PyObject *mod, PyObject *args) -{ - do_all_constants(NULL, do_printmod); - (void)mod; - (void)args; - Py_RETURN_NONE; -} diff --git a/src/convert.c b/src/convert.c deleted file mode 100644 index 9e50148a7..000000000 --- a/src/convert.c +++ /dev/null @@ -1,588 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -/** - * Conversion functions - */ - -/** - * This is only called if 'o' is not bytes - */ -static PyObject* -convert_to_bytesobj(PyObject *o) -{ - PyObject *bytesobj = NULL; - pycbc_assert(!PyBytes_Check(o)); - - if (PyUnicode_Check(o)) { - bytesobj = PyUnicode_AsUTF8String(o); - } - - if (!bytesobj) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ENCODING, - 0, "Couldn't convert object to bytes", - o); - } - return bytesobj; -} - -enum { - CONVERT_MODE_UTF8_FIRST, - CONVERT_MODE_UTF8_ONLY, - CONVERT_MODE_BYTES_ONLY -}; - -static PyObject * -convert_to_string(const char *buf, size_t nbuf, int mode) -{ - PyObject *ret = NULL; - - if (mode == CONVERT_MODE_BYTES_ONLY) { - goto GT_BYTES; - } - - ret = PyUnicode_DecodeUTF8(buf, nbuf, "strict"); - - if (ret) { - return ret; - } - - if (mode == CONVERT_MODE_UTF8_ONLY) { - PYCBC_EXC_WRAP(PYCBC_EXC_ENCODING, 0, "Couldn't decode as UTF-8"); - return NULL; - } - - PyErr_Clear(); - GT_BYTES: - - return PyBytes_FromStringAndSize(buf, nbuf); -} - -static int -encode_common(PyObject **o, void **buf, size_t *nbuf, lcb_uint32_t flags) -{ - PyObject *bytesobj; - Py_ssize_t plen; - int rv; - - if (flags == PYCBC_FMT_UTF8) { -#if PY_MAJOR_VERSION == 2 - if (PyString_Check(*o)) { -#else - if (0) { -#endif - bytesobj = *o; - Py_INCREF(*o); - } else { - if (!PyUnicode_Check(*o)) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ENCODING, - 0, "Must be unicode or string", *o); - return -1; - } - bytesobj = PyUnicode_AsUTF8String(*o); - } - - } else if (flags == PYCBC_FMT_BYTES) { - if (PyBytes_Check(*o) || PyByteArray_Check(*o)) { - bytesobj = *o; - Py_INCREF(*o); - - } else { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ENCODING, 0, - "Must be bytes or bytearray", *o); - return -1; - } - - } else { - PyObject *args = NULL; - PyObject *helper; - - if (flags == PYCBC_FMT_PICKLE) { - helper = pycbc_helpers.pickle_encode; - - } else if (flags == PYCBC_FMT_JSON) { - helper = pycbc_helpers.json_encode; - - } else { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Unrecognized format"); - return -1; - } - - args = PyTuple_Pack(1, *o); - bytesobj = PyObject_CallObject(helper, args); - Py_DECREF(args); - - if (!bytesobj) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ENCODING, - 0, "Couldn't encode value", *o); - return -1; - } - - if (!PyBytes_Check(bytesobj)) { - PyObject *old = bytesobj; - bytesobj = convert_to_bytesobj(old); - Py_DECREF(old); - if (!bytesobj) { - return -1; - } - } - } - - if (PyByteArray_Check(bytesobj)) { - *buf = PyByteArray_AS_STRING(bytesobj); - plen = PyByteArray_GET_SIZE(bytesobj); - rv = 0; - } else { - rv = PyBytes_AsStringAndSize(bytesobj, (char**)buf, &plen); - } - - if (rv < 0) { - Py_DECREF(bytesobj); - PYCBC_EXC_WRAP(PYCBC_EXC_ENCODING, 0, "Couldn't encode value"); - return -1; - } - - *nbuf = plen; - *o = bytesobj; - return 0; -} - - -static int -decode_common(PyObject **vp, const char *buf, size_t nbuf, lcb_uint32_t flags) -{ - PyObject *decoded = NULL; - - /* Strip away non-common-flag info if we are indeed common flags */ - if (flags & PYCBC_FMT_COMMON_MASK) { - flags &= PYCBC_FMT_COMMON_MASK; - } - - #define FMT_MATCHES(fmtbase) \ - (flags == PYCBC_FMT_COMMON_##fmtbase) || \ - (flags == PYCBC_FMT_LEGACY_##fmtbase) - - if (FMT_MATCHES(UTF8)) { - decoded = convert_to_string(buf, nbuf, CONVERT_MODE_UTF8_ONLY); - if (!decoded) { - return -1; - } - - } else if (FMT_MATCHES(BYTES)) { - GT_BYTES: - decoded = convert_to_string(buf, nbuf, CONVERT_MODE_BYTES_ONLY); - pycbc_assert(decoded); - - } else { - PyObject *converter = NULL; - PyObject *args = NULL; - PyObject *first_arg = NULL; - - if (FMT_MATCHES(PICKLE)) { - converter = pycbc_helpers.pickle_decode; - first_arg = convert_to_string(buf, nbuf, CONVERT_MODE_BYTES_ONLY); - pycbc_assert(first_arg); - - } else if (FMT_MATCHES(JSON)) { - converter = pycbc_helpers.json_decode; - first_arg = convert_to_string(buf, nbuf, CONVERT_MODE_UTF8_ONLY); - - if (!first_arg) { - return -1; - } - - } else { - PyErr_Warn(PyExc_UserWarning, "Unrecognized flags. Forcing bytes"); - goto GT_BYTES; - } - - pycbc_assert(first_arg); - args = PyTuple_Pack(1, first_arg); - decoded = PyObject_CallObject(converter, args); - - Py_DECREF(args); - Py_DECREF(first_arg); - } - - if (!decoded) { - PyObject *bytes_tmp = PyBytes_FromStringAndSize(buf, nbuf); - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ENCODING, 0, "Failed to decode bytes", - bytes_tmp); - Py_XDECREF(bytes_tmp); - return -1; - } - - *vp = decoded; - return 0; - - #undef FMT_MATCHES -} - -int -pycbc_tc_simple_encode(PyObject **p, - void *buf, - size_t *nbuf, - lcb_uint32_t flags) -{ - return encode_common(p, buf, nbuf, flags); -} - -int -pycbc_tc_simple_decode(PyObject **vp, - const char *buf, - size_t nbuf, - lcb_uint32_t flags) -{ - return decode_common(vp, buf, nbuf, flags); -} - -enum { - ENCODE_KEY = 1, - ENCODE_VALUE, - DECODE_KEY, - DECODE_VALUE -}; -static int -do_call_tc(pycbc_Bucket *conn, - PyObject *obj, - PyObject *flags, - PyObject **result, - int mode) -{ - PyObject *meth = NULL; - PyObject *args = NULL; - PyObject *strlookup = NULL; - int ret = -1; - - switch (mode) { - case ENCODE_KEY: - strlookup = pycbc_helpers.tcname_encode_key; - args = PyTuple_Pack(1, obj); - break; - case DECODE_KEY: - strlookup = pycbc_helpers.tcname_decode_key; - args = PyTuple_Pack(1, obj); - break; - - case ENCODE_VALUE: - strlookup = pycbc_helpers.tcname_encode_value; - args = PyTuple_Pack(2, obj, flags); - break; - - case DECODE_VALUE: - strlookup = pycbc_helpers.tcname_decode_value; - args = PyTuple_Pack(2, obj, flags); - break; - } - if (args == NULL) { - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "Couldn't build arguments"); - goto GT_DONE; - } - - meth = PyObject_GetAttr(conn->tc, strlookup); - if (!meth) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ENCODING, 0, - "Couldn't find transcoder method", - conn->tc); - goto GT_DONE; - } - *result = PyObject_Call(meth, args, NULL); - if (*result) { - ret = 0; - } else { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ENCODING, 0, - "User-Defined transcoder failed", - obj); - } - - GT_DONE: - Py_XDECREF(meth); - Py_XDECREF(args); - return ret; -} - - -int -pycbc_tc_encode_key(pycbc_Bucket *conn, - PyObject **key, - void **buf, - size_t *nbuf) -{ - int rv; - Py_ssize_t plen; - - PyObject *orig_key; - PyObject *new_key = NULL; - - if (!conn->tc) { - return encode_common(key, buf, nbuf, PYCBC_FMT_UTF8); - } - - orig_key = *key; - pycbc_assert(orig_key); - - rv = do_call_tc(conn, orig_key, NULL, &new_key, ENCODE_KEY); - - if (new_key == NULL || rv < 0) { - return -1; - } - - rv = PyBytes_AsStringAndSize(new_key, (char**)buf, &plen); - - if (rv == -1) { - PYCBC_EXC_WRAP_KEY(PYCBC_EXC_ENCODING, - 0, - "Couldn't convert encoded key to bytes. It is " - "possible that the Transcoder.encode_key method " - "returned an unexpected value", - new_key); - - Py_XDECREF(new_key); - return -1; - } - - if (plen == 0) { - PYCBC_EXC_WRAP_KEY(PYCBC_EXC_ENCODING, - 0, - "Transcoder.encode_key returned an empty string", - new_key); - Py_XDECREF(new_key); - return -1; - } - - *nbuf = plen; - *key = new_key; - return 0; -} - -int -pycbc_tc_decode_key(pycbc_Bucket *conn, - const void *key, - size_t nkey, - PyObject **pobj) -{ - PyObject *bobj; - int rv = 0; - if (conn->data_passthrough) { - bobj = PyBytes_FromStringAndSize(key, nkey); - *pobj = bobj; - - } else if (!conn->tc) { - return decode_common(pobj, key, nkey, PYCBC_FMT_UTF8); - - } else { - bobj = PyBytes_FromStringAndSize(key, nkey); - if (bobj) { - rv = do_call_tc(conn, bobj, NULL, pobj, DECODE_KEY); - Py_XDECREF(bobj); - - } else { - rv = -1; - } - - if (rv < 0) { - return -1; - } - } - - if (*pobj == NULL) { - return -1; - } - - if (PyObject_Hash(*pobj) == -1) { - PYCBC_EXC_WRAP_KEY(PYCBC_EXC_ENCODING, 0, - "Transcoder.decode_key must return a hashable object", - *pobj); - Py_XDECREF(*pobj); - return -1; - } - - return 0; -} - -PyObject * -pycbc_tc_determine_format(PyObject *value) -{ - if (PyUnicode_Check(value)) { - return pycbc_helpers.fmt_utf8_flags; - - } else if (PyBytes_Check(value) || PyByteArray_Check(value)) { - return pycbc_helpers.fmt_bytes_flags; - - } else if (PyList_Check(value) || - PyTuple_Check(value) || - PyDict_Check(value) || - value == Py_True || - value == Py_False || - value == Py_None) { - return pycbc_helpers.fmt_json_flags; - - } else { - return pycbc_helpers.fmt_pickle_flags; - } -} - -int -pycbc_tc_encode_value(pycbc_Bucket *conn, - PyObject **value, - PyObject *flag_v, - void **buf, - size_t *nbuf, - lcb_uint32_t *flags) -{ - PyObject *flags_obj; - PyObject *orig_value; - PyObject *new_value = NULL; - PyObject *result_tuple = NULL; - lcb_uint32_t flags_stackval; - int rv; - Py_ssize_t plen; - - orig_value = *value; - - if (!flag_v) { - flag_v = conn->dfl_fmt; - } - - if (!conn->tc) { - - if (flag_v == pycbc_helpers.fmt_auto) { - flag_v = pycbc_tc_determine_format(*value); - } - - rv = pycbc_get_u32(flag_v, &flags_stackval); - if (rv < 0) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Bad value for flags", - flag_v); - return -1; - } - - *flags = flags_stackval; - return encode_common(value, buf, nbuf, flags_stackval); - } - - /** - * Calling into Transcoder - */ - - rv = do_call_tc(conn, orig_value, flag_v, &result_tuple, ENCODE_VALUE); - if (rv < 0) { - return -1; - } - - if (!PyTuple_Check(result_tuple) || PyTuple_GET_SIZE(result_tuple) != 2) { - PYCBC_EXC_WRAP_EX(PYCBC_EXC_ENCODING, 0, - "Expected return of (bytes, flags)", - orig_value, - result_tuple); - - Py_XDECREF(result_tuple); - return -1; - - } - - new_value = PyTuple_GET_ITEM(result_tuple, 0); - flags_obj = PyTuple_GET_ITEM(result_tuple, 1); - - if (new_value == NULL || flags_obj == NULL) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_INTERNAL, 0, "Tuple GET_ITEM had NULL", - result_tuple); - - Py_XDECREF(result_tuple); - return -1; - } - - rv = pycbc_get_u32(flags_obj, &flags_stackval); - if (rv < 0) { - Py_XDECREF(result_tuple); - PYCBC_EXC_WRAP_VALUE(PYCBC_EXC_ENCODING, 0, - "Transcoder.encode_value() returned a bad " - "value for flags", orig_value); - return -1; - } - - *flags = flags_stackval; - rv = PyBytes_AsStringAndSize(new_value, (char**)buf, &plen); - if (rv == -1) { - Py_XDECREF(result_tuple); - - PYCBC_EXC_WRAP_VALUE(PYCBC_EXC_ENCODING, 0, - "Value returned by Transcoder.encode_value() " - "could not be converted to bytes", - orig_value); - return -1; - } - - *value = new_value; - *nbuf = plen; - - Py_INCREF(new_value); - Py_XDECREF(result_tuple); - - return 0; -} - -int -pycbc_tc_decode_value(pycbc_Bucket *conn, - const void *value, - size_t nvalue, - lcb_uint32_t flags, - PyObject **pobj) -{ - PyObject *result = NULL; - PyObject *pint = NULL; - PyObject *pbuf = NULL; - int rv; - - if (conn->data_passthrough == 0 && conn->tc == NULL) { - return decode_common(pobj, value, nvalue, flags); - } - - if (conn->data_passthrough) { - *pobj = PyBytes_FromStringAndSize(value, nvalue); - if (*pobj) { - return 0; - } - return -1; - } - - pbuf = PyBytes_FromStringAndSize(value, nvalue); - if (!pbuf) { - pbuf = PyBytes_FromString(""); - } - - pint = pycbc_IntFromUL(flags); - if (!pint) { - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "Couldn't create flags object"); - rv = -1; - goto GT_DONE; - } - - rv = do_call_tc(conn, pbuf, pint, &result, DECODE_VALUE); - - GT_DONE: - Py_XDECREF(pint); - Py_XDECREF(pbuf); - - if (rv < 0) { - return -1; - } - - *pobj = result; - return 0; -} diff --git a/src/counter.c b/src/counter.c deleted file mode 100644 index 2a112695e..000000000 --- a/src/counter.c +++ /dev/null @@ -1,179 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "oputil.h" - -struct arithmetic_common_vars { - lcb_S64 delta; - lcb_U64 initial; - unsigned long ttl; - int create; -}; - -static int -handle_single_arith(pycbc_Bucket *self, struct pycbc_common_vars *cv, - int optype, PyObject *curkey, PyObject *curvalue, PyObject *options, - pycbc_Item *item, void *arg) -{ - void *key; - size_t nkey; - int rv = 0; - lcb_error_t err; - lcb_CMDCOUNTER cmd; - struct arithmetic_common_vars my_params; - static char *kwlist[] = { "delta", "initial", "ttl", NULL }; - my_params = *(struct arithmetic_common_vars *)arg; - - (void)item; - memset(&cmd, 0, sizeof cmd); - - rv = pycbc_tc_encode_key(self, &curkey, &key, &nkey); - if (rv < 0) { - return -1; - } - - if (!nkey) { - PYCBC_EXCTHROW_EMPTYKEY(); - rv = -1; - goto GT_DONE; - } - - if (options) { - curvalue = options; - } - - if (curvalue) { - if (PyDict_Check(curvalue)) { - PyObject *initial_O = NULL; - rv = PyArg_ParseTupleAndKeywords(pycbc_DummyTuple, curvalue, "L|Ok", - kwlist, &my_params.delta, &initial_O, &my_params.ttl); - if (!rv) { - PYCBC_EXC_WRAP_KEY(PYCBC_EXC_ARGUMENTS, 0, "Couldn't parse parameter for key", curkey); - rv = -1; - goto GT_DONE; - } - - if (initial_O) { - if (PyNumber_Check(initial_O)) { - my_params.create = 1; - my_params.initial = pycbc_IntAsULL(initial_O); - } else { - my_params.create = 0; - } - } - - } else if (PyNumber_Check(curvalue)) { - my_params.delta = pycbc_IntAsLL(curvalue); - } else { - PYCBC_EXC_WRAP_KEY(PYCBC_EXC_ARGUMENTS, 0, "value for key must be an integer amount or a dict of parameters", curkey); - return -1; - } - } - - LCB_CMD_SET_KEY(&cmd, key, nkey); - cmd.delta = my_params.delta; - cmd.create = my_params.create; - cmd.initial = my_params.initial; - cmd.exptime = my_params.ttl; - err = lcb_counter3(self->instance, cv->mres, &cmd); - if (err != LCB_SUCCESS) { - PYCBC_EXCTHROW_SCHED(err); - rv = -1; - } else { - rv = 0; - } - - GT_DONE: - Py_XDECREF(curkey); - return rv; -} - -PyObject * -arithmetic_common(pycbc_Bucket *self, PyObject *args, PyObject *kwargs, - int optype, int argopts) -{ - int rv; - Py_ssize_t ncmds; - struct arithmetic_common_vars global_params = { 0 }; - pycbc_seqtype_t seqtype; - PyObject *all_initial_O = NULL; - PyObject *all_ttl_O = NULL; - PyObject *collection; - struct pycbc_common_vars cv = PYCBC_COMMON_VARS_STATIC_INIT; - - static char *kwlist[] = { "keys", "delta", "initial", "ttl", NULL }; - - global_params.delta = 1; - - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "O|LOO", kwlist, - &collection, &global_params.delta, &all_initial_O, &all_ttl_O); - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - rv = pycbc_get_ttl(all_ttl_O, &global_params.ttl, 1); - if (rv < 0) { - return NULL; - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_check_sequence(collection, 1, &ncmds, &seqtype); - if (rv < 0) { - return NULL; - } - } else { - ncmds = 1; - } - - - if (all_initial_O && PyNumber_Check(all_initial_O)) { - global_params.create = 1; - global_params.initial = pycbc_IntAsULL(all_initial_O); - } - - rv = pycbc_common_vars_init(&cv, self, argopts, ncmds, 0); - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_iter_multi(self, seqtype, collection, &cv, optype, - handle_single_arith, &global_params); - - } else { - rv = handle_single_arith(self, &cv, optype, collection, NULL, NULL, NULL, - &global_params); - } - - if (rv < 0) { - goto GT_DONE; - } - - if (-1 == pycbc_common_vars_wait(&cv, self)) { - goto GT_DONE; - } - - GT_DONE: - pycbc_common_vars_finalize(&cv, self); - return cv.ret; -} - -#define DECLFUNC(name, operation, mode) \ - PyObject *pycbc_Bucket_##name(pycbc_Bucket *self, \ - PyObject *args, PyObject *kwargs) { \ - return arithmetic_common(self, args, kwargs, operation, mode); \ -} - -DECLFUNC(counter, PYCBC_CMD_COUNTER, PYCBC_ARGOPT_SINGLE) -DECLFUNC(counter_multi, PYCBC_CMD_COUNTER, PYCBC_ARGOPT_MULTI) diff --git a/src/ctranscoder.c b/src/ctranscoder.c deleted file mode 100644 index 23e6de924..000000000 --- a/src/ctranscoder.c +++ /dev/null @@ -1,275 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -/** - * An optimized Trancoder class. Users may subclass this object and only use - * some of the transcoder methods - */ -#include "pycbc.h" -#include "structmember.h" - -static PyObject * -encode_key(PyObject *self, PyObject *args) -{ - int rv; - char *buf; - size_t nbuf; - PyObject *kobj; - - rv = PyArg_ParseTuple(args, "O", &kobj); - if (!rv) { - return NULL; - } - - rv = pycbc_tc_simple_encode(&kobj, &buf, &nbuf, PYCBC_FMT_UTF8); - if (rv < 0) { - return NULL; - } - - (void)self; - return kobj; -} - -static PyObject * -decode_key(PyObject *self, PyObject *args) -{ - int rv; - char *buf; - PyObject *bobj; - Py_ssize_t plen; - - rv = PyArg_ParseTuple(args, "O", &bobj); - if (!rv) { - return NULL; - } - - rv = PyBytes_AsStringAndSize(bobj, &buf, &plen); - if (rv < 0) { - return NULL; - } - - rv = pycbc_tc_simple_decode(&bobj, buf, plen, PYCBC_FMT_UTF8); - if (rv < 0) { - return NULL; - } - - (void)self; - return bobj; -} - -static PyObject * -encode_value(PyObject *self, PyObject *args) -{ - lcb_uint32_t flags; - int rv; - PyObject *vobj; - PyObject *flagsobj; - char *buf; - PyObject *ret; - size_t nbuf; - - rv = PyArg_ParseTuple(args, "OO", &vobj, &flagsobj); - if (!rv) { - return NULL; - } - - rv = pycbc_get_u32(flagsobj, &flags); - if (rv < 0) { - return NULL; - } - - rv = pycbc_tc_simple_encode(&vobj, &buf, &nbuf, flags); - if (rv < 0) { - return NULL; - } - - ret = PyTuple_New(2); - PyTuple_SET_ITEM(ret, 0, vobj); - PyTuple_SET_ITEM(ret, 1, flagsobj); - - /** INCREF flags because we got it as an argument */ - Py_INCREF(flagsobj); - - (void)self; - return ret; -} - -static PyObject * -decode_value(PyObject *self, PyObject *args) -{ - PyObject *flagsobj; - PyObject *vobj; - char *buf; - Py_ssize_t nbuf; - int rv; - lcb_uint32_t flags; - - rv = PyArg_ParseTuple(args, "OO", &vobj, &flagsobj); - if (!rv) { - return NULL; - } - - rv = PyBytes_AsStringAndSize(vobj, &buf, &nbuf); - if (rv < 0) { - return NULL; - } - - rv = pycbc_get_u32(flagsobj, &flags); - if (rv < 0) { - return NULL; - } - - rv = pycbc_tc_simple_decode(&vobj, buf, nbuf, flags); - if (rv < 0) { - return NULL; - } - - (void)self; - return vobj; -} - -static PyObject* -determine_format(PyObject *self, PyObject *args) -{ - int rv; - PyObject *orig; - - rv = PyArg_ParseTuple(args, "O", &orig); - if (!rv) { - return NULL; - } - - (void)self; - return pycbc_tc_determine_format(orig); -} - -static PyTypeObject TranscoderType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -typedef struct { - PyObject_HEAD -} TranscoderObject; - -PyDoc_STRVAR(encode_key_doc, -"Encode the key as a bytes object.\n" -"\n" -":param key: This is an object passed as a string key.\n" -" There is no restriction on this type\n" -"\n" -":return: a bytes object\n" -" The default implementation encodes the key as UTF-8.\n" -" On Python 2.x, ``bytes`` is a synonym for ``str``. On Python 3.x,\n" -" ``bytes`` and ``str`` are distinct objects, in which one must first\n" -" *encode* a string to a specific encoding\n" -"\n"); - -PyDoc_STRVAR(decode_key_doc, -"Convert the key from bytes into something else.\n" -"\n" -":param bytes key: The key, in the form of a bytearray\n" -"\n" -":return: a string or other object your application will use\n" -" The returned key *must* be hashable\n" -"\n" -"The default implementation decodes the keys from UTF-8.\n" -"\n"); - -PyDoc_STRVAR(encode_value_doc, -"Encode the value into something meaningful\n" -"\n" -":param any value: A value. This may be a string or a complex python\n" -" object.\n" -":param any format: The `format` argument as passed to the mutator\n" -"\n" -":return: A tuple of ``(value, flags)``\n" -" ``value`` must be a ``bytes`` object. ``flags`` must be an integer type\n" -" whose value does not exceed 32 bits\n" -"\n"); - -PyDoc_STRVAR(decode_value_doc, -"Decode the value from the raw bytes representation into something\n" -"meaningful\n" -"\n" -":param bytes value: Raw bytes, as stored on the server\n" -":param int flags: The flags for the value\n" -"\n" -":return: Something meaningful to be used as a value within the\n" -" application\n" -"\n"); - -PyDoc_STRVAR(determine_format_doc, -"Guess the suitable format for the object specified.\n" -" .. versionadded:: 1.1.0\n" -"\n" -"Used primarily if received a :data:`~couchbase.FMT_AUTO` for the\n" -"`format` parameter in one of the encode methods\n" -"\n" -":param object value: The value whose format should be guessed\n" -":return: An integer representing the guessed format.\n" -"\n" -"Note that this function is provided as a convenience. It is not called\n" -"by the Connection object\n" -"\n" -"This function always succeeds\n" -"" -); - -static PyMethodDef cTranscoder_methods[] = { - { PYCBC_TCNAME_ENCODE_KEY, (PyCFunction)encode_key, - METH_VARARGS, encode_key_doc - }, - { PYCBC_TCNAME_DECODE_KEY, (PyCFunction)decode_key, - METH_VARARGS, decode_key_doc - }, - { PYCBC_TCNAME_ENCODE_VALUE, (PyCFunction)encode_value, - METH_VARARGS, encode_value_doc - }, - { PYCBC_TCNAME_DECODE_VALUE, (PyCFunction)decode_value, - METH_VARARGS, decode_value_doc - }, - { "determine_format", (PyCFunction)determine_format, - METH_VARARGS, determine_format_doc - }, - { NULL } -}; - -static void -transcoder_dealloc(PyObject *o) -{ - Py_TYPE(o)->tp_free(o); -} - -int -pycbc_TranscoderType_init(PyObject **ptr) -{ - PyTypeObject *p = &TranscoderType; - - *ptr = (PyObject*)p; - if (p->tp_name) { - return 0; - } - - p->tp_name = "Transcoder"; - p->tp_doc = PyDoc_STR("Efficient, subclassable transcoder interface/class"); - p->tp_dealloc = transcoder_dealloc; - p->tp_basicsize = sizeof(TranscoderObject); - p->tp_methods = cTranscoder_methods; - p->tp_new = PyType_GenericNew; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - return PyType_Ready(p); -} diff --git a/src/diagnostics.cxx b/src/diagnostics.cxx new file mode 100644 index 000000000..0087d2706 --- /dev/null +++ b/src/diagnostics.cxx @@ -0,0 +1,407 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "diagnostics.hxx" +#include "exceptions.hxx" +#include "result.hxx" +#include "utils.hxx" + +template<typename T> +void +add_extras_to_service_endpoint([[maybe_unused]] const T& t, [[maybe_unused]] PyObject* dict) +{ +} + +template<> +void +add_extras_to_service_endpoint<couchbase::core::diag::endpoint_ping_info>( + const couchbase::core::diag::endpoint_ping_info& e, + PyObject* pyObj_dict) +{ + + long duration = e.latency.count(); + PyObject* pyObj_tmp = PyLong_FromLong(duration); + if (-1 == PyDict_SetItemString(pyObj_dict, "latency_us", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + std::string ping_state = std::string(); + switch (e.state) { + case couchbase::core::diag::ping_state::ok: { + ping_state = "ok"; + break; + } + case couchbase::core::diag::ping_state::timeout: { + ping_state = "timeout"; + break; + } + case couchbase::core::diag::ping_state::error: { + ping_state = "error"; + break; + } + default: { + break; + } + }; + + if (ping_state.length() > 0) { + pyObj_tmp = PyUnicode_FromString(ping_state.c_str()); + if (-1 == PyDict_SetItemString(pyObj_dict, "state", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + } + + if (e.error.has_value()) { + pyObj_tmp = PyUnicode_FromString(e.error.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_dict, "error", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + } +} + +template<> +void +add_extras_to_service_endpoint<couchbase::core::diag::endpoint_diag_info>( + const couchbase::core::diag::endpoint_diag_info& e, + PyObject* pyObj_dict) +{ + PyObject* pyObj_tmp = nullptr; + + if (e.last_activity.has_value()) { + long duration = e.last_activity.value().count(); + pyObj_tmp = PyLong_FromLong(duration); + if (-1 == PyDict_SetItemString(pyObj_dict, "last_activity_us", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + } + + std::string endpoint_state = std::string(); + switch (e.state) { + case couchbase::core::diag::endpoint_state::disconnected: { + endpoint_state = "disconnected"; + break; + } + case couchbase::core::diag::endpoint_state::connecting: { + endpoint_state = "connecting"; + break; + } + case couchbase::core::diag::endpoint_state::connected: { + endpoint_state = "connected"; + break; + } + case couchbase::core::diag::endpoint_state::disconnecting: { + endpoint_state = "disconnecting"; + break; + } + default: { + break; + } + }; + + if (endpoint_state.length() > 0) { + pyObj_tmp = PyUnicode_FromString(endpoint_state.c_str()); + if (-1 == PyDict_SetItemString(pyObj_dict, "state", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + } +} + +template<typename T> +PyObject* +get_service_endpoints(const T& resp) +{ + PyObject* pyObj_services_dict = PyDict_New(); + for (auto const& service : resp.services) { + PyObject* pyObj_endpoints = PyList_New(static_cast<Py_ssize_t>(0)); + std::string service_type = service_type_to_str(service.first); + for (auto e : service.second) { + PyObject* pyObj_service_dict = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(e.id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_service_dict, "id", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + Py_XDECREF(pyObj_endpoints); + Py_XDECREF(pyObj_services_dict); + Py_XDECREF(pyObj_service_dict); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(e.local.c_str()); + if (-1 == PyDict_SetItemString(pyObj_service_dict, "local", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + Py_XDECREF(pyObj_endpoints); + Py_XDECREF(pyObj_services_dict); + Py_DECREF(pyObj_service_dict); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(e.remote.c_str()); + if (-1 == PyDict_SetItemString(pyObj_service_dict, "remote", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + Py_XDECREF(pyObj_endpoints); + Py_XDECREF(pyObj_services_dict); + Py_DECREF(pyObj_service_dict); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (e.bucket.has_value()) { + pyObj_tmp = PyUnicode_FromString(e.bucket.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_service_dict, "namespace", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + Py_XDECREF(pyObj_endpoints); + Py_XDECREF(pyObj_services_dict); + Py_DECREF(pyObj_service_dict); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + add_extras_to_service_endpoint(e, pyObj_service_dict); + PyList_Append(pyObj_endpoints, pyObj_service_dict); + Py_DECREF(pyObj_service_dict); + } + + if (-1 == PyDict_SetItemString(pyObj_services_dict, service_type.c_str(), pyObj_endpoints)) { + Py_XDECREF(pyObj_endpoints); + Py_DECREF(pyObj_services_dict); + return nullptr; + } + Py_DECREF(pyObj_endpoints); + } + + return pyObj_services_dict; +} + +template<typename T> +result* +create_diagnostics_op_result(const T& resp) +{ + PyObject* result_obj = create_result_obj(); + result* res = reinterpret_cast<result*>(result_obj); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.id.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "id", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.sdk.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "sdk", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(resp.version); + if (-1 == PyDict_SetItemString(res->dict, "version", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (resp.services.size() > 0) { + PyObject* pyObj_services_dict = get_service_endpoints(resp); + if (pyObj_services_dict == nullptr) { + return nullptr; + } + + if (-1 == PyDict_SetItemString(res->dict, "endpoints", pyObj_services_dict)) { + Py_XDECREF(pyObj_services_dict); + return nullptr; + } + Py_DECREF(pyObj_services_dict); + } + return res; +} + +template<typename T> +void +create_diagnostics_op_response(const T& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + + auto res = create_diagnostics_op_result(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Diagnostic operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to catch exception here? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +PyObject* +handle_diagnostics_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + Operations::OperationType op_type = Operations::UNKNOWN; + char* bucket = nullptr; + uint64_t timeout = 0; + char* report_id = nullptr; + PyObject* pyObj_service_types = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + + static const char* kw_list[] = { "conn", "op_type", "bucket", "timeout", "report_id", + "service_types", "callback", "errback", nullptr }; + + const char* kw_format = "O!I|sLsOOO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &op_type, + &bucket, + &timeout, + &report_id, + &pyObj_service_types, + &pyObj_callback, + &pyObj_errback); + if (!ret) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform diagnostics operation. Unable to parse args/kwargs."); + return nullptr; + } + + connection* conn = nullptr; + std::optional<std::chrono::milliseconds> timeout_ms{}; + + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + if (0 < timeout) { + timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + } + + auto reportId = report_id ? std::optional<std::string>{ report_id } : std::nullopt; + auto bucketName = bucket ? std::optional<std::string>{ bucket } : std::nullopt; + std::set<couchbase::core::service_type> services; + if (pyObj_service_types && PyList_Check(pyObj_service_types)) { + for (Py_ssize_t i = 0; i < PyList_Size(pyObj_service_types); i++) { + PyObject* pyObj_svc = PyList_GetItem(pyObj_service_types, i); + // PyList_GetItem returns borrowed ref, inc while using, decr after done + Py_INCREF(pyObj_svc); + if (PyUnicode_Check(pyObj_svc)) { + auto res = std::string(PyUnicode_AsUTF8(pyObj_svc)); + auto svc_type = str_to_service_type(res); + services.insert(svc_type); + } + Py_DECREF(pyObj_svc); + pyObj_svc = nullptr; + } + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + + if (op_type == Operations::DIAGNOSTICS) { + Py_BEGIN_ALLOW_THREADS conn->cluster_.diagnostics( + reportId, + [pyObj_callback, pyObj_errback, barrier](couchbase::core::diag::diagnostics_result r) { + create_diagnostics_op_response(r, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS + } else { + couchbase::core::diag::ping_result resp; + Py_BEGIN_ALLOW_THREADS conn->cluster_.ping( + reportId, + bucketName, + services, + timeout_ms, + [pyObj_callback, pyObj_errback, barrier](couchbase::core::diag::ping_result r) { + create_diagnostics_op_response(r, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS + } + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} diff --git a/src/diagnostics.hxx b/src/diagnostics.hxx new file mode 100644 index 000000000..e7d6b775c --- /dev/null +++ b/src/diagnostics.hxx @@ -0,0 +1,26 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef DIAGNOSTICS_H_ +#define DIAGNOSTICS_H_ + +#include "client.hxx" + +PyObject* +handle_diagnostics_op(PyObject* self, PyObject* args, PyObject* kwargs); + +#endif diff --git a/src/exceptions.c b/src/exceptions.c deleted file mode 100644 index d197aa152..000000000 --- a/src/exceptions.c +++ /dev/null @@ -1,188 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" - -int pycbc_handle_assert(const char *msg, const char* file, int line) -{ - const char *assert_props = getenv("PYCBC_ASSERT_CONTINUE"); - if (assert_props == NULL || *assert_props == '\0') { - fprintf(stderr, - "python-couchbase: %s at %s:%d. Abort", msg, file, line); - abort(); - } - - fprintf(stderr, - "!!! python-couchbase: Assertion failure detected.. \n" - "!!! Not aborting because os.environ['PYCBC_ASSERT_CONTINUE'] was set\n" - "!!! Depending on what went wrong, further exceptions may \n" - "!!! still be raised, or the program may abort due to \n" - "!!! invalid state\n" - "!!! (debuggers should break at pycbc_handle_assert in exceptions.c)\n"); - - fprintf(stderr, "!!! Assertion: '%s' at %s:%d\n", msg, file, line); - return 0; -} - -void -pycbc_exc_wrap_REAL(int mode, struct pycbc_exception_params *p) -{ - PyObject *type = NULL, *value = NULL, *traceback = NULL; - PyObject *excls; - PyObject *excparams; - PyObject *excinstance; - PyObject *ctor_args; - - PyErr_Fetch(&type, &value, &traceback); - PyErr_Clear(); - - excls = pycbc_exc_map(mode, p->err); - - excparams = PyDict_New(); - pycbc_assert(excparams); - - if (p->err) { - PyObject *errtmp = pycbc_IntFromL(p->err); - PyDict_SetItemString(excparams, "rc", errtmp); - Py_DECREF(errtmp); - } - - if (type) { - PyErr_NormalizeException(&type, &value, &traceback); - PyDict_SetItemString(excparams, "inner_cause", value); - Py_XDECREF(type); - Py_XDECREF(value); - } - - if (p->msg) { - PyObject *msgstr = pycbc_SimpleStringZ(p->msg); - PyDict_SetItemString(excparams, "message", msgstr); - Py_DECREF(msgstr); - } - - if (p->key) { - PyDict_SetItemString(excparams, "key", p->key); - } - - if (p->objextra) { - PyDict_SetItemString(excparams, "objextra", p->objextra); - } - - { - PyObject *csrc_info = Py_BuildValue("(s,i)", p->file, p->line); - PyDict_SetItemString(excparams, "csrc_info", csrc_info); - Py_DECREF(csrc_info); - } - - ctor_args = Py_BuildValue("(O)", excparams); - excinstance = PyObject_CallObject(excls, ctor_args); - Py_XDECREF(ctor_args); - Py_XDECREF(excparams); - - if (!excinstance) { - Py_XDECREF(traceback); - - } else { - Py_INCREF(Py_TYPE(excinstance)); - PyErr_Restore((PyObject*)Py_TYPE(excinstance), excinstance, traceback); - PYCBC_REFCNT_ASSERT(Py_REFCNT(excinstance) == 1); - } -} - -PyObject * -pycbc_exc_map(int mode, lcb_error_t err) -{ - PyObject *ikey; - PyObject *excls; - - if (mode == PYCBC_EXC_LCBERR) { - ikey = pycbc_IntFromL(err); - excls = PyDict_GetItem(pycbc_helpers.lcb_errno_map, ikey); - if (!excls) { - excls = PyObject_CallMethod(pycbc_helpers.default_exception, - "rc_to_exctype", "O", ikey); - } - } else { - ikey = pycbc_IntFromL(mode); - excls = PyDict_GetItem(pycbc_helpers.misc_errno_map, ikey); - } - - if (!excls) { - excls = pycbc_helpers.default_exception; - } - - Py_DECREF(ikey); - return excls; -} - -PyObject * -pycbc_exc_message(int mode, lcb_error_t err, const char *msg) -{ - PyObject *instance; - PyObject *args; - PyObject *excls = pycbc_exc_map(mode, err); - - args = PyTuple_New(1); - PyTuple_SET_ITEM(args, 0, pycbc_SimpleStringZ(msg)); - - instance = PyObject_CallObject(excls, args); - Py_DECREF(args); - - pycbc_assert(instance); - return instance; -} - -PyObject * -pycbc_exc_get_categories(PyObject *self, PyObject *arg) -{ - int rv = 0; - int rc = 0; - - (void)self; - rv = PyArg_ParseTuple(arg, "i", &rc); - if (!rv) { - return NULL; - } - rv = lcb_get_errtype(rc); - return pycbc_IntFromL(rv); -} - -PyObject * -pycbc_exc_mktuple(void) -{ - PyObject *type, *value, *traceback; - PyObject *ret; - - pycbc_assert(PyErr_Occurred()); - - PyErr_Fetch(&type, &value, &traceback); - PyErr_Clear(); - - if (value == NULL) { - value = Py_None; Py_INCREF(value); - } - if (traceback == NULL) { - traceback = Py_None; Py_INCREF(traceback); - } - - ret = PyTuple_New(3); - /** Steal references from PyErr_Fetch() */ - PyTuple_SET_ITEM(ret, 0, type); - PyTuple_SET_ITEM(ret, 1, value); - PyTuple_SET_ITEM(ret, 2, traceback); - - return ret; -} diff --git a/src/exceptions.cxx b/src/exceptions.cxx new file mode 100644 index 000000000..b592cc9a9 --- /dev/null +++ b/src/exceptions.cxx @@ -0,0 +1,593 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "exceptions.hxx" + +#include "result.hxx" + +static PyObject* +exception_base__category__(exception_base* self, [[maybe_unused]] PyObject* args) +{ + return PyUnicode_FromString(self->ec.category().name()); +} + +static PyObject* +exception_base__err__(exception_base* self, [[maybe_unused]] PyObject* args) +{ + return PyLong_FromLong(self->ec.value()); +} +static PyObject* +exception_base__strerror__(exception_base* self, [[maybe_unused]] PyObject* args) +{ + if (self->ec) { + return PyUnicode_FromString(self->ec.message().c_str()); + } + Py_RETURN_NONE; +} + +static PyObject* +exception_base__context__(exception_base* self, [[maybe_unused]] PyObject* args) +{ + if (self->error_context) { + PyObject* pyObj_error_context = PyDict_Copy(self->error_context); + return pyObj_error_context; + } + Py_RETURN_NONE; +} + +static PyObject* +exception_base__info__(exception_base* self, [[maybe_unused]] PyObject* args) +{ + if (self->exc_info) { + PyObject* pyObj_exc_info = PyDict_Copy(self->exc_info); + return pyObj_exc_info; + } + Py_RETURN_NONE; +} + +static void +exception_base_dealloc(exception_base* self) +{ + if (self->error_context) { + if (PyDict_Check(self->error_context)) { + PyDict_Clear(self->error_context); + } + Py_DECREF(self->error_context); + } + if (self->exc_info) { + if (PyDict_Check(self->exc_info)) { + PyDict_Clear(self->exc_info); + } + Py_DECREF(self->exc_info); + } + Py_TYPE(self)->tp_free((PyObject*)self); + CB_LOG_DEBUG("{}: exception_base_dealloc completed", "PYCBC"); +} + +static PyObject* +exception_base__new__(PyTypeObject* type, PyObject* args, PyObject* kwargs) +{ + auto self = reinterpret_cast<exception_base*>(type->tp_alloc(type, 0)); + self->ec = std::error_code(); + return reinterpret_cast<PyObject*>(self); +} + +static PyMethodDef exception_base_methods[] = { + { "strerror", + (PyCFunction)exception_base__strerror__, + METH_NOARGS, + PyDoc_STR("String description of error") }, + { "err", (PyCFunction)exception_base__err__, METH_NOARGS, PyDoc_STR("Integer error code") }, + { "err_category", + (PyCFunction)exception_base__category__, + METH_NOARGS, + PyDoc_STR("error category, expressed as a string") }, + { "error_context", + (PyCFunction)exception_base__context__, + METH_NOARGS, + PyDoc_STR("error context dict") }, + { "error_info", (PyCFunction)exception_base__info__, METH_NOARGS, PyDoc_STR("error info dict") }, + { nullptr, nullptr, 0, nullptr } +}; + +/* +Initializing the PyTypeObject w/ the C99-style designated initializers fails to compile when using +Python 3.12 on alpine3.18 (error below). The container I tested was using gcc 12.2 (don't know how +other versions might behave). + +Error: +/tmp/couchbase-python-client/src/exceptions.cxx:130:1: error: designator order for field +'_typeobject::tp_basicsize' does not match declaration order in 'PyTypeObject' {aka '_typeobject'} +130 | }; + +This is what the docs show, so odd there is an issue... +https://docs.python.org/3.12/extending/newtypes_tutorial.html#the-basics +*/ +// static PyTypeObject exception_base_type = { +// .ob_base = PyVarObject_HEAD_INIT(NULL, 0) +// .tp_name = "pycbc_core.exception", +// .tp_doc = PyDoc_STR("Base class for exceptions coming from pycbc_core"), +// .tp_basicsize = sizeof(exception_base), +// .tp_itemsize = 0, +// .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, +// .tp_new = exception_base__new__, +// .tp_dealloc = (destructor)exception_base_dealloc, +// .tp_methods = exception_base_methods, +// }; + +static PyTypeObject +init_exception_base_type() +{ + PyTypeObject obj = {}; + obj.ob_base = PyVarObject_HEAD_INIT(NULL, 0) obj.tp_name = "pycbc_core.exception"; + obj.tp_doc = PyDoc_STR("Base class for exceptions coming from pycbc_core"); + obj.tp_basicsize = sizeof(exception_base); + obj.tp_itemsize = 0; + obj.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + obj.tp_new = exception_base__new__; + obj.tp_dealloc = (destructor)exception_base_dealloc; + obj.tp_methods = exception_base_methods; + return obj; +} + +static PyTypeObject exception_base_type = init_exception_base_type(); + +exception_base* +create_exception_base_obj() +{ + PyObject* exc = PyObject_CallObject(reinterpret_cast<PyObject*>(&exception_base_type), nullptr); + return reinterpret_cast<exception_base*>(exc); +} + +std::string +retry_reason_to_string(couchbase::retry_reason reason) +{ + switch (reason) { + case couchbase::retry_reason::socket_not_available: + return "socket_not_available"; + case couchbase::retry_reason::service_not_available: + return "service_not_available"; + case couchbase::retry_reason::node_not_available: + return "node_not_available"; + case couchbase::retry_reason::key_value_not_my_vbucket: + return "key_value_not_my_vbucket"; + case couchbase::retry_reason::key_value_collection_outdated: + return "key_value_collection_outdated"; + case couchbase::retry_reason::key_value_error_map_retry_indicated: + return "key_value_error_map_retry_indicated"; + case couchbase::retry_reason::key_value_locked: + return "key_value_locked"; + case couchbase::retry_reason::key_value_temporary_failure: + return "key_value_temporary_failure"; + case couchbase::retry_reason::key_value_sync_write_in_progress: + return "key_value_sync_write_in_progress"; + case couchbase::retry_reason::key_value_sync_write_re_commit_in_progress: + return "key_value_sync_write_re_commit_in_progress"; + case couchbase::retry_reason::service_response_code_indicated: + return "service_response_code_indicated"; + case couchbase::retry_reason::circuit_breaker_open: + return "circuit_breaker_open"; + case couchbase::retry_reason::query_prepared_statement_failure: + return "query_prepared_statement_failure"; + case couchbase::retry_reason::query_index_not_found: + return "query_index_not_found"; + case couchbase::retry_reason::analytics_temporary_failure: + return "analytics_temporary_failure"; + case couchbase::retry_reason::search_too_many_requests: + return "search_too_many_requests"; + case couchbase::retry_reason::views_temporary_failure: + return "views_temporary_failure"; + case couchbase::retry_reason::views_no_active_partition: + return "views_no_active_partition"; + case couchbase::retry_reason::do_not_retry: + return "do_not_retry"; + case couchbase::retry_reason::socket_closed_while_in_flight: + return "socket_closed_while_in_flight"; + case couchbase::retry_reason::unknown: + return "unknown"; + default: + return "unknown"; + } +} + +PyObject* +build_kv_error_map_info(couchbase::core::key_value_error_map_info error_info) +{ + PyObject* err_info = PyDict_New(); + PyObject* pyObj_tmp = PyLong_FromLong(static_cast<uint16_t>(error_info.code())); + if (-1 == PyDict_SetItemString(err_info, "code", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(error_info.name().c_str()); + if (-1 == PyDict_SetItemString(err_info, "name", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(error_info.description().c_str()); + if (-1 == PyDict_SetItemString(err_info, "description", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + PyObject* attr_set = PySet_New(nullptr); + for (auto attr : error_info.attributes()) { + pyObj_tmp = PyLong_FromLong(static_cast<uint16_t>(attr)); + if (-1 == PySet_Add(attr_set, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + Py_ssize_t set_size = PySet_Size(attr_set); + if (set_size > 0) { + if (-1 == PyDict_SetItemString(err_info, "attributes", attr_set)) { + PyErr_Print(); + PyErr_Clear(); + } + } + Py_XDECREF(attr_set); + + return err_info; +} + +void +build_kv_error_context(const couchbase::core::key_value_error_context& ctx, + PyObject* pyObj_error_ctx) +{ + PyObject* pyObj_tmp = nullptr; + pyObj_tmp = PyUnicode_FromString(ctx.id().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_ctx, KV_DOCUMENT_ID, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.bucket().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_ctx, KV_DOCUMENT_BUCKET, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.scope().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_ctx, KV_DOCUMENT_SCOPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.collection().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_ctx, KV_DOCUMENT_COLLECTION, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromLong(ctx.opaque()); + if (-1 == PyDict_SetItemString(pyObj_error_ctx, KV_OPAQUE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (ctx.status_code().has_value()) { + pyObj_tmp = PyLong_FromLong(static_cast<uint16_t>(ctx.status_code().value())); + if (-1 == PyDict_SetItemString(pyObj_error_ctx, KV_STATUS_CODE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (ctx.error_map_info().has_value()) { + PyObject* err_info = build_kv_error_map_info(ctx.error_map_info().value()); + if (-1 == PyDict_SetItemString(pyObj_error_ctx, KV_ERROR_MAP_INFO, err_info)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(err_info); + } + + if (ctx.extended_error_info().has_value()) { + PyObject* enhanced_err_info = PyDict_New(); + pyObj_tmp = PyUnicode_FromString(ctx.extended_error_info().value().reference().c_str()); + if (-1 == PyDict_SetItemString(enhanced_err_info, "reference", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.extended_error_info().value().context().c_str()); + if (-1 == PyDict_SetItemString(enhanced_err_info, "context", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString(pyObj_error_ctx, KV_EXTENDED_ERROR_INFO, enhanced_err_info)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(enhanced_err_info); + } +} + +struct PycbcErrorCategory : std::error_category { + const char* name() const noexcept override; + std::string message(int ec) const override; +}; + +const char* +PycbcErrorCategory::name() const noexcept +{ + return "pycbc"; +} + +std::string +PycbcErrorCategory::message(int ec) const +{ + switch (static_cast<PycbcError>(ec)) { + case PycbcError::InvalidArgument: + return "Invalid argument"; + case PycbcError::BucketNotFound: + return "Bucket not found"; + case PycbcError::HTTPError: + return "HTTP Error"; + case PycbcError::UnsuccessfulOperation: + return "Unsuccessful operation"; + case PycbcError::UnableToBuildResult: + return "Unable to build operation's result"; + case PycbcError::CallbackUnsuccessful: + return "Async callback failed"; + case PycbcError::InternalSDKError: + return "Internal SDK error occurred"; + default: + return "(Unrecognized error)"; + } +} + +const PycbcErrorCategory defaultPycbcErrorCategory{}; + +std::error_code +make_error_code(PycbcError ec) +{ + return { static_cast<int>(ec), defaultPycbcErrorCategory }; +} + +PyObject* +get_pycbc_exception_class(PyObject* pyObj_exc_module, std::error_code ec) +{ + switch (static_cast<PycbcError>(ec.value())) { + case PycbcError::InvalidArgument: + return PyObject_GetAttrString(pyObj_exc_module, "InvalidArgumentException"); + case PycbcError::BucketNotFound: + return PyObject_GetAttrString(pyObj_exc_module, "BucketNotFoundException"); + case PycbcError::HTTPError: + return PyObject_GetAttrString(pyObj_exc_module, "HTTPException"); + case PycbcError::UnsuccessfulOperation: + return PyObject_GetAttrString(pyObj_exc_module, "UnsuccessfulOperationException"); + case PycbcError::FeatureUnavailable: + return PyObject_GetAttrString(pyObj_exc_module, "FeatureUnavailableException"); + case PycbcError::UnableToBuildResult: + case PycbcError::CallbackUnsuccessful: + case PycbcError::InternalSDKError: + default: + return PyObject_GetAttrString(pyObj_exc_module, "InternalSDKException"); + } + + return PyObject_GetAttrString(pyObj_exc_module, "InternalSDKException"); +} + +void +pycbc_set_python_exception(std::error_code ec, const char* file, int line, const char* msg) +{ + PyObject *pyObj_type = nullptr, *pyObj_value = nullptr, *pyObj_traceback = nullptr; + PyObject* pyObj_exc_class = nullptr; + PyObject* pyObj_exc_instance = nullptr; + + PyErr_Fetch(&pyObj_type, &pyObj_value, &pyObj_traceback); + PyErr_Clear(); + + PyObject* pyObj_exc_params = PyDict_New(); + + if (pyObj_type != nullptr) { + PyErr_NormalizeException(&pyObj_type, &pyObj_value, &pyObj_traceback); + if (-1 == PyDict_SetItemString(pyObj_exc_params, "inner_cause", pyObj_value)) { + PyErr_Print(); + Py_DECREF(pyObj_type); + Py_XDECREF(pyObj_value); + Py_XDECREF(pyObj_traceback); + Py_DECREF(pyObj_exc_params); + return; + } + Py_XDECREF(pyObj_type); + Py_XDECREF(pyObj_value); + } + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_params, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + Py_DECREF(pyObj_exc_params); + return; + } + Py_DECREF(pyObj_cinfo); + + PyObject* pyObj_exc_module = PyImport_ImportModule("couchbase.exceptions"); + if (pyObj_exc_module == nullptr) { + PyErr_Print(); + Py_DECREF(pyObj_exc_params); + return; + } + + pyObj_exc_class = get_pycbc_exception_class(pyObj_exc_module, ec); + if (pyObj_exc_class == nullptr) { + PyErr_Print(); + Py_XDECREF(pyObj_exc_params); + Py_DECREF(pyObj_exc_module); + return; + } + Py_DECREF(pyObj_exc_module); + + PyObject* pyObj_args = PyTuple_New(0); + PyObject* pyObj_kwargs = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(msg); + if (-1 == PyDict_SetItemString(pyObj_kwargs, "message", pyObj_tmp)) { + PyErr_Print(); + Py_XDECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_tmp); + Py_DECREF(pyObj_exc_params); + Py_DECREF(pyObj_exc_class); + return; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromLong(ec.value()); + if (-1 == PyDict_SetItemString(pyObj_kwargs, "error_code", pyObj_tmp)) { + PyErr_Print(); + Py_XDECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_tmp); + Py_DECREF(pyObj_exc_params); + Py_DECREF(pyObj_exc_class); + return; + } + Py_DECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString(pyObj_kwargs, "exc_info", pyObj_exc_params)) { + PyErr_Print(); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + Py_DECREF(pyObj_exc_params); + Py_DECREF(pyObj_exc_class); + return; + } + Py_DECREF(pyObj_exc_params); + + pyObj_exc_instance = PyObject_Call(pyObj_exc_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + Py_DECREF(pyObj_exc_class); + + if (pyObj_exc_instance == nullptr) { + Py_XDECREF(pyObj_traceback); + return; + } + + Py_INCREF(Py_TYPE(pyObj_exc_instance)); + PyErr_Restore((PyObject*)Py_TYPE(pyObj_exc_instance), pyObj_exc_instance, pyObj_traceback); +} + +PyObject* +pycbc_build_exception(std::error_code ec, const char* file, int line, std::string msg) +{ + PyObject *pyObj_type = nullptr, *pyObj_value = nullptr, *pyObj_traceback = nullptr; + + PyErr_Fetch(&pyObj_type, &pyObj_value, &pyObj_traceback); + PyErr_Clear(); + + PyObject* pyObj_exc_info = PyDict_New(); + + if (pyObj_type != nullptr) { + PyErr_NormalizeException(&pyObj_type, &pyObj_value, &pyObj_traceback); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "inner_cause", pyObj_value)) { + PyErr_Print(); + Py_DECREF(pyObj_type); + Py_XDECREF(pyObj_value); + + Py_XDECREF(pyObj_exc_info); + return nullptr; + } + Py_DECREF(pyObj_type); + Py_XDECREF(pyObj_value); + // Py_XDECREF(pyObj_traceback); + } + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + Py_XDECREF(pyObj_exc_info); + return nullptr; + } + Py_DECREF(pyObj_cinfo); + + if (!msg.empty()) { + PyObject* pyObj_msg = PyUnicode_FromString(msg.c_str()); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "error_msg", pyObj_msg)) { + PyErr_Print(); + Py_DECREF(pyObj_exc_info); + Py_XDECREF(pyObj_msg); + return nullptr; + } + Py_DECREF(pyObj_msg); + } + + exception_base* exc = create_exception_base_obj(); + exc->ec = ec; + + exc->exc_info = pyObj_exc_info; + Py_INCREF(exc->exc_info); + + return reinterpret_cast<PyObject*>(exc); +} + +void +pycbc_add_exception_info(PyObject* pyObj_exc_base, const char* key, PyObject* pyObj_value) +{ + exception_base* exc = reinterpret_cast<exception_base*>(pyObj_exc_base); + + if (exc->exc_info) { + if (-1 == PyDict_SetItemString(exc->exc_info, key, pyObj_value)) { + PyErr_Print(); + return; + } + Py_DECREF(pyObj_value); + } else { + PyObject* pyObj_exc_info = PyDict_New(); + if (-1 == PyDict_SetItemString(pyObj_exc_info, key, pyObj_value)) { + PyErr_Print(); + Py_XDECREF(pyObj_exc_info); + return; + } + Py_DECREF(pyObj_value); + exc->exc_info = pyObj_exc_info; + Py_INCREF(exc->exc_info); + } +} + +PyObject* +add_exception_objects(PyObject* pyObj_module) +{ + if (PyType_Ready(&exception_base_type) < 0) { + return nullptr; + } + Py_INCREF(&exception_base_type); + if (PyModule_AddObject( + pyObj_module, "exception", reinterpret_cast<PyObject*>(&exception_base_type)) < 0) { + Py_DECREF(&exception_base_type); + return nullptr; + } + return pyObj_module; +} diff --git a/src/exceptions.hxx b/src/exceptions.hxx new file mode 100644 index 000000000..76b481b31 --- /dev/null +++ b/src/exceptions.hxx @@ -0,0 +1,751 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <exception> + +#include "client.hxx" +#include <core/error_context/analytics.hxx> +#include <core/error_context/http.hxx> +#include <core/error_context/key_value.hxx> +#include <core/error_context/query.hxx> +#include <core/error_context/search.hxx> +#include <core/error_context/view.hxx> + +#define DISPATCHED_TO "last_dispatched_to" +#define DISPATCHED_FROM "last_dispatched_from" +#define RETRY_ATTEMPTS "retry_attempts" +#define RETRY_REASONS "retry_reasons" + +#define CONTEXT_TYPE "context_type" +#define CONTEXT_DETAIL_TYPE "context_detail_type" +#define CLIENT_CONTEXT_ID "client_context_id" + +#define KV_DOCUMENT_ID "key" +#define KV_DOCUMENT_BUCKET "bucket_name" +#define KV_DOCUMENT_SCOPE "scope_name" +#define KV_DOCUMENT_COLLECTION "collection_name" +#define KV_OPAQUE "opaque" +#define KV_STATUS_CODE "status_code" +#define KV_ERROR_MAP_INFO "error_map_info" +#define KV_EXTENDED_ERROR_INFO "extended_error_info" + +#define MGMT_CONTENT "content" +#define MGMT_PATH "path" +#define MGMT_STATUS "http_status" + +#define HTTP_STATUS "http_status" +#define HTTP_METHOD "method" +#define HTTP_PATH "path" +#define HTTP_BODY "http_body" + +#define QUERY_FIRST_ERROR_CODE "first_error_code" +#define QUERY_FIRST_ERROR_MSG "first_error_message" +#define QUERY_STATEMENT "statement" +#define QUERY_PARAMETERS "parameters" + +#define SEARCH_INDEX_NAME "index_name" +#define SEARCH_QUERY "query" +#define SEARCH_PARAMETERS "parameters" + +#define SUBDOC_PATH "first_error_path" +#define SUBDOC_INDEX "first_error_index" +#define SUBDOC_DELETED "deleted" + +#define VIEW_DDOC_NAME "design_document_name" +#define VIEW_NAME "view_name" +#define VIEW_QUERY "query_string" + +#define NULL_CONN_OBJECT "Received a null connection." + +struct exception_base { + PyObject_HEAD std::error_code ec; + PyObject* error_context = nullptr; + PyObject* exc_info = nullptr; +}; + +int +pycbc_exception_base_type_init(PyObject** ptr); + +exception_base* +create_exception_base_obj(); + +std::string +retry_reason_to_string(couchbase::retry_reason reason); + +// start - needed for Pycbc error code +enum class PycbcError { + InvalidArgument = 3, + BucketNotFound = 10, + FeatureUnavailable = 15, + InternalSDKError = 5000, + HTTPError = 5001, + UnsuccessfulOperation, + UnableToBuildResult, + CallbackUnsuccessful, +}; + +namespace std +{ +template<> +struct is_error_code_enum<PycbcError> : true_type { +}; +} // namespace std + +std::error_code +make_error_code(PycbcError ec); +// end - needed for Pycbc error code + +PyObject* +build_kv_error_map_info(couchbase::core::key_value_error_map_info error_info); + +void +build_kv_error_context(const couchbase::core::key_value_error_context& ctx, PyObject* pyObj_ctx); + +/* + +Build exceptions via error context + +*/ +template<class T> +PyObject* +build_base_error_context(const T& ctx) +{ + PyObject* pyObj_error_context = PyDict_New(); + + PyObject* pyObj_tmp = nullptr; + if (ctx.last_dispatched_to.has_value()) { + pyObj_tmp = PyUnicode_FromString(ctx.last_dispatched_to.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, DISPATCHED_TO, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + if (ctx.last_dispatched_from.has_value()) { + pyObj_tmp = PyUnicode_FromString(ctx.last_dispatched_from.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, DISPATCHED_FROM, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = PyLong_FromLong(ctx.retry_attempts); + if (-1 == PyDict_SetItemString(pyObj_error_context, RETRY_ATTEMPTS, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + PyObject* rr_set = PySet_New(nullptr); + for (auto rr : ctx.retry_reasons) { + std::string reason = retry_reason_to_string(rr); + pyObj_tmp = PyUnicode_FromString(reason.c_str()); + if (-1 == PySet_Add(rr_set, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + Py_ssize_t set_size = PySet_Size(rr_set); + if (set_size > 0) { + if (-1 == PyDict_SetItemString(pyObj_error_context, RETRY_REASONS, rr_set)) { + PyErr_Print(); + PyErr_Clear(); + } + } + Py_DECREF(rr_set); + + return pyObj_error_context; +} + +template<typename Context> +inline PyObject* +build_base_error_context_new(const Context& ctx) +{ + PyObject* pyObj_error_context = PyDict_New(); + + PyObject* pyObj_tmp = nullptr; + if (ctx.last_dispatched_to().has_value()) { + pyObj_tmp = PyUnicode_FromString(ctx.last_dispatched_to().value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, DISPATCHED_TO, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + if (ctx.last_dispatched_from().has_value()) { + pyObj_tmp = PyUnicode_FromString(ctx.last_dispatched_from().value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, DISPATCHED_FROM, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = PyLong_FromLong(ctx.retry_attempts()); + if (-1 == PyDict_SetItemString(pyObj_error_context, RETRY_ATTEMPTS, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + PyObject* rr_set = PySet_New(nullptr); + for (auto rr : ctx.retry_reasons()) { + std::string reason = retry_reason_to_string(rr); + pyObj_tmp = PyUnicode_FromString(reason.c_str()); + if (-1 == PySet_Add(rr_set, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + Py_ssize_t set_size = PySet_Size(rr_set); + if (set_size > 0) { + if (-1 == PyDict_SetItemString(pyObj_error_context, RETRY_REASONS, rr_set)) { + PyErr_Print(); + PyErr_Clear(); + } + } + Py_DECREF(rr_set); + + return pyObj_error_context; +} + +template<typename T> +void +build_base_http_error_context(const T& ctx, PyObject* pyObj_error_context) +{ + PyObject* pyObj_tmp = nullptr; + pyObj_tmp = PyUnicode_FromString(ctx.client_context_id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CLIENT_CONTEXT_ID, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.method.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, HTTP_METHOD, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.path.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, HTTP_PATH, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromLong(static_cast<uint32_t>(ctx.http_status)); + if (-1 == PyDict_SetItemString(pyObj_error_context, HTTP_STATUS, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.http_body.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, HTTP_BODY, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); +} + +template<typename T> +PyObject* +build_exception_from_context(const T& ctx, + const char* file = __FILE__, + int line = __LINE__, + std::string error_msg = std::string(), + std::string context_detail_type = std::string()) +{ + exception_base* exc = create_exception_base_obj(); + exc->ec = ctx.ec(); + exc->error_context = build_base_error_context(ctx); + + return reinterpret_cast<PyObject*>(exc); +} + +template<> +inline PyObject* +build_exception_from_context(const couchbase::core::key_value_error_context& ctx, + const char* file, + int line, + std::string error_msg, + std::string context_detail_type) +{ + exception_base* exc = create_exception_base_obj(); + exc->ec = ctx.ec(); + PyObject* pyObj_error_context = build_base_error_context_new(ctx); + + build_kv_error_context(ctx, pyObj_error_context); + + std::string context_type = "KeyValueErrorContext"; + PyObject* pyObj_tmp = PyUnicode_FromString(context_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CONTEXT_TYPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + exc->error_context = pyObj_error_context; + + PyObject* pyObj_exc_info = PyDict_New(); + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + } + Py_DECREF(pyObj_cinfo); + + if (!error_msg.empty()) { + PyObject* pyObj_error_msg = PyUnicode_FromString(error_msg.c_str()); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "error_message", pyObj_error_msg)) { + PyErr_Print(); + Py_XDECREF(pyObj_error_msg); + } + Py_DECREF(pyObj_error_msg); + } + + exc->exc_info = pyObj_exc_info; + + return reinterpret_cast<PyObject*>(exc); +} + +template<> +inline PyObject* +build_exception_from_context(const couchbase::core::subdocument_error_context& ctx, + const char* file, + int line, + std::string error_msg, + std::string context_detail_type) +{ + exception_base* exc = create_exception_base_obj(); + exc->ec = ctx.ec(); + PyObject* pyObj_error_context = build_base_error_context_new(ctx); + + build_kv_error_context(ctx, pyObj_error_context); + + std::string context_type = "SubdocumentErrorContext"; + PyObject* pyObj_tmp = PyUnicode_FromString(context_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CONTEXT_TYPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + exc->error_context = pyObj_error_context; + + PyObject* pyObj_exc_info = PyDict_New(); + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + } + Py_DECREF(pyObj_cinfo); + + if (!error_msg.empty()) { + PyObject* pyObj_error_msg = PyUnicode_FromString(error_msg.c_str()); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "error_message", pyObj_error_msg)) { + PyErr_Print(); + Py_XDECREF(pyObj_error_msg); + } + Py_DECREF(pyObj_error_msg); + } + + exc->exc_info = pyObj_exc_info; + + return reinterpret_cast<PyObject*>(exc); +} + +template<> +inline PyObject* +build_exception_from_context(const couchbase::core::error_context::http& ctx, + const char* file, + int line, + std::string error_msg, + std::string context_detail_type) +{ + exception_base* exc = create_exception_base_obj(); + exc->ec = ctx.ec; + + PyObject* pyObj_error_context = build_base_error_context(ctx); + build_base_http_error_context(ctx, pyObj_error_context); + + std::string context_type = "HTTPErrorContext"; + PyObject* pyObj_tmp = nullptr; + pyObj_tmp = PyUnicode_FromString(context_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CONTEXT_TYPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (!context_detail_type.empty()) { + pyObj_tmp = PyUnicode_FromString(context_detail_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CONTEXT_DETAIL_TYPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + exc->error_context = pyObj_error_context; + + PyObject* pyObj_exc_info = PyDict_New(); + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + } + Py_DECREF(pyObj_cinfo); + + if (!error_msg.empty()) { + PyObject* pyObj_error_msg = PyUnicode_FromString(error_msg.c_str()); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "error_message", pyObj_error_msg)) { + PyErr_Print(); + Py_XDECREF(pyObj_error_msg); + } + Py_DECREF(pyObj_error_msg); + } + + exc->exc_info = pyObj_exc_info; + + return reinterpret_cast<PyObject*>(exc); +} + +template<> +inline PyObject* +build_exception_from_context(const couchbase::core::error_context::query& ctx, + const char* file, + int line, + std::string error_msg, + std::string context_detail_type) +{ + exception_base* exc = create_exception_base_obj(); + exc->ec = ctx.ec; + PyObject* pyObj_error_context = build_base_error_context(ctx); + build_base_http_error_context(ctx, pyObj_error_context); + + PyObject* pyObj_tmp = nullptr; + pyObj_tmp = PyLong_FromLongLong(ctx.first_error_code); + if (-1 == PyDict_SetItemString(pyObj_error_context, QUERY_FIRST_ERROR_CODE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.first_error_message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, QUERY_FIRST_ERROR_MSG, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.statement.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, QUERY_STATEMENT, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (ctx.parameters.has_value()) { + pyObj_tmp = PyUnicode_FromString(ctx.parameters.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, QUERY_PARAMETERS, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + std::string context_type = "QueryErrorContext"; + pyObj_tmp = PyUnicode_FromString(context_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CONTEXT_TYPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + exc->error_context = pyObj_error_context; + + PyObject* pyObj_exc_info = PyDict_New(); + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + } + Py_DECREF(pyObj_cinfo); + + if (!error_msg.empty()) { + PyObject* pyObj_error_msg = PyUnicode_FromString(error_msg.c_str()); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "error_message", pyObj_error_msg)) { + PyErr_Print(); + Py_XDECREF(pyObj_error_msg); + } + Py_DECREF(pyObj_error_msg); + } + + exc->exc_info = pyObj_exc_info; + + return reinterpret_cast<PyObject*>(exc); +} + +template<> +inline PyObject* +build_exception_from_context(const couchbase::core::error_context::analytics& ctx, + const char* file, + int line, + std::string error_msg, + std::string context_detail_type) +{ + exception_base* exc = create_exception_base_obj(); + exc->ec = ctx.ec; + PyObject* pyObj_error_context = build_base_error_context(ctx); + build_base_http_error_context(ctx, pyObj_error_context); + + PyObject* pyObj_tmp = nullptr; + pyObj_tmp = PyLong_FromLongLong(ctx.first_error_code); + if (-1 == PyDict_SetItemString(pyObj_error_context, QUERY_FIRST_ERROR_CODE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.first_error_message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, QUERY_FIRST_ERROR_MSG, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.statement.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, QUERY_STATEMENT, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (ctx.parameters.has_value()) { + pyObj_tmp = PyUnicode_FromString(ctx.parameters.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, QUERY_PARAMETERS, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + std::string context_type = "AnalyticsErrorContext"; + pyObj_tmp = PyUnicode_FromString(context_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CONTEXT_TYPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + exc->error_context = pyObj_error_context; + + PyObject* pyObj_exc_info = PyDict_New(); + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + } + Py_DECREF(pyObj_cinfo); + + if (!error_msg.empty()) { + PyObject* pyObj_error_msg = PyUnicode_FromString(error_msg.c_str()); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "error_message", pyObj_error_msg)) { + PyErr_Print(); + Py_XDECREF(pyObj_error_msg); + } + Py_DECREF(pyObj_error_msg); + } + + exc->exc_info = pyObj_exc_info; + + return reinterpret_cast<PyObject*>(exc); +} + +template<> +inline PyObject* +build_exception_from_context(const couchbase::core::error_context::search& ctx, + const char* file, + int line, + std::string error_msg, + std::string context_detail_type) +{ + exception_base* exc = create_exception_base_obj(); + exc->ec = ctx.ec; + PyObject* pyObj_error_context = build_base_error_context(ctx); + build_base_http_error_context(ctx, pyObj_error_context); + + PyObject* pyObj_tmp = nullptr; + pyObj_tmp = PyUnicode_FromString(ctx.index_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, SEARCH_INDEX_NAME, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.query.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, SEARCH_QUERY, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (ctx.parameters.has_value()) { + pyObj_tmp = PyUnicode_FromString(ctx.parameters.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, SEARCH_PARAMETERS, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + std::string context_type = "SearchErrorContext"; + pyObj_tmp = PyUnicode_FromString(context_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CONTEXT_TYPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + exc->error_context = pyObj_error_context; + + PyObject* pyObj_exc_info = PyDict_New(); + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + } + Py_DECREF(pyObj_cinfo); + + if (!error_msg.empty()) { + PyObject* pyObj_error_msg = PyUnicode_FromString(error_msg.c_str()); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "error_message", pyObj_error_msg)) { + PyErr_Print(); + Py_XDECREF(pyObj_error_msg); + } + Py_DECREF(pyObj_error_msg); + } + + exc->exc_info = pyObj_exc_info; + + return reinterpret_cast<PyObject*>(exc); +} + +template<> +inline PyObject* +build_exception_from_context(const couchbase::core::error_context::view& ctx, + const char* file, + int line, + std::string error_msg, + std::string context_detail_type) +{ + exception_base* exc = create_exception_base_obj(); + exc->ec = ctx.ec; + PyObject* pyObj_error_context = build_base_error_context(ctx); + build_base_http_error_context(ctx, pyObj_error_context); + + PyObject* pyObj_tmp = nullptr; + pyObj_tmp = PyUnicode_FromString(ctx.design_document_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, VIEW_DDOC_NAME, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(ctx.view_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, VIEW_NAME, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_query_string = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& query : ctx.query_string) { + pyObj_tmp = PyUnicode_FromString(query.c_str()); + if (-1 == PyList_Append(pyObj_query_string, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyDict_SetItemString(pyObj_error_context, VIEW_QUERY, pyObj_query_string)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_query_string); + + std::string context_type = "ViewErrorContext"; + pyObj_tmp = PyUnicode_FromString(context_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error_context, CONTEXT_TYPE, pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + exc->error_context = pyObj_error_context; + + PyObject* pyObj_exc_info = PyDict_New(); + + PyObject* pyObj_cinfo = Py_BuildValue("(s,i)", file, line); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "cinfo", pyObj_cinfo)) { + PyErr_Print(); + Py_XDECREF(pyObj_cinfo); + } + Py_DECREF(pyObj_cinfo); + + if (!error_msg.empty()) { + PyObject* pyObj_error_msg = PyUnicode_FromString(error_msg.c_str()); + if (-1 == PyDict_SetItemString(pyObj_exc_info, "error_message", pyObj_error_msg)) { + PyErr_Print(); + Py_XDECREF(pyObj_error_msg); + } + Py_DECREF(pyObj_error_msg); + } + + exc->exc_info = pyObj_exc_info; + + return reinterpret_cast<PyObject*>(exc); +} + +void +pycbc_set_python_exception(std::error_code ec, const char* file, int line, const char* msg); + +PyObject* +pycbc_build_exception(std::error_code ec, const char* file, int line, std::string msg); + +void +pycbc_add_exception_info(PyObject* pyObj_exc_base, const char* key, PyObject* pyObj_value); + +PyObject* +add_exception_objects(PyObject* pyObj_module); diff --git a/src/ext.c b/src/ext.c deleted file mode 100644 index e969ef140..000000000 --- a/src/ext.c +++ /dev/null @@ -1,437 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "structmember.h" -/** - * This file contains boilerplate for the module itself - */ - -struct pycbc_helpers_ST pycbc_helpers; - -PyObject *pycbc_log_handler = NULL; - -static void log_handler(struct lcb_logprocs_st *procs, unsigned int iid, - const char *subsys, int severity, const char *srcfile, int srcline, - const char *fmt, va_list ap); - -struct lcb_logprocs_st pycbc_lcb_logprocs = { 0 }; - -static PyObject * -_libcouchbase_init_helpers(PyObject *self, PyObject *args, PyObject *kwargs) { - -#define X(n) \ - pycbc_helpers.n = PyDict_GetItemString(kwargs, #n); \ - if (!pycbc_helpers.n) { \ - PyErr_SetString(PyExc_EnvironmentError, "Can't find " #n); \ - return NULL; \ - } \ - - PYCBC_XHELPERS(X); -#undef X - -#define X(n) \ - Py_XINCREF(pycbc_helpers.n); - PYCBC_XHELPERS(X) -#undef X - - (void)self; - (void)args; - - Py_RETURN_NONE; -} - -static void -get_helper_field(const char *s, PyObject *key, PyObject **cand, PyObject ***out) -{ - PyObject *sk = pycbc_SimpleStringZ(s); - if (PyUnicode_Compare(sk, key) == 0) { - *out = cand; - } - Py_DECREF(sk); -} - -static PyObject * -_libcouchbase_modify_helpers(PyObject *self, PyObject *args, PyObject *kwargs) -{ - Py_ssize_t dictpos = 0; - PyObject *curkey; - PyObject *curval; - PyObject *ret; - - if (kwargs == NULL || PyDict_Check(kwargs) == 0) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - ret = PyDict_New(); - - while (PyDict_Next(kwargs, &dictpos, &curkey, &curval)) { - PyObject **field = NULL; - PyObject *dent = curval; - - #define X(name) \ - if (!field) { \ - get_helper_field(#name, \ - curkey, \ - &pycbc_helpers.name, \ - &field); \ - } - - PYCBC_XHELPERS(X) - #undef X - - if (!field) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "Unknown helper", curkey); - Py_DECREF(ret); - return NULL; - } - - if (*field) { - dent = *field; - } else { - dent = Py_None; - Py_INCREF(dent); - } - - PyDict_SetItem(ret, curkey, dent); - Py_DECREF(dent); - - Py_INCREF(curval); - *field = curval; - } - - (void)args; - (void)self; - return ret; -} - -static PyObject * -_libcouchbase_get_helper(PyObject *self, PyObject *arg) -{ - PyObject *key = NULL; - PyObject **field = NULL; - int rv; - (void)self; - - rv = PyArg_ParseTuple(arg, "O", &key); - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - #define X(name) \ - if (!field) { \ - get_helper_field(#name, key, &pycbc_helpers.name, &field); \ - } - - PYCBC_XHELPERS(X) - #undef X - - if (!field) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "Unknown helper", key); - return NULL; - } - if (*field) { - Py_INCREF(*field); - return *field; - } else { - Py_RETURN_NONE; - } - -} - -static PyObject * -_libcouchbase_strerror(PyObject *self, PyObject *args, PyObject *kw) -{ - int rv; - int rc = 0; - rv = PyArg_ParseTuple(args, "i", &rc); - if (!rv) { - return NULL; - } - - (void)self; - (void)kw; - - return pycbc_lcb_errstr(NULL, rc); -} - -static PyObject * -pycbc_lcb_version(pycbc_Bucket *self) -{ - const char *verstr; - lcb_uint32_t vernum; - PyObject *ret; - - verstr = lcb_get_version(&vernum); - ret = Py_BuildValue("(s,k)", verstr, vernum); - - (void)self; - - return ret; -} - -static PyObject * -set_log_handler(PyObject *self, PyObject *args) -{ - PyObject *val_O = NULL; - PyObject *oldval = pycbc_log_handler; - - int rv; - - rv = PyArg_ParseTuple(args, "|O", &val_O); - if (!rv) { - return NULL; - } - - if (val_O) { - pycbc_log_handler = val_O; - if (val_O != Py_None) { - Py_INCREF(val_O); - pycbc_log_handler = val_O; - } else { - pycbc_log_handler = NULL; - } - - if (oldval) { - return oldval; - } else { - Py_RETURN_NONE; - } - } else { - /* Simple GET */ - if (oldval) { - Py_INCREF(oldval); - return oldval; - } else { - Py_RETURN_NONE; - } - } -} - -static PyMethodDef _libcouchbase_methods[] = { - { "_init_helpers", (PyCFunction)_libcouchbase_init_helpers, - METH_VARARGS|METH_KEYWORDS, - "internal function to initialize python-language helpers" - }, - { "_strerror", (PyCFunction)_libcouchbase_strerror, - METH_VARARGS|METH_KEYWORDS, - "Internal function to map errors" - }, - { "_modify_helpers", (PyCFunction)_libcouchbase_modify_helpers, - METH_VARARGS|METH_KEYWORDS, - "Internal function to modify helpers during runtime" - }, - { "_get_helper", (PyCFunction)_libcouchbase_get_helper, - METH_VARARGS, - "Get a helper by name" - }, - { "_get_errtype", (PyCFunction) pycbc_exc_get_categories, - METH_VARARGS, - "Get error categories for a given code" - }, - { "lcb_version", - (PyCFunction)pycbc_lcb_version, - METH_NOARGS, - PyDoc_STR( - "Get `libcouchbase` version information\n" - "\n" - ":return: a tuple of ``(version_string, version_number)``\n" - " corresponding to the underlying libcouchbase version\n" - - "Show the versions ::\n" \ - " \n" - " verstr, vernum = Connection.lcb_version()\n" - " print('0x{0:x}'.format(vernum))\n" - " # 0x020005\n" - " \n" - " print(verstr)\n" - " # 2.0.5\n" - "\n" - "\n") - }, - { "lcb_logging", (PyCFunction)set_log_handler, METH_VARARGS, - PyDoc_STR("Get/Set logging callback") - }, - { "dump_constants", - (PyCFunction)pycbc_print_constants, METH_NOARGS, - PyDoc_STR("Print the constants to standard output") - }, - { NULL } -}; - - -#if PY_MAJOR_VERSION >= 3 - -#define PyString_FromString PyUnicode_FromString - -static struct PyModuleDef moduledef = { - PyModuleDef_HEAD_INIT, - PYCBC_MODULE_NAME, - NULL, - 0, - NULL, - NULL, - NULL, - NULL, - NULL -}; -#endif - -#if PY_MAJOR_VERSION >= 3 -PyObject *PyInit__libcouchbase(void) -#define INITERROR return NULL - -#else -#define INITERROR return -PyMODINIT_FUNC -init_libcouchbase(void) -#endif -{ - PyObject *m = NULL; - -#ifndef PYCBC_CPYCHECKER - - /** - * Each of our types has an initializer function that accepts a single - * PyObject** pointer which is set to the newly created class object. - * - * More types should be added to this list. The first field is the type - * name (not a string but a literal) and the second is the name of the - * function used to initialize it - */ -#define X_PYTYPES(X) \ - X(Bucket, pycbc_BucketType_init) \ - /** Remember to keep base classes in order */ \ - X(Result, pycbc_ResultType_init) \ - X(OperationResult, pycbc_OperationResultType_init) \ - X(ValueResult, pycbc_ValueResultType_init) \ - X(MultiResult, pycbc_MultiResultType_init) \ - X(HttpResult, pycbc_HttpResultType_init) \ - X(ViewResult, pycbc_ViewResultType_init) \ - X(Transcoder, pycbc_TranscoderType_init) \ - X(ObserveInfo, pycbc_ObserveInfoType_init) \ - X(Item, pycbc_ItemType_init) \ - X(Event, pycbc_EventType_init) \ - X(IOEvent, pycbc_IOEventType_init) \ - X(TimerEvent, pycbc_TimerEventType_init) \ - X(AsyncResult, pycbc_AsyncResultType_init) \ - X(_IOPSWrapper, pycbc_IOPSWrapperType_init) - -#define X(name, inf) PyObject *cls_##name; - X_PYTYPES(X) -#undef X - -#define X(name, infunc) \ - if (infunc(&cls_##name) < 0) { INITERROR; } - X_PYTYPES(X) -#undef X - -#endif /* PYCBC_CPYCHECKER */ - -#if PY_MAJOR_VERSION >= 3 - moduledef.m_methods = _libcouchbase_methods; - m = PyModule_Create(&moduledef); -#else - m = Py_InitModule(PYCBC_MODULE_NAME, _libcouchbase_methods); -#endif - if (m == NULL) { - INITERROR; - } - -#ifndef PYCBC_CPYCHECKER - /** - * Add the type: - */ -#define X(name, infunc) \ - PyModule_AddObject(m, #name, cls_##name); - X_PYTYPES(X) -#undef X -#endif /* PYCBC_CPYCHECKER */ - - /** - * Initialize the helper names - */ -#define X(var, val) pycbc_helpers.var = pycbc_SimpleStringZ(val); - PYCBC_XHELPERS_STRS(X); -#undef X - - pycbc_helpers.fmt_auto = - PyObject_CallFunction((PyObject*)&PyBaseObject_Type, NULL, NULL); - PyModule_AddObject(m, "FMT_AUTO", pycbc_helpers.fmt_auto); - - pycbc_init_pyconstants(m); - - /* Add various implementation specific flags */ - PyModule_AddIntConstant(m, "_IMPL_INCLUDE_DOCS", 0); - - /* Initialize the logging routines */ - pycbc_lcb_logprocs.v.v0.callback = log_handler; - -#if PY_MAJOR_VERSION >= 3 - return m; -#endif -} - -#ifndef va_copy -#define va_copy(a, b) (a) = (b) -#endif - -/* Logging functionality */ -static void log_handler(struct lcb_logprocs_st *procs, - unsigned int iid, - const char *subsys, - int severity, - const char *srcfile, - int srcline, - const char *fmt, - va_list ap) -{ - PyGILState_STATE gil_prev; - PyObject *tmp; - PyObject *kwargs; - va_list vacp; - - if (!pycbc_log_handler) { - return; - } - - gil_prev = PyGILState_Ensure(); - - kwargs = PyDict_New(); - va_copy(vacp, ap); - tmp = PyUnicode_FromFormatV(fmt, vacp); - va_end(ap); - - PyDict_SetItemString(kwargs, "message", tmp); Py_DECREF(tmp); - - tmp = pycbc_IntFromL(iid); - PyDict_SetItemString(kwargs, "id", tmp); Py_DECREF(tmp); - - tmp = pycbc_IntFromL(severity); - PyDict_SetItemString(kwargs, "level", tmp); Py_DECREF(tmp); - - tmp = Py_BuildValue("(s,i)", srcfile, srcline); - PyDict_SetItemString(kwargs, "c_src", tmp); Py_DECREF(tmp); - - tmp = pycbc_SimpleStringZ(subsys); - PyDict_SetItemString(kwargs, "subsys", tmp); Py_DECREF(tmp); - - PyObject_Call(pycbc_log_handler, pycbc_DummyTuple, kwargs); - Py_DECREF(kwargs); - PyGILState_Release(gil_prev); -} diff --git a/src/get.c b/src/get.c deleted file mode 100644 index 07126da86..000000000 --- a/src/get.c +++ /dev/null @@ -1,327 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "oputil.h" -/** - * Covers 'lock', 'touch', and 'get_and_touch' - */ - -struct getcmd_vars_st { - int optype; - int allow_dval; - union { - unsigned long ttl; - struct { - int strategy; - short index; - } replica; - } u; -}; - -static int -handle_single_key(pycbc_Bucket *self, struct pycbc_common_vars *cv, int optype, - PyObject *curkey, PyObject *curval, PyObject *options, pycbc_Item *itm, - void *arg) -{ - int rv; - char *key; - size_t nkey; - unsigned int lock = 0; - struct getcmd_vars_st *gv = (struct getcmd_vars_st *)arg; - unsigned long ttl = gv->u.ttl; - lcb_error_t err; - - union { - lcb_CMDBASE base; - lcb_CMDGET get; - lcb_CMDTOUCH touch; - lcb_CMDGETREPLICA rget; - } u_cmd; - - memset(&u_cmd, 0, sizeof u_cmd); - (void)itm; - - rv = pycbc_tc_encode_key(self, &curkey, (void**)&key, &nkey); - if (rv == -1) { - return -1; - } - - if (!nkey) { - PYCBC_EXCTHROW_EMPTYKEY(); - rv = -1; - goto GT_DONE; - } - - LCB_CMD_SET_KEY(&u_cmd.base, key, nkey); - - if (curval && gv->allow_dval && options == NULL) { - options = curval; - } - if (options) { - static char *kwlist[] = { "ttl", NULL }; - PyObject *ttl_O = NULL; - if (gv->u.ttl) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Both global and single TTL specified"); - rv = -1; - goto GT_DONE; - } - - if (PyDict_Check(curval)) { - rv = PyArg_ParseTupleAndKeywords(pycbc_DummyTuple, - curval, "|O", kwlist, &ttl_O); - if (!rv) { - PYCBC_EXC_WRAP_KEY(PYCBC_EXC_ARGUMENTS, 0, "Couldn't get sub-parmeters for key", curkey); - rv = -1; - goto GT_DONE; - } - } else { - ttl_O = curval; - } - - rv = pycbc_get_ttl(ttl_O, &ttl, 1); - if (rv < 0) { - rv = -1; - goto GT_DONE; - } - } - u_cmd.base.exptime = ttl; - - switch (optype) { - case PYCBC_CMD_GAT: - if (!ttl) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "GAT must have positive TTL"); - rv = -1; - goto GT_DONE; - } - goto GT_GET; - - case PYCBC_CMD_LOCK: - if (!ttl) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Lock must have an expiry"); - rv = -1; - goto GT_DONE; - } - lock = 1; - goto GT_GET; - - case PYCBC_CMD_GET: - GT_GET: - u_cmd.get.lock = lock; - err = lcb_get3(self->instance, cv->mres, &u_cmd.get); - break; - - case PYCBC_CMD_TOUCH: - u_cmd.touch.exptime = ttl; - err = lcb_touch3(self->instance, cv->mres, &u_cmd.touch); - break; - - case PYCBC_CMD_GETREPLICA: - case PYCBC_CMD_GETREPLICA_INDEX: - case PYCBC_CMD_GETREPLICA_ALL: - u_cmd.rget.strategy = gv->u.replica.strategy; - u_cmd.rget.index = gv->u.replica.index; - err = lcb_rget3(self->instance, cv->mres, &u_cmd.rget); - break; - default: - err = LCB_ERROR; - abort(); - break; - } - - if (err != LCB_SUCCESS) { - PYCBC_EXCTHROW_SCHED(err); - rv = -1; - goto GT_DONE; - } else { - rv = 0; - } - - GT_DONE: - Py_XDECREF(curkey); - return rv; -} - -static int -handle_replica_options(int *optype, struct getcmd_vars_st *gv, PyObject *replica_O) -{ - switch (*optype) { - case PYCBC_CMD_GET: - *optype = PYCBC_CMD_GETREPLICA; - if (gv->u.ttl) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "TTL specified along with replica"); - return -1; - } - gv->u.replica.strategy = LCB_REPLICA_FIRST; - return 0; - - case PYCBC_CMD_GETREPLICA: - gv->u.replica.strategy = LCB_REPLICA_FIRST; - return 0; - - case PYCBC_CMD_GETREPLICA_INDEX: - gv->u.replica.strategy = LCB_REPLICA_SELECT; - if (replica_O == NULL || replica_O == Py_None) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "rgetix must have a valid replica index"); - return -1; - } - gv->u.replica.index = (short)pycbc_IntAsL(replica_O); - if (PyErr_Occurred()) { - return -1; - } - return 0; - - case PYCBC_CMD_GETREPLICA_ALL: - gv->u.replica.strategy = LCB_REPLICA_ALL; - return 0; - - default: - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Replica option not supported for this operation"); - return -1; - } - return -1; -} - - -static PyObject* -get_common(pycbc_Bucket *self, PyObject *args, PyObject *kwargs, int optype, - int argopts) -{ - int rv; - Py_ssize_t ncmds = 0; - pycbc_seqtype_t seqtype; - PyObject *kobj = NULL; - PyObject *is_quiet = NULL; - PyObject *ttl_O = NULL; - PyObject *replica_O = NULL; - PyObject *nofmt_O = NULL; - - struct pycbc_common_vars cv = PYCBC_COMMON_VARS_STATIC_INIT; - struct getcmd_vars_st gv = { 0 }; - static char *kwlist[] = { - "keys", "ttl", "quiet", "replica", "no_format", NULL - }; - - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOOO", kwlist, - &kobj, &ttl_O, &is_quiet, &replica_O, &nofmt_O); - - if (!rv) { - PYCBC_EXCTHROW_ARGS() - return NULL; - } - - gv.optype = optype; - - rv = pycbc_get_ttl(ttl_O, &gv.u.ttl, 1); - if (rv < 0) { - return NULL; - } - - if (replica_O && replica_O != Py_None && replica_O != Py_False) { - if (-1 == handle_replica_options(&optype, &gv, replica_O)) { - return NULL; - } - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_check_sequence(kobj, optype, &ncmds, &seqtype); - if (rv < 0) { - return NULL; - } - - } else { - ncmds = 1; - } - - gv.allow_dval = 1; - - switch (optype) { - case PYCBC_CMD_GET: - case PYCBC_CMD_LOCK: - case PYCBC_CMD_GAT: - gv.allow_dval = 1; - break; - - case PYCBC_CMD_TOUCH: - gv.allow_dval = 1; - break; - - case PYCBC_CMD_GETREPLICA: - case PYCBC_CMD_GETREPLICA_INDEX: - case PYCBC_CMD_GETREPLICA_ALL: - gv.allow_dval = 0; - break; - - default: - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "Unrecognized optype"); - return NULL; - - } - - rv = pycbc_common_vars_init(&cv, self, argopts, ncmds, 0); - - if (rv < 0) { - return NULL; - } - - if (nofmt_O && nofmt_O != Py_None) { - cv.mres->mropts |= PyObject_IsTrue(nofmt_O) - ? PYCBC_MRES_F_FORCEBYTES : 0; - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_iter_multi(self, seqtype, kobj, &cv, optype, - handle_single_key, &gv); - - } else { - rv = handle_single_key(self, &cv, optype, kobj, NULL, NULL, NULL, &gv); - } - if (rv < 0) { - goto GT_DONE; - } - - if (pycbc_maybe_set_quiet(cv.mres, is_quiet) == -1) { - goto GT_DONE; - } - - if (-1 == pycbc_common_vars_wait(&cv, self)) { - goto GT_DONE; - } - -GT_DONE: - pycbc_common_vars_finalize(&cv, self); - return cv.ret; -} - -#define DECLFUNC(name, operation, mode) \ - PyObject *pycbc_Bucket_##name(pycbc_Bucket *self, \ - PyObject *args, PyObject *kwargs) { \ - return get_common(self, args, kwargs, operation, mode); \ -} - - -DECLFUNC(get, PYCBC_CMD_GET, PYCBC_ARGOPT_SINGLE) -DECLFUNC(touch, PYCBC_CMD_TOUCH, PYCBC_ARGOPT_SINGLE) -DECLFUNC(lock, PYCBC_CMD_LOCK, PYCBC_ARGOPT_SINGLE) -DECLFUNC(get_multi, PYCBC_CMD_GET, PYCBC_ARGOPT_MULTI) -DECLFUNC(touch_multi, PYCBC_CMD_TOUCH, PYCBC_ARGOPT_MULTI) -DECLFUNC(lock_multi, PYCBC_CMD_LOCK, PYCBC_ARGOPT_MULTI) - -DECLFUNC(_rget, PYCBC_CMD_GETREPLICA, PYCBC_ARGOPT_SINGLE) -DECLFUNC(_rget_multi, PYCBC_CMD_GETREPLICA, PYCBC_ARGOPT_MULTI) -DECLFUNC(_rgetix, PYCBC_CMD_GETREPLICA_INDEX, PYCBC_ARGOPT_SINGLE) -DECLFUNC(_rgetix_multi, PYCBC_CMD_GETREPLICA_INDEX, PYCBC_ARGOPT_MULTI) -DECLFUNC(_rgetall, PYCBC_CMD_GETREPLICA_ALL, PYCBC_ARGOPT_SINGLE) -DECLFUNC(_rgetall_multi, PYCBC_CMD_GETREPLICA_ALL, PYCBC_ARGOPT_MULTI) diff --git a/src/htresult.c b/src/htresult.c deleted file mode 100644 index 02139c861..000000000 --- a/src/htresult.c +++ /dev/null @@ -1,151 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "structmember.h" - -int -pycbc_httpresult_ok(pycbc_HttpResult *self) -{ - if (self->rc == LCB_SUCCESS && - ((self->htcode < 300 && self->htcode > 199) || self->htcode == 0)) { - return 1; - } - return 0; -} - -static PyObject * -HttpResult_success(pycbc_HttpResult *self, void *unused) -{ - PyObject *ret = NULL; - if (pycbc_httpresult_ok(self)) { - ret = Py_True; - } else { - ret = Py_False; - } - - Py_INCREF(ret); - - (void)unused; - return ret; -} - -static PyObject * -HttpResult_headers(pycbc_HttpResult *self, void *unused) -{ - (void)unused; - if (!self->headers) { - Py_RETURN_NONE; - } - Py_INCREF(self->headers); - return self->headers; -} - -static void -HttpResult_dealloc(pycbc_HttpResult *self) -{ - if (self->u.htreq) { - if (self->parent) { - if (self->htype == PYCBC_HTTP_HVIEW) { - lcb_view_cancel(self->parent->instance, self->u.vh); - } else if (self->htype == PYCBC_HTTP_HN1QL) { - lcb_n1ql_cancel(self->parent->instance, self->u.nq); - } else { - lcb_cancel_http_request(self->parent->instance, self->u.htreq); - } - } - self->u.htreq = NULL; - } - Py_XDECREF(self->http_data); - Py_XDECREF(self->headers); - Py_XDECREF(self->parent); - pycbc_Result_dealloc((pycbc_Result*)self); -} - - -static struct PyMemberDef HttpResult_TABLE_members[] = { - { "http_status", - T_USHORT, offsetof(pycbc_HttpResult, htcode), - READONLY, PyDoc_STR("HTTP Status Code") - }, - - { "value", - T_OBJECT_EX, offsetof(pycbc_HttpResult, http_data), - READONLY, PyDoc_STR("HTTP Payload") - }, - - { "url", - T_OBJECT_EX, offsetof(pycbc_HttpResult, key), - READONLY, PyDoc_STR("HTTP URI") - }, - { "done", - T_BOOL, offsetof(pycbc_HttpResult, done), - READONLY, PyDoc_STR("If the result is done") - }, - { NULL } -}; - -static PyGetSetDef HttpResult_TABLE_getset[] = { - { "success", (getter)HttpResult_success, NULL, - PyDoc_STR("Whether the HTTP request was successful") - }, - - { "headers", (getter)HttpResult_headers, NULL, - PyDoc_STR("Headers dict for the request. ") - }, - - { NULL } -}; - -static PyMethodDef HttpResult_TABLE_methods[] = { - { NULL } -}; - -PyTypeObject pycbc_HttpResultType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -int -pycbc_HttpResultType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_HttpResultType; - *ptr = (PyObject*)p; - - if (p->tp_name) { - return 0; - } - p->tp_name = "HttpResult"; - p->tp_doc = PyDoc_STR("Generic object returned for HTTP operations\n"); - p->tp_new = PyType_GenericNew; - p->tp_basicsize = sizeof(pycbc_HttpResult); - p->tp_base = &pycbc_ResultType; - p->tp_getset = HttpResult_TABLE_getset; - p->tp_members = HttpResult_TABLE_members; - p->tp_methods = HttpResult_TABLE_methods; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_dealloc = (destructor)HttpResult_dealloc; - return pycbc_ResultType_ready(p, PYCBC_HTRESULT_BASEFLDS); -} - -void -pycbc_httpresult_init(pycbc_HttpResult *self, pycbc_MultiResult *mres) -{ - PyDict_SetItem((PyObject*)mres, Py_None, (PyObject*)self); - Py_DECREF(self); - self->parent = mres->parent; - Py_INCREF(self->parent); -} diff --git a/src/http.c b/src/http.c deleted file mode 100644 index 43ca446d9..000000000 --- a/src/http.c +++ /dev/null @@ -1,280 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "oputil.h" - -static void -get_headers(pycbc_HttpResult *htres, const char * const *headers) -{ - const char * const *p; - htres->headers = PyDict_New(); - - if (!headers) { - return; - } - - for (p = headers; *p; p += 2) { - PyObject *hval = pycbc_SimpleStringZ(p[1]); - PyDict_SetItemString(htres->headers, p[0], hval); - Py_DECREF(hval); - } -} - -void -pycbc_httpresult_add_data(pycbc_MultiResult *mres, pycbc_HttpResult *htres, - const void *bytes, size_t nbytes) -{ - PyObject *newbuf; - if (!nbytes) { - return; - } - - newbuf = PyBytes_FromStringAndSize(bytes, nbytes); - if (htres->http_data) { - PyObject *old_s = htres->http_data; - PyBytes_ConcatAndDel(&htres->http_data, newbuf); - if (!htres->http_data) { - htres->http_data = old_s; - Py_XDECREF(newbuf); - pycbc_multiresult_adderr(mres); - } - } else { - htres->http_data = newbuf; - } -} - -static void -decode_data(pycbc_MultiResult *mres, pycbc_HttpResult *htres) -{ - int rv; - lcb_U32 format = htres->format; - const void *data; - Py_ssize_t ndata; - PyObject *tmp; - int is_success = 1; - - if (!format) { - /* Already bytes */ - return; - } - - if (!htres->http_data) { - htres->http_data = Py_None; - Py_INCREF(Py_None); - return; - } - - - if (htres->htcode < 200 || htres->htcode > 299) { - /* Not a successful response. */ - is_success = 0; - } - - /* Handle cases where we already have a failure. In this case failure should - * be for the actual content or HTTP code, rather than on encoding. */ - #define MAYBE_ADD_ERR() \ - if (is_success) { pycbc_multiresult_adderr(mres); } \ - else { PyErr_Clear(); } - - - rv = PyBytes_AsStringAndSize(htres->http_data, (char**)&data, &ndata); - if (rv != 0) { - MAYBE_ADD_ERR(); - return; - } - rv = pycbc_tc_simple_decode(&tmp, data, ndata, format); - if (rv != 0) { - MAYBE_ADD_ERR(); - return; - } - #undef MAYBE_ADD_ERR - - Py_DECREF(htres->http_data); - htres->http_data = tmp; -} - -#define HTTP_IS_OK(st) (st > 199 && st < 300) - -void -pycbc_httpresult_complete(pycbc_HttpResult *htres, pycbc_MultiResult *mres, - lcb_error_t err, short status, - const char * const *headers) -{ - int should_raise = 0; - pycbc_Bucket *bucket = htres->parent; - - if (htres->rc == LCB_SUCCESS) { - htres->rc = err; - } - - htres->htcode = status; - htres->done = 1; - htres->u.htreq = NULL; - Py_XDECREF(htres->parent); - htres->parent = NULL; - - if (err != LCB_SUCCESS) { - should_raise = 1; - } else if (status && !HTTP_IS_OK(status) && - (mres->mropts & PYCBC_MRES_F_QUIET) == 0) { - should_raise = 1; - } - - if (should_raise) { - PYCBC_EXC_WRAP_EX(err ? PYCBC_EXC_LCBERR : PYCBC_EXC_HTTP, err, - "HTTP Request failed. Examine 'objextra' for " - "full result", htres->key, (PyObject*)htres); - pycbc_multiresult_adderr(mres); - } - - get_headers(htres, headers); - decode_data(mres, htres); - - if ((bucket->flags & PYCBC_CONN_F_ASYNC) == 0) { - if (!bucket->nremaining) { - lcb_breakout(bucket->instance); - } - PYCBC_CONN_THR_BEGIN(bucket); - } else { - pycbc_AsyncResult *ares = (pycbc_AsyncResult *)mres; - ares->nops--; - Py_INCREF(ares); - pycbc_asyncresult_invoke(ares); - /* We don't handle the GIL in async mode */ - } -} - -static void -complete_callback(lcb_t instance, int cbtype, const lcb_RESPBASE *rb) -{ - pycbc_MultiResult *mres; - pycbc_Bucket *bucket; - pycbc_HttpResult *htres; - const lcb_RESPHTTP *resp = (const lcb_RESPHTTP *)rb; - - mres = (pycbc_MultiResult *)resp->cookie; - bucket = mres->parent; - - PYCBC_CONN_THR_END(bucket); - - htres = (pycbc_HttpResult*)PyDict_GetItem((PyObject*)mres, Py_None); - pycbc_httpresult_add_data(mres, htres, resp->body, resp->nbody); - pycbc_httpresult_complete(htres, mres, resp->rc, resp->htstatus, resp->headers); - - /* CONN_THR_BEGIN called by httpresult_complete() */ - (void)instance; (void)cbtype; -} - -void -pycbc_http_callbacks_init(lcb_t instance) -{ - lcb_install_callback3(instance, LCB_CALLBACK_HTTP, complete_callback); - pycbc_views_callbacks_init(instance); -} - -PyObject * -pycbc_Bucket__http_request(pycbc_Bucket *self, PyObject *args, PyObject *kwargs) -{ - int rv; - int method; - int reqtype; - unsigned value_format = PYCBC_FMT_JSON; - lcb_error_t err; - - const char *body = NULL; - PyObject *ret = NULL; - PyObject *quiet_O = NULL; - pycbc_strlen_t nbody = 0; - const char *path = NULL; - const char *content_type = NULL; - pycbc_HttpResult *htres = NULL; - pycbc_MultiResult *mres = NULL; - lcb_CMDHTTP htcmd = { 0 }; - - static char *kwlist[] = { - "type", "method", "path", "content_type", "post_data", - "response_format", "quiet", NULL - }; - - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "iis|zz#IO", kwlist, - &reqtype, &method, &path, - &content_type, &body, &nbody, - &value_format, &quiet_O); - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - if (-1 == pycbc_oputil_conn_lock(self)) { - return NULL; - } - - if (self->pipeline_queue) { - PYCBC_EXC_WRAP(PYCBC_EXC_PIPELINE, 0, - "HTTP/View Requests cannot be executed in " - "pipeline context"); - goto GT_DONE; - } - - mres = (pycbc_MultiResult*)pycbc_multiresult_new(self); - htres = (pycbc_HttpResult*)PYCBC_TYPE_CTOR(&pycbc_HttpResultType); - pycbc_httpresult_init(htres, mres); - - htres->key = pycbc_SimpleStringZ(path); - htres->format = value_format; - - if (quiet_O != NULL && quiet_O != Py_None && PyObject_IsTrue(quiet_O)) { - mres->mropts |= PYCBC_MRES_F_QUIET; - } - mres->mropts |= PYCBC_MRES_F_SINGLE; - - LCB_CMD_SET_KEY(&htcmd, path, strlen(path)); - htcmd.body = body; - htcmd.nbody = nbody; - htcmd.content_type = content_type; - htcmd.method = method; - htcmd.reqhandle = &htres->u.htreq; - htcmd.type = reqtype; - - err = lcb_http3(self->instance, mres, &htcmd); - - if (err != LCB_SUCCESS) { - PYCBC_EXCTHROW_SCHED(err); - goto GT_DONE; - } - - if (!(self->flags & PYCBC_CONN_F_ASYNC)) { - pycbc_oputil_wait_common(self); - /* RC=1 (decref on done) */ - if (pycbc_multiresult_maybe_raise(mres)) { - goto GT_DONE; - } - - ret = pycbc_multiresult_get_result(mres); - Py_DECREF(mres); /* Don't need multiresult anymore. Use ret */ - } else { - ret = (PyObject*)mres; - } - - mres = NULL; /* Avoid the DECREF on success */ - - GT_DONE: - Py_XDECREF(mres); - pycbc_oputil_conn_unlock(self); - return ret; -} - diff --git a/src/iops.c b/src/iops.c deleted file mode 100644 index 30b8916f4..000000000 --- a/src/iops.c +++ /dev/null @@ -1,633 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "iops.h" -#include "structmember.h" - -#define XIONAME_CACHENTRIES(X) \ - X(modevent, 0) \ - X(modtimer, 0) \ - X(startwatch, 0) \ - X(stopwatch, 0) \ - X(mkevent, 1) \ - X(mktimer, 1) - -static PyTypeObject pycbc_EventType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static PyTypeObject pycbc_IOEventType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static PyTypeObject pycbc_TimerEventType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static PyTypeObject pycbc_IOPSWrapperType = { - PYCBC_POBJ_HEAD_INIT(NULL) -}; - -static struct PyMemberDef pycbc_Event_TABLE_members[] = { - - { "__dict__", T_OBJECT_EX, offsetof(pycbc_Event, vdict), 0 }, - { "evtype", T_INT, offsetof(pycbc_Event, type), READONLY }, - { "state", T_INT, offsetof(pycbc_Event, state), READONLY }, - { NULL } -}; - -static struct PyMemberDef pycbc_IOEvent_TABLE_members[] = { - { "fd", T_LONGLONG, offsetof(pycbc_IOEvent, fd), READONLY }, - { "flags", T_SHORT, offsetof(pycbc_IOEvent, flags), READONLY }, - { NULL } -}; - - -static struct PyMemberDef pycbc_TimerEvent_TABLE_members[] = { - { NULL } -}; - - -static void event_fire_common(pycbc_Event *ev, short which) -{ - lcb_socket_t fd = 0; - PyObject *parent; - - if (ev->state == PYCBC_EVSTATE_FREED) { - return; - } - - if (ev->type == PYCBC_EVTYPE_IO) { - fd = (lcb_socket_t)((pycbc_IOEvent*)ev)->fd; - } - Py_INCREF(ev); - parent = ev->parent; - Py_XINCREF(parent); - ev->cb.handler(fd, which, ev->cb.data); - Py_XDECREF(parent); - Py_DECREF(ev); -} - -#define READY_RETURN() \ - if (PyErr_Occurred()) { return NULL; } Py_RETURN_NONE - -/** - * e.g.: - * event.event_received(PYCBC_LCB_READ_EVENT) - */ -static PyObject * -Event_on_ready(pycbc_Event *ev, PyObject *args) -{ - short flags; - int rv; - - rv = PyArg_ParseTuple(args, "h", &flags); - if (!rv) { - return NULL; - } - - event_fire_common(ev, flags); - READY_RETURN(); -} - -static PyObject * -Event_on_read(pycbc_Event *ev) -{ - event_fire_common(ev, LCB_READ_EVENT); - READY_RETURN(); -} - -static PyObject * -Event_on_write(pycbc_Event *ev) -{ - event_fire_common(ev, LCB_WRITE_EVENT); - READY_RETURN(); -} - -static PyObject * -Event_on_readwrite(pycbc_Event *ev) -{ - event_fire_common(ev, LCB_RW_EVENT); - READY_RETURN(); -} - -static PyObject * -IOEvent_fileno(pycbc_IOEvent *self, PyObject *args) -{ - (void)args; - return pycbc_IntFromL((lcb_socket_t)self->fd); -} - -static PyObject * -IOEvent__repr__(pycbc_IOEvent *self) -{ - return PyUnicode_FromFormat("%s<fd=%lu,flags=0x%x @%p>", - Py_TYPE(self)->tp_name, - (unsigned long)self->fd, (int)self->flags, self); -} - - -static struct PyMethodDef pycbc_Event_TABLE_methods[] = { - { "ready", - (PyCFunction)Event_on_ready, - METH_VARARGS, - PyDoc_STR("Called when an event is ready") - }, - - { "ready_r", - (PyCFunction)Event_on_read, - METH_NOARGS, - PyDoc_STR("Called for read events. This is the efficient \n" - "form of ``ready(LCB_READ_EVENT)``\n") - }, - - { "ready_w", - (PyCFunction)Event_on_write, - METH_NOARGS, - PyDoc_STR("Called for write events. This is equivalent to\n" - "``ready(LCB_WRITE_EVENT)``\n") - }, - - { "ready_rw", - (PyCFunction)Event_on_readwrite, - METH_NOARGS, - PyDoc_STR("Called for rw events. This is equivalent to\n" - "``ready(LCB_READ_EVENT|LCB_WRITE_EVENT)``\n") - }, - - { NULL } -}; - -static struct PyMethodDef pycbc_IOEvent_TABLE_methods[] = { - { "fileno", (PyCFunction)IOEvent_fileno, METH_VARARGS }, - { NULL } -}; - -static int -Event__init__(pycbc_Event *self, PyObject *args, PyObject *kwargs) -{ - if (PyBaseObject_Type.tp_init((PyObject *)self, args, kwargs) != 0) { - return -1; - } - if (!self->vdict) { - self->vdict = PyDict_New(); - } - return 0; -} - -static int -Event_gc_traverse(pycbc_Event *ev, visitproc visit, void *arg) -{ - Py_VISIT(ev->vdict); - Py_VISIT(ev->parent); - return 0; -} - -static void -Event_gc_clear(pycbc_Event *ev) -{ - Py_CLEAR(ev->vdict); - Py_CLEAR(ev->parent); -} - -static void -Event_dealloc(pycbc_Event *self) -{ - Event_gc_clear(self); - Py_TYPE(self)->tp_free((PyObject*)self); -} - -#define SET_EVENT_GCFUNCS(type) do {\ - (type)->tp_flags |= Py_TPFLAGS_HAVE_GC; \ - (type)->tp_traverse = (traverseproc)Event_gc_traverse; \ - (type)->tp_clear = (inquiry)Event_gc_clear; \ - (type)->tp_dealloc = (destructor)Event_dealloc; \ -} while (0); - -int -pycbc_EventType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_EventType; - *ptr = (PyObject*)p; - - if (p->tp_name) { - return 0; - } - - p->tp_name = "Event"; - p->tp_doc = PyDoc_STR("Internal event handle"); - p->tp_new = PyType_GenericNew; - p->tp_basicsize = sizeof(pycbc_Event); - p->tp_members = pycbc_Event_TABLE_members; - p->tp_methods = pycbc_Event_TABLE_methods; - p->tp_flags = Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE; - p->tp_init = (initproc)Event__init__; - p->tp_dictoffset = offsetof(pycbc_Event, vdict); - - SET_EVENT_GCFUNCS(p); - return PyType_Ready(p); -} - -int -pycbc_IOEventType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_IOEventType; - *ptr = (PyObject*)p; - if (p->tp_name) { - return 0; - } - p->tp_name = "IOEvent"; - p->tp_new = PyType_GenericNew; - p->tp_basicsize = sizeof(pycbc_IOEvent); - p->tp_members = pycbc_IOEvent_TABLE_members; - p->tp_methods = pycbc_IOEvent_TABLE_methods; - p->tp_repr = (reprfunc)IOEvent__repr__; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_base = &pycbc_EventType; - - SET_EVENT_GCFUNCS(p); - return PyType_Ready(p); -} - -int -pycbc_TimerEventType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_TimerEventType; - *ptr = (PyObject*)p; - if (p->tp_name) { - return 0; - } - p->tp_name = "TimerEvent"; - p->tp_new = PyType_GenericNew; - p->tp_basicsize = sizeof(pycbc_TimerEvent); - p->tp_members = pycbc_TimerEvent_TABLE_members; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_base = &pycbc_EventType; - - SET_EVENT_GCFUNCS(p); - return PyType_Ready(p); -} - -static int -IOPSWrapper_traverse(pycbc_IOPSWrapper *self, visitproc visit, void *arg) -{ - #define X(n, ign) Py_VISIT(self->n); - XIONAME_CACHENTRIES(X); - #undef X - Py_VISIT(self->parent); - Py_VISIT(self->pyio); - return 0; -} - -static void -IOPSWrapper_clear(pycbc_IOPSWrapper *self) -{ - #define X(n, ign) Py_CLEAR(self->n); - XIONAME_CACHENTRIES(X); - #undef X - Py_CLEAR(self->parent); - Py_CLEAR(self->pyio); -} - -static void -IOPSWrapper_dealloc(pycbc_IOPSWrapper *self) -{ - IOPSWrapper_clear(self); - free(self->iops); - - Py_TYPE(self)->tp_free((PyObject*)self); -} - -int -pycbc_IOPSWrapperType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_IOPSWrapperType; - *ptr = (PyObject *)p; - if (p->tp_name) { - return 0; - } - - p->tp_name = "_IOPSWrapper"; - p->tp_new = PyType_GenericNew; - p->tp_basicsize = sizeof(pycbc_IOPSWrapper); - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC; - p->tp_dealloc = (destructor)IOPSWrapper_dealloc; - p->tp_traverse = (traverseproc)IOPSWrapper_traverse; - p->tp_clear = (inquiry)IOPSWrapper_clear; - - return PyType_Ready(p); -} - -static int -modify_event_python(pycbc_IOPSWrapper *pio, pycbc_Event *ev, - pycbc_evaction_t action, lcb_socket_t newsock, void *arg) -{ - int ret; - PyObject *result; - PyObject *o_arg; - PyObject *meth = NULL; - PyObject *argtuple; - - short flags = 0; - unsigned long usecs = 0; - - argtuple = PyTuple_New(3); - Py_INCREF((PyObject *)ev); - PyTuple_SET_ITEM(argtuple, 0, (PyObject *)ev); - PyTuple_SET_ITEM(argtuple, 1, pycbc_IntFromL(action)); - - if (ev->type == PYCBC_EVTYPE_IO) { - flags = *(short*)arg; - o_arg = pycbc_IntFromL(flags); - ((pycbc_IOEvent *)ev)->fd = newsock; - meth = pio->modevent; - - } else { - usecs = *(lcb_uint32_t*)arg; - o_arg = pycbc_IntFromL(usecs); - meth = pio->modtimer; - } - PyTuple_SET_ITEM(argtuple, 2, o_arg); - - result = PyObject_CallObject(meth, argtuple); - Py_DECREF(argtuple); - Py_XDECREF(result); - - if (ev->type == PYCBC_EVTYPE_IO) { - pycbc_IOEvent *evio = (pycbc_IOEvent*)ev; - evio->flags = flags; - } - - if (action == PYCBC_EVACTION_WATCH) { - ev->state = PYCBC_EVSTATE_ACTIVE; - } else { - ev->state = PYCBC_EVSTATE_SUSPENDED; - } - - if (!result) { - ret = -1; - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "Couldn't invoke IO Function"); - } else { - ret = 0; - } - - return ret; -} - -/** - * Begin GLUE - */ -static void * -create_event_python(lcb_io_opt_t io, pycbc_evtype_t evtype) -{ - PyObject *meth, *ret; - PyTypeObject *defltype; - pycbc_IOPSWrapper *pio = PYCBC_IOW_FROM_IOPS(io); - - if (evtype == PYCBC_EVTYPE_IO) { - defltype = &pycbc_IOEventType; - meth = pio->mkevent; - - } else { - defltype = &pycbc_TimerEventType; - meth = pio->mktimer; - } - - if (meth) { - ret = PyObject_CallObject(meth, NULL); - if (!ret) { - PyErr_PrintEx(0); - abort(); - } - - } else { - PyErr_Clear(); - ret = PYCBC_TYPE_CTOR(defltype); - } - - ((pycbc_Event *)ret)->type = evtype; - ((pycbc_Event *)ret)->parent = (PyObject *)pio; - Py_INCREF(pio); - return ret; -} - -static void * -create_event(lcb_io_opt_t io) -{ - return create_event_python(io, PYCBC_EVTYPE_IO); -} - -static void * -create_timer(lcb_io_opt_t io) -{ - return create_event_python(io, PYCBC_EVTYPE_TIMER); -} - -static void -destroy_event_common(lcb_io_opt_t io, void *arg) -{ - pycbc_Event *ev = arg; - lcb_U32 dummy = 0; - pycbc_assert(ev->state != PYCBC_EVSTATE_ACTIVE); - - modify_event_python(PYCBC_IOW_FROM_IOPS(io), ev, PYCBC_EVACTION_CLEANUP, - 0, &dummy); - - ev->state = PYCBC_EVSTATE_FREED; - Py_DECREF(ev); -} - -static int -update_event(lcb_io_opt_t io, lcb_socket_t sock, void *event, short flags, - void *data, pycbc_lcb_cb_t handler) -{ - pycbc_IOEvent *ev = (pycbc_IOEvent*)event; - pycbc_evaction_t action; - pycbc_evstate_t new_state; - - if (!flags) { - action = PYCBC_EVACTION_UNWATCH; - new_state = PYCBC_EVSTATE_SUSPENDED; - - } else { - action = PYCBC_EVACTION_WATCH; - new_state = PYCBC_EVSTATE_ACTIVE; - } - - ev->cb.handler = handler; - ev->cb.data = data; - - if (ev->flags == flags && new_state == ev->state && ev->fd == sock) { - return 0; - } - - return modify_event_python(PYCBC_IOW_FROM_IOPS(io), (pycbc_Event*)ev, - action, sock, &flags); -} - -static void -delete_event(lcb_io_opt_t io, lcb_socket_t sock, void *event) -{ - pycbc_Event *ev = (pycbc_Event*)event; - pycbc_IOPSWrapper *pio = PYCBC_IOW_FROM_IOPS(io); - short tmp = 0; - - modify_event_python(pio, ev, PYCBC_EVACTION_UNWATCH, sock, &tmp); -} - -static void -delete_timer(lcb_io_opt_t io, void *timer) -{ - lcb_U32 dummy = 0; - pycbc_IOPSWrapper *pio = PYCBC_IOW_FROM_IOPS(io); - modify_event_python(pio, (pycbc_Event*)timer, PYCBC_EVACTION_UNWATCH, -1, - &dummy); -} - -static int -update_timer(lcb_io_opt_t io, void *timer, lcb_U32 usec, void *data, - pycbc_lcb_cb_t handler) -{ - pycbc_TimerEvent *ev = (pycbc_TimerEvent*)timer; - ev->cb.data = data; - ev->cb.handler = handler; - - return modify_event_python(PYCBC_IOW_FROM_IOPS(io), (pycbc_Event*)ev, - PYCBC_EVACTION_WATCH, -1, &usec); -} - -static void -run_event_loop(lcb_io_opt_t io) -{ - pycbc_IOPSWrapper *pio = PYCBC_IOW_FROM_IOPS(io); - pio->in_loop = 1; - PyObject_CallFunctionObjArgs(pio->startwatch, NULL); -} - -static void -stop_event_loop(lcb_io_opt_t io) -{ - pycbc_IOPSWrapper *pio = PYCBC_IOW_FROM_IOPS(io); - pio->in_loop = 0; - PyObject_CallFunctionObjArgs(pio->stopwatch, NULL); -} - -static void -iops_destructor(lcb_io_opt_t io) -{ - /* Empty. The IOPS object is not scoped by the library */ - (void)io; -} - -static int -load_cached_method(PyObject *obj, - PyObject *attr, PyObject **target, int optional) -{ - *target = PyObject_GetAttr(obj, attr); - if (*target) { - if (!PyCallable_Check(*target)) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Invalid IOPS object", obj); - return -1; - } - return 0; - } - - if (optional) { - PyErr_Clear(); - return 0; - } - - return -1; -} - -static int -cache_io_methods(pycbc_IOPSWrapper *pio, PyObject *obj) -{ - -#define X(b, is_optional) \ - if (load_cached_method(obj, pycbc_helpers.ioname_##b, &pio->b, \ - is_optional) == -1) { \ - return -1; \ - } - - XIONAME_CACHENTRIES(X) - return 0; -#undef X -#undef XIONAME_CACHENTRIES -} - -static void iops_getprocs(int version, - lcb_loop_procs *loop_procs, lcb_timer_procs *timer_procs, - lcb_bsd_procs *bsd_procs, lcb_ev_procs *ev_procs, - lcb_completion_procs *completion_procs, lcb_iomodel_t *iomodel) { - - /* Call the parent function */ - lcb_iops_wire_bsd_impl2(bsd_procs, version); - - /* Now apply our new I/O functionality */ - ev_procs->create = create_event; - ev_procs->destroy = destroy_event_common; - ev_procs->watch = update_event; - ev_procs->cancel = delete_event; - - timer_procs->create = create_timer; - timer_procs->destroy = destroy_event_common; - timer_procs->schedule = update_timer; - timer_procs->cancel = delete_timer; - - loop_procs->start = run_event_loop; - loop_procs->stop = stop_event_loop; -} - - -PyObject * -pycbc_iowrap_new(pycbc_Bucket *unused, PyObject *pyio) -{ - lcb_io_opt_t iops = NULL; - pycbc_IOPSWrapper *wrapper; - - (void)unused; - - wrapper = (pycbc_IOPSWrapper *)PYCBC_TYPE_CTOR(&pycbc_IOPSWrapperType); - wrapper->pyio = pyio; - Py_INCREF(pyio); - - iops = calloc(1, sizeof(*iops)); - iops->dlhandle = NULL; - iops->destructor = iops_destructor; - iops->version = 2; - iops->v.v2.get_procs = iops_getprocs; - - LCB_IOPS_BASEFLD(iops, cookie) = wrapper; - wrapper->iops = iops; - - if (-1 == cache_io_methods(wrapper, pyio)) { - return NULL; - } - return (PyObject *)wrapper; -} - -lcb_io_opt_t -pycbc_iowrap_getiops(PyObject *iowrap) -{ - pycbc_assert(Py_TYPE(iowrap) == &pycbc_IOPSWrapperType); - return ((pycbc_IOPSWrapper*)iowrap)->iops; -} diff --git a/src/iops.h b/src/iops.h deleted file mode 100644 index 2b00f9935..000000000 --- a/src/iops.h +++ /dev/null @@ -1,101 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#ifndef PYCBC_IOPS_H -#define PYCBC_IOPS_H -#include "pycbc.h" - -typedef enum { - /** Activate the event so that it may fire upon the trigger */ - PYCBC_EVACTION_WATCH = 1 << 0, - - /** Deactive the event; ignoring the trigger */ - PYCBC_EVACTION_UNWATCH = 1 << 1, - - /** Unused for now */ - PYCBC_EVACTION_SUSPEND = 1 << 2, - - /** Unused for now */ - PYCBC_EVACTION_RESUME = 1 << 3, - - /** Cleanup the event, removing all references of it */ - PYCBC_EVACTION_CLEANUP = 1 << 4 -} pycbc_evaction_t; - -typedef enum { - PYCBC_EVSTATE_INITIALIZED, - PYCBC_EVSTATE_ACTIVE, - PYCBC_EVSTATE_SUSPENDED, - PYCBC_EVSTATE_FREED -} pycbc_evstate_t; - -typedef enum { - PYCBC_EVTYPE_IO, - PYCBC_EVTYPE_TIMER -} pycbc_evtype_t; - -typedef void (*pycbc_lcb_cb_t)(lcb_socket_t,short,void*); - -#define pycbc_Event_HEAD \ - PyObject_HEAD \ - struct { \ - pycbc_lcb_cb_t handler; \ - void *data; \ - } cb; \ - PyObject *vdict; \ - PyObject *parent; \ - pycbc_evstate_t state; \ - pycbc_evtype_t type; - -typedef struct { - pycbc_Event_HEAD -} pycbc_Event; - -typedef struct { - pycbc_Event_HEAD - PY_LONG_LONG fd; - short flags; -} pycbc_IOEvent; - -typedef struct { - pycbc_Event_HEAD -} pycbc_TimerEvent; - -/** Wrapper around the IOPS structure. */ -typedef struct pycbc_IOPSWrapper_st { - PyObject_HEAD - - /** LCB's iops */ - struct lcb_io_opt_st *iops; - - PyObject *pyio; - pycbc_Bucket *parent; - - /** Whether the loop is currently active */ - int in_loop; - - /** Cached method references */ - PyObject *mkevent; - PyObject *mktimer; - PyObject *modevent; - PyObject *modtimer; - PyObject *startwatch; - PyObject *stopwatch; -} pycbc_IOPSWrapper; - -#define PYCBC_IOW_FROM_IOPS(p) LCB_IOPS_BASEFLD(p, cookie) - -#endif diff --git a/src/kv_ops.cxx b/src/kv_ops.cxx new file mode 100644 index 000000000..5ddaa65dd --- /dev/null +++ b/src/kv_ops.cxx @@ -0,0 +1,1375 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_ops.hxx" + +#include "exceptions.hxx" +#include "result.hxx" +#include "tracing.hxx" +#include "utils.hxx" + +template<typename T> +result* +add_extras_to_result([[maybe_unused]] const T& resp, result* res) +{ + return res; +} + +template<typename T> +result* +add_flags_and_value_to_result(const T& resp, result* res) +{ + PyObject* pyObj_tmp = PyLong_FromUnsignedLong(resp.flags); + if (-1 == PyDict_SetItemString(res->dict, RESULT_FLAGS, pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_XDECREF(pyObj_tmp); + + try { + pyObj_tmp = binary_to_PyObject(resp.value); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + return nullptr; + } + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + return res; +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::exists_response>( + const couchbase::core::operations::exists_response& resp, + result* res) +{ + PyObject* pyObj_tmp = PyBool_FromLong(static_cast<long>(resp.exists())); + if (-1 == PyDict_SetItemString(res->dict, RESULT_EXISTS, pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + return res; +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::get_projected_response>( + const couchbase::core::operations::get_projected_response& resp, + result* res) +{ + if (resp.expiry) { + PyObject* pyObj_tmp = PyLong_FromUnsignedLong(resp.expiry.value()); + if (-1 == PyDict_SetItemString(res->dict, RESULT_EXPIRY, pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + return add_flags_and_value_to_result(resp, res); +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::get_response>( + const couchbase::core::operations::get_response& resp, + result* res) +{ + return add_flags_and_value_to_result(resp, res); +} +template<> +result* +add_extras_to_result<couchbase::core::operations::get_and_lock_response>( + const couchbase::core::operations::get_and_lock_response& resp, + result* res) +{ + return add_flags_and_value_to_result(resp, res); +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::get_and_touch_response>( + const couchbase::core::operations::get_and_touch_response& resp, + result* res) +{ + return add_flags_and_value_to_result(resp, res); +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::get_any_replica_response>( + const couchbase::core::operations::get_any_replica_response& resp, + result* res) +{ + if (-1 == PyDict_SetItemString(res->dict, "is_replica", resp.replica ? Py_True : Py_False)) { + return nullptr; + } + + return add_flags_and_value_to_result(resp, res); +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::get_all_replicas_response::entry>( + const couchbase::core::operations::get_all_replicas_response::entry& resp, + result* res) +{ + if (-1 == PyDict_SetItemString(res->dict, "is_replica", resp.replica ? Py_True : Py_False)) { + return nullptr; + } + + return add_flags_and_value_to_result(resp, res); +} + +template<typename T> +result* +create_base_result_from_get_operation_response(const char* key, const T& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(resp.cas.value()); + if (-1 == PyDict_SetItemString(res->dict, RESULT_CAS, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (nullptr != key) { + pyObj_tmp = PyUnicode_FromString(key); + if (-1 == PyDict_SetItemString(res->dict, RESULT_KEY, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + return res; +} + +template<> +result* +create_base_result_from_get_operation_response< + couchbase::core::operations::get_all_replicas_response::entry>( + const char* key, + const couchbase::core::operations::get_all_replicas_response::entry& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(resp.cas.value()); + if (-1 == PyDict_SetItemString(res->dict, RESULT_CAS, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (nullptr != key) { + pyObj_tmp = PyUnicode_FromString(key); + if (-1 == PyDict_SetItemString(res->dict, RESULT_KEY, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + return res; +} + +template<typename T> +void +create_result_from_get_operation_response(const char* key, + const T& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* pyObj_args = NULL; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + if (resp.ctx.ec()) { + pyObj_exc = + build_exception_from_context(resp.ctx, __FILE__, __LINE__, "KV read operation error."); + if (pyObj_errback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_False); + barrier->set_value(Py_False); + if (-1 == PyDict_SetItemString(multi_result->dict, key, pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + // won't fall into logic path where pyObj_exc is decremented later + Py_DECREF(pyObj_exc); + } else { + barrier->set_value(pyObj_exc); + } + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_base_result_from_get_operation_response(key, resp); + if (res != nullptr) { + res = add_extras_to_result(resp, res); + } + + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_True); + barrier->set_value(Py_True); + if (-1 == + PyDict_SetItemString(multi_result->dict, key, reinterpret_cast<PyObject*>(res))) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(reinterpret_cast<PyObject*>(res)); + } else { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "KV read operation error."); + if (pyObj_errback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_False); + barrier->set_value(Py_False); + if (-1 == PyDict_SetItemString(multi_result->dict, key, pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + // won't fall into logic path where pyObj_exc is decremented later + Py_DECREF(pyObj_exc); + } else { + barrier->set_value(pyObj_exc); + } + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<> +void +create_result_from_get_operation_response<couchbase::core::operations::get_all_replicas_response>( + const char* key, + const couchbase::core::operations::get_all_replicas_response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* pyObj_args = NULL; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + streamed_result* streamed_res = nullptr; + + if (resp.ctx.ec()) { + pyObj_exc = + build_exception_from_context(resp.ctx, __FILE__, __LINE__, "KV read operation error."); + if (pyObj_errback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_False); + barrier->set_value(Py_False); + if (-1 == PyDict_SetItemString(multi_result->dict, key, pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + // won't fall into logic path where pyObj_exc is decremented later + Py_DECREF(pyObj_exc); + } else { + barrier->set_value(pyObj_exc); + } + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto streamed_res = + create_streamed_result_obj(couchbase::core::timeout_defaults::key_value_durable_timeout); + for (auto const& entry : resp.entries) { + auto res = create_base_result_from_get_operation_response(key, entry); + if (res == nullptr) { + set_exception = true; + break; + } + res = add_extras_to_result(entry, res); + streamed_res->rows->put(reinterpret_cast<PyObject*>(res)); + } + + if (PyErr_Occurred() != nullptr) { + set_exception = true; + } else if (!set_exception) { + Py_INCREF(Py_None); + streamed_res->rows->put(Py_None); + if (pyObj_callback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_True); + barrier->set_value(Py_True); + if (-1 == PyDict_SetItemString( + multi_result->dict, key, reinterpret_cast<PyObject*>(streamed_res))) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(reinterpret_cast<PyObject*>(streamed_res)); + } else { + barrier->set_value(reinterpret_cast<PyObject*>(streamed_res)); + } + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(streamed_res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "KV read operation error."); + streamed_res->rows->put(pyObj_exc); + if (pyObj_errback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_False); + barrier->set_value(Py_False); + if (-1 == PyDict_SetItemString( + multi_result->dict, key, reinterpret_cast<PyObject*>(streamed_res))) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + // won't fall into logic path where pyObj_exc is decremented later + Py_DECREF(pyObj_exc); + } else { + barrier->set_value(reinterpret_cast<PyObject*>(streamed_res)); + } + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(streamed_res)); + } + } else if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<typename Request> +void +do_get(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, + [key = req.id.key(), pyObj_callback, pyObj_errback, barrier, multi_result](response_type resp) { + create_result_from_get_operation_response( + key.c_str(), resp, pyObj_callback, pyObj_errback, barrier, multi_result); + }); + Py_END_ALLOW_THREADS +} + +PyObject* +prepare_and_execute_read_op(struct read_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + switch (options->op_type) { + case Operations::GET: { + couchbase::core::operations::get_request req{ options->id }; + req.timeout = options->timeout_ms; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_get<couchbase::core::operations::get_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::GET_PROJECTED: { + std::vector<std::string> projections; + if (nullptr != options->project) { + // this needs to be a list... + bool ok = !!PyList_Check(options->project); + if (ok) { + // and the list needs to all be strings + for (Py_ssize_t i = 0; ok && i < PyList_Size(options->project); i++) { + PyObject* pyObj_projection = PyList_GetItem(options->project, i); + // PyList_GetItem returns borrowed ref, inc while using, decr after done + Py_INCREF(pyObj_projection); + ok = !!PyUnicode_Check(pyObj_projection); + // lets build the projections vector we will use in the request, since + // we are iterating... + projections.emplace_back(PyUnicode_AsUTF8(pyObj_projection)); + Py_DECREF(pyObj_projection); + pyObj_projection = nullptr; + } + } + if (!ok) { + if (multi_result != nullptr) { + PyObject* pyObj_exc = pycbc_build_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Project must be a list of strings."); + if (-1 == + PyDict_SetItemString(multi_result->dict, options->id.key().c_str(), pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_exc); + Py_INCREF(Py_False); + barrier->set_value(Py_False); + } else { + if (barrier) { + barrier->set_value(nullptr); + } + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Project must be a list of strings."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + } + + couchbase::core::operations::get_projected_request req{ options->id }; + req.timeout = options->timeout_ms; + req.with_expiry = !!options->with_expiry; + req.projections = projections; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_get<couchbase::core::operations::get_projected_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::GET_ANY_REPLICA: { + couchbase::core::operations::get_any_replica_request req{ options->id, + options->timeout_ms, + options->read_preference }; + do_get<couchbase::core::operations::get_any_replica_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::GET_ALL_REPLICAS: { + couchbase::core::operations::get_all_replicas_request req{ options->id, + options->timeout_ms, + options->read_preference }; + do_get<couchbase::core::operations::get_all_replicas_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::GET_AND_TOUCH: { + couchbase::core::operations::get_and_touch_request req{ options->id }; + req.expiry = options->expiry; + req.timeout = options->timeout_ms; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_get<couchbase::core::operations::get_and_touch_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::GET_AND_LOCK: { + couchbase::core::operations::get_and_lock_request req{ options->id }; + req.lock_time = options->lock_time; + req.timeout = options->timeout_ms; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_get<couchbase::core::operations::get_and_lock_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::EXISTS: { + couchbase::core::operations::exists_request req{ options->id }; + req.timeout = options->timeout_ms; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_get<couchbase::core::operations::exists_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::TOUCH: { + couchbase::core::operations::touch_request req{ options->id }; + req.expiry = options->expiry; + req.timeout = options->timeout_ms; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_get<couchbase::core::operations::touch_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::UNLOCK: { + couchbase::core::operations::unlock_request req{ options->id }; + req.cas = options->cas; + req.timeout = options->timeout_ms; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_get<couchbase::core::operations::unlock_request>( + *(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + default: { + if (multi_result != nullptr) { + PyObject* pyObj_exc = pycbc_build_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unrecognized get operation passed in."); + if (-1 == PyDict_SetItemString(multi_result->dict, options->id.key().c_str(), pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_exc); + Py_INCREF(Py_False); + barrier->set_value(Py_False); + break; + } else { + if (barrier) { + barrier->set_value(nullptr); + } + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unrecognized get operation passed in."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + }; + Py_RETURN_NONE; +} + +template<typename Response> +result* +create_base_result_from_mutation_operation_response(const char* key, const Response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(resp.cas.value()); + if (-1 == PyDict_SetItemString(res->dict, RESULT_CAS, pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (nullptr != key) { + pyObj_tmp = PyUnicode_FromString(key); + if (-1 == PyDict_SetItemString(res->dict, RESULT_KEY, pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyObject* pyObj_mutation_token = create_mutation_token_obj(resp.token); + if (-1 == PyDict_SetItemString(res->dict, RESULT_MUTATION_TOKEN, pyObj_mutation_token)) { + Py_XDECREF(pyObj_mutation_token); + return nullptr; + } + Py_DECREF(pyObj_mutation_token); + + return res; +} + +template<typename Response> +void +create_result_from_mutation_operation_response(const char* key, + const Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + if (resp.ctx.ec()) { + pyObj_exc = + build_exception_from_context(resp.ctx, __FILE__, __LINE__, "KV mutation operation error."); + if (pyObj_errback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_False); + barrier->set_value(Py_False); + if (-1 == PyDict_SetItemString(multi_result->dict, key, pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + // won't fall into logic path where pyObj_exc is decremented later + Py_DECREF(pyObj_exc); + } else { + barrier->set_value(pyObj_exc); + } + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_base_result_from_mutation_operation_response(key, resp); + if (res != nullptr) { + res = add_extras_to_result(resp, res); + } + + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_True); + barrier->set_value(Py_True); + if (-1 == + PyDict_SetItemString(multi_result->dict, key, reinterpret_cast<PyObject*>(res))) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(reinterpret_cast<PyObject*>(res)); + } else { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "KV mutation operation error."); + if (pyObj_errback == nullptr) { + if (multi_result != nullptr) { + Py_INCREF(Py_False); + barrier->set_value(Py_False); + if (-1 == PyDict_SetItemString(multi_result->dict, key, pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + // won't fall into logic path where pyObj_exc is decremented later + Py_DECREF(pyObj_exc); + } else { + barrier->set_value(pyObj_exc); + } + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } else if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + pycbc_set_python_exception( + PycbcError::InternalSDKError, __FILE__, __LINE__, "Mutation operation callback failed."); + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + // CB_LOG_DEBUG("{}: create mutation callback completed", "PYCBC"); + PyGILState_Release(state); +} + +template<typename Request> +void +do_mutation(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + result* multi_result = nullptr) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, + [key = req.id.key(), pyObj_callback, pyObj_errback, barrier, multi_result](response_type resp) { + create_result_from_mutation_operation_response( + key.c_str(), resp, pyObj_callback, pyObj_errback, barrier, multi_result); + }); + Py_END_ALLOW_THREADS +} + +PyObject* +prepare_and_execute_mutation_op(struct mutation_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier = nullptr, + result* multi_result = nullptr) +{ + // **DO NOT DECREF** these -- content from tuples are borrowed references!! + PyObject* pyObj_flags = nullptr; + PyObject* pyObj_value = nullptr; + couchbase::core::utils::binary value; + + if (options->value) { + pyObj_value = PyTuple_GET_ITEM(options->value, 0); + pyObj_flags = PyTuple_GET_ITEM(options->value, 1); + try { + value = PyObject_to_binary(pyObj_value); + } catch (const std::exception& e) { + if (multi_result != nullptr) { + PyObject* pyObj_exc = + pycbc_build_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, e.what()); + if (-1 == PyDict_SetItemString(multi_result->dict, options->id.key().c_str(), pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_exc); + Py_INCREF(Py_False); + barrier->set_value(Py_False); + Py_RETURN_NONE; + } + if (barrier) { + barrier->set_value(nullptr); + } + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, e.what()); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + + switch (options->op_type) { + case Operations::INSERT: { + auto req = couchbase::core::operations::insert_request{ options->id }; + req.timeout = options->timeout_ms; + req.value = value; + req.flags = static_cast<uint32_t>(PyLong_AsLong(pyObj_flags)); + if (options->expiry > 0) { + req.expiry = options->expiry; + } + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::insert_request_with_legacy_durability{ + req, options->persist_to, options->replicate_to + }; + do_mutation(*(options->conn), + req_legacy_durability, + pyObj_callback, + pyObj_errback, + barrier, + multi_result); + break; + } + req.durability_level = options->durability_level; + do_mutation(*(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::UPSERT: { + auto req = couchbase::core::operations::upsert_request{ options->id }; + req.timeout = options->timeout_ms; + req.value = value; + req.flags = static_cast<uint32_t>(PyLong_AsLong(pyObj_flags)); + if (options->expiry > 0) { + req.expiry = options->expiry; + } + if (options->preserve_expiry) { + req.preserve_expiry = options->preserve_expiry; + } + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::upsert_request_with_legacy_durability{ + req, options->persist_to, options->replicate_to + }; + do_mutation(*(options->conn), + req_legacy_durability, + pyObj_callback, + pyObj_errback, + barrier, + multi_result); + break; + } + req.durability_level = options->durability_level; + do_mutation(*(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::REPLACE: { + auto req = couchbase::core::operations::replace_request{ options->id }; + req.timeout = options->timeout_ms; + req.cas = options->cas; + req.value = value; + req.flags = static_cast<uint32_t>(PyLong_AsLong(pyObj_flags)); + if (options->expiry > 0) { + req.expiry = options->expiry; + } + if (options->preserve_expiry) { + req.preserve_expiry = options->preserve_expiry; + } + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::replace_request_with_legacy_durability{ + req, options->persist_to, options->replicate_to + }; + do_mutation(*(options->conn), + req_legacy_durability, + pyObj_callback, + pyObj_errback, + barrier, + multi_result); + break; + } + req.durability_level = options->durability_level; + do_mutation(*(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + case Operations::REMOVE: { + auto req = couchbase::core::operations::remove_request{ options->id }; + req.timeout = options->timeout_ms; + req.cas = options->cas; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::remove_request_with_legacy_durability{ + req, options->persist_to, options->replicate_to + }; + do_mutation(*(options->conn), + req_legacy_durability, + pyObj_callback, + pyObj_errback, + barrier, + multi_result); + break; + } + req.durability_level = options->durability_level; + do_mutation(*(options->conn), req, pyObj_callback, pyObj_errback, barrier, multi_result); + break; + } + default: { + if (multi_result != nullptr) { + PyObject* pyObj_exc = pycbc_build_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized mutation operation passed in."); + if (-1 == PyDict_SetItemString(multi_result->dict, options->id.key().c_str(), pyObj_exc)) { + // TODO: not much we can do here...maybe? + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_exc); + Py_INCREF(Py_False); + barrier->set_value(Py_False); + break; + } + + if (barrier) { + barrier->set_value(nullptr); + } + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized mutation operation passed in."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + Py_RETURN_NONE; +} + +struct read_options +get_read_options(PyObject* op_args) +{ + struct read_options opts { + }; + + PyObject* pyObj_span = PyDict_GetItemString(op_args, "span"); + if (pyObj_span != nullptr) { + opts.span = pyObj_span; + } + + PyObject* pyObj_expiry = PyDict_GetItemString(op_args, "expiry"); + if (pyObj_expiry != nullptr) { + auto expiry = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_expiry)); + opts.expiry = expiry; + } + + PyObject* pyObj_cas = PyDict_GetItemString(op_args, "cas"); + couchbase::cas cas = couchbase::cas{ 0 }; + if (pyObj_cas != nullptr) { + auto cas_int = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_cas)); + if (cas_int != 0) { + cas = couchbase::cas{ cas_int }; + } + } + opts.cas = cas; + + PyObject* pyObj_lock_time = PyDict_GetItemString(op_args, "lock_time"); + if (pyObj_lock_time != nullptr) { + auto lock_time = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_lock_time)); + opts.lock_time = lock_time; + } + + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (pyObj_timeout != nullptr) { + auto timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_timeout)); + timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + if (0 < timeout) { + opts.timeout_ms = timeout_ms; + } + } + + PyObject* pyObj_with_expiry = PyDict_GetItemString(op_args, "with_expiry"); + opts.with_expiry = pyObj_with_expiry != nullptr && pyObj_with_expiry == Py_True ? true : false; + + PyObject* pyObj_read_preference = PyDict_GetItemString(op_args, "read_preference"); + if (pyObj_read_preference != nullptr) { + opts.read_preference = PyObject_to_read_preference(pyObj_read_preference); + } + + return opts; +} + +mutation_options +get_mutation_options(PyObject* op_args) +{ + struct mutation_options opts; + + PyObject* pyObj_span = PyDict_GetItemString(op_args, "span"); + if (pyObj_span != nullptr) { + opts.span = pyObj_span; + } + + PyObject* pyObj_expiry = PyDict_GetItemString(op_args, "expiry"); + if (pyObj_expiry != nullptr) { + opts.expiry = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_expiry)); + } + + PyObject* pyObj_cas = PyDict_GetItemString(op_args, "cas"); + couchbase::cas cas = couchbase::cas{ 0 }; + if (pyObj_cas != nullptr) { + auto cas_int = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_cas)); + if (cas_int != 0) { + cas = couchbase::cas{ cas_int }; + } + } + opts.cas = cas; + + PyObject* pyObj_preserve_expiry = PyDict_GetItemString(op_args, "preserve_expiry"); + opts.preserve_expiry = + pyObj_preserve_expiry != nullptr && pyObj_preserve_expiry == Py_True ? true : false; + + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (pyObj_timeout != nullptr) { + auto timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_timeout)); + timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + if (0 < timeout) { + opts.timeout_ms = timeout_ms; + } + } + + PyObject* pyObj_durability = PyDict_GetItemString(op_args, "durability"); + if (pyObj_durability) { + if (PyDict_Check(pyObj_durability)) { + auto durability = PyObject_to_durability(pyObj_durability); + opts.use_legacy_durability = true; + opts.persist_to = durability.first; + opts.replicate_to = durability.second; + } else if (PyLong_Check(pyObj_durability)) { + opts.durability_level = PyObject_to_durability_level(pyObj_durability); + } + } + + return opts; +} + +PyObject* +handle_kv_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + char* bucket = nullptr; + char* scope = nullptr; + char* collection = nullptr; + char* key = nullptr; + Operations::OperationType op_type = Operations::UNKNOWN; + PyObject* pyObj_value = nullptr; + PyObject* pyObj_op_args = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + + static const char* kw_list[] = { "conn", "bucket", "scope", "collection_name", "key", "op_type", + "value", "op_args", nullptr }; + + const char* kw_format = "O!ssssI|OO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &bucket, + &scope, + &collection, + &key, + &op_type, + &pyObj_value, + &pyObj_op_args); + if (!ret) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform kv operation. Unable to parse args/kwargs."); + return nullptr; + } + + connection* conn = nullptr; + + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback, pyObj_transcoder }; + pyObj_callback = PyDict_GetItemString(pyObj_op_args, "callback"); + pyObj_errback = PyDict_GetItemString(pyObj_op_args, "errback"); + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + + PyObject* pyObj_op_response = nullptr; + std::shared_ptr<std::promise<PyObject*>> barrier = nullptr; + std::future<PyObject*> fut; + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + barrier = std::make_shared<std::promise<PyObject*>>(); + fut = barrier->get_future(); + } + + switch (op_type) { + case Operations::INSERT: + case Operations::UPSERT: + case Operations::REPLACE: + case Operations::REMOVE: { + auto opts = get_mutation_options(pyObj_op_args); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, key }; + opts.op_type = op_type; + if (pyObj_value != nullptr) { + opts.value = pyObj_value; + } + try { + pyObj_op_response = + prepare_and_execute_mutation_op(&opts, pyObj_callback, pyObj_errback, barrier); + } catch (const std::system_error& e) { + if (barrier) { + barrier->set_value(nullptr); + } + pycbc_set_python_exception(e.code(), __FILE__, __LINE__, e.what()); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + pyObj_op_response = nullptr; + } + break; + } + case Operations::GET: + case Operations::GET_PROJECTED: + case Operations::GET_ANY_REPLICA: + case Operations::GET_ALL_REPLICAS: + case Operations::GET_AND_LOCK: + case Operations::GET_AND_TOUCH: + case Operations::TOUCH: + case Operations::EXISTS: + case Operations::UNLOCK: { + auto opts = get_read_options(pyObj_op_args); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, key }; + PyObject* pyObj_project = PyDict_GetItemString(pyObj_op_args, "project"); + if (pyObj_project != nullptr || opts.with_expiry) { + op_type = Operations::GET_PROJECTED; + opts.project = pyObj_project; + } + opts.op_type = op_type; + try { + pyObj_op_response = + prepare_and_execute_read_op(&opts, pyObj_callback, pyObj_errback, barrier); + } catch (const std::system_error& e) { + if (barrier) { + barrier->set_value(nullptr); + } + pycbc_set_python_exception(e.code(), __FILE__, __LINE__, e.what()); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + pyObj_op_response = nullptr; + } + break; + } + default: { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unrecognized KV operation passed in."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = fut.get(); + Py_END_ALLOW_THREADS return ret; + } + return pyObj_op_response; +} + +PyObject* +handle_kv_multi_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_conn = nullptr; + char* bucket = nullptr; + char* scope = nullptr; + char* collection = nullptr; + Operations::OperationType op_type = Operations::UNKNOWN; + PyObject* pyObj_op_args = nullptr; + + static const char* kw_list[] = { "conn", "bucket", "scope", "collection_name", + "op_type", "op_args", nullptr }; + + const char* kw_format = "O!sssIO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &bucket, + &scope, + &collection, + &op_type, + &pyObj_op_args); + if (!ret) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform kv multi operation. Unable to parse args/kwargs."); + return nullptr; + } + + connection* conn = nullptr; + + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + std::vector<std::future<PyObject*>> op_results{}; + + PyObject* pyObj_multi_result = create_result_obj(); + result* multi_result = reinterpret_cast<result*>(pyObj_multi_result); + + if (pyObj_op_args && PyDict_Check(pyObj_op_args)) { + PyObject *pyObj_doc_key, *pyObj_op_dict; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_op_args, &pos, &pyObj_doc_key, &pyObj_op_dict)) { + std::string k; + PyObject* pyObj_op_response = nullptr; + if (PyUnicode_Check(pyObj_doc_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_doc_key)); + } + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + if (PyDict_Check(pyObj_op_dict) && !k.empty()) { + PyObject* pyObj_value = PyDict_GetItemString(pyObj_op_dict, "value"); + + switch (op_type) { + case Operations::INSERT: + case Operations::UPSERT: + case Operations::REPLACE: + case Operations::REMOVE: { + auto opts = get_mutation_options(pyObj_op_dict); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, k }; + opts.op_type = op_type; + if (pyObj_value != nullptr) { + opts.value = pyObj_value; + } + + try { + pyObj_op_response = + prepare_and_execute_mutation_op(&opts, nullptr, nullptr, barrier, multi_result); + } catch (const std::system_error& e) { + PyObject* pyObj_exc = pycbc_build_exception(e.code(), __FILE__, __LINE__, e.what()); + barrier->set_value(pyObj_exc); + } + break; + } + case Operations::GET: + case Operations::GET_PROJECTED: + case Operations::GET_ANY_REPLICA: + case Operations::GET_ALL_REPLICAS: + case Operations::GET_AND_LOCK: + case Operations::GET_AND_TOUCH: + case Operations::TOUCH: + case Operations::EXISTS: + case Operations::UNLOCK: { + auto opts = get_read_options(pyObj_op_dict); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, k }; + PyObject* pyObj_project = PyDict_GetItemString(pyObj_op_dict, "project"); + if (pyObj_project != nullptr || opts.with_expiry) { + op_type = Operations::GET_PROJECTED; + opts.project = pyObj_project; + } + opts.op_type = op_type; + try { + pyObj_op_response = + prepare_and_execute_read_op(&opts, nullptr, nullptr, barrier, multi_result); + } catch (const std::system_error& e) { + PyObject* pyObj_exc = pycbc_build_exception(e.code(), __FILE__, __LINE__, e.what()); + barrier->set_value(pyObj_exc); + } + break; + } + default: { + PyObject* pyObj_exc = pycbc_build_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized KV operation passed in."); + barrier->set_value(pyObj_exc); + break; + } + }; + } + + Py_XDECREF(pyObj_op_response); + op_results.emplace_back(std::move(f)); + } + } + + auto all_okay = true; + for (auto i = 0; i < op_results.size(); i++) { + PyObject* res = nullptr; + Py_BEGIN_ALLOW_THREADS res = op_results[i].get(); + Py_END_ALLOW_THREADS if (res == Py_False) + { + all_okay = false; + } + Py_XDECREF(res); + } + + if (all_okay) { + PyDict_SetItemString(multi_result->dict, "all_okay", Py_True); + } else { + PyDict_SetItemString(multi_result->dict, "all_okay", Py_False); + } + + return reinterpret_cast<PyObject*>(multi_result); +} diff --git a/src/kv_ops.hxx b/src/kv_ops.hxx new file mode 100644 index 000000000..d53357ce0 --- /dev/null +++ b/src/kv_ops.hxx @@ -0,0 +1,94 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <future> + +#include "client.hxx" +#include <core/document_id.hxx> +#include <couchbase/cas.hxx> +#include <couchbase/persist_to.hxx> +#include <couchbase/read_preference.hxx> +#include <couchbase/replicate_to.hxx> + +/** + * GET, GET_PROJECTED, GET_AND_LOCK, GET_AND_TOUCH + * EXISTS, TOUCH, UNLOCK + */ +struct read_options { + // common - required + connection* conn; + couchbase::core::document_id id; + Operations::OperationType op_type{ Operations::UNKNOWN }; + + // common - options + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + + // optional + bool with_expiry{ false }; + uint32_t expiry{}; + uint32_t lock_time{}; + couchbase::cas cas; + PyObject* span{ nullptr }; + PyObject* project{ nullptr }; + + // optional - replica reads + couchbase::read_preference read_preference{ couchbase::read_preference::no_preference }; + + // TODO: + // retries? + // partition? +}; + +/** + * INSERT, UPSERT, REPLACE, REMOVE + */ +struct mutation_options { + // common - required + connection* conn; + couchbase::core::document_id id; + Operations::OperationType op_type{ Operations::UNKNOWN }; + PyObject* value{ nullptr }; // not for REMOVE + + // common - optional + couchbase::durability_level durability_level{ couchbase::durability_level::none }; + bool use_legacy_durability{ false }; + couchbase::replicate_to replicate_to{ couchbase::replicate_to::none }; + couchbase::persist_to persist_to{ couchbase::persist_to::none }; + uint32_t expiry{ 0 }; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + PyObject* span = nullptr; + + // optional: REPLACE + couchbase::cas cas; + bool preserve_expiry{ false }; + + // TODO: + // retries? + // partition? + // durability_timeout +}; + +PyObject* +handle_kv_op(PyObject* self, PyObject* args, PyObject* kwargs); + +PyObject* +handle_kv_multi_op(PyObject* self, PyObject* args, PyObject* kwargs); + +PyObject* +handle_kv_blocking_result(std::future<PyObject*>&& fut); diff --git a/src/kv_range_scan.cxx b/src/kv_range_scan.cxx new file mode 100644 index 000000000..e4a2a611c --- /dev/null +++ b/src/kv_range_scan.cxx @@ -0,0 +1,283 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "kv_range_scan.hxx" + +#include <core/agent_group.hxx> + +#include "exceptions.hxx" +#include "utils.hxx" + +std::optional<couchbase::core::scan_term> +get_scan_term(PyObject* pyObj_scan_term) +{ + if (pyObj_scan_term == nullptr) { + return std::nullopt; + } + PyObject* pyObj_term = PyDict_GetItemString(pyObj_scan_term, "term"); + if (pyObj_term == nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Must provide term for ScanTerm."); + return {}; + } + if (!PyUnicode_Check(pyObj_term)) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Term should be a string."); + return {}; + } + + couchbase::core::scan_term scan_term; + + try { + scan_term = couchbase::core::scan_term{ PyUnicode_AsUTF8(pyObj_term) }; + } catch (const std::exception& e) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, e.what()); + return {}; + } + PyObject* pyObj_exclusive = PyDict_GetItemString(pyObj_scan_term, "exclusive"); + if (pyObj_exclusive != nullptr && pyObj_exclusive != Py_None) { + if (pyObj_exclusive == Py_True) { + scan_term.exclusive = true; + } else if (pyObj_exclusive == Py_False) { + scan_term.exclusive = false; + } else { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Exclusive must be a boolean or None."); + } + } + return scan_term; +} + +couchbase::core::range_scan +get_range_scan(PyObject* op_args) +{ + PyObject* pyObj_start_term = PyDict_GetItemString(op_args, "start"); + auto start_term = get_scan_term(pyObj_start_term); + + PyObject* pyObj_end_term = PyDict_GetItemString(op_args, "end"); + auto end_term = get_scan_term(pyObj_end_term); + + return couchbase::core::range_scan{ start_term, end_term }; +} + +couchbase::core::sampling_scan +get_sampling_scan(PyObject* op_args) +{ + PyObject* pyObj_limit = PyDict_GetItemString(op_args, "limit"); + if (pyObj_limit == nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Must provide limit for sample scan."); + return {}; + } + auto scan_type = + couchbase::core::sampling_scan{ static_cast<std::size_t>(PyLong_AsUnsignedLong(pyObj_limit)) }; + PyObject* pyObj_seed = PyDict_GetItemString(op_args, "seed"); + if (pyObj_seed != nullptr && pyObj_seed != Py_None) { + scan_type.seed = static_cast<std::uint64_t>(PyLong_AsUnsignedLong(pyObj_seed)); + } + + return scan_type; +} + +couchbase::core::prefix_scan +get_prefix_scan(PyObject* op_args) +{ + PyObject* pyObj_prefix = PyDict_GetItemString(op_args, "prefix"); + if (pyObj_prefix == nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Must provide prefix for prefix scan."); + return {}; + } + if (!PyUnicode_Check(pyObj_prefix)) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Prefix should be a string."); + return {}; + } + return couchbase::core::prefix_scan{ PyUnicode_AsUTF8(pyObj_prefix) }; +} + +couchbase::core::range_scan_orchestrator_options +get_range_scan_orchestrator_options(PyObject* op_args) +{ + couchbase::core::range_scan_orchestrator_options opts{}; + + PyObject* pyObj_ids_only = PyDict_GetItemString(op_args, "ids_only"); + opts.ids_only = pyObj_ids_only != nullptr && pyObj_ids_only == Py_True; + + PyObject* pyObj_consistent_with = PyDict_GetItemString(op_args, "consistent_with"); + if (pyObj_consistent_with != nullptr && PyList_Check(pyObj_consistent_with)) { + auto mutation_state = get_mutation_state(pyObj_consistent_with); + opts.consistent_with = couchbase::core::mutation_state{ mutation_state }; + } + + PyObject* pyObj_batch_byte_limit = PyDict_GetItemString(op_args, "batch_byte_limit"); + if (pyObj_batch_byte_limit != nullptr) { + opts.batch_byte_limit = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_batch_byte_limit)); + } + + PyObject* pyObj_batch_item_limit = PyDict_GetItemString(op_args, "batch_item_limit"); + if (pyObj_batch_item_limit != nullptr) { + opts.batch_item_limit = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_batch_item_limit)); + } + + PyObject* pyObj_concurrency = PyDict_GetItemString(op_args, "concurrency"); + if (pyObj_concurrency != nullptr) { + opts.concurrency = static_cast<uint16_t>(PyLong_AsUnsignedLong(pyObj_concurrency)); + } + + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (pyObj_timeout != nullptr) { + auto timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_timeout)); + auto timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + if (0 < timeout) { + opts.timeout = timeout_ms; + } + } + + PyObject* pyObj_span = PyDict_GetItemString(op_args, "span"); + if (pyObj_span != nullptr) { + opts.parent_span = std::make_shared<pycbc::request_span>(pyObj_span); + } + + return opts; +} + +scan_iterator* +handle_kv_range_scan_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_conn = nullptr; + char* bucket_name = nullptr; + char* scope_name = nullptr; + char* collection_name = nullptr; + Operations::OperationType op_type = Operations::UNKNOWN; + PyObject* pyObj_op_args = nullptr; + + static const char* kw_list[] = { "conn", "bucket", "scope", "collection_name", + "op_type", "op_args", nullptr }; + + const char* kw_format = "O!sssI|O"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &bucket_name, + &scope_name, + &collection_name, + &op_type, + &pyObj_op_args); + if (!ret) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform kv range scan operation. Unable to parse args/kwargs."); + return nullptr; + } + + connection* conn = nullptr; + + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + auto barrier = std::make_shared<std::promise< + std::pair<std::error_code, std::shared_ptr<couchbase::core::topology::configuration>>>>(); + auto f = barrier->get_future(); + conn->cluster_.with_bucket_configuration(bucket_name, + [barrier](std::error_code ec, auto config) mutable { + barrier->set_value({ ec, std::move(config) }); + }); + auto [ec, config] = f.get(); + if (ec) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Cannot perform kv range scan operation. Unable to get bucket configuration."); + return nullptr; + } + if (!config->capabilities.supports_range_scan()) { + pycbc_set_python_exception(PycbcError::FeatureUnavailable, + __FILE__, + __LINE__, + "The server does not support key-value scan operations."); + return nullptr; + } + if (!config->vbmap || config->vbmap->empty()) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Cannot perform kv range scan operation. Unable to get vbucket map."); + return nullptr; + } + auto vbucket_map = config->vbmap.value(); + + auto agent_group = couchbase::core::agent_group( + conn->io_, couchbase::core::agent_group_config{ { conn->cluster_ } }); + agent_group.open_bucket(bucket_name); + auto agent = agent_group.get_agent(bucket_name); + + if (!agent.has_value()) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Cannot perform kv range scan operation. Unable to get operation agent."); + return nullptr; + } + + auto options = get_range_scan_orchestrator_options(pyObj_op_args); + if (PyErr_Occurred() != nullptr) { + return nullptr; + } + + std::variant<std::monostate, + couchbase::core::range_scan, + couchbase::core::prefix_scan, + couchbase::core::sampling_scan> + scan_type{}; + if (op_type == Operations::KV_RANGE_SCAN) { + scan_type = get_range_scan(pyObj_op_args); + } else if (op_type == Operations::KV_PREFIX_SCAN) { + scan_type = get_prefix_scan(pyObj_op_args); + } else { + scan_type = get_sampling_scan(pyObj_op_args); + } + + if (PyErr_Occurred() != nullptr) { + return nullptr; + } + + auto orchestrator = couchbase::core::range_scan_orchestrator( + conn->io_, agent.value(), vbucket_map, scope_name, collection_name, scan_type, options); + auto scan_result = orchestrator.scan(); + if (!scan_result.has_value()) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Cannot perform kv scan operation. Unable to start scan operation."); + return nullptr; + } + + return create_scan_iterator_obj(scan_result.value()); +} diff --git a/src/kv_range_scan.hxx b/src/kv_range_scan.hxx new file mode 100644 index 000000000..fcc3eb8db --- /dev/null +++ b/src/kv_range_scan.hxx @@ -0,0 +1,66 @@ +/* + * Copyright 2016-2023. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" +#include "result.hxx" +#include <core/range_scan_options.hxx> +#include <core/range_scan_orchestrator.hxx> +#include <core/range_scan_orchestrator_options.hxx> + +struct range_scan_create_options { + // common - required + connection* conn; + + // common - options + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_scan_timeout; + std::string collection_name; + std::string scope_name; + std::uint32_t collection_id{ 0 }; + std::variant<couchbase::core::range_scan, + couchbase::core::prefix_scan, + couchbase::core::sampling_scan> + scan_type; + std::optional<couchbase::core::range_snapshot_requirements> snapshot_requirements{}; + bool ids_only{ false }; + + // optional + PyObject* span{ nullptr }; +}; + +struct range_scan_continue_options { + // common - required + connection* conn; + + // common - options + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_scan_timeout; + std::uint32_t batch_item_limit{ 0 }; + std::uint32_t batch_byte_limit{ 0 }; + bool ids_only{ false }; +}; + +struct range_scan_cancel_options { + // common - required + connection* conn; + + // common - options + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_scan_timeout; +}; + +scan_iterator* +handle_kv_range_scan_op(PyObject* self, PyObject* args, PyObject* kwargs); diff --git a/src/logger.cxx b/src/logger.cxx new file mode 100644 index 000000000..6d2dd9989 --- /dev/null +++ b/src/logger.cxx @@ -0,0 +1,278 @@ +/* + * Copyright 2016-2023. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "logger.hxx" + +#include "exceptions.hxx" + +static void +pycbc_logger_dealloc(pycbc_logger* self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +PyObject* +pycbc_logger__configure_logging_sink__(PyObject* self, PyObject* args, PyObject* kwargs) +{ + auto logger = reinterpret_cast<pycbc_logger*>(self); + PyObject* pyObj_logger = nullptr; + PyObject* pyObj_level = nullptr; + const char* kw_list[] = { "logger", "level", nullptr }; + const char* kw_format = "OO"; + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, kw_format, const_cast<char**>(kw_list), &pyObj_logger, &pyObj_level)) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot set pycbc_logger sink. Unable to parse args/kwargs."); + return nullptr; + } + + if (couchbase::core::logger::is_initialized()) { + pycbc_set_python_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Cannot create logger. Another logger has already been " + "initialized. Make sure the PYCBC_LOG_LEVEL env " + "variable is not set if using configure_logging."); + return nullptr; + } + + if (pyObj_logger != nullptr) { + logger->logger_sink_ = std::make_shared<pycbc_logger_sink>(pyObj_logger); + } + + couchbase::core::logger::configuration logger_settings; + logger_settings.console = false; + logger_settings.sink = logger->logger_sink_; + auto level = convert_python_log_level(pyObj_level); + logger_settings.log_level = level; + couchbase::core::logger::create_file_logger(logger_settings); + Py_RETURN_NONE; +} + +PyObject* +pycbc_logger__create_logger__(PyObject* self, PyObject* args, PyObject* kwargs) +{ + auto logger = reinterpret_cast<pycbc_logger*>(self); + char* log_level = nullptr; + char* log_filename = nullptr; + int enable_console = 0; + const char* kw_list[] = { "level", "filename", "enable_console", nullptr }; + const char* kw_format = "s|si"; + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &log_level, + &log_filename, + &enable_console)) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot create logger. Unable to parse args/kwargs."); + return nullptr; + } + + if (couchbase::core::logger::is_initialized()) { + pycbc_set_python_exception( + PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Cannot create logger. Another logger has already been initialized."); + return nullptr; + } + + if (log_level == nullptr) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot create logger. Unable to determine log level."); + return nullptr; + } + auto level = couchbase::core::logger::level_from_str(log_level); + if (log_filename != nullptr) { + couchbase::core::logger::configuration configuration{}; + configuration.filename = std::string{ log_filename }; + configuration.log_level = level; + configuration.console = enable_console > 0; + couchbase::core::logger::create_file_logger(configuration); + logger->is_file_logger = true; + } else { + couchbase::core::logger::create_console_logger(); + couchbase::core::logger::set_log_levels(level); + logger->is_console_logger = true; + } + Py_RETURN_NONE; +} + +PyObject* +pycbc_logger__enable_protocol_logger__(PyObject* self, PyObject* args, PyObject* kwargs) +{ + char* filename = nullptr; + const char* kw_list[] = { "filename", nullptr }; + const char* kw_format = "s"; + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, kw_format, const_cast<char**>(kw_list), &filename)) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot enable the protocol logger. Unable to parse args/kwargs."); + return nullptr; + } + couchbase::core::logger::configuration configuration{}; + configuration.filename = std::string{ filename }; + couchbase::core::logger::create_protocol_logger(configuration); + Py_RETURN_NONE; +} + +PyObject* +pycbc_logger__is_console_logger__(PyObject* self, PyObject* Py_UNUSED(ignored)) +{ + auto logger = reinterpret_cast<pycbc_logger*>(self); + if (logger->is_console_logger) { + Py_INCREF(Py_True); + return Py_True; + } else { + Py_INCREF(Py_False); + return Py_False; + } +} + +PyObject* +pycbc_logger__is_file_logger__(PyObject* self, PyObject* Py_UNUSED(ignored)) +{ + auto logger = reinterpret_cast<pycbc_logger*>(self); + if (logger->is_file_logger) { + Py_INCREF(Py_True); + return Py_True; + } else { + Py_INCREF(Py_False); + return Py_False; + } +} + +static PyMethodDef pycbc_logger_methods[] = { + { "configure_logging_sink", + (PyCFunction)pycbc_logger__configure_logging_sink__, + METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Configure logger's logging sink") }, + { "create_logger", + (PyCFunction)pycbc_logger__create_logger__, + METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Create a C++ core logger") }, + { "enable_protocol_logger", + (PyCFunction)pycbc_logger__enable_protocol_logger__, + METH_VARARGS | METH_KEYWORDS, + PyDoc_STR("Enables the protocol logger") }, + { "is_console_logger", + (PyCFunction)pycbc_logger__is_console_logger__, + METH_NOARGS, + PyDoc_STR("Check if logger is console logger or not") }, + { "is_file_logger", + (PyCFunction)pycbc_logger__is_file_logger__, + METH_NOARGS, + PyDoc_STR("Check if logger is file logger or not") }, + { NULL } +}; + +static PyObject* +pycbc_logger_new(PyTypeObject* type, PyObject*, PyObject*) +{ + pycbc_logger* self = reinterpret_cast<pycbc_logger*>(type->tp_alloc(type, 0)); + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_pycbc_logger_type() +{ + PyTypeObject obj = {}; + obj.ob_base = PyVarObject_HEAD_INIT(NULL, 0) obj.tp_name = "pycbc_core.pycbc_logger"; + obj.tp_doc = PyDoc_STR("Python SDK Logger"); + obj.tp_basicsize = sizeof(pycbc_logger); + obj.tp_itemsize = 0; + obj.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + obj.tp_new = pycbc_logger_new; + obj.tp_dealloc = (destructor)pycbc_logger_dealloc; + obj.tp_methods = pycbc_logger_methods; + return obj; +} + +static PyTypeObject pycbc_logger_type = init_pycbc_logger_type(); + +size_t +convert_spdlog_level(spdlog::level::level_enum lvl) +{ + // TODO: support trace level in the python logger + switch (lvl) { + case spdlog::level::level_enum::off: + return 0; + case spdlog::level::level_enum::trace: + return 5; + case spdlog::level::level_enum::debug: + return 10; + case spdlog::level::level_enum::info: + return 20; + case spdlog::level::level_enum::warn: + return 30; + case spdlog::level::level_enum::err: + return 40; + case spdlog::level::level_enum::critical: + return 50; + default: + return 0; + } +} + +couchbase::core::logger::level +convert_python_log_level(PyObject* level) +{ + auto lvl = PyLong_AsSize_t(level); + switch (lvl) { + case 0: + return couchbase::core::logger::level::off; + case 5: + return couchbase::core::logger::level::trace; + case 10: + return couchbase::core::logger::level::debug; + case 20: + return couchbase::core::logger::level::info; + case 30: + return couchbase::core::logger::level::warn; + case 40: + return couchbase::core::logger::level::err; + case 50: + return couchbase::core::logger::level::critical; + default: + return couchbase::core::logger::level::off; + } +} + +PyObject* +add_logger_objects(PyObject* pyObj_module) +{ + if (PyType_Ready(&pycbc_logger_type) < 0) { + return nullptr; + } + Py_INCREF(&pycbc_logger_type); + if (PyModule_AddObject( + pyObj_module, "pycbc_logger", reinterpret_cast<PyObject*>(&pycbc_logger_type)) < 0) { + Py_DECREF(&pycbc_logger_type); + return nullptr; + } + return pyObj_module; +} diff --git a/src/logger.hxx b/src/logger.hxx new file mode 100644 index 000000000..901f42a8c --- /dev/null +++ b/src/logger.hxx @@ -0,0 +1,249 @@ +/* + * Copyright 2022 Couchbase, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Python.h" +#include <core/logger/configuration.hxx> +#include <core/logger/logger.hxx> +#include <core/transactions.hxx> +#include <queue> +#include <spdlog/details/log_msg.h> +#include <spdlog/sinks/base_sink.h> + +// gh-108014 added Py_IsFinalizing() to Python 3.13.0a1 +// PR: https://github.com/python/cpython/pull/108032/files +// bpo-1856 added _Py_Finalizing to Python 3.2.1b1. +#if (0x030201B1 <= PY_VERSION_HEX && PY_VERSION_HEX < 0x030D00A1) +inline int +Py_IsFinalizing(void) +{ +// _Py_IsFinalizing() was added to Python 3.7.0a1. +#if PY_VERSION_HEX >= 0x30700A1 + return _Py_IsFinalizing(); +#else + return (_Py_IsFinalizing != NULL); +#endif +} +#endif + +// the spdlog::log_msg uses string_view, since it doesn't want +// copies. Since we consume the log_msg then asych process it, +// the string_view can be pointing to data that is gone already, +// so lets copy into this struct. + +struct log_msg_copy { + std::string logger_name; + spdlog::level::level_enum level; + std::chrono::system_clock::time_point time; + spdlog::source_loc source; + std::string payload; + + log_msg_copy(const spdlog::details::log_msg& msg) + { + logger_name = std::string(msg.logger_name.data(), msg.logger_name.size()); + payload = std::string(msg.payload.data(), msg.payload.size()); + level = msg.level; + time = msg.time; + source = msg.source; + } +}; + +size_t +convert_spdlog_level(spdlog::level::level_enum lvl); + +couchbase::core::logger::level +convert_python_log_level(PyObject* level); + +// Moved to implementing a spdlog::sinks::sink instead of a base_sink. Allows us to not +// worry about the mutex w/in the base_sink. The GIL is the locking mechanism that makes +// sure logging is thread safe as we acquire the GIL prior to passing the log message to +// Python's Logging module. +// +// Still probably the better way to do logging: asynchronous logger (see note below). +// +// A third way would be to use asynchronous logger. However the txns lib only creates synchronous +// loggers now. This is probably the best solution, which we can do when we merge the txn lib +// into the client lib. +// +class pycbc_logger_sink : public spdlog::sinks::sink +{ +public: + pycbc_logger_sink(PyObject* pyObj_logger) + : pyObj_logger_(pyObj_logger) + { + Py_INCREF(pyObj_logger_); + } + + // no copy or move constructor or assignment + pycbc_logger_sink(const pycbc_logger_sink&) = delete; + pycbc_logger_sink(pycbc_logger_sink&&) = delete; + + pycbc_logger_sink& operator=(const pycbc_logger_sink&) = delete; + pycbc_logger_sink& operator=(pycbc_logger_sink&&) = delete; + + ~pycbc_logger_sink() + { + if (0 == Py_IsFinalizing()) { + auto state = PyGILState_Ensure(); + Py_DECREF(pyObj_logger_); + PyGILState_Release(state); + } + } + + void log(const spdlog::details::log_msg& msg) final + { + if (0 == Py_IsFinalizing()) { + log_it_(msg); + } + } + + void flush() final {}; + + void set_pattern(const std::string& pattern) final {}; + void set_formatter(std::unique_ptr<spdlog::formatter> sink_formatter) final {}; + +protected: + void log_it_(const spdlog::details::log_msg& msg) + { + PyGILState_STATE state = PyGILState_Ensure(); + try { + + // static initialize the type and method once. These 'leak' a single + // object, but that is fine. Same for an empty tuple we will on each call. + static PyObject* pyObj_log_record_type = init_log_record_type(); + static PyObject* pyObj_logger_handle_method = init_logger_handle_method(); + + // convert the log_msg_copy to a dict first... + auto pyObj_log_record_details = convert_log_msg(msg); + + // now, create an actual LogRecord from it... + auto pyObj_log_record = PyObject_CallObject(pyObj_log_record_type, pyObj_log_record_details); + Py_DECREF(pyObj_log_record_details); + if (nullptr != pyObj_log_record) { + // we need to fixup the created time, which cannot be passed in the constructor... + // The created member is a float containing a float expressed as seconds since the epoch, in + // UTC. + PyObject* log_time = convert_time_to_float(msg.time); + PyObject_SetAttrString(pyObj_log_record, "created", log_time); + Py_DECREF(log_time); + + // now, we want to hand this record to the logger... + PyObject* pyObj_args = PyTuple_Pack(1, pyObj_log_record); + PyObject_CallObject(pyObj_logger_handle_method, pyObj_args); + + // that's it, now cleanup. + Py_DECREF(pyObj_log_record); + Py_DECREF(pyObj_args); + } else { + PyErr_Print(); + } + PyGILState_Release(state); + } catch (...) { + // There is still a possibility we hit this after the interpret has started to finalize + if (0 == Py_IsFinalizing()) { + PyGILState_Release(state); + } + throw; + } + } + + PyObject* convert_time_to_float(std::chrono::system_clock::time_point tm) + { + auto duration_us = std::chrono::duration_cast<std::chrono::microseconds>(tm.time_since_epoch()); + auto time = static_cast<double>(duration_us.count()) / 1000000; + return PyFloat_FromDouble(time); + } + + PyObject* convert_log_msg(const log_msg_copy& msg) + { + // convert to a python dict, assuming we already have the GIL + // We need to supply the following keys/values: + // name: str + // level: int ( CRITICAL = 50, DEBUG=10, ERROR=40, FATAL=50, INFO=20, WARNING=30, NOTSET=0) + // TODO: map trace from spdlog - can start with making it debug as well, but really + // should add TRACE to python logging levels + // pathname: str (path to file that did the logging) + // lineno: int (line number of line that logged in that file) + // msg: str (text of the message) + // args: Dict (extras - probably we will not use that for now) + // exc_info: str (python exception tuple if there is one) + PyObject* retval = PyTuple_New(8); + // name + PyObject* pyObj_value = + PyUnicode_FromStringAndSize(msg.logger_name.data(), msg.logger_name.size()); + PyTuple_SetItem(retval, 0, pyObj_value); + // level + pyObj_value = PyLong_FromSize_t(convert_spdlog_level(msg.level)); + PyTuple_SetItem(retval, 1, pyObj_value); + // pathname + if (nullptr != msg.source.filename) { + pyObj_value = PyUnicode_FromString(msg.source.filename); + } else { + pyObj_value = PyUnicode_FromString("transactions"); + } + PyTuple_SetItem(retval, 2, pyObj_value); + // lineno + pyObj_value = PyLong_FromSize_t(static_cast<size_t>(msg.source.line)); + PyTuple_SetItem(retval, 3, pyObj_value); + // msg + pyObj_value = PyUnicode_FromStringAndSize(msg.payload.data(), msg.payload.size()); + PyTuple_SetItem(retval, 4, pyObj_value); + // args + Py_INCREF(Py_None); + PyTuple_SetItem(retval, 5, Py_None); + // exc_info + Py_INCREF(Py_None); + PyTuple_SetItem(retval, 6, Py_None); + // func + if (nullptr != msg.source.funcname) { + pyObj_value = PyUnicode_FromString(msg.source.funcname); + } else { + pyObj_value = Py_None; + Py_INCREF(pyObj_value); + } + PyTuple_SetItem(retval, 7, pyObj_value); + + return retval; + } + + PyObject* init_log_record_type() + { + static PyObject* logging = PyImport_ImportModule("logging"); + assert(nullptr != logging); + return PyObject_GetAttrString(logging, "LogRecord"); + } + + PyObject* init_logger_handle_method() + { + // we want the 'handle' method on the pyObj_logger_, so... + PyObject* meth = PyObject_GetAttrString(pyObj_logger_, "handle"); + assert(nullptr != meth); + return meth; + } + +private: + PyObject* pyObj_logger_; +}; + +struct pycbc_logger { + PyObject_HEAD std::shared_ptr<pycbc_logger_sink> logger_sink_; + bool is_console_logger{ false }; + bool is_file_logger{ false }; +}; + +PyObject* +add_logger_objects(PyObject* pyObj_module); diff --git a/src/management/analytics_link.hxx b/src/management/analytics_link.hxx new file mode 100644 index 000000000..64c9ac1a4 --- /dev/null +++ b/src/management/analytics_link.hxx @@ -0,0 +1,580 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "analytics_management.hxx" +#include <core/management/analytics_link_couchbase_remote.hxx> + +PyObject* +build_couchbase_remote_link_encryption_settings( + couchbase::core::management::analytics::couchbase_link_encryption_settings settings) +{ + PyObject* pyObj_encryption = PyDict_New(); + PyObject* pyObj_tmp = nullptr; + + auto level = couchbase::core::management::analytics::to_string(settings.level); + pyObj_tmp = PyUnicode_FromString(level.c_str()); + if (-1 == PyDict_SetItemString(pyObj_encryption, "encryption_level", pyObj_tmp)) { + Py_XDECREF(pyObj_encryption); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (settings.certificate.has_value()) { + pyObj_tmp = PyUnicode_FromString(settings.certificate.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_encryption, "certificate", pyObj_tmp)) { + Py_DECREF(pyObj_encryption); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.client_certificate.has_value()) { + pyObj_tmp = PyUnicode_FromString(settings.client_certificate.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_encryption, "client_certificate", pyObj_tmp)) { + Py_DECREF(pyObj_encryption); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + // no need to read in the client_key + + return pyObj_encryption; +} + +PyObject* +build_couchbase_remote_link(couchbase::core::management::analytics::couchbase_remote_link link) +{ + PyObject* pyObj_link = PyDict_New(); + PyObject* pyObj_tmp = nullptr; + + pyObj_tmp = PyUnicode_FromString(link.link_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "link_name", pyObj_tmp)) { + Py_XDECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(link.dataverse.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "dataverse", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(link.hostname.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "hostname", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(link.hostname.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "hostname", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (link.username.has_value()) { + pyObj_tmp = PyUnicode_FromString(link.username.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "username", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + // no need to read in pw + + PyObject* pyObj_encryption_settings = + build_couchbase_remote_link_encryption_settings(link.encryption); + if (pyObj_encryption_settings == nullptr) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_link, "encryption_settings", pyObj_encryption_settings)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_encryption_settings); + + return pyObj_link; +} + +PyObject* +build_s3_link(couchbase::core::management::analytics::s3_external_link link) +{ + PyObject* pyObj_link = PyDict_New(); + PyObject* pyObj_tmp = nullptr; + + pyObj_tmp = PyUnicode_FromString(link.link_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "link_name", pyObj_tmp)) { + Py_XDECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(link.dataverse.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "dataverse", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(link.access_key_id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "access_key_id", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(link.region.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "region", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (link.service_endpoint.has_value()) { + pyObj_tmp = PyUnicode_FromString(link.service_endpoint.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "service_endpoint", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + return pyObj_link; +} + +PyObject* +build_azure_blob_link(couchbase::core::management::analytics::azure_blob_external_link link) +{ + PyObject* pyObj_link = PyDict_New(); + PyObject* pyObj_tmp = nullptr; + + pyObj_tmp = PyUnicode_FromString(link.link_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "link_name", pyObj_tmp)) { + Py_XDECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(link.dataverse.c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "dataverse", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (link.account_name.has_value()) { + pyObj_tmp = PyUnicode_FromString(link.account_name.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "account_name", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (link.blob_endpoint.has_value()) { + pyObj_tmp = PyUnicode_FromString(link.blob_endpoint.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "blob_endpoint", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (link.endpoint_suffix.has_value()) { + pyObj_tmp = PyUnicode_FromString(link.endpoint_suffix.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_link, "endpoint_suffix", pyObj_tmp)) { + Py_DECREF(pyObj_link); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + return pyObj_link; +} + +couchbase::core::management::analytics::couchbase_link_encryption_level +str_to_encryption_level(PyObject* pyObj_level) +{ + auto level = std::string(PyUnicode_AsUTF8(pyObj_level)); + if (level.compare("none") == 0) { + return couchbase::core::management::analytics::couchbase_link_encryption_level::none; + } + if (level.compare("half") == 0) { + return couchbase::core::management::analytics::couchbase_link_encryption_level::half; + } + if (level.compare("full") == 0) { + return couchbase::core::management::analytics::couchbase_link_encryption_level::full; + } + // TODO: better exception + PyErr_SetString(PyExc_ValueError, "Invalid couchbase remote link encryption level."); + return {}; +} + +couchbase::core::management::analytics::couchbase_link_encryption_settings +get_couchbase_remote_link_encryption_settings(PyObject* pyObj_settings) +{ + couchbase::core::management::analytics::couchbase_link_encryption_settings settings{}; + + PyObject* pyObj_encryption_level = PyDict_GetItemString(pyObj_settings, "encryption_level"); + auto encryption_level = str_to_encryption_level(pyObj_encryption_level); + settings.level = encryption_level; + + PyObject* pyObj_certificate = PyDict_GetItemString(pyObj_settings, "certificate"); + if (pyObj_certificate != nullptr) { + auto certificate = std::string(PyUnicode_AsUTF8(pyObj_certificate)); + settings.certificate = certificate; + } + + PyObject* pyObj_client_certificate = PyDict_GetItemString(pyObj_settings, "client_certificate"); + if (pyObj_client_certificate != nullptr) { + auto client_certificate = std::string(PyUnicode_AsUTF8(pyObj_client_certificate)); + settings.client_certificate = client_certificate; + } + + PyObject* pyObj_client_key = PyDict_GetItemString(pyObj_settings, "client_key"); + if (pyObj_client_key != nullptr) { + auto client_key = std::string(PyUnicode_AsUTF8(pyObj_client_key)); + settings.client_key = client_key; + } + + return settings; +} + +template<typename analytics_link_type> +analytics_link_type +get_link([[maybe_unused]] PyObject* pyObj_link) +{ + analytics_link_type link{}; + return link; +} + +template<> +couchbase::core::management::analytics::couchbase_remote_link +get_link([[maybe_unused]] PyObject* pyObj_link) +{ + couchbase::core::management::analytics::couchbase_remote_link link{}; + PyObject* pyObj_link_name = PyDict_GetItemString(pyObj_link, "link_name"); + auto link_name = std::string(PyUnicode_AsUTF8(pyObj_link_name)); + link.link_name = link_name; + + PyObject* pyObj_dataverse = PyDict_GetItemString(pyObj_link, "dataverse"); + auto dataverse = std::string(PyUnicode_AsUTF8(pyObj_dataverse)); + link.dataverse = dataverse; + + PyObject* pyObj_hostname = PyDict_GetItemString(pyObj_link, "hostname"); + auto hostname = std::string(PyUnicode_AsUTF8(pyObj_hostname)); + link.hostname = hostname; + + PyObject* pyObj_username = PyDict_GetItemString(pyObj_link, "username"); + if (pyObj_username != nullptr) { + auto username = std::string(PyUnicode_AsUTF8(pyObj_username)); + link.username = username; + } + + PyObject* pyObj_password = PyDict_GetItemString(pyObj_link, "password"); + if (pyObj_password != nullptr) { + auto password = std::string(PyUnicode_AsUTF8(pyObj_password)); + link.password = password; + } + + PyObject* pyObj_encryption = PyDict_GetItemString(pyObj_link, "encryption"); + link.encryption = get_couchbase_remote_link_encryption_settings(pyObj_encryption); + + return link; +} + +template<> +couchbase::core::management::analytics::s3_external_link +get_link([[maybe_unused]] PyObject* pyObj_link) +{ + couchbase::core::management::analytics::s3_external_link link{}; + PyObject* pyObj_link_name = PyDict_GetItemString(pyObj_link, "link_name"); + auto link_name = std::string(PyUnicode_AsUTF8(pyObj_link_name)); + link.link_name = link_name; + + PyObject* pyObj_dataverse = PyDict_GetItemString(pyObj_link, "dataverse"); + auto dataverse = std::string(PyUnicode_AsUTF8(pyObj_dataverse)); + link.dataverse = dataverse; + + PyObject* pyObj_access_key_id = PyDict_GetItemString(pyObj_link, "access_key_id"); + auto access_key_id = std::string(PyUnicode_AsUTF8(pyObj_access_key_id)); + link.access_key_id = access_key_id; + + PyObject* pyObj_secret_access_key = PyDict_GetItemString(pyObj_link, "secret_access_key"); + auto secret_access_key = std::string(PyUnicode_AsUTF8(pyObj_secret_access_key)); + link.secret_access_key = secret_access_key; + + PyObject* pyObj_session_token = PyDict_GetItemString(pyObj_link, "session_token"); + if (pyObj_session_token != nullptr) { + auto session_token = std::string(PyUnicode_AsUTF8(pyObj_session_token)); + link.session_token = session_token; + } + + PyObject* pyObj_region = PyDict_GetItemString(pyObj_link, "region"); + auto region = std::string(PyUnicode_AsUTF8(pyObj_region)); + link.region = region; + + PyObject* pyObj_service_endpoint = PyDict_GetItemString(pyObj_link, "service_endpoint"); + if (pyObj_service_endpoint != nullptr) { + auto service_endpoint = std::string(PyUnicode_AsUTF8(pyObj_service_endpoint)); + link.service_endpoint = service_endpoint; + } + + return link; +} + +template<> +couchbase::core::management::analytics::azure_blob_external_link +get_link([[maybe_unused]] PyObject* pyObj_link) +{ + couchbase::core::management::analytics::azure_blob_external_link link{}; + PyObject* pyObj_link_name = PyDict_GetItemString(pyObj_link, "link_name"); + auto link_name = std::string(PyUnicode_AsUTF8(pyObj_link_name)); + link.link_name = link_name; + + PyObject* pyObj_dataverse = PyDict_GetItemString(pyObj_link, "dataverse"); + auto dataverse = std::string(PyUnicode_AsUTF8(pyObj_dataverse)); + link.dataverse = dataverse; + + PyObject* pyObj_connection_string = PyDict_GetItemString(pyObj_link, "connection_string"); + if (pyObj_connection_string != nullptr) { + auto connection_string = std::string(PyUnicode_AsUTF8(pyObj_connection_string)); + link.connection_string = connection_string; + } + + PyObject* pyObj_account_name = PyDict_GetItemString(pyObj_link, "account_name"); + if (pyObj_account_name != nullptr) { + auto account_name = std::string(PyUnicode_AsUTF8(pyObj_account_name)); + link.account_name = account_name; + } + + PyObject* pyObj_account_key = PyDict_GetItemString(pyObj_link, "account_key"); + if (pyObj_account_key != nullptr) { + auto account_key = std::string(PyUnicode_AsUTF8(pyObj_account_key)); + link.account_key = account_key; + } + + PyObject* pyObj_shared_access_signature = + PyDict_GetItemString(pyObj_link, "shared_access_signature"); + if (pyObj_shared_access_signature != nullptr) { + auto shared_access_signature = std::string(PyUnicode_AsUTF8(pyObj_shared_access_signature)); + link.shared_access_signature = shared_access_signature; + } + + PyObject* pyObj_blob_endpoint = PyDict_GetItemString(pyObj_link, "blob_endpoint"); + if (pyObj_blob_endpoint != nullptr) { + auto blob_endpoint = std::string(PyUnicode_AsUTF8(pyObj_blob_endpoint)); + link.blob_endpoint = blob_endpoint; + } + + PyObject* pyObj_endpoint_suffix = PyDict_GetItemString(pyObj_link, "endpoint_suffix"); + if (pyObj_endpoint_suffix != nullptr) { + auto endpoint_suffix = std::string(PyUnicode_AsUTF8(pyObj_endpoint_suffix)); + link.endpoint_suffix = endpoint_suffix; + } + + return link; +} + +template<typename analytics_link_type> +couchbase::core::operations::management::analytics_link_create_request<analytics_link_type> +get_analytics_link_create_request(struct analytics_mgmt_options* options) +{ + couchbase::core::operations::management::analytics_link_create_request<analytics_link_type> req{}; + + PyObject* pyObj_link = PyDict_GetItemString(options->op_args, "link"); + req.link = get_link<analytics_link_type>(pyObj_link); + + PyObject* pyObj_client_context_id = PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + return req; +} + +template<typename analytics_link_type> +couchbase::core::operations::management::analytics_link_replace_request<analytics_link_type> +get_analytics_link_replace_request(struct analytics_mgmt_options* options) +{ + couchbase::core::operations::management::analytics_link_replace_request<analytics_link_type> + req{}; + + PyObject* pyObj_link = PyDict_GetItemString(options->op_args, "link"); + req.link = get_link<analytics_link_type>(pyObj_link); + + PyObject* pyObj_client_context_id = PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + return req; +} + +couchbase::core::operations::management::analytics_link_get_all_request +get_link_get_all_request(struct analytics_mgmt_options* options) +{ + couchbase::core::operations::management::analytics_link_get_all_request req{}; + + PyObject* pyObj_link_type = PyDict_GetItemString(options->op_args, "link_type"); + if (pyObj_link_type != nullptr) { + auto link_type = std::string(PyUnicode_AsUTF8(pyObj_link_type)); + req.link_type = link_type; + } + + PyObject* pyObj_link_name = PyDict_GetItemString(options->op_args, "link_name"); + if (pyObj_link_name != nullptr) { + auto link_name = std::string(PyUnicode_AsUTF8(pyObj_link_name)); + req.link_name = link_name; + } + + PyObject* pyObj_dataverse_name = PyDict_GetItemString(options->op_args, "dataverse_name"); + if (pyObj_dataverse_name != nullptr) { + auto dataverse_name = std::string(PyUnicode_AsUTF8(pyObj_dataverse_name)); + req.dataverse_name = dataverse_name; + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + return req; +} + +couchbase::core::operations::management::analytics_link_drop_request +get_link_drop_request(struct analytics_mgmt_options* options) +{ + couchbase::core::operations::management::analytics_link_drop_request req{}; + + PyObject* pyObj_link_name = PyDict_GetItemString(options->op_args, "link_name"); + auto link_name = std::string(PyUnicode_AsUTF8(pyObj_link_name)); + req.link_name = link_name; + + PyObject* pyObj_dataverse_name = PyDict_GetItemString(options->op_args, "dataverse_name"); + auto dataverse_name = std::string(PyUnicode_AsUTF8(pyObj_dataverse_name)); + req.dataverse_name = dataverse_name; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + return req; +} + +couchbase::core::operations::management::analytics_link_disconnect_request +get_link_disconnect_request(struct analytics_mgmt_options* options) +{ + couchbase::core::operations::management::analytics_link_disconnect_request req{}; + + PyObject* pyObj_dataverse_name = PyDict_GetItemString(options->op_args, "dataverse_name"); + if (pyObj_dataverse_name != nullptr) { + auto dataverse_name = std::string(PyUnicode_AsUTF8(pyObj_dataverse_name)); + req.dataverse_name = dataverse_name; + } + + PyObject* pyObj_link_name = PyDict_GetItemString(options->op_args, "link_name"); + if (pyObj_link_name != nullptr) { + auto link_name = std::string(PyUnicode_AsUTF8(pyObj_link_name)); + req.link_name = link_name; + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + return req; +} + +couchbase::core::operations::management::analytics_link_connect_request +get_link_connect_request(struct analytics_mgmt_options* options) +{ + couchbase::core::operations::management::analytics_link_connect_request req{}; + + PyObject* pyObj_dataverse_name = PyDict_GetItemString(options->op_args, "dataverse_name"); + if (pyObj_dataverse_name != nullptr) { + auto dataverse_name = std::string(PyUnicode_AsUTF8(pyObj_dataverse_name)); + req.dataverse_name = dataverse_name; + } + + PyObject* pyObj_link_name = PyDict_GetItemString(options->op_args, "link_name"); + if (pyObj_link_name != nullptr) { + auto link_name = std::string(PyUnicode_AsUTF8(pyObj_link_name)); + req.link_name = link_name; + } + + PyObject* pyObj_force = PyDict_GetItemString(options->op_args, "force"); + if (pyObj_force) { + if (pyObj_force == Py_True) { + req.force = true; + } + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + return req; +} diff --git a/src/management/analytics_management.cxx b/src/management/analytics_management.cxx new file mode 100644 index 000000000..daa20f7c9 --- /dev/null +++ b/src/management/analytics_management.cxx @@ -0,0 +1,843 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "analytics_management.hxx" + +#include <core/analytics_scan_consistency.hxx> +#include <core/management/analytics_dataset.hxx> +#include <core/management/analytics_index.hxx> +#include <core/management/analytics_link.hxx> +#include <core/operations/management/analytics.hxx> + +#include "../exceptions.hxx" +#include "../result.hxx" +#include "analytics_link.hxx" + +/* couchbase::core::operations::management::analytics_* request building methods */ + +template<typename T> +T +get_index_request_base(struct analytics_mgmt_options* options) +{ + T req{}; + + PyObject* pyObj_dataverse_name = PyDict_GetItemString(options->op_args, "dataverse_name"); + if (pyObj_dataverse_name != nullptr) { + auto dataverse_name = std::string(PyUnicode_AsUTF8(pyObj_dataverse_name)); + req.dataverse_name = dataverse_name; + } + + PyObject* pyObj_dataset_name = PyDict_GetItemString(options->op_args, "dataset_name"); + auto dataset_name = std::string(PyUnicode_AsUTF8(pyObj_dataset_name)); + req.dataset_name = dataset_name; + + PyObject* pyObj_index_name = PyDict_GetItemString(options->op_args, "index_name"); + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + return req; +} + +couchbase::core::operations::management::analytics_index_drop_request +get_index_drop_request(struct analytics_mgmt_options* options) +{ + auto req = + get_index_request_base<couchbase::core::operations::management::analytics_index_drop_request>( + options); + + PyObject* pyObj_ignore_if_does_not_exist = + PyDict_GetItemString(options->op_args, "ignore_if_does_not_exist"); + if (pyObj_ignore_if_does_not_exist) { + if (pyObj_ignore_if_does_not_exist == Py_True) { + req.ignore_if_does_not_exist = true; + } + } + + return req; +} + +couchbase::core::operations::management::analytics_index_create_request +get_index_create_request(struct analytics_mgmt_options* options) +{ + auto req = + get_index_request_base<couchbase::core::operations::management::analytics_index_create_request>( + options); + + PyObject* pyObj_fields = PyDict_GetItemString(options->op_args, "fields"); + + if (pyObj_fields != nullptr && PyDict_Check(pyObj_fields)) { + std::map<std::string, std::string> fields{}; + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + while (PyDict_Next(pyObj_fields, &pos, &pyObj_key, &pyObj_value)) { + std::string k; + if (PyUnicode_Check(pyObj_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_key)); + } + if (PyUnicode_Check(pyObj_value) && !k.empty()) { + auto value = std::string(PyUnicode_AsUTF8(pyObj_value)); + fields.emplace(k, value); + } + } + if (fields.size() > 0) { + req.fields = fields; + } + } + + PyObject* pyObj_ignore_if_exists = PyDict_GetItemString(options->op_args, "ignore_if_exists"); + if (pyObj_ignore_if_exists) { + if (pyObj_ignore_if_exists == Py_True) { + req.ignore_if_exists = true; + } + } + + return req; +} + +template<typename T> +T +get_dataset_request_base(struct analytics_mgmt_options* options) +{ + T req{}; + + PyObject* pyObj_dataverse_name = PyDict_GetItemString(options->op_args, "dataverse_name"); + if (pyObj_dataverse_name != nullptr) { + auto dataverse_name = std::string(PyUnicode_AsUTF8(pyObj_dataverse_name)); + req.dataverse_name = dataverse_name; + } + + PyObject* pyObj_dataset_name = PyDict_GetItemString(options->op_args, "dataset_name"); + auto dataset_name = std::string(PyUnicode_AsUTF8(pyObj_dataset_name)); + req.dataset_name = dataset_name; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + return req; +} + +couchbase::core::operations::management::analytics_dataset_drop_request +get_dataset_drop_request(struct analytics_mgmt_options* options) +{ + auto req = get_dataset_request_base< + couchbase::core::operations::management::analytics_dataset_drop_request>(options); + + PyObject* pyObj_ignore_if_does_not_exist = + PyDict_GetItemString(options->op_args, "ignore_if_does_not_exist"); + if (pyObj_ignore_if_does_not_exist) { + if (pyObj_ignore_if_does_not_exist == Py_True) { + req.ignore_if_does_not_exist = true; + } + } + + return req; +} + +couchbase::core::operations::management::analytics_dataset_create_request +get_dataset_create_request(struct analytics_mgmt_options* options) +{ + auto req = get_dataset_request_base< + couchbase::core::operations::management::analytics_dataset_create_request>(options); + + PyObject* pyObj_bucket_name = PyDict_GetItemString(options->op_args, "bucket_name"); + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + + PyObject* pyObj_condition = PyDict_GetItemString(options->op_args, "condition"); + if (pyObj_condition != nullptr) { + auto condition = std::string(PyUnicode_AsUTF8(pyObj_condition)); + req.condition = condition; + } + + PyObject* pyObj_ignore_if_exists = PyDict_GetItemString(options->op_args, "ignore_if_exists"); + if (pyObj_ignore_if_exists) { + if (pyObj_ignore_if_exists == Py_True) { + req.ignore_if_exists = true; + } + } + + return req; +} + +template<typename T> +T +get_dataverse_request_base(struct analytics_mgmt_options* options) +{ + T req{}; + + PyObject* pyObj_dataverse_name = PyDict_GetItemString(options->op_args, "dataverse_name"); + auto dataverse_name = std::string(PyUnicode_AsUTF8(pyObj_dataverse_name)); + + req.dataverse_name = dataverse_name; + req.timeout = options->timeout_ms; + + return req; +} + +couchbase::core::operations::management::analytics_dataverse_drop_request +get_dataverse_drop_request(struct analytics_mgmt_options* options) +{ + auto req = get_dataverse_request_base< + couchbase::core::operations::management::analytics_dataverse_drop_request>(options); + + PyObject* pyObj_ignore_if_does_not_exist = + PyDict_GetItemString(options->op_args, "ignore_if_does_not_exist"); + if (pyObj_ignore_if_does_not_exist) { + if (pyObj_ignore_if_does_not_exist == Py_True) { + req.ignore_if_does_not_exist = true; + } + } + + return req; +} + +couchbase::core::operations::management::analytics_dataverse_create_request +get_dataverse_create_request(struct analytics_mgmt_options* options) +{ + auto req = get_dataverse_request_base< + couchbase::core::operations::management::analytics_dataverse_create_request>(options); + + PyObject* pyObj_ignore_if_exists = PyDict_GetItemString(options->op_args, "ignore_if_exists"); + if (pyObj_ignore_if_exists) { + if (pyObj_ignore_if_exists == Py_True) { + req.ignore_if_exists = true; + } + } + + return req; +} + +/* couchbase::core::operations::management::analytics_* response building methods */ + +template<typename T> +result* +create_base_result_from_analytics_mgmt_response(const T& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_query_problems = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& error : resp.errors) { + PyObject* pyObj_query_problem = PyDict_New(); + pyObj_tmp = PyLong_FromUnsignedLongLong(error.code); + if (-1 == PyDict_SetItemString(pyObj_query_problem, "code", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_query_problems); + Py_XDECREF(pyObj_query_problem); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(error.message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_query_problem, "message", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_query_problems); + Py_DECREF(pyObj_query_problem); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + Py_ssize_t set_size = PyList_Size(pyObj_query_problems); + if (set_size > 0) { + if (-1 == PyDict_SetItemString(res->dict, "errors", pyObj_query_problems)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_query_problems); + return nullptr; + } + } + Py_DECREF(pyObj_query_problems); + + return res; +} + +template<typename T> +result* +create_result_from_analytics_mgmt_response(const T& resp) +{ + return create_base_result_from_analytics_mgmt_response(resp); +} + +template<> +result* +create_result_from_analytics_mgmt_response( + const couchbase::core::operations::management::analytics_dataset_get_all_response& resp) +{ + auto res = create_base_result_from_analytics_mgmt_response(resp); + if (res == nullptr) { + return nullptr; + } + PyObject* pyObj_datasets = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& dataset : resp.datasets) { + PyObject* pyObj_dataset = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(dataset.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_dataset, "dataset_name", pyObj_tmp)) { + Py_XDECREF(pyObj_datasets); + Py_XDECREF(pyObj_dataset); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(dataset.dataverse_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_dataset, "dataverse_name", pyObj_tmp)) { + Py_XDECREF(pyObj_datasets); + Py_DECREF(pyObj_dataset); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(dataset.link_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_dataset, "link_name", pyObj_tmp)) { + Py_XDECREF(pyObj_datasets); + Py_DECREF(pyObj_dataset); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(dataset.bucket_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_dataset, "bucket_name", pyObj_tmp)) { + Py_XDECREF(pyObj_datasets); + Py_DECREF(pyObj_dataset); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyList_Append(pyObj_datasets, pyObj_dataset); + Py_DECREF(pyObj_dataset); + } + + if (-1 == PyDict_SetItemString(res->dict, "datasets", pyObj_datasets)) { + Py_XDECREF(pyObj_datasets); + return nullptr; + } + Py_DECREF(pyObj_datasets); + return res; +} + +template<> +result* +create_result_from_analytics_mgmt_response( + const couchbase::core::operations::management::analytics_index_get_all_response& resp) +{ + auto res = create_base_result_from_analytics_mgmt_response(resp); + if (res == nullptr) { + return nullptr; + } + PyObject* pyObj_indexes = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& index : resp.indexes) { + PyObject* pyObj_index = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(index.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_indexes); + Py_XDECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(index.dataverse_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "dataverse_name", pyObj_tmp)) { + Py_XDECREF(pyObj_indexes); + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(index.dataset_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "dataset_name", pyObj_tmp)) { + Py_XDECREF(pyObj_indexes); + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (index.is_primary) { + if (-1 == PyDict_SetItemString(pyObj_index, "is_primary", Py_True)) { + Py_XDECREF(pyObj_indexes); + Py_DECREF(pyObj_index); + return nullptr; + } + } else { + if (-1 == PyDict_SetItemString(pyObj_index, "is_primary", Py_False)) { + Py_XDECREF(pyObj_indexes); + Py_DECREF(pyObj_index); + return nullptr; + } + } + + PyList_Append(pyObj_indexes, pyObj_index); + Py_DECREF(pyObj_index); + } + + if (-1 == PyDict_SetItemString(res->dict, "indexes", pyObj_indexes)) { + Py_XDECREF(pyObj_indexes); + return nullptr; + } + Py_DECREF(pyObj_indexes); + return res; +} + +template<> +result* +create_result_from_analytics_mgmt_response( + const couchbase::core::operations::management::analytics_get_pending_mutations_response& resp) +{ + auto res = create_base_result_from_analytics_mgmt_response(resp); + if (res == nullptr) { + return nullptr; + } + PyObject* pyObj_stats = PyDict_New(); + for (auto const& stat : resp.stats) { + PyObject* pyObj_key = PyUnicode_FromString(stat.first.c_str()); + PyObject* pyObj_value = PyLong_FromUnsignedLongLong(stat.second); + if (-1 == PyDict_SetItem(pyObj_stats, pyObj_key, pyObj_value)) { + Py_XDECREF(pyObj_stats); + Py_XDECREF(pyObj_key); + Py_XDECREF(pyObj_value); + return nullptr; + } + Py_DECREF(pyObj_key); + Py_DECREF(pyObj_value); + } + + if (-1 == PyDict_SetItemString(res->dict, "stats", pyObj_stats)) { + Py_XDECREF(pyObj_stats); + return nullptr; + } + Py_DECREF(pyObj_stats); + return res; +} + +template<> +result* +create_result_from_analytics_mgmt_response( + const couchbase::core::operations::management::analytics_link_get_all_response& resp) +{ + auto res = create_base_result_from_analytics_mgmt_response(resp); + if (res == nullptr) { + return nullptr; + } + PyObject* pyObj_couchbase_links = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& link : resp.couchbase) { + PyObject* pyObj_link = build_couchbase_remote_link(link); + if (pyObj_link == nullptr) { + Py_XDECREF(pyObj_couchbase_links); + return nullptr; + } + PyList_Append(pyObj_couchbase_links, pyObj_link); + Py_DECREF(pyObj_link); + } + if (-1 == PyDict_SetItemString(res->dict, "couchbase_links", pyObj_couchbase_links)) { + Py_XDECREF(pyObj_couchbase_links); + return nullptr; + } + Py_DECREF(pyObj_couchbase_links); + + PyObject* pyObj_s3_links = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& link : resp.s3) { + PyObject* pyObj_link = build_s3_link(link); + if (pyObj_link == nullptr) { + Py_XDECREF(pyObj_couchbase_links); + Py_XDECREF(pyObj_s3_links); + return nullptr; + } + PyList_Append(pyObj_s3_links, pyObj_link); + Py_DECREF(pyObj_link); + } + if (-1 == PyDict_SetItemString(res->dict, "s3_links", pyObj_s3_links)) { + Py_XDECREF(pyObj_couchbase_links); + Py_XDECREF(pyObj_s3_links); + return nullptr; + } + Py_DECREF(pyObj_s3_links); + + PyObject* pyObj_azure_blob_links = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& link : resp.azure_blob) { + PyObject* pyObj_link = build_azure_blob_link(link); + if (pyObj_link == nullptr) { + Py_XDECREF(pyObj_couchbase_links); + Py_XDECREF(pyObj_s3_links); + Py_XDECREF(pyObj_azure_blob_links); + return nullptr; + } + PyList_Append(pyObj_azure_blob_links, pyObj_link); + Py_DECREF(pyObj_link); + } + if (-1 == PyDict_SetItemString(res->dict, "azure_blob_links", pyObj_azure_blob_links)) { + Py_XDECREF(pyObj_couchbase_links); + Py_XDECREF(pyObj_s3_links); + Py_XDECREF(pyObj_azure_blob_links); + return nullptr; + } + Py_DECREF(pyObj_azure_blob_links); + + return res; +} + +template<typename Response> +void +create_result_from_analytics_mgmt_op_response(const Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context(resp.ctx, + __FILE__, + __LINE__, + "Error doing analytics index mgmt operation.", + "AnalyticsIndexMgmt"); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_analytics_mgmt_response(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Analytics index mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<typename Request> +PyObject* +do_analytics_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_analytics_mgmt_op_response(resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_analytics_mgmt_op(connection* conn, + struct analytics_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + PyObject* res = nullptr; + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + switch (options->op_type) { + case AnalyticsManagementOperations::CREATE_DATAVERSE: { + auto req = get_dataverse_create_request(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_dataverse_create_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::CREATE_DATASET: { + auto req = get_dataset_create_request(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_dataset_create_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::CREATE_INDEX: { + auto req = get_index_create_request(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_index_create_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::GET_ALL_DATASETS: { + couchbase::core::operations::management::analytics_dataset_get_all_request req{}; + PyObject* pyObj_client_context_id = + PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_dataset_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::GET_ALL_INDEXES: { + couchbase::core::operations::management::analytics_index_get_all_request req{}; + PyObject* pyObj_client_context_id = + PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_index_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::DROP_DATAVERSE: { + auto req = get_dataverse_drop_request(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_dataverse_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::DROP_DATASET: { + auto req = get_dataset_drop_request(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_dataset_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::DROP_INDEX: { + auto req = get_index_drop_request(options); + res = + do_analytics_mgmt_op<couchbase::core::operations::management::analytics_index_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::GET_PENDING_MUTATIONS: { + couchbase::core::operations::management::analytics_get_pending_mutations_request req{}; + PyObject* pyObj_client_context_id = + PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_get_pending_mutations_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::LINK_CREATE: { + PyObject* pyObj_link_type = PyDict_GetItemString(options->op_args, "link_type"); + if (pyObj_link_type == nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Invalid analytics link type."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + auto link_type = std::string(PyUnicode_AsUTF8(pyObj_link_type)); + if (link_type.compare("couchbase") == 0) { + auto req = get_analytics_link_create_request< + couchbase::core::management::analytics::couchbase_remote_link>(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_create_request< + couchbase::core::management::analytics::couchbase_remote_link>>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + } else if (link_type.compare("s3") == 0) { + auto req = get_analytics_link_create_request< + couchbase::core::management::analytics::s3_external_link>(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_create_request< + couchbase::core::management::analytics::s3_external_link>>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + } else if (link_type.compare("azureblob") == 0) { + auto req = get_analytics_link_create_request< + couchbase::core::management::analytics::azure_blob_external_link>(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_create_request< + couchbase::core::management::analytics::azure_blob_external_link>>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + } + break; + } + case AnalyticsManagementOperations::LINK_CONNECT: { + auto req = get_link_connect_request(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_connect_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::GET_ALL_LINKS: { + auto req = get_link_get_all_request(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::LINK_DISCONNECT: { + auto req = get_link_disconnect_request(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_disconnect_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case AnalyticsManagementOperations::LINK_REPLACE: { + PyObject* pyObj_link_type = PyDict_GetItemString(options->op_args, "link_type"); + if (pyObj_link_type == nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Invalid analytics link type."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + auto link_type = std::string(PyUnicode_AsUTF8(pyObj_link_type)); + if (link_type.compare("couchbase") == 0) { + auto req = get_analytics_link_replace_request< + couchbase::core::management::analytics::couchbase_remote_link>(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_replace_request< + couchbase::core::management::analytics::couchbase_remote_link>>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + } else if (link_type.compare("s3") == 0) { + auto req = get_analytics_link_replace_request< + couchbase::core::management::analytics::s3_external_link>(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_replace_request< + couchbase::core::management::analytics::s3_external_link>>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + } else if (link_type.compare("azureblob") == 0) { + auto req = get_analytics_link_replace_request< + couchbase::core::management::analytics::azure_blob_external_link>(options); + res = do_analytics_mgmt_op< + couchbase::core::operations::management::analytics_link_replace_request< + couchbase::core::management::analytics::azure_blob_external_link>>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + } + break; + } + case AnalyticsManagementOperations::DROP_LINK: { + auto req = get_link_drop_request(options); + res = + do_analytics_mgmt_op<couchbase::core::operations::management::analytics_link_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized analytics index mgmt operation passed in."); + barrier->set_value(nullptr); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + return res; +} + +void +add_analytics_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = + PyUnicode_FromString(AnalyticsManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("AnalyticsManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "analytics_mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/analytics_management.hxx b/src/management/analytics_management.hxx new file mode 100644 index 000000000..45b2c4609 --- /dev/null +++ b/src/management/analytics_management.hxx @@ -0,0 +1,106 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class AnalyticsManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + CREATE_DATAVERSE, + CREATE_DATASET, + CREATE_INDEX, + GET_ALL_DATASETS, + GET_ALL_INDEXES, + DROP_DATAVERSE, + DROP_DATASET, + DROP_INDEX, + GET_PENDING_MUTATIONS, + LINK_CREATE, + LINK_CONNECT, + GET_ALL_LINKS, + LINK_DISCONNECT, + LINK_REPLACE, + DROP_LINK + }; + + AnalyticsManagementOperations() + : AnalyticsManagementOperations{ UNKNOWN } + { + } + constexpr AnalyticsManagementOperations(AnalyticsManagementOperations::OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(AnalyticsManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(AnalyticsManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "CREATE_DATAVERSE " + "CREATE_DATASET " + "CREATE_INDEX " + "GET_ALL_DATASETS " + "GET_ALL_INDEXES " + "DROP_DATAVERSE " + "DROP_DATASET " + "DROP_INDEX " + "GET_PENDING_MUTATIONS " + "LINK_CREATE " + "LINK_CONNECT " + "GET_ALL_LINKS " + "LINK_DISCONNECT " + "LINK_REPLACE " + "DROP_LINK "; + + return ops; + } + +private: + OperationType operation; +}; + +struct analytics_mgmt_options { + PyObject* op_args; + AnalyticsManagementOperations::OperationType op_type = AnalyticsManagementOperations::UNKNOWN; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; +}; + +PyObject* +handle_analytics_mgmt_op(connection* conn, + struct analytics_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback); + +void +add_analytics_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/management/bucket_management.cxx b/src/management/bucket_management.cxx new file mode 100644 index 000000000..9371b156b --- /dev/null +++ b/src/management/bucket_management.cxx @@ -0,0 +1,905 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "bucket_management.hxx" + +#include <core/management/bucket_settings.hxx> +#include <core/operations/management/bucket.hxx> +#include <core/operations/management/bucket_describe.hxx> // should be in the include above, but isn't + +#include "../exceptions.hxx" +#include "../result.hxx" +#include "../utils.hxx" + +PyObject* +build_bucket_settings(couchbase::core::management::cluster::bucket_settings settings) +{ + PyObject* pyObj_bucket_settings = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(settings.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + switch (settings.bucket_type) { + case couchbase::core::management::cluster::bucket_type::couchbase: { + pyObj_tmp = PyUnicode_FromString("membase"); + break; + } + case couchbase::core::management::cluster::bucket_type::memcached: { + pyObj_tmp = PyUnicode_FromString("memcached"); + break; + } + case couchbase::core::management::cluster::bucket_type::ephemeral: { + pyObj_tmp = PyUnicode_FromString("ephemeral"); + break; + } + case couchbase::core::management::cluster::bucket_type::unknown: { + pyObj_tmp = PyUnicode_FromString("unknown"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("unknown"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "bucketType", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(settings.ram_quota_mb); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "ramQuotaMB", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (settings.max_expiry.has_value()) { + // TODO: this is deprecated in the Python client, should remove at some point in the future + pyObj_tmp = PyLong_FromUnsignedLong(settings.max_expiry.value()); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "maxTTL", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(settings.max_expiry.value()); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "maxExpiry", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + switch (settings.compression_mode) { + case couchbase::core::management::cluster::bucket_compression::off: { + pyObj_tmp = PyUnicode_FromString("off"); + break; + } + case couchbase::core::management::cluster::bucket_compression::active: { + pyObj_tmp = PyUnicode_FromString("active"); + break; + } + case couchbase::core::management::cluster::bucket_compression::passive: { + pyObj_tmp = PyUnicode_FromString("passive"); + break; + } + case couchbase::core::management::cluster::bucket_compression::unknown: { + Py_INCREF(Py_None); + pyObj_tmp = Py_None; + break; + } + default: { + Py_INCREF(Py_None); + pyObj_tmp = Py_None; + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "compressionMode", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (settings.minimum_durability_level.has_value()) { + switch (settings.minimum_durability_level.value()) { + case couchbase::durability_level::majority_and_persist_to_active: { + pyObj_tmp = PyUnicode_FromString("majorityAndPersistActive"); + break; + } + case couchbase::durability_level::majority: { + pyObj_tmp = PyUnicode_FromString("majority"); + break; + } + case couchbase::durability_level::persist_to_majority: { + pyObj_tmp = PyUnicode_FromString("persistToMajority"); + break; + } + case couchbase::durability_level::none: { + pyObj_tmp = PyUnicode_FromString("none"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("none"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "durabilityMinLevel", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.num_replicas.has_value()) { + pyObj_tmp = PyLong_FromUnsignedLong(settings.num_replicas.value()); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "numReplicas", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.replica_indexes.has_value()) { + pyObj_tmp = PyBool_FromLong(settings.replica_indexes.value()); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "replicaIndex", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.flush_enabled.has_value()) { + pyObj_tmp = PyBool_FromLong(settings.flush_enabled.value()); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "flushEnabled", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + switch (settings.eviction_policy) { + case couchbase::core::management::cluster::bucket_eviction_policy::full: { + pyObj_tmp = PyUnicode_FromString("fullEviction"); + break; + } + case couchbase::core::management::cluster::bucket_eviction_policy::value_only: { + pyObj_tmp = PyUnicode_FromString("valueOnly"); + break; + } + case couchbase::core::management::cluster::bucket_eviction_policy::no_eviction: { + pyObj_tmp = PyUnicode_FromString("noEviction"); + break; + } + case couchbase::core::management::cluster::bucket_eviction_policy::not_recently_used: { + pyObj_tmp = PyUnicode_FromString("nruEviction"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("noEviction"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "evictionPolicy", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + switch (settings.conflict_resolution_type) { + case couchbase::core::management::cluster::bucket_conflict_resolution::timestamp: { + pyObj_tmp = PyUnicode_FromString("lww"); + break; + } + case couchbase::core::management::cluster::bucket_conflict_resolution::sequence_number: { + pyObj_tmp = PyUnicode_FromString("seqno"); + break; + } + case couchbase::core::management::cluster::bucket_conflict_resolution::custom: { + pyObj_tmp = PyUnicode_FromString("custom"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("seqno"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "conflictResolutionType", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + switch (settings.storage_backend) { + case couchbase::core::management::cluster::bucket_storage_backend::couchstore: { + pyObj_tmp = PyUnicode_FromString("couchstore"); + break; + } + case couchbase::core::management::cluster::bucket_storage_backend::magma: { + pyObj_tmp = PyUnicode_FromString("magma"); + break; + } + case couchbase::core::management::cluster::bucket_storage_backend::unknown: { + pyObj_tmp = PyUnicode_FromString("undefined"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("undefined"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "storageBackend", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (settings.history_retention_collection_default.has_value()) { + pyObj_tmp = PyBool_FromLong(settings.history_retention_collection_default.value()); + if (-1 == PyDict_SetItemString( + pyObj_bucket_settings, "historyRetentionCollectionDefault", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.history_retention_bytes.has_value()) { + pyObj_tmp = PyLong_FromUnsignedLong(settings.history_retention_bytes.value()); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "historyRetentionBytes", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.history_retention_duration.has_value()) { + pyObj_tmp = PyLong_FromUnsignedLong(settings.history_retention_duration.value()); + if (-1 == PyDict_SetItemString(pyObj_bucket_settings, "historyRetentionDuration", pyObj_tmp)) { + Py_DECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + return pyObj_bucket_settings; +} + +template<typename Response> +result* +create_result_from_bucket_mgmt_response([[maybe_unused]] const Response& resp) +{ + PyObject* result_obj = create_result_obj(); + result* res = reinterpret_cast<result*>(result_obj); + return res; +} + +template<> +result* +create_result_from_bucket_mgmt_response( + const couchbase::core::operations::management::bucket_update_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_bucket_settings = build_bucket_settings(resp.bucket); + if (pyObj_bucket_settings == nullptr) { + Py_XDECREF(pyObj_result); + return nullptr; + } + + if (-1 == PyDict_SetItemString(res->dict, "bucket_settings", pyObj_bucket_settings)) { + Py_XDECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_result); + return nullptr; + } + Py_DECREF(pyObj_bucket_settings); + return res; +} + +template<> +result* +create_result_from_bucket_mgmt_response( + const couchbase::core::operations::management::bucket_get_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_bucket_settings = build_bucket_settings(resp.bucket); + if (pyObj_bucket_settings == nullptr) { + Py_XDECREF(pyObj_result); + return nullptr; + } + + if (-1 == PyDict_SetItemString(res->dict, "bucket_settings", pyObj_bucket_settings)) { + Py_XDECREF(pyObj_bucket_settings); + Py_XDECREF(pyObj_result); + return nullptr; + } + Py_DECREF(pyObj_bucket_settings); + return res; +} + +template<> +result* +create_result_from_bucket_mgmt_response( + const couchbase::core::operations::management::bucket_get_all_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_buckets = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& bucket : resp.buckets) { + PyObject* pyObj_bucket_settings = build_bucket_settings(bucket); + PyList_Append(pyObj_buckets, pyObj_bucket_settings); + Py_DECREF(pyObj_bucket_settings); + } + if (-1 == PyDict_SetItemString(res->dict, "buckets", pyObj_buckets)) { + Py_XDECREF(pyObj_buckets); + Py_XDECREF(pyObj_result); + return nullptr; + } + Py_DECREF(pyObj_buckets); + return res; +} + +template<> +result* +create_result_from_bucket_mgmt_response( + const couchbase::core::operations::management::bucket_describe_response& resp) +{ + PyObject* result_obj = create_result_obj(); + result* res = reinterpret_cast<result*>(result_obj); + + PyObject* pyObj_bucket_info = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(resp.info.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_bucket_info, "name", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_XDECREF(pyObj_bucket_info); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.info.uuid.c_str()); + if (-1 == PyDict_SetItemString(pyObj_bucket_info, "uuid", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_DECREF(pyObj_bucket_info); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromLong(static_cast<int>(resp.info.number_of_nodes)); + if (-1 == PyDict_SetItemString(pyObj_bucket_info, "number_of_nodes", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_DECREF(pyObj_bucket_info); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromLong(static_cast<int>(resp.info.number_of_replicas)); + if (-1 == PyDict_SetItemString(pyObj_bucket_info, "number_of_replicas", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_DECREF(pyObj_bucket_info); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_capabilities = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& capability : resp.info.bucket_capabilities) { + pyObj_tmp = PyUnicode_FromString(capability.c_str()); + PyList_Append(pyObj_capabilities, pyObj_tmp); + Py_DECREF(pyObj_tmp); + } + if (-1 == PyDict_SetItemString(pyObj_bucket_info, "bucket_capabilities", pyObj_capabilities)) { + Py_XDECREF(result_obj); + Py_DECREF(pyObj_bucket_info); + Py_XDECREF(pyObj_capabilities); + return nullptr; + } + Py_DECREF(pyObj_capabilities); + + switch (resp.info.storage_backend) { + case couchbase::core::management::cluster::bucket_storage_backend::couchstore: { + pyObj_tmp = PyUnicode_FromString("couchstore"); + break; + } + case couchbase::core::management::cluster::bucket_storage_backend::magma: { + pyObj_tmp = PyUnicode_FromString("magma"); + break; + } + case couchbase::core::management::cluster::bucket_storage_backend::unknown: { + pyObj_tmp = PyUnicode_FromString("undefined"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("undefined"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_bucket_info, "storage_backend", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_DECREF(pyObj_bucket_info); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString(res->dict, "bucket_info", pyObj_bucket_info)) { + Py_XDECREF(result_obj); + Py_DECREF(pyObj_bucket_info); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_bucket_info); + + return res; +} + +template<typename Response> +std::string +get_bucket_mgmt_error_msg(const Response& resp) +{ + return std::string(); +} + +template<> +std::string +get_bucket_mgmt_error_msg( + const couchbase::core::operations::management::bucket_create_response& resp) +{ + return resp.error_message; +} + +template<> +std::string +get_bucket_mgmt_error_msg( + const couchbase::core::operations::management::bucket_update_response& resp) +{ + return resp.error_message; +} + +template<typename Response> +void +create_result_from_bucket_mgmt_op_response(Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + // update and create responses might provide an erorr message + auto error_msg = get_bucket_mgmt_error_msg(resp); + if (error_msg.empty()) { + error_msg = std::string("Error doing bucket mgmt operation."); + } + // make sure this is an HTTPException + pyObj_exc = build_exception_from_context(resp.ctx, __FILE__, __LINE__, error_msg, "BucketMgmt"); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_bucket_mgmt_response(resp); + + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Bucket mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +couchbase::core::management::cluster::bucket_settings +get_bucket_settings(PyObject* settings) +{ + couchbase::core::management::cluster::bucket_settings bucket_settings{}; + + PyObject* pyObj_name = PyDict_GetItemString(settings, "name"); + if (pyObj_name == nullptr) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Expected bucket settings name to be provided."); + throw std::invalid_argument("name"); + } + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_name)); + bucket_settings.name = bucket_name; + + PyObject* pyObj_bucket_type = PyDict_GetItemString(settings, "bucketType"); + if (pyObj_bucket_type) { + auto b_type = std::string(PyUnicode_AsUTF8(pyObj_bucket_type)); + if (b_type.compare("membase") == 0 || b_type.compare("couchbase") == 0) { + bucket_settings.bucket_type = couchbase::core::management::cluster::bucket_type::couchbase; + } else if (b_type.compare("memcached") == 0) { + bucket_settings.bucket_type = couchbase::core::management::cluster::bucket_type::memcached; + } else if (b_type.compare("ephemeral") == 0) { + bucket_settings.bucket_type = couchbase::core::management::cluster::bucket_type::ephemeral; + } + } + + PyObject* pyObj_ram = PyDict_GetItemString(settings, "ramQuotaMB"); + if (pyObj_ram != nullptr) { + bucket_settings.ram_quota_mb = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_ram)); + } + + PyObject* pyObj_max_expiry = PyDict_GetItemString(settings, "maxExpiry"); + if (pyObj_max_expiry != nullptr) { + bucket_settings.max_expiry = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_max_expiry)); + } + + PyObject* pyObj_compression_mode = PyDict_GetItemString(settings, "compressionMode"); + if (pyObj_compression_mode) { + auto comp_mode = std::string(PyUnicode_AsUTF8(pyObj_compression_mode)); + if (comp_mode.compare("off") == 0) { + bucket_settings.compression_mode = + couchbase::core::management::cluster::bucket_compression::off; + } else if (comp_mode.compare("active") == 0) { + bucket_settings.compression_mode = + couchbase::core::management::cluster::bucket_compression::active; + } else if (comp_mode.compare("passive") == 0) { + bucket_settings.compression_mode = + couchbase::core::management::cluster::bucket_compression::passive; + } + } + + PyObject* pyObj_durability_level = PyDict_GetItemString(settings, "durabilityMinLevel"); + if (pyObj_durability_level != nullptr) { + bucket_settings.minimum_durability_level = PyObject_to_durability_level(pyObj_durability_level); + } + + PyObject* pyObj_num_rep = PyDict_GetItemString(settings, "numReplicas"); + if (pyObj_num_rep != nullptr) { + bucket_settings.num_replicas = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_num_rep)); + } + + PyObject* pyObj_replica_indexes = PyDict_GetItemString(settings, "replicaIndex"); + if (pyObj_replica_indexes != nullptr) { + if (pyObj_replica_indexes == Py_True) { + bucket_settings.replica_indexes = true; + } else { + bucket_settings.replica_indexes = false; + } + } + + PyObject* pyObj_flush_enabled = PyDict_GetItemString(settings, "flushEnabled"); + if (pyObj_flush_enabled != nullptr) { + if (pyObj_flush_enabled == Py_True) { + bucket_settings.flush_enabled = true; + } else { + bucket_settings.flush_enabled = false; + } + } + + PyObject* pyObj_eviction_policy = PyDict_GetItemString(settings, "evictionPolicy"); + if (pyObj_eviction_policy != nullptr) { + auto evict = std::string(PyUnicode_AsUTF8(pyObj_eviction_policy)); + if (evict.compare("fullEviction") == 0) { + bucket_settings.eviction_policy = + couchbase::core::management::cluster::bucket_eviction_policy::full; + } else if (evict.compare("valueOnly") == 0) { + bucket_settings.eviction_policy = + couchbase::core::management::cluster::bucket_eviction_policy::value_only; + } else if (evict.compare("noEviction") == 0) { + bucket_settings.eviction_policy = + couchbase::core::management::cluster::bucket_eviction_policy::no_eviction; + } else if (evict.compare("nruEviction") == 0) { + bucket_settings.eviction_policy = + couchbase::core::management::cluster::bucket_eviction_policy::not_recently_used; + } + } + + PyObject* pyObj_conflict_res_type = PyDict_GetItemString(settings, "conflictResolutionType"); + if (pyObj_conflict_res_type != nullptr) { + auto crt = std::string(PyUnicode_AsUTF8(pyObj_conflict_res_type)); + if (crt.compare("lww") == 0) { + bucket_settings.conflict_resolution_type = + couchbase::core::management::cluster::bucket_conflict_resolution::timestamp; + } else if (crt.compare("seqno") == 0) { + bucket_settings.conflict_resolution_type = + couchbase::core::management::cluster::bucket_conflict_resolution::sequence_number; + } else if (crt.compare("custom") == 0) { + bucket_settings.conflict_resolution_type = + couchbase::core::management::cluster::bucket_conflict_resolution::custom; + } + } + + PyObject* pyObj_storage_backend = PyDict_GetItemString(settings, "storageBackend"); + if (pyObj_storage_backend != nullptr) { + auto backend = std::string(PyUnicode_AsUTF8(pyObj_storage_backend)); + if (backend.compare("couchstore") == 0) { + bucket_settings.storage_backend = + couchbase::core::management::cluster::bucket_storage_backend::couchstore; + } + if (backend.compare("magma") == 0) { + bucket_settings.storage_backend = + couchbase::core::management::cluster::bucket_storage_backend::magma; + } + } + + PyObject* pyObj_history_retention_collection_default = + PyDict_GetItemString(settings, "historyRetentionCollectionDefault"); + if (pyObj_history_retention_collection_default != nullptr) { + if (pyObj_history_retention_collection_default == Py_True) { + bucket_settings.history_retention_collection_default = true; + } else { + bucket_settings.history_retention_collection_default = false; + } + } + + PyObject* pyObj_history_retention_bytes = PyDict_GetItemString(settings, "historyRetentionBytes"); + if (pyObj_history_retention_bytes != nullptr) { + bucket_settings.history_retention_bytes = + static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_history_retention_bytes)); + } + + PyObject* pyObj_history_retention_duration = + PyDict_GetItemString(settings, "historyRetentionDuration"); + if (pyObj_history_retention_duration != nullptr) { + bucket_settings.history_retention_duration = + static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_history_retention_duration)); + } + + return bucket_settings; +} + +template<typename Request> +Request +get_bucket_mgmt_with_bucket_settings_req(PyObject* op_args) +{ + Request req{}; + + PyObject* pyObj_bucket_settings = PyDict_GetItemString(op_args, "bucket_settings"); + if (pyObj_bucket_settings == nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Expected bucket settings to be provided."); + throw std::invalid_argument("bucket_settings"); + } + req.bucket = get_bucket_settings(pyObj_bucket_settings); + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + return req; +} + +template<typename Request> +Request +get_bucket_mgmt_with_bucket_name_req(PyObject* op_args) +{ + Request req{}; + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name == nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Expected bucket_name to be provided."); + throw std::invalid_argument("bucket_name"); + } + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.name = bucket_name; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + return req; +} + +template<typename Request> +PyObject* +do_bucket_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_bucket_mgmt_op_response(resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_bucket_mgmt_op(connection* conn, + struct bucket_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + PyObject* res = nullptr; + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + try { + switch (options->op_type) { + case BucketManagementOperations::CREATE_BUCKET: { + auto req = get_bucket_mgmt_with_bucket_settings_req< + couchbase::core::operations::management::bucket_create_request>(options->op_args); + req.timeout = options->timeout_ms; + res = do_bucket_mgmt_op<couchbase::core::operations::management::bucket_create_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case BucketManagementOperations::UPDATE_BUCKET: { + auto req = get_bucket_mgmt_with_bucket_settings_req< + couchbase::core::operations::management::bucket_update_request>(options->op_args); + req.timeout = options->timeout_ms; + res = do_bucket_mgmt_op<couchbase::core::operations::management::bucket_update_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case BucketManagementOperations::DROP_BUCKET: { + auto req = get_bucket_mgmt_with_bucket_name_req< + couchbase::core::operations::management::bucket_drop_request>(options->op_args); + req.timeout = options->timeout_ms; + res = do_bucket_mgmt_op<couchbase::core::operations::management::bucket_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case BucketManagementOperations::GET_BUCKET: { + auto req = get_bucket_mgmt_with_bucket_name_req< + couchbase::core::operations::management::bucket_get_request>(options->op_args); + req.timeout = options->timeout_ms; + res = do_bucket_mgmt_op<couchbase::core::operations::management::bucket_get_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case BucketManagementOperations::GET_ALL_BUCKETS: { + couchbase::core::operations::management::bucket_get_all_request req{}; + req.timeout = options->timeout_ms; + res = do_bucket_mgmt_op<couchbase::core::operations::management::bucket_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case BucketManagementOperations::FLUSH_BUCKET: { + auto req = get_bucket_mgmt_with_bucket_name_req< + couchbase::core::operations::management::bucket_flush_request>(options->op_args); + req.timeout = options->timeout_ms; + res = do_bucket_mgmt_op<couchbase::core::operations::management::bucket_flush_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case BucketManagementOperations::BUCKET_DESCRIBE: { + auto req = get_bucket_mgmt_with_bucket_name_req< + couchbase::core::operations::management::bucket_describe_request>(options->op_args); + req.timeout = options->timeout_ms; + res = do_bucket_mgmt_op<couchbase::core::operations::management::bucket_describe_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized bucket mgmt operation passed in."); + barrier->set_value(nullptr); + break; + } + }; + } catch (const std::invalid_argument&) { + } + + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + + if (res == nullptr) { + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + return res; +} + +void +add_bucket_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = PyUnicode_FromString(BucketManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("BucketManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "bucket_mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/bucket_management.hxx b/src/management/bucket_management.hxx new file mode 100644 index 000000000..6e42f3156 --- /dev/null +++ b/src/management/bucket_management.hxx @@ -0,0 +1,90 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class BucketManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + CREATE_BUCKET, + UPDATE_BUCKET, + DROP_BUCKET, + GET_BUCKET, + GET_ALL_BUCKETS, + FLUSH_BUCKET, + BUCKET_DESCRIBE + }; + + BucketManagementOperations() + : BucketManagementOperations{ UNKNOWN } + { + } + constexpr BucketManagementOperations(BucketManagementOperations::OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(BucketManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(BucketManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "CREATE_BUCKET " + "UPDATE_BUCKET " + "DROP_BUCKET " + "GET_BUCKET " + "GET_ALL_BUCKETS " + "FLUSH_BUCKET " + "BUCKET_DESCRIBE"; + + return ops; + } + +private: + OperationType operation; +}; + +struct bucket_mgmt_options { + PyObject* op_args; + BucketManagementOperations::OperationType op_type = BucketManagementOperations::UNKNOWN; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; +}; + +PyObject* +handle_bucket_mgmt_op(connection* conn, + struct bucket_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback); + +void +add_bucket_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/management/collection_management.cxx b/src/management/collection_management.cxx new file mode 100644 index 000000000..3fa322903 --- /dev/null +++ b/src/management/collection_management.cxx @@ -0,0 +1,399 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "collection_management.hxx" + +#include <core/operations/management/collections.hxx> + +#include "../exceptions.hxx" +#include "../result.hxx" + +template<typename T> +result* +create_result_from_collection_mgmt_response([[maybe_unused]] const T& resp) +{ + PyObject* result_obj = create_result_obj(); + result* res = reinterpret_cast<result*>(result_obj); + return res; +} + +template<> +result* +create_result_from_collection_mgmt_response< + couchbase::core::operations::management::scope_get_all_response>( + const couchbase::core::operations::management::scope_get_all_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_scopes = PyList_New(static_cast<Py_ssize_t>(0)); + PyObject* pyObj_tmp = nullptr; + + for (auto const& scope : resp.manifest.scopes) { + PyObject* pyObj_scope_spec = PyDict_New(); + pyObj_tmp = PyUnicode_FromString(scope.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_scope_spec, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_scopes); + Py_XDECREF(pyObj_scope_spec); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_collections = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& collection : scope.collections) { + PyObject* pyObj_collection_spec = PyDict_New(); + pyObj_tmp = PyUnicode_FromString(collection.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_collection_spec, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_scopes); + Py_XDECREF(pyObj_collections); + Py_DECREF(pyObj_scope_spec); + Py_DECREF(pyObj_collection_spec); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(scope.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_collection_spec, "scope_name", pyObj_tmp)) { + Py_XDECREF(pyObj_scopes); + Py_XDECREF(pyObj_collections); + Py_DECREF(pyObj_scope_spec); + Py_DECREF(pyObj_collection_spec); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromLong(collection.max_expiry); + if (-1 == PyDict_SetItemString(pyObj_collection_spec, "max_expiry", pyObj_tmp)) { + Py_XDECREF(pyObj_scopes); + Py_XDECREF(pyObj_collections); + Py_DECREF(pyObj_scope_spec); + Py_DECREF(pyObj_collection_spec); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (collection.history.has_value()) { + pyObj_tmp = PyBool_FromLong(collection.history.value()); + if (-1 == PyDict_SetItemString(pyObj_collection_spec, "history", pyObj_tmp)) { + Py_XDECREF(pyObj_scopes); + Py_XDECREF(pyObj_collections); + Py_DECREF(pyObj_scope_spec); + Py_DECREF(pyObj_collection_spec); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyList_Append(pyObj_collections, pyObj_collection_spec); + Py_DECREF(pyObj_collection_spec); + } + + if (-1 == PyDict_SetItemString(pyObj_scope_spec, "collections", pyObj_collections)) { + Py_XDECREF(pyObj_scopes); + Py_XDECREF(pyObj_collections); + Py_DECREF(pyObj_scope_spec); + return nullptr; + } + Py_DECREF(pyObj_collections); + + PyList_Append(pyObj_scopes, pyObj_scope_spec); + Py_DECREF(pyObj_scope_spec); + } + if (-1 == PyDict_SetItemString(res->dict, "scopes", pyObj_scopes)) { + Py_XDECREF(pyObj_scopes); + return nullptr; + } + Py_DECREF(pyObj_scopes); + + return res; +} + +template<typename T> +void +create_result_from_collection_mgmt_op_response(const T& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing collection mgmt operation.", "CollectionMgmt"); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_collection_mgmt_response(resp); + if (res == nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Collection mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to catch exception here? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +template<typename Request> +PyObject* +do_collection_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_collection_mgmt_op_response(resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_collection_mgmt_op(connection* conn, + struct collection_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + PyObject* res = nullptr; + PyObject* pyObj_bucket_name = PyDict_GetItemString(options->op_args, "bucket_name"); + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + + switch (options->op_type) { + case CollectionManagementOperations::CREATE_SCOPE: { + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + couchbase::core::operations::management::scope_create_request req{}; + req.bucket_name = bucket_name; + req.scope_name = scope_name; + req.timeout = options->timeout_ms; + + res = do_collection_mgmt_op<couchbase::core::operations::management::scope_create_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case CollectionManagementOperations::DROP_SCOPE: { + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + couchbase::core::operations::management::scope_drop_request req{}; + req.bucket_name = bucket_name; + req.scope_name = scope_name; + req.timeout = options->timeout_ms; + + res = do_collection_mgmt_op<couchbase::core::operations::management::scope_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case CollectionManagementOperations::GET_ALL_SCOPES: { + couchbase::core::operations::management::scope_get_all_request req{}; + req.bucket_name = bucket_name; + req.timeout = options->timeout_ms; + + res = do_collection_mgmt_op<couchbase::core::operations::management::scope_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case CollectionManagementOperations::CREATE_COLLECTION: { + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + PyObject* pyObj_collection_name = PyDict_GetItemString(options->op_args, "collection_name"); + auto collection_name = std::string(PyUnicode_AsUTF8(pyObj_collection_name)); + // optional + PyObject* pyObj_max_expiry = PyDict_GetItemString(options->op_args, "max_expiry"); + PyObject* pyObj_history = PyDict_GetItemString(options->op_args, "history"); + + couchbase::core::operations::management::collection_create_request req{}; + req.bucket_name = bucket_name; + req.scope_name = scope_name; + req.collection_name = collection_name; + if (pyObj_max_expiry != nullptr) { + req.max_expiry = static_cast<int32_t>(PyLong_AsLong(pyObj_max_expiry)); + // PyLong_AsLong() returns -1 on error, need to use PyErr_Occurred() to disambiguate. + if (PyErr_Occurred() != nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unabled to parse max_expiry."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return res; + } + } + if (pyObj_history != nullptr) { + if (pyObj_history == Py_True) { + req.history = true; + } else { + req.history = false; + } + } + req.timeout = options->timeout_ms; + + res = + do_collection_mgmt_op<couchbase::core::operations::management::collection_create_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case CollectionManagementOperations::UPDATE_COLLECTION: { + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + PyObject* pyObj_collection_name = PyDict_GetItemString(options->op_args, "collection_name"); + auto collection_name = std::string(PyUnicode_AsUTF8(pyObj_collection_name)); + // optional + PyObject* pyObj_max_expiry = PyDict_GetItemString(options->op_args, "max_expiry"); + PyObject* pyObj_history = PyDict_GetItemString(options->op_args, "history"); + + couchbase::core::operations::management::collection_update_request req{}; + req.bucket_name = bucket_name; + req.scope_name = scope_name; + req.collection_name = collection_name; + if (pyObj_max_expiry != nullptr) { + req.max_expiry = static_cast<int32_t>(PyLong_AsLong(pyObj_max_expiry)); + // PyLong_AsLong() returns -1 on error, need to use PyErr_Occurred() to disambiguate. + if (PyErr_Occurred() != nullptr) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unabled to parse max_expiry."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return res; + } + } + if (pyObj_history != nullptr) { + if (pyObj_history == Py_True) { + req.history = true; + } else { + req.history = false; + } + } + req.timeout = options->timeout_ms; + + res = + do_collection_mgmt_op<couchbase::core::operations::management::collection_update_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + + case CollectionManagementOperations::DROP_COLLECTION: { + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + PyObject* pyObj_collection_name = PyDict_GetItemString(options->op_args, "collection_name"); + auto collection_name = std::string(PyUnicode_AsUTF8(pyObj_collection_name)); + + couchbase::core::operations::management::collection_drop_request req{}; + req.bucket_name = bucket_name; + req.scope_name = scope_name; + req.collection_name = collection_name; + req.timeout = options->timeout_ms; + + res = do_collection_mgmt_op<couchbase::core::operations::management::collection_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized collection mgmt operation passed in."); + barrier->set_value(nullptr); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + + return res; +} + +void +add_collection_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = + PyUnicode_FromString(CollectionManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("CollectionManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "collection_mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/collection_management.hxx b/src/management/collection_management.hxx new file mode 100644 index 000000000..04312f4f0 --- /dev/null +++ b/src/management/collection_management.hxx @@ -0,0 +1,88 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class CollectionManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + CREATE_SCOPE, + DROP_SCOPE, + GET_ALL_SCOPES, + CREATE_COLLECTION, + DROP_COLLECTION, + UPDATE_COLLECTION + }; + + CollectionManagementOperations() + : CollectionManagementOperations{ UNKNOWN } + { + } + constexpr CollectionManagementOperations(CollectionManagementOperations::OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(CollectionManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(CollectionManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "CREATE_SCOPE " + "DROP_SCOPE " + "GET_ALL_SCOPES " + "CREATE_COLLECTION " + "DROP_COLLECTION " + "UPDATE_COLLECTION"; + + return ops; + } + +private: + OperationType operation; +}; + +struct collection_mgmt_options { + PyObject* op_args; + CollectionManagementOperations::OperationType op_type = CollectionManagementOperations::UNKNOWN; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; +}; + +PyObject* +handle_collection_mgmt_op(connection* conn, + struct collection_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback); + +void +add_collection_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/management/eventing_function_management.cxx b/src/management/eventing_function_management.cxx new file mode 100644 index 000000000..e7d9fac9d --- /dev/null +++ b/src/management/eventing_function_management.cxx @@ -0,0 +1,2013 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "eventing_function_management.hxx" + +#include <core/management/eventing_function.hxx> +#include <core/management/eventing_status.hxx> +#include <core/operations/management/eventing.hxx> +#include <core/operations/management/eventing_problem.hxx> +#include <couchbase/query_scan_consistency.hxx> + +#include "../exceptions.hxx" +#include "../result.hxx" + +PyObject* +build_eventing_function_status_functions( + std::vector<couchbase::core::management::eventing::function_state> functions) +{ + PyObject* pyObj_functions = PyList_New(static_cast<Py_ssize_t>(0)); + PyObject* pyObj_tmp = nullptr; + + for (auto const& function : functions) { + PyObject* pyObj_function = PyDict_New(); + + pyObj_tmp = PyUnicode_FromString(function.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_function, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + switch (function.status) { + case couchbase::core::management::eventing::function_status::undeployed: { + pyObj_tmp = PyUnicode_FromString("undeployed"); + break; + } + case couchbase::core::management::eventing::function_status::undeploying: { + pyObj_tmp = PyUnicode_FromString("undeploying"); + break; + } + case couchbase::core::management::eventing::function_status::deploying: { + pyObj_tmp = PyUnicode_FromString("deploying"); + break; + } + case couchbase::core::management::eventing::function_status::deployed: { + pyObj_tmp = PyUnicode_FromString("deployed"); + break; + } + case couchbase::core::management::eventing::function_status::pausing: { + pyObj_tmp = PyUnicode_FromString("pausing"); + break; + } + case couchbase::core::management::eventing::function_status::paused: { + pyObj_tmp = PyUnicode_FromString("paused"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("undeployed"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_function, "status", pyObj_tmp)) { + Py_DECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(function.num_bootstrapping_nodes); + if (-1 == PyDict_SetItemString(pyObj_function, "num_bootstrapping_nodes", pyObj_tmp)) { + Py_DECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(function.num_deployed_nodes); + if (-1 == PyDict_SetItemString(pyObj_function, "num_deployed_nodes", pyObj_tmp)) { + Py_DECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + switch (function.deployment_status) { + case couchbase::core::management::eventing::function_deployment_status::deployed: { + pyObj_tmp = PyUnicode_FromString("deployed"); + break; + } + case couchbase::core::management::eventing::function_deployment_status::undeployed: { + pyObj_tmp = PyUnicode_FromString("undeployed"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("undeployed"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_function, "deployment_status", pyObj_tmp)) { + Py_DECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + switch (function.processing_status) { + case couchbase::core::management::eventing::function_processing_status::paused: { + pyObj_tmp = PyUnicode_FromString("paused"); + break; + } + case couchbase::core::management::eventing::function_processing_status::running: { + pyObj_tmp = PyUnicode_FromString("running"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("paused"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_function, "processing_status", pyObj_tmp)) { + Py_DECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (function.redeploy_required.has_value()) { + if (function.redeploy_required.value()) { + if (-1 == PyDict_SetItemString(pyObj_function, "redeploy_required", Py_True)) { + Py_DECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + } else { + if (-1 == PyDict_SetItemString(pyObj_function, "redeploy_required", Py_False)) { + Py_DECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + } + } + + if (-1 == PyList_Append(pyObj_functions, pyObj_function)) { + Py_XDECREF(pyObj_function); + Py_XDECREF(pyObj_functions); + return nullptr; + } + Py_DECREF(pyObj_function); + } + + return pyObj_functions; +} + +PyObject* +build_eventing_function_status(const couchbase::core::management::eventing::status& status) +{ + PyObject* pyObj_status = PyDict_New(); + PyObject* pyObj_tmp = nullptr; + + pyObj_tmp = PyLong_FromLongLong(status.num_eventing_nodes); + if (-1 == PyDict_SetItemString(pyObj_status, "num_eventing_nodes", pyObj_tmp)) { + Py_XDECREF(pyObj_status); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = build_eventing_function_status_functions(status.functions); + if (pyObj_tmp == nullptr) { + Py_DECREF(pyObj_status); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_status, "functions", pyObj_tmp)) { + Py_DECREF(pyObj_status); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + return pyObj_status; +} + +PyObject* +build_eventing_function_settings( + const couchbase::core::management::eventing::function_settings& settings) +{ + PyObject* pyObj_settings = PyDict_New(); + PyObject* pyObj_tmp = nullptr; + + if (settings.cpp_worker_count.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.cpp_worker_count.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "cpp_worker_count", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.dcp_stream_boundary.has_value()) { + switch (settings.dcp_stream_boundary.value()) { + case couchbase::core::management::eventing::function_dcp_boundary::everything: { + pyObj_tmp = PyUnicode_FromString("everything"); + break; + } + case couchbase::core::management::eventing::function_dcp_boundary::from_now: { + pyObj_tmp = PyUnicode_FromString("from_now"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("everything"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_settings, "dcp_stream_boundary", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.description.has_value()) { + pyObj_tmp = PyUnicode_FromString(settings.description.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_settings, "description", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.deployment_status.has_value()) { + switch (settings.deployment_status.value()) { + case couchbase::core::management::eventing::function_deployment_status::deployed: { + pyObj_tmp = PyUnicode_FromString("deployed"); + break; + } + case couchbase::core::management::eventing::function_deployment_status::undeployed: { + pyObj_tmp = PyUnicode_FromString("undeployed"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("undeployed"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_settings, "deployment_status", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.processing_status.has_value()) { + switch (settings.processing_status.value()) { + case couchbase::core::management::eventing::function_processing_status::running: { + pyObj_tmp = PyUnicode_FromString("running"); + break; + } + case couchbase::core::management::eventing::function_processing_status::paused: { + pyObj_tmp = PyUnicode_FromString("paused"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("running"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_settings, "processing_status", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.log_level.has_value()) { + switch (settings.log_level.value()) { + case couchbase::core::management::eventing::function_log_level::info: { + pyObj_tmp = PyUnicode_FromString("info"); + break; + } + case couchbase::core::management::eventing::function_log_level::error: { + pyObj_tmp = PyUnicode_FromString("error"); + break; + } + case couchbase::core::management::eventing::function_log_level::warning: { + pyObj_tmp = PyUnicode_FromString("warning"); + break; + } + case couchbase::core::management::eventing::function_log_level::debug: { + pyObj_tmp = PyUnicode_FromString("debug"); + break; + } + case couchbase::core::management::eventing::function_log_level::trace: { + pyObj_tmp = PyUnicode_FromString("trace"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("info"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_settings, "log_level", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.language_compatibility.has_value()) { + switch (settings.language_compatibility.value()) { + case couchbase::core::management::eventing::function_language_compatibility::version_6_0_0: { + pyObj_tmp = PyUnicode_FromString("version_6_0_0"); + break; + } + case couchbase::core::management::eventing::function_language_compatibility::version_6_5_0: { + pyObj_tmp = PyUnicode_FromString("version_6_5_0"); + break; + } + case couchbase::core::management::eventing::function_language_compatibility::version_6_6_2: { + pyObj_tmp = PyUnicode_FromString("version_6_6_2"); + break; + } + case couchbase::core::management::eventing::function_language_compatibility::version_7_2_0: { + pyObj_tmp = PyUnicode_FromString("version_7_2_0"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("version_6_6_2"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_settings, "language_compatibility", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.execution_timeout.has_value()) { + std::chrono::duration<unsigned long long> int_sec = settings.execution_timeout.value(); + pyObj_tmp = PyLong_FromUnsignedLongLong(int_sec.count()); + if (-1 == PyDict_SetItemString(pyObj_settings, "execution_timeout", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.lcb_inst_capacity.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.lcb_inst_capacity.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "lcb_inst_capacity", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.lcb_retry_count.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.lcb_retry_count.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "lcb_retry_count", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.lcb_timeout.has_value()) { + std::chrono::duration<unsigned long long> int_sec = settings.lcb_timeout.value(); + pyObj_tmp = PyLong_FromUnsignedLongLong(int_sec.count()); + if (-1 == PyDict_SetItemString(pyObj_settings, "lcb_timeout", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.query_consistency.has_value()) { + switch (settings.query_consistency.value()) { + case couchbase::query_scan_consistency::not_bounded: { + pyObj_tmp = PyUnicode_FromString("not_bounded"); + break; + } + case couchbase::query_scan_consistency::request_plus: { + pyObj_tmp = PyUnicode_FromString("request_plus"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("not_bounded"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_settings, "query_consistency", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.num_timer_partitions.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.num_timer_partitions.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "num_timer_partitions", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.sock_batch_size.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.sock_batch_size.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "sock_batch_size", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.tick_duration.has_value()) { + std::chrono::duration<unsigned long long, std::milli> int_msec = settings.tick_duration.value(); + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_settings, "tick_duration", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.timer_context_size.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.timer_context_size.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "timer_context_size", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.user_prefix.has_value()) { + pyObj_tmp = PyUnicode_FromString(settings.user_prefix.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_settings, "user_prefix", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.bucket_cache_size.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.bucket_cache_size.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "bucket_cache_size", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.bucket_cache_age.has_value()) { + std::chrono::duration<unsigned long long, std::milli> int_msec = + settings.bucket_cache_age.value(); + pyObj_tmp = PyLong_FromUnsignedLongLong(int_msec.count()); + if (-1 == PyDict_SetItemString(pyObj_settings, "bucket_cache_age", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.curl_max_allowed_resp_size.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.curl_max_allowed_resp_size.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "curl_max_allowed_resp_size", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.query_prepare_all.has_value()) { + if (settings.query_prepare_all.value()) { + if (-1 == PyDict_SetItemString(pyObj_settings, "query_prepare_all", Py_True)) { + Py_DECREF(pyObj_settings); + return nullptr; + } + } else { + if (-1 == PyDict_SetItemString(pyObj_settings, "query_prepare_all", Py_False)) { + Py_DECREF(pyObj_settings); + return nullptr; + } + } + } + + if (settings.worker_count.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.worker_count.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "worker_count", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.handler_headers.size() > 0) { + PyObject* pyObj_handler_headers = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& header : settings.handler_headers) { + pyObj_tmp = PyUnicode_FromString(header.c_str()); + if (-1 == PyList_Append(pyObj_handler_headers, pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_handler_headers); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + if (-1 == PyDict_SetItemString(pyObj_settings, "handler_headers", pyObj_handler_headers)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_handler_headers); + return nullptr; + } + Py_DECREF(pyObj_handler_headers); + } + + if (settings.handler_footers.size() > 0) { + PyObject* pyObj_handler_footers = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& footer : settings.handler_footers) { + pyObj_tmp = PyUnicode_FromString(footer.c_str()); + if (-1 == PyList_Append(pyObj_handler_footers, pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_handler_footers); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + if (-1 == PyDict_SetItemString(pyObj_settings, "handler_footers", pyObj_handler_footers)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_handler_footers); + return nullptr; + } + Py_DECREF(pyObj_handler_footers); + } + + if (settings.enable_app_log_rotation.has_value()) { + if (settings.enable_app_log_rotation.value()) { + if (-1 == PyDict_SetItemString(pyObj_settings, "enable_app_log_rotation", Py_True)) { + Py_DECREF(pyObj_settings); + return nullptr; + } + } else { + if (-1 == PyDict_SetItemString(pyObj_settings, "enable_app_log_rotation", Py_False)) { + Py_DECREF(pyObj_settings); + return nullptr; + } + } + } + + if (settings.app_log_dir.has_value()) { + pyObj_tmp = PyUnicode_FromString(settings.app_log_dir.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_settings, "app_log_dir", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.app_log_max_size.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.app_log_max_size.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "app_log_max_size", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.app_log_max_files.has_value()) { + pyObj_tmp = PyLong_FromLongLong(settings.app_log_max_files.value()); + if (-1 == PyDict_SetItemString(pyObj_settings, "app_log_max_files", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (settings.checkpoint_interval.has_value()) { + std::chrono::duration<unsigned long long> int_sec = settings.checkpoint_interval.value(); + pyObj_tmp = PyLong_FromUnsignedLongLong(int_sec.count()); + if (-1 == PyDict_SetItemString(pyObj_settings, "checkpoint_interval", pyObj_tmp)) { + Py_DECREF(pyObj_settings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + return pyObj_settings; +} + +PyObject* +build_eventing_function_keyspace( + const couchbase::core::management::eventing::function_keyspace& keyspace) +{ + PyObject* pyObj_keyspace = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(keyspace.bucket.c_str()); + if (-1 == PyDict_SetItemString(pyObj_keyspace, "bucket", pyObj_tmp)) { + Py_XDECREF(pyObj_keyspace); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (keyspace.scope.has_value()) { + pyObj_tmp = PyUnicode_FromString(keyspace.scope.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_keyspace, "scope", pyObj_tmp)) { + Py_DECREF(pyObj_keyspace); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (keyspace.collection.has_value()) { + pyObj_tmp = PyUnicode_FromString(keyspace.collection.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_keyspace, "collection", pyObj_tmp)) { + Py_DECREF(pyObj_keyspace); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + return pyObj_keyspace; +} + +PyObject* +build_function_bucket_bindings( + std::vector<couchbase::core::management::eventing::function_bucket_binding> bucket_bindings) +{ + PyObject* pyObj_bucket_bindings = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& binding : bucket_bindings) { + PyObject* pyObj_binding = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(binding.alias.c_str()); + if (-1 == PyDict_SetItemString(pyObj_binding, "alias", pyObj_tmp)) { + Py_XDECREF(pyObj_binding); + Py_XDECREF(pyObj_bucket_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = build_eventing_function_keyspace(binding.name); + if (pyObj_tmp == nullptr) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_bucket_bindings); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_binding, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_binding); + Py_XDECREF(pyObj_bucket_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + switch (binding.access) { + case couchbase::core::management::eventing::function_bucket_access::read_only: { + pyObj_tmp = PyUnicode_FromString("read_only"); + break; + } + case couchbase::core::management::eventing::function_bucket_access::read_write: { + pyObj_tmp = PyUnicode_FromString("read_write"); + break; + } + default: { + pyObj_tmp = PyUnicode_FromString("read_write"); + break; + } + } + if (-1 == PyDict_SetItemString(pyObj_binding, "access", pyObj_tmp)) { + Py_XDECREF(pyObj_binding); + Py_XDECREF(pyObj_bucket_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + PyList_Append(pyObj_bucket_bindings, pyObj_binding); + Py_DECREF(pyObj_binding); + } + + return pyObj_bucket_bindings; +} + +PyObject* +build_function_url_bindings( + std::vector<couchbase::core::management::eventing::function_url_binding> url_bindings) +{ + PyObject* pyObj_url_bindings = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& binding : url_bindings) { + PyObject* pyObj_binding = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(binding.alias.c_str()); + if (-1 == PyDict_SetItemString(pyObj_binding, "alias", pyObj_tmp)) { + Py_XDECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(binding.hostname.c_str()); + if (-1 == PyDict_SetItemString(pyObj_binding, "hostname", pyObj_tmp)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (binding.allow_cookies) { + if (-1 == PyDict_SetItemString(pyObj_binding, "allow_cookies", Py_True)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + return nullptr; + } + } else { + if (-1 == PyDict_SetItemString(pyObj_binding, "allow_cookies", Py_False)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + return nullptr; + } + } + + if (binding.validate_ssl_certificate) { + if (-1 == PyDict_SetItemString(pyObj_binding, "validate_ssl_certificate", Py_True)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + return nullptr; + } + } else { + if (-1 == PyDict_SetItemString(pyObj_binding, "validate_ssl_certificate", Py_False)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + return nullptr; + } + } + + if (std::holds_alternative<couchbase::core::management::eventing::function_url_no_auth>( + binding.auth)) { + pyObj_tmp = PyUnicode_FromString("no-auth"); + if (-1 == PyDict_SetItemString(pyObj_binding, "auth_type", pyObj_tmp)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } else if (std::holds_alternative< + couchbase::core::management::eventing::function_url_auth_basic>(binding.auth)) { + pyObj_tmp = PyUnicode_FromString("basic"); + if (-1 == PyDict_SetItemString(pyObj_binding, "auth_type", pyObj_tmp)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString( + std::get<couchbase::core::management::eventing::function_url_auth_basic>(binding.auth) + .username.c_str()); + if (-1 == PyDict_SetItemString(pyObj_binding, "username", pyObj_tmp)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } else if (std::holds_alternative< + couchbase::core::management::eventing::function_url_auth_digest>(binding.auth)) { + pyObj_tmp = PyUnicode_FromString("digest"); + if (-1 == PyDict_SetItemString(pyObj_binding, "auth_type", pyObj_tmp)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString( + std::get<couchbase::core::management::eventing::function_url_auth_digest>(binding.auth) + .username.c_str()); + if (-1 == PyDict_SetItemString(pyObj_binding, "username", pyObj_tmp)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } else if (std::holds_alternative< + couchbase::core::management::eventing::function_url_auth_bearer>(binding.auth)) { + pyObj_tmp = PyUnicode_FromString("bearer"); + if (-1 == PyDict_SetItemString(pyObj_binding, "auth_type", pyObj_tmp)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_url_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyList_Append(pyObj_url_bindings, pyObj_binding); + Py_DECREF(pyObj_binding); + } + + return pyObj_url_bindings; +} + +PyObject* +build_function_constant_bindings( + std::vector<couchbase::core::management::eventing::function_constant_binding> constant_bindings) +{ + PyObject* pyObj_constant_bindings = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& binding : constant_bindings) { + PyObject* pyObj_binding = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(binding.alias.c_str()); + if (-1 == PyDict_SetItemString(pyObj_binding, "alias", pyObj_tmp)) { + Py_XDECREF(pyObj_binding); + Py_XDECREF(pyObj_constant_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(binding.literal.c_str()); + if (-1 == PyDict_SetItemString(pyObj_binding, "literal", pyObj_tmp)) { + Py_DECREF(pyObj_binding); + Py_XDECREF(pyObj_constant_bindings); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyList_Append(pyObj_constant_bindings, pyObj_binding); + Py_DECREF(pyObj_binding); + } + return pyObj_constant_bindings; +} + +PyObject* +build_eventing_function(const couchbase::core::management::eventing::function& function) +{ + PyObject* pyObj_eventing_function = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(function.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(function.code.c_str()); + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "code", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = build_eventing_function_keyspace(function.metadata_keyspace); + if (pyObj_tmp == nullptr) { + Py_DECREF(pyObj_eventing_function); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "metadata_keyspace", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = build_eventing_function_keyspace(function.source_keyspace); + if (pyObj_tmp == nullptr) { + Py_DECREF(pyObj_eventing_function); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "source_keyspace", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (function.version.has_value()) { + pyObj_tmp = PyUnicode_FromString(function.version.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "version", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (function.enforce_schema.has_value()) { + if (function.enforce_schema.value()) { + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "enforce_schema", Py_True)) { + Py_DECREF(pyObj_eventing_function); + return nullptr; + } + } else { + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "enforce_schema", Py_False)) { + Py_DECREF(pyObj_eventing_function); + return nullptr; + } + } + } + + if (function.handler_uuid.has_value()) { + pyObj_tmp = PyLong_FromLongLong(function.handler_uuid.value()); + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "handler_uuid", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (function.function_instance_id.has_value()) { + pyObj_tmp = PyUnicode_FromString(function.function_instance_id.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "function_instance_id", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = build_function_bucket_bindings(function.bucket_bindings); + if (pyObj_tmp == nullptr) { + Py_DECREF(pyObj_eventing_function); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "bucket_bindings", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = build_function_url_bindings(function.url_bindings); + if (pyObj_tmp == nullptr) { + Py_DECREF(pyObj_eventing_function); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "url_bindings", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = build_function_constant_bindings(function.constant_bindings); + if (pyObj_tmp == nullptr) { + Py_DECREF(pyObj_eventing_function); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "constant_bindings", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = build_eventing_function_settings(function.settings); + if (pyObj_tmp == nullptr) { + Py_DECREF(pyObj_eventing_function); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_eventing_function, "settings", pyObj_tmp)) { + Py_DECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + return pyObj_eventing_function; +} + +template<typename Response> +result* +create_result_from_eventing_function_mgmt_response([[maybe_unused]] const Response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + return res; +} + +template<> +result* +create_result_from_eventing_function_mgmt_response< + couchbase::core::operations::management::eventing_get_function_response>( + const couchbase::core::operations::management::eventing_get_function_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_eventing_function = build_eventing_function(resp.function); + if (-1 == PyDict_SetItemString(res->dict, "function", pyObj_eventing_function)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_eventing_function); + return nullptr; + } + Py_DECREF(pyObj_eventing_function); + + return res; +} + +template<> +result* +create_result_from_eventing_function_mgmt_response< + couchbase::core::operations::management::eventing_get_all_functions_response>( + const couchbase::core::operations::management::eventing_get_all_functions_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_functions = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& function : resp.functions) { + PyObject* pyObj_eventing_function = build_eventing_function(function); + if (pyObj_eventing_function == nullptr) { + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_result); + return nullptr; + } + if (-1 == PyList_Append(pyObj_functions, pyObj_eventing_function)) { + Py_XDECREF(pyObj_functions); + Py_XDECREF(pyObj_eventing_function); + Py_XDECREF(pyObj_result); + return nullptr; + } + Py_DECREF(pyObj_eventing_function); + } + if (-1 == PyDict_SetItemString(res->dict, "function", pyObj_functions)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_functions); + return nullptr; + } + Py_DECREF(pyObj_functions); + + return res; +} + +template<> +result* +create_result_from_eventing_function_mgmt_response< + couchbase::core::operations::management::eventing_get_status_response>( + const couchbase::core::operations::management::eventing_get_status_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_eventing_function_status = build_eventing_function_status(resp.status); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_eventing_function_status)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_eventing_function_status); + return nullptr; + } + Py_DECREF(pyObj_eventing_function_status); + + return res; +} + +PyObject* +build_eventing_function_mgmt_problem( + const couchbase::core::operations::management::eventing_problem& problem) +{ + PyObject* pyObj_problem = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(problem.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_problem, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_problem); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(problem.description.c_str()); + if (-1 == PyDict_SetItemString(pyObj_problem, "description", pyObj_tmp)) { + Py_XDECREF(pyObj_problem); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(problem.code); + if (-1 == PyDict_SetItemString(pyObj_problem, "code", pyObj_tmp)) { + Py_XDECREF(pyObj_problem); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + return pyObj_problem; +} + +template<typename Response> +void +create_result_from_eventing_function_mgmt_op_response( + Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + PyObject* pyObj_problem = nullptr; + if (resp.error.has_value()) { + pyObj_problem = build_eventing_function_mgmt_problem(resp.error.value()); + } + pyObj_exc = build_exception_from_context(resp.ctx, + __FILE__, + __LINE__, + "Error doing eventing function mgmt operation.", + "EventingFunctionMgmt"); + if (pyObj_problem != nullptr) { + pycbc_add_exception_info(pyObj_exc, "eventing_problem", pyObj_problem); + } + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_eventing_function_mgmt_response(resp); + + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception(PycbcError::UnableToBuildResult, + __FILE__, + __LINE__, + "Eventing function mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +couchbase::core::management::eventing::function_settings +get_event_function_settings(PyObject* pyObj_settings) +{ + couchbase::core::management::eventing::function_settings settings{}; + + PyObject* pyObj_cpp_worker_count = PyDict_GetItemString(pyObj_settings, "cpp_worker_count"); + if (pyObj_cpp_worker_count != nullptr) { + settings.cpp_worker_count = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_cpp_worker_count)); + } + + PyObject* pyObj_dcp_stream_boundary = PyDict_GetItemString(pyObj_settings, "dcp_stream_boundary"); + if (pyObj_dcp_stream_boundary != nullptr) { + auto dcp_boundary = std::string(PyUnicode_AsUTF8(pyObj_dcp_stream_boundary)); + if (dcp_boundary.compare("everything") == 0) { + settings.dcp_stream_boundary = + couchbase::core::management::eventing::function_dcp_boundary::everything; + } else if (dcp_boundary.compare("from_now") == 0) { + settings.dcp_stream_boundary = + couchbase::core::management::eventing::function_dcp_boundary::from_now; + } + } + + PyObject* pyObj_description = PyDict_GetItemString(pyObj_settings, "description"); + if (pyObj_description != nullptr) { + auto description = std::string(PyUnicode_AsUTF8(pyObj_description)); + settings.description = description; + } + + PyObject* pyObj_deployment_status = PyDict_GetItemString(pyObj_settings, "deployment_status"); + if (pyObj_deployment_status != nullptr) { + auto deployment_status = std::string(PyUnicode_AsUTF8(pyObj_deployment_status)); + if (deployment_status.compare("deployed") == 0) { + settings.deployment_status = + couchbase::core::management::eventing::function_deployment_status::deployed; + } else if (deployment_status.compare("undeployed") == 0) { + settings.deployment_status = + couchbase::core::management::eventing::function_deployment_status::undeployed; + } + } + + PyObject* pyObj_processing_status = PyDict_GetItemString(pyObj_settings, "processing_status"); + if (pyObj_processing_status != nullptr) { + auto processing_status = std::string(PyUnicode_AsUTF8(pyObj_processing_status)); + if (processing_status.compare("running") == 0) { + settings.processing_status = + couchbase::core::management::eventing::function_processing_status::running; + } else if (processing_status.compare("paused") == 0) { + settings.processing_status = + couchbase::core::management::eventing::function_processing_status::paused; + } + } + + PyObject* pyObj_log_level = PyDict_GetItemString(pyObj_settings, "log_level"); + if (pyObj_log_level != nullptr) { + auto log_level = std::string(PyUnicode_AsUTF8(pyObj_log_level)); + if (log_level.compare("info") == 0) { + settings.log_level = couchbase::core::management::eventing::function_log_level::info; + } else if (log_level.compare("error") == 0) { + settings.log_level = couchbase::core::management::eventing::function_log_level::error; + } else if (log_level.compare("warning") == 0) { + settings.log_level = couchbase::core::management::eventing::function_log_level::warning; + } else if (log_level.compare("debug") == 0) { + settings.log_level = couchbase::core::management::eventing::function_log_level::debug; + } else if (log_level.compare("trace") == 0) { + settings.log_level = couchbase::core::management::eventing::function_log_level::trace; + } + } + + PyObject* pyObj_language_compatibility = + PyDict_GetItemString(pyObj_settings, "language_compatibility"); + if (pyObj_language_compatibility != nullptr) { + auto language_compatibility = std::string(PyUnicode_AsUTF8(pyObj_language_compatibility)); + if (language_compatibility.compare("version_6_0_0") == 0) { + settings.language_compatibility = + couchbase::core::management::eventing::function_language_compatibility::version_6_0_0; + } else if (language_compatibility.compare("version_6_5_0") == 0) { + settings.language_compatibility = + couchbase::core::management::eventing::function_language_compatibility::version_6_5_0; + } else if (language_compatibility.compare("version_6_6_2") == 0) { + settings.language_compatibility = + couchbase::core::management::eventing::function_language_compatibility::version_6_6_2; + } else if (language_compatibility.compare("version_7_2_0") == 0) { + settings.language_compatibility = + couchbase::core::management::eventing::function_language_compatibility::version_7_2_0; + } + } + + PyObject* pyObj_execution_timeout = PyDict_GetItemString(pyObj_settings, "execution_timeout"); + if (pyObj_execution_timeout != nullptr) { + auto execution_timeout = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_execution_timeout)); + settings.execution_timeout = std::chrono::seconds(execution_timeout); + } + + PyObject* pyObj_lcb_inst_capacity = PyDict_GetItemString(pyObj_settings, "lcb_inst_capacity"); + if (pyObj_lcb_inst_capacity != nullptr) { + settings.lcb_inst_capacity = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_lcb_inst_capacity)); + } + + PyObject* pyObj_lcb_retry_count = PyDict_GetItemString(pyObj_settings, "lcb_retry_count"); + if (pyObj_lcb_retry_count != nullptr) { + settings.lcb_retry_count = static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_lcb_retry_count)); + } + + PyObject* pyObj_lcb_timeout = PyDict_GetItemString(pyObj_settings, "lcb_timeout"); + if (pyObj_execution_timeout != nullptr) { + auto lcb_timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_lcb_timeout)); + settings.lcb_timeout = std::chrono::seconds(lcb_timeout); + } + + PyObject* pyObj_query_consistency = PyDict_GetItemString(pyObj_settings, "query_consistency"); + if (pyObj_query_consistency != nullptr) { + auto query_consistency = std::string(PyUnicode_AsUTF8(pyObj_query_consistency)); + if (query_consistency.compare("not_bounded") == 0) { + settings.query_consistency = couchbase::query_scan_consistency::not_bounded; + } else if (query_consistency.compare("request_plus") == 0) { + settings.query_consistency = couchbase::query_scan_consistency::request_plus; + } + } + + PyObject* pyObj_num_timer_partitions = + PyDict_GetItemString(pyObj_settings, "num_timer_partitions"); + if (pyObj_num_timer_partitions != nullptr) { + settings.num_timer_partitions = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_num_timer_partitions)); + } + + PyObject* pyObj_sock_batch_size = PyDict_GetItemString(pyObj_settings, "sock_batch_size"); + if (pyObj_sock_batch_size != nullptr) { + settings.sock_batch_size = static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_sock_batch_size)); + } + + PyObject* pyObj_tick_duration = PyDict_GetItemString(pyObj_settings, "tick_duration"); + if (pyObj_tick_duration != nullptr) { + auto tick_duration = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_tick_duration)); + settings.tick_duration = std::chrono::milliseconds(std::max(0ULL, tick_duration / 1000ULL)); + } + + PyObject* pyObj_timer_context_size = PyDict_GetItemString(pyObj_settings, "timer_context_size"); + if (pyObj_timer_context_size != nullptr) { + settings.timer_context_size = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_timer_context_size)); + } + + PyObject* pyObj_user_prefix = PyDict_GetItemString(pyObj_settings, "user_prefix"); + if (pyObj_user_prefix != nullptr) { + settings.user_prefix = std::string(PyUnicode_AsUTF8(pyObj_user_prefix)); + } + + PyObject* pyObj_bucket_cache_size = PyDict_GetItemString(pyObj_settings, "bucket_cache_size"); + if (pyObj_bucket_cache_size != nullptr) { + settings.bucket_cache_size = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_bucket_cache_size)); + } + + PyObject* pyObj_bucket_cache_age = PyDict_GetItemString(pyObj_settings, "bucket_cache_age"); + if (pyObj_bucket_cache_age != nullptr) { + auto bucket_cache_age = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_bucket_cache_age)); + settings.bucket_cache_age = + std::chrono::milliseconds(std::max(0ULL, bucket_cache_age / 1000ULL)); + } + + PyObject* pyObj_curl_max_allowed_resp_size = + PyDict_GetItemString(pyObj_settings, "curl_max_allowed_resp_size"); + if (pyObj_curl_max_allowed_resp_size != nullptr) { + settings.curl_max_allowed_resp_size = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_curl_max_allowed_resp_size)); + } + + PyObject* pyObj_query_prepare_all = PyDict_GetItemString(pyObj_settings, "query_prepare_all"); + if (pyObj_query_prepare_all != nullptr) { + if (pyObj_query_prepare_all == Py_True) { + settings.query_prepare_all = true; + } else { + settings.query_prepare_all = false; + } + } + + PyObject* pyObj_worker_count = PyDict_GetItemString(pyObj_settings, "worker_count"); + if (pyObj_worker_count != nullptr) { + settings.worker_count = static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_worker_count)); + } + + PyObject* pyObj_handler_headers = PyDict_GetItemString(pyObj_settings, "handler_headers"); + if (pyObj_handler_headers != nullptr && PyList_Check(pyObj_handler_headers)) { + size_t nheaders = static_cast<size_t>(PyList_Size(pyObj_handler_headers)); + size_t ii; + std::vector<std::string> headers{}; + for (ii = 0; ii < nheaders; ++ii) { + PyObject* pyObj_header = PyList_GetItem(pyObj_handler_headers, ii); + headers.emplace_back(std::string(PyUnicode_AsUTF8(pyObj_header))); + } + if (headers.size() > 0) { + settings.handler_headers = headers; + } + } + + PyObject* pyObj_handler_footers = PyDict_GetItemString(pyObj_settings, "handler_footers"); + if (pyObj_handler_footers != nullptr) { + size_t nfooters = static_cast<size_t>(PyList_Size(pyObj_handler_footers)); + size_t ii; + std::vector<std::string> footers{}; + for (ii = 0; ii < nfooters; ++ii) { + PyObject* pyObj_footer = PyList_GetItem(pyObj_handler_footers, ii); + footers.emplace_back(std::string(PyUnicode_AsUTF8(pyObj_footer))); + } + if (footers.size() > 0) { + settings.handler_footers = footers; + } + } + + PyObject* pyObj_enable_app_log_rotation = + PyDict_GetItemString(pyObj_settings, "enable_app_log_rotation"); + if (pyObj_enable_app_log_rotation != nullptr) { + if (pyObj_enable_app_log_rotation == Py_True) { + settings.enable_app_log_rotation = true; + } else { + settings.enable_app_log_rotation = false; + } + } + + PyObject* pyObj_app_log_dir = PyDict_GetItemString(pyObj_settings, "app_log_dir"); + if (pyObj_app_log_dir != nullptr) { + settings.app_log_dir = std::string(PyUnicode_AsUTF8(pyObj_app_log_dir)); + } + + PyObject* pyObj_app_log_max_size = PyDict_GetItemString(pyObj_settings, "app_log_max_size"); + if (pyObj_app_log_max_size != nullptr) { + settings.app_log_max_size = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_app_log_max_size)); + } + + PyObject* pyObj_app_log_max_files = PyDict_GetItemString(pyObj_settings, "app_log_max_files"); + if (pyObj_app_log_max_files != nullptr) { + settings.app_log_max_files = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_app_log_max_files)); + } + + PyObject* pyObj_checkpoint_interval = PyDict_GetItemString(pyObj_settings, "checkpoint_interval"); + if (pyObj_checkpoint_interval != nullptr) { + auto checkpoint_interval = + static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_checkpoint_interval)); + settings.checkpoint_interval = std::chrono::seconds(checkpoint_interval); + } + + return settings; +} + +couchbase::core::management::eventing::function_keyspace +get_eventing_function_keyspace(PyObject* pyObj_keyspace) +{ + couchbase::core::management::eventing::function_keyspace keyspace{}; + + PyObject* pyObj_bucket = PyDict_GetItemString(pyObj_keyspace, "bucket"); + if (pyObj_bucket == nullptr) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Expected eventing function keyspace bucket to be provided."); + throw std::invalid_argument("bucket name"); + } + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket)); + keyspace.bucket = bucket_name; + + PyObject* pyObj_scope = PyDict_GetItemString(pyObj_keyspace, "scope"); + if (pyObj_scope != nullptr) { + keyspace.scope = std::string(PyUnicode_AsUTF8(pyObj_scope)); + } + + PyObject* pyObj_collection = PyDict_GetItemString(pyObj_keyspace, "collection"); + if (pyObj_collection != nullptr) { + keyspace.collection = std::string(PyUnicode_AsUTF8(pyObj_collection)); + } + + return keyspace; +} + +std::vector<couchbase::core::management::eventing::function_constant_binding> +get_function_constant_bindings(PyObject* pyObj_function_constant_bindings) +{ + std::vector<couchbase::core::management::eventing::function_constant_binding> bindings{}; + if (pyObj_function_constant_bindings && PyList_Check(pyObj_function_constant_bindings)) { + size_t nbindings = static_cast<size_t>(PyList_Size(pyObj_function_constant_bindings)); + size_t ii; + for (ii = 0; ii < nbindings; ++ii) { + PyObject* pyObj_binding = PyList_GetItem(pyObj_function_constant_bindings, ii); + if (!pyObj_binding) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Could not determine constant binding."); + throw std::invalid_argument("constant binding"); + } + // PyList_GetItem returns borrowed ref, inc while using, decr after done + Py_INCREF(pyObj_binding); + couchbase::core::management::eventing::function_constant_binding constant_binding{}; + PyObject* pyObj_alias = PyDict_GetItemString(pyObj_binding, "alias"); + if (pyObj_alias != nullptr) { + constant_binding.alias = std::string(PyUnicode_AsUTF8(pyObj_alias)); + } + + PyObject* pyObj_literal = PyDict_GetItemString(pyObj_binding, "literal"); + if (pyObj_literal != nullptr) { + constant_binding.literal = std::string(PyUnicode_AsUTF8(pyObj_literal)); + } + + bindings.emplace_back(constant_binding); + Py_DECREF(pyObj_binding); + pyObj_binding = nullptr; + } + } + return bindings; +} + +std::vector<couchbase::core::management::eventing::function_url_binding> +get_function_url_bindings(PyObject* pyObj_function_url_bindings) +{ + std::vector<couchbase::core::management::eventing::function_url_binding> bindings{}; + if (pyObj_function_url_bindings && PyList_Check(pyObj_function_url_bindings)) { + size_t nbindings = static_cast<size_t>(PyList_Size(pyObj_function_url_bindings)); + size_t ii; + for (ii = 0; ii < nbindings; ++ii) { + PyObject* pyObj_binding = PyList_GetItem(pyObj_function_url_bindings, ii); + if (!pyObj_binding) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Could not determine url binding."); + throw std::invalid_argument("url binding"); + } + // PyList_GetItem returns borrowed ref, inc while using, decr after done + Py_INCREF(pyObj_binding); + couchbase::core::management::eventing::function_url_binding url_binding{}; + PyObject* pyObj_alias = PyDict_GetItemString(pyObj_binding, "alias"); + if (pyObj_alias != nullptr) { + url_binding.alias = std::string(PyUnicode_AsUTF8(pyObj_alias)); + } + + PyObject* pyObj_hostname = PyDict_GetItemString(pyObj_binding, "hostname"); + if (pyObj_hostname != nullptr) { + url_binding.hostname = std::string(PyUnicode_AsUTF8(pyObj_hostname)); + } + + PyObject* pyObj_allow_cookies = PyDict_GetItemString(pyObj_binding, "allow_cookies"); + if (pyObj_allow_cookies != nullptr) { + if (pyObj_allow_cookies == Py_True) { + url_binding.allow_cookies = true; + } + } + + PyObject* pyObj_validate_ssl_certificate = + PyDict_GetItemString(pyObj_binding, "validate_ssl_certificate"); + if (pyObj_validate_ssl_certificate != nullptr) { + if (pyObj_validate_ssl_certificate == Py_False) { + url_binding.validate_ssl_certificate = false; + } + } + + PyObject* pyObj_auth_type = PyDict_GetItemString(pyObj_binding, "auth_type"); + auto auth_type = std::string(PyUnicode_AsUTF8(pyObj_auth_type)); + if (auth_type.compare("basic") == 0) { + couchbase::core::management::eventing::function_url_auth_basic auth{}; + PyObject* pyObj_username = PyDict_GetItemString(pyObj_binding, "username"); + auth.username = std::string(PyUnicode_AsUTF8(pyObj_username)); + PyObject* pyObj_password = PyDict_GetItemString(pyObj_binding, "password"); + auth.password = std::string(PyUnicode_AsUTF8(pyObj_password)); + url_binding.auth = auth; + } else if (auth_type.compare("digest") == 0) { + couchbase::core::management::eventing::function_url_auth_digest auth{}; + PyObject* pyObj_username = PyDict_GetItemString(pyObj_binding, "username"); + auth.username = std::string(PyUnicode_AsUTF8(pyObj_username)); + PyObject* pyObj_password = PyDict_GetItemString(pyObj_binding, "password"); + auth.password = std::string(PyUnicode_AsUTF8(pyObj_password)); + url_binding.auth = auth; + } else if (auth_type.compare("bearer") == 0) { + couchbase::core::management::eventing::function_url_auth_bearer auth{}; + PyObject* pyObj_key = PyDict_GetItemString(pyObj_binding, "bearer_key"); + auth.key = std::string(PyUnicode_AsUTF8(pyObj_key)); + url_binding.auth = auth; + } + + bindings.emplace_back(url_binding); + Py_DECREF(pyObj_binding); + pyObj_binding = nullptr; + } + } + return bindings; +} + +std::vector<couchbase::core::management::eventing::function_bucket_binding> +get_function_bucket_bindings(PyObject* pyObj_bucket_bindings) +{ + std::vector<couchbase::core::management::eventing::function_bucket_binding> bindings{}; + if (pyObj_bucket_bindings && PyList_Check(pyObj_bucket_bindings)) { + size_t nbindings = static_cast<size_t>(PyList_Size(pyObj_bucket_bindings)); + size_t ii; + for (ii = 0; ii < nbindings; ++ii) { + PyObject* pyObj_binding = PyList_GetItem(pyObj_bucket_bindings, ii); + if (!pyObj_binding) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Could not determine bucket binding."); + throw std::invalid_argument("bucket binding"); + } + // PyList_GetItem returns borrowed ref, inc while using, decr after done + Py_INCREF(pyObj_binding); + couchbase::core::management::eventing::function_bucket_binding bucket_binding{}; + PyObject* pyObj_alias = PyDict_GetItemString(pyObj_binding, "alias"); + if (pyObj_alias != nullptr) { + bucket_binding.alias = std::string(PyUnicode_AsUTF8(pyObj_alias)); + } + + PyObject* pyObj_keyspace = PyDict_GetItemString(pyObj_binding, "name"); + bucket_binding.name = get_eventing_function_keyspace(pyObj_keyspace); + + PyObject* pyObj_access = PyDict_GetItemString(pyObj_binding, "access"); + if (pyObj_access != nullptr) { + auto access = std::string(PyUnicode_AsUTF8(pyObj_access)); + if (access.compare("read_only") == 0) { + bucket_binding.access = + couchbase::core::management::eventing::function_bucket_access::read_only; + } + } + + bindings.emplace_back(bucket_binding); + Py_DECREF(pyObj_binding); + pyObj_binding = nullptr; + } + } + return bindings; +} + +couchbase::core::management::eventing::function +get_eventing_function(PyObject* pyObj_eventing_function) +{ + couchbase::core::management::eventing::function eventing_function{}; + + PyObject* pyObj_name = PyDict_GetItemString(pyObj_eventing_function, "name"); + if (pyObj_name == nullptr) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Expected eventing function name to be provided."); + throw std::invalid_argument("name"); + } + auto name = std::string(PyUnicode_AsUTF8(pyObj_name)); + eventing_function.name = name; + + PyObject* pyObj_code = PyDict_GetItemString(pyObj_eventing_function, "code"); + if (pyObj_code == nullptr) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Expected eventing function code to be provided."); + throw std::invalid_argument("code"); + } + auto code = std::string(PyUnicode_AsUTF8(pyObj_code)); + eventing_function.code = code; + + PyObject* pyObj_metadata_keyspace = + PyDict_GetItemString(pyObj_eventing_function, "metadata_keyspace"); + eventing_function.metadata_keyspace = get_eventing_function_keyspace(pyObj_metadata_keyspace); + + PyObject* pyObj_source_keyspace = + PyDict_GetItemString(pyObj_eventing_function, "source_keyspace"); + eventing_function.source_keyspace = get_eventing_function_keyspace(pyObj_source_keyspace); + + PyObject* pyObj_version = PyDict_GetItemString(pyObj_eventing_function, "version"); + if (pyObj_version != nullptr) { + eventing_function.version = std::string(PyUnicode_AsUTF8(pyObj_version)); + } + + PyObject* pyObj_enforce_schema = PyDict_GetItemString(pyObj_eventing_function, "enforce_schema"); + if (pyObj_enforce_schema != nullptr) { + if (pyObj_enforce_schema == Py_True) { + eventing_function.enforce_schema = true; + } else { + eventing_function.enforce_schema = false; + } + } + + PyObject* pyObj_handler_uuid = PyDict_GetItemString(pyObj_eventing_function, "handler_uuid"); + if (pyObj_handler_uuid != nullptr) { + eventing_function.handler_uuid = + static_cast<std::int64_t>(PyLong_AsLongLong(pyObj_handler_uuid)); + } + + PyObject* pyObj_function_instance_id = + PyDict_GetItemString(pyObj_eventing_function, "function_instance_id"); + if (pyObj_function_instance_id != nullptr) { + eventing_function.function_instance_id = + std::string(PyUnicode_AsUTF8(pyObj_function_instance_id)); + } + + PyObject* pyObj_bucket_bindings = + PyDict_GetItemString(pyObj_eventing_function, "bucket_bindings"); + auto bucket_bindings = get_function_bucket_bindings(pyObj_bucket_bindings); + if (bucket_bindings.size() > 0) { + eventing_function.bucket_bindings = bucket_bindings; + } + + PyObject* pyObj_url_bindings = PyDict_GetItemString(pyObj_eventing_function, "url_bindings"); + auto url_bindings = get_function_url_bindings(pyObj_url_bindings); + if (url_bindings.size() > 0) { + eventing_function.url_bindings = url_bindings; + } + + PyObject* pyObj_constant_bindings = + PyDict_GetItemString(pyObj_eventing_function, "constant_bindings"); + auto constant_bindings = get_function_constant_bindings(pyObj_constant_bindings); + if (constant_bindings.size() > 0) { + eventing_function.constant_bindings = constant_bindings; + } + + PyObject* pyObj_settings = PyDict_GetItemString(pyObj_eventing_function, "settings"); + auto function_settings = get_event_function_settings(pyObj_settings); + eventing_function.settings = function_settings; + + return eventing_function; +} + +template<typename Request> +Request +get_eventing_function_mgmt_req(PyObject* op_args) +{ + Request req{}; + + PyObject* pyObj_name = PyDict_GetItemString(op_args, "name"); + auto name = std::string(PyUnicode_AsUTF8(pyObj_name)); + req.name = name; + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + return req; +} + +template<typename Request> +PyObject* +do_eventing_function_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_eventing_function_mgmt_op_response( + resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_eventing_function_mgmt_op(connection* conn, + struct eventing_function_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + PyObject* res = nullptr; + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + try { + switch (options->op_type) { + case EventingFunctionManagementOperations::UPSERT_FUNCTION: { + couchbase::core::operations::management::eventing_upsert_function_request req{}; + + PyObject* pyObj_bucket_name = PyDict_GetItemString(options->op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + PyObject* pyObj_client_context_id = + PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_eventing_function = + PyDict_GetItemString(options->op_args, "eventing_function"); + req.function = get_eventing_function(pyObj_eventing_function); + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_upsert_function_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case EventingFunctionManagementOperations::DEPLOY_FUNCTION: { + auto req = get_eventing_function_mgmt_req< + couchbase::core::operations::management::eventing_deploy_function_request>( + options->op_args); + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_deploy_function_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case EventingFunctionManagementOperations::GET_FUNCTION: { + auto req = get_eventing_function_mgmt_req< + couchbase::core::operations::management::eventing_get_function_request>(options->op_args); + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_get_function_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case EventingFunctionManagementOperations::PAUSE_FUNCTION: { + auto req = get_eventing_function_mgmt_req< + couchbase::core::operations::management::eventing_pause_function_request>( + options->op_args); + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_pause_function_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case EventingFunctionManagementOperations::RESUME_FUNCTION: { + auto req = get_eventing_function_mgmt_req< + couchbase::core::operations::management::eventing_resume_function_request>( + options->op_args); + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_resume_function_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case EventingFunctionManagementOperations::UNDEPLOY_FUNCTION: { + auto req = get_eventing_function_mgmt_req< + couchbase::core::operations::management::eventing_undeploy_function_request>( + options->op_args); + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_undeploy_function_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case EventingFunctionManagementOperations::DROP_FUNCTION: { + auto req = get_eventing_function_mgmt_req< + couchbase::core::operations::management::eventing_drop_function_request>( + options->op_args); + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_drop_function_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case EventingFunctionManagementOperations::GET_ALL_FUNCTIONS: { + couchbase::core::operations::management::eventing_get_all_functions_request req{}; + + PyObject* pyObj_bucket_name = PyDict_GetItemString(options->op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + PyObject* pyObj_client_context_id = + PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_get_all_functions_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case EventingFunctionManagementOperations::GET_STATUS: { + couchbase::core::operations::management::eventing_get_status_request req{}; + + PyObject* pyObj_bucket_name = PyDict_GetItemString(options->op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + PyObject* pyObj_client_context_id = + PyDict_GetItemString(options->op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + req.timeout = options->timeout_ms; + + res = do_eventing_function_mgmt_op< + couchbase::core::operations::management::eventing_get_status_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized eventing function mgmt operation passed in."); + barrier->set_value(nullptr); + break; + } + }; + } catch (const std::invalid_argument&) { + } + + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + + if (res == nullptr) { + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + return res; +} + +void +add_eventing_function_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = + PyUnicode_FromString(EventingFunctionManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("EventingFunctionManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "eventing_function_mgmt_operations", pyObj_mgmt_operations) < + 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/eventing_function_management.hxx b/src/management/eventing_function_management.hxx new file mode 100644 index 000000000..c08bcc86f --- /dev/null +++ b/src/management/eventing_function_management.hxx @@ -0,0 +1,96 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class EventingFunctionManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + UPSERT_FUNCTION, + DEPLOY_FUNCTION, + GET_FUNCTION, + PAUSE_FUNCTION, + RESUME_FUNCTION, + UNDEPLOY_FUNCTION, + DROP_FUNCTION, + GET_ALL_FUNCTIONS, + GET_STATUS + }; + + EventingFunctionManagementOperations() + : EventingFunctionManagementOperations{ UNKNOWN } + { + } + constexpr EventingFunctionManagementOperations( + EventingFunctionManagementOperations::OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(EventingFunctionManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(EventingFunctionManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "UPSERT_FUNCTION " + "DEPLOY_FUNCTION " + "GET_FUNCTION " + "PAUSE_FUNCTION " + "RESUME_FUNCTION " + "UNDEPLOY_FUNCTION " + "DROP_FUNCTION " + "GET_ALL_FUNCTIONS " + "GET_STATUS"; + + return ops; + } + +private: + OperationType operation; +}; + +struct eventing_function_mgmt_options { + PyObject* op_args; + EventingFunctionManagementOperations::OperationType op_type = + EventingFunctionManagementOperations::UNKNOWN; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; +}; + +PyObject* +handle_eventing_function_mgmt_op(connection* conn, + struct eventing_function_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback); + +void +add_eventing_function_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/management/management.cxx b/src/management/management.cxx new file mode 100644 index 000000000..02c5db936 --- /dev/null +++ b/src/management/management.cxx @@ -0,0 +1,483 @@ + +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "management.hxx" + +#include <core/operations/management/cluster_describe.hxx> +#include <core/operations/management/cluster_developer_preview_enable.hxx> + +#include "analytics_management.hxx" +#include "bucket_management.hxx" +#include "collection_management.hxx" +#include "eventing_function_management.hxx" +#include "query_index_management.hxx" +#include "search_index_management.hxx" +#include "user_management.hxx" +#include "view_index_management.hxx" + +#include "../exceptions.hxx" +#include "../result.hxx" +#include "../utils.hxx" + +template<typename T> +result* +create_result_from_mgmt_response(const T& resp) +{ + PyObject* result_obj = create_result_obj(); + result* res = reinterpret_cast<result*>(result_obj); + return res; +} + +template<> +result* +create_result_from_mgmt_response< + couchbase::core::operations::management::cluster_describe_response>( + const couchbase::core::operations::management::cluster_describe_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_tmp = nullptr; + + PyObject* pyObj_nodes = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& node : resp.info.nodes) { + PyObject* pyObj_node = PyDict_New(); + + pyObj_tmp = PyUnicode_FromString(node.uuid.c_str()); + if (-1 == PyDict_SetItemString(pyObj_node, "uuid", pyObj_tmp)) { + Py_XDECREF(pyObj_nodes); + Py_XDECREF(pyObj_node); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(node.otp_node.c_str()); + if (-1 == PyDict_SetItemString(pyObj_node, "otp_node", pyObj_tmp)) { + Py_XDECREF(pyObj_nodes); + Py_DECREF(pyObj_node); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(node.status.c_str()); + if (-1 == PyDict_SetItemString(pyObj_node, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_nodes); + Py_DECREF(pyObj_node); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(node.hostname.c_str()); + if (-1 == PyDict_SetItemString(pyObj_node, "hostname", pyObj_tmp)) { + Py_XDECREF(pyObj_nodes); + Py_DECREF(pyObj_node); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(node.os.c_str()); + if (-1 == PyDict_SetItemString(pyObj_node, "os", pyObj_tmp)) { + Py_XDECREF(pyObj_nodes); + Py_DECREF(pyObj_node); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(node.version.c_str()); + if (-1 == PyDict_SetItemString(pyObj_node, "version", pyObj_tmp)) { + Py_XDECREF(pyObj_nodes); + Py_DECREF(pyObj_node); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_node_svcs = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& svc : node.services) { + PyObject* pyObj_node_svc = nullptr; + pyObj_node_svc = PyUnicode_FromString(svc.c_str()); + if (pyObj_node_svc) { + PyList_Append(pyObj_node_svcs, pyObj_node_svc); + Py_DECREF(pyObj_node_svc); + } + } + + if (-1 == PyDict_SetItemString(pyObj_node, "services", pyObj_node_svcs)) { + Py_XDECREF(pyObj_nodes); + Py_DECREF(pyObj_node); + Py_XDECREF(pyObj_node_svcs); + return nullptr; + } + Py_DECREF(pyObj_node_svcs); + PyList_Append(pyObj_nodes, pyObj_node); + Py_DECREF(pyObj_node); + } + + if (-1 == PyDict_SetItemString(res->dict, "nodes", pyObj_nodes)) { + Py_XDECREF(pyObj_nodes); + return nullptr; + } + Py_DECREF(pyObj_nodes); + + PyObject* pyObj_buckets = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& bucket : resp.info.buckets) { + PyObject* pyObj_bucket = PyDict_New(); + + pyObj_tmp = PyUnicode_FromString(bucket.uuid.c_str()); + if (-1 == PyDict_SetItemString(pyObj_bucket, "uuid", pyObj_tmp)) { + Py_XDECREF(pyObj_buckets); + Py_XDECREF(pyObj_bucket); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(bucket.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_bucket, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_buckets); + Py_DECREF(pyObj_bucket); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyList_Append(pyObj_buckets, pyObj_bucket); + Py_DECREF(pyObj_bucket); + } + + if (-1 == PyDict_SetItemString(res->dict, "buckets", pyObj_buckets)) { + Py_DECREF(pyObj_buckets); + return nullptr; + } + Py_DECREF(pyObj_buckets); + + PyObject* pyObj_svc_type_set = PySet_New(nullptr); + for (auto const& svc_type : resp.info.services) { + std::string service_type = service_type_to_str(svc_type); + pyObj_tmp = PyUnicode_FromString(service_type.c_str()); + if (-1 == PySet_Add(pyObj_svc_type_set, pyObj_tmp)) { + Py_XDECREF(pyObj_svc_type_set); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyDict_SetItemString(res->dict, "service_types", pyObj_svc_type_set)) { + Py_DECREF(pyObj_svc_type_set); + return nullptr; + } + Py_DECREF(pyObj_svc_type_set); + + return res; +} + +template<typename T> +void +create_result_from_mgmt_op_response(const T& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing collection mgmt operation.", "ClusterMgmt"); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_mgmt_response(resp); + if (res == nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Cluster mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to catch exception here? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +template<typename Request> +PyObject* +do_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_mgmt_op_response(resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_mgmt_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + ManagementOperations::OperationType mgmt_op = ManagementOperations::UNKNOWN; + int op_type = 0; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* op_args = nullptr; + + uint64_t timeout = 0; + + static const char* kw_list[] = { "conn", "mgmt_op", "op_type", "callback", + "errback", "timeout", "op_args", nullptr }; + + const char* kw_format = "O!II|OOLO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &mgmt_op, + &op_type, + &pyObj_callback, + &pyObj_errback, + &timeout, + &op_args); + if (!ret) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform management operation. Unable to parse args/kwargs."); + return nullptr; + } + + connection* conn = nullptr; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; + + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + PyErr_Clear(); + if (0 < timeout) { + timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { callback, errback }; + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + PyObject* res = nullptr; + + switch (mgmt_op) { + case ManagementOperations::CLUSTER: { + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + + auto op = static_cast<ClusterManagementOperations::OperationType>(op_type); + if (op == ClusterManagementOperations::GET_CLUSTER_INFO) { + couchbase::core::operations::management::cluster_describe_request req{}; + req.timeout = timeout_ms; + res = do_mgmt_op<couchbase::core::operations::management::cluster_describe_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + } else if (op == ClusterManagementOperations::ENABLE_DP) { + couchbase::core::operations::management::cluster_developer_preview_enable_request req{}; + req.timeout = timeout_ms; + res = do_mgmt_op< + couchbase::core::operations::management::cluster_developer_preview_enable_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + } + + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + break; + } + case ManagementOperations::BUCKET: { + struct bucket_mgmt_options opts = { + op_args, static_cast<BucketManagementOperations::OperationType>(op_type), timeout_ms + }; + res = handle_bucket_mgmt_op(conn, &opts, pyObj_callback, pyObj_errback); + break; + } + case ManagementOperations::COLLECTION: { + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + struct collection_mgmt_options opts = { + op_args, static_cast<CollectionManagementOperations::OperationType>(op_type), timeout_ms + }; + res = handle_collection_mgmt_op(conn, &opts, pyObj_callback, pyObj_errback); + break; + } + case ManagementOperations::USER: { + struct user_mgmt_options opts = { + op_args, static_cast<UserManagementOperations::OperationType>(op_type), timeout_ms + }; + res = handle_user_mgmt_op(conn, &opts, pyObj_callback, pyObj_errback); + break; + } + case ManagementOperations::QUERY_INDEX: { + struct query_index_mgmt_options opts = { + op_args, static_cast<QueryIndexManagementOperations::OperationType>(op_type), timeout_ms + }; + res = handle_query_index_mgmt_op(conn, &opts, pyObj_callback, pyObj_errback); + break; + } + case ManagementOperations::ANALYTICS: { + struct analytics_mgmt_options opts = { + op_args, static_cast<AnalyticsManagementOperations::OperationType>(op_type), timeout_ms + }; + res = handle_analytics_mgmt_op(conn, &opts, pyObj_callback, pyObj_errback); + break; + } + case ManagementOperations::SEARCH_INDEX: { + struct search_index_mgmt_options opts = { + op_args, static_cast<SearchIndexManagementOperations::OperationType>(op_type), timeout_ms + }; + res = handle_search_index_mgmt_op(conn, &opts, pyObj_callback, pyObj_errback); + break; + } + case ManagementOperations::VIEW_INDEX: { + struct view_index_mgmt_options opts = { + op_args, static_cast<ViewIndexManagementOperations::OperationType>(op_type), timeout_ms + }; + res = handle_view_index_mgmt_op(conn, &opts, pyObj_callback, pyObj_errback); + break; + } + case ManagementOperations::EVENTING_FUNCTION: { + struct eventing_function_mgmt_options opts = { + op_args, + static_cast<EventingFunctionManagementOperations::OperationType>(op_type), + timeout_ms + }; + res = handle_eventing_function_mgmt_op(conn, &opts, pyObj_callback, pyObj_errback); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized management operation passed in."); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + return res; +} + +void +add_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = PyUnicode_FromString(ManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("ManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} + +void +add_cluster_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = PyUnicode_FromString(ClusterManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("ClusterManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "cluster_mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/management.hxx b/src/management/management.hxx new file mode 100644 index 000000000..333952104 --- /dev/null +++ b/src/management/management.hxx @@ -0,0 +1,133 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class ManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + CLUSTER, + BUCKET, + COLLECTION, + QUERY_INDEX, + SEARCH_INDEX, + USER, + ANALYTICS, + VIEW_INDEX, + EVENTING_FUNCTION + }; + + ManagementOperations() + : ManagementOperations{ UNKNOWN } + { + } + constexpr ManagementOperations(OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(ManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(ManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "CLUSTER " + "BUCKET " + "COLLECTION " + "QUERY_INDEX " + "SEARCH_INDEX " + "USER " + "ANALYTICS " + "VIEW_INDEX " + "EVENTING_FUNCTION"; + + return ops; + } + +private: + OperationType operation; +}; + +class ClusterManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + GET_CLUSTER_INFO, + ENABLE_DP + }; + + ClusterManagementOperations() + : ClusterManagementOperations{ UNKNOWN } + { + } + constexpr ClusterManagementOperations(OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(ClusterManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(ClusterManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "GET_CLUSTER_INFO " + "ENABLE_DP"; + + return ops; + } + +private: + OperationType operation; +}; + +PyObject* +handle_mgmt_op(PyObject* self, PyObject* args, PyObject* kwargs); + +void +add_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); + +void +add_cluster_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/management/query_index_management.cxx b/src/management/query_index_management.cxx new file mode 100644 index 000000000..12c33a80a --- /dev/null +++ b/src/management/query_index_management.cxx @@ -0,0 +1,567 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "query_index_management.hxx" + +#include <core/operations/management/query.hxx> +#include <couchbase/management/query_index.hxx> + +#include "../exceptions.hxx" +#include "../result.hxx" + +PyObject* +build_query_index(const couchbase::management::query_index& index) +{ + PyObject* pyObj_index = PyDict_New(); + if (index.is_primary) { + //@TODO: I do not think an increment is necessary since adding to the + // dict will increment the ref + // Py_INCREF(Py_True); + if (-1 == PyDict_SetItemString(pyObj_index, "is_primary", Py_True)) { + Py_DECREF(pyObj_index); + return nullptr; + } + } else { + // Py_INCREF(Py_False); + if (-1 == PyDict_SetItemString(pyObj_index, "is_primary", Py_False)) { + Py_DECREF(pyObj_index); + return nullptr; + } + } + + PyObject* pyObj_tmp = PyUnicode_FromString(index.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "name", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(index.state.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "state", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (index.collection_name.has_value()) { + pyObj_tmp = PyUnicode_FromString(index.collection_name->c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "collection_name", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = PyUnicode_FromString(index.type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "type", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (index.index_key.size() > 0) { + PyObject* pyObj_index_keys = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& index_key : index.index_key) { + PyObject* pyObj_index_key = PyUnicode_FromString(index_key.c_str()); + PyList_Append(pyObj_index_keys, pyObj_index_key); + Py_DECREF(pyObj_index_key); + } + + if (-1 == PyDict_SetItemString(pyObj_index, "index_key", pyObj_index_keys)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_index_keys); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_index_keys); + } + + if (index.partition.has_value()) { + pyObj_tmp = PyUnicode_FromString(index.partition.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "partition", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (index.condition.has_value()) { + pyObj_tmp = PyUnicode_FromString(index.condition.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "condition", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = PyUnicode_FromString(index.bucket_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "bucket_name", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (index.scope_name.has_value()) { + pyObj_tmp = PyUnicode_FromString(index.scope_name.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "scope_name", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + return pyObj_index; +} + +template<typename T> +result* +create_result_from_query_index_mgmt_response(const T& resp) +{ + PyObject* result_obj = create_result_obj(); + result* res = reinterpret_cast<result*>(result_obj); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_query_problems = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& error : resp.errors) { + PyObject* pyObj_query_problem = PyDict_New(); + pyObj_tmp = PyLong_FromUnsignedLongLong(error.code); + if (-1 == PyDict_SetItemString(pyObj_query_problem, "code", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_XDECREF(pyObj_query_problems); + Py_XDECREF(pyObj_query_problem); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(error.message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_query_problem, "message", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_XDECREF(pyObj_query_problems); + Py_DECREF(pyObj_query_problem); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + Py_ssize_t set_size = PyList_Size(pyObj_query_problems); + if (set_size > 0) { + if (-1 == PyDict_SetItemString(res->dict, "errors", pyObj_query_problems)) { + Py_XDECREF(result_obj); + Py_XDECREF(pyObj_query_problems); + return nullptr; + } + } + Py_DECREF(pyObj_query_problems); + + return res; +} + +template<> +result* +create_result_from_query_index_mgmt_response( + const couchbase::core::operations::management::query_index_get_all_response& resp) +{ + PyObject* result_obj = create_result_obj(); + result* res = reinterpret_cast<result*>(result_obj); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(result_obj); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_indexes = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& index : resp.indexes) { + PyObject* pyObj_index = build_query_index(index); + if (pyObj_index == nullptr) { + Py_XDECREF(result_obj); + Py_XDECREF(pyObj_indexes); + return nullptr; + } + PyList_Append(pyObj_indexes, pyObj_index); + Py_DECREF(pyObj_index); + } + + if (-1 == PyDict_SetItemString(res->dict, "indexes", pyObj_indexes)) { + Py_XDECREF(result_obj); + Py_XDECREF(pyObj_indexes); + return nullptr; + } + Py_DECREF(pyObj_indexes); + + return res; +} + +template<typename Response> +void +create_result_from_query_index_mgmt_op_response(const Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing query index mgmt operation.", "QueryIndexMgmt"); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_query_index_mgmt_response(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Collection mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } else if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +couchbase::core::operations::management::query_index_create_request +get_create_query_index_req(PyObject* op_args) +{ + couchbase::core::operations::management::query_index_create_request req{}; + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + PyObject* pyObj_collection_name = PyDict_GetItemString(op_args, "collection_name"); + if (pyObj_collection_name != nullptr) { + auto collection_name = std::string(PyUnicode_AsUTF8(pyObj_collection_name)); + req.collection_name = collection_name; + } + + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + if (pyObj_index_name != nullptr) { + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + } + + PyObject* pyObj_is_primary = PyDict_GetItemString(op_args, "is_primary"); + if (pyObj_is_primary != nullptr) { + if (pyObj_is_primary == Py_True) { + req.is_primary = true; + } else { + req.is_primary = false; + } + } + + PyObject* pyObj_ignore_if_exists = PyDict_GetItemString(op_args, "ignore_if_exists"); + if (pyObj_ignore_if_exists != nullptr) { + if (pyObj_ignore_if_exists == Py_True) { + req.ignore_if_exists = true; + } else { + req.ignore_if_exists = false; + } + } + + PyObject* pyObj_deferred = PyDict_GetItemString(op_args, "deferred"); + if (pyObj_deferred != nullptr) { + if (pyObj_deferred == Py_True) { + req.deferred = true; + } else { + req.deferred = false; + } + } + + PyObject* pyObj_condition = PyDict_GetItemString(op_args, "condition"); + if (pyObj_condition != nullptr) { + auto condition = std::string(PyUnicode_AsUTF8(pyObj_condition)); + req.condition = condition; + } + + PyObject* pyObj_num_replicas = PyDict_GetItemString(op_args, "num_replicas"); + if (pyObj_num_replicas != nullptr) { + auto num_replicas = static_cast<int>(PyLong_AsLong(pyObj_num_replicas)); + req.num_replicas = num_replicas; + } + + PyObject* pyObj_keys = PyDict_GetItemString(op_args, "keys"); + if (pyObj_keys != nullptr) { + size_t nkeys = static_cast<size_t>(PyList_GET_SIZE(pyObj_keys)); + std::vector<std::string> keys{}; + size_t ii; + for (ii = 0; ii < nkeys; ++ii) { + PyObject* pyObj_key = PyList_GetItem(pyObj_keys, ii); + auto key = std::string(PyUnicode_AsUTF8(pyObj_key)); + keys.push_back(key); + } + + req.keys = keys; + } + + return req; +} + +couchbase::core::operations::management::query_index_drop_request +get_drop_query_index_req(PyObject* op_args) +{ + couchbase::core::operations::management::query_index_drop_request req{}; + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + PyObject* pyObj_collection_name = PyDict_GetItemString(op_args, "collection_name"); + if (pyObj_collection_name != nullptr) { + auto collection_name = std::string(PyUnicode_AsUTF8(pyObj_collection_name)); + req.collection_name = collection_name; + } + + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + if (pyObj_index_name != nullptr) { + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + } + + PyObject* pyObj_is_primary = PyDict_GetItemString(op_args, "is_primary"); + if (pyObj_is_primary != nullptr) { + if (pyObj_is_primary == Py_True) { + req.is_primary = true; + } else { + req.is_primary = false; + } + } + + PyObject* pyObj_ignore_if_does_not_exist = + PyDict_GetItemString(op_args, "ignore_if_does_not_exist"); + if (pyObj_ignore_if_does_not_exist != nullptr) { + if (pyObj_ignore_if_does_not_exist == Py_True) { + req.ignore_if_does_not_exist = true; + } else { + req.ignore_if_does_not_exist = false; + } + } + + return req; +} + +template<typename Request> +PyObject* +do_query_index_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_query_index_mgmt_op_response(resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_query_index_mgmt_op(connection* conn, + struct query_index_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + PyObject* res = nullptr; + std::shared_ptr<std::promise<PyObject*>> barrier = nullptr; + std::future<PyObject*> fut; + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + barrier = std::make_shared<std::promise<PyObject*>>(); + fut = barrier->get_future(); + } + PyObject* pyObj_bucket_name = PyDict_GetItemString(options->op_args, "bucket_name"); + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + std::string scope_name{}; + PyObject* pyObj_scope_name = PyDict_GetItemString(options->op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + } + std::string collection_name{}; + PyObject* pyObj_collection_name = PyDict_GetItemString(options->op_args, "collection_name"); + if (pyObj_collection_name != nullptr) { + collection_name = std::string(PyUnicode_AsUTF8(pyObj_collection_name)); + } + + switch (options->op_type) { + case QueryIndexManagementOperations::CREATE_INDEX: { + auto req = get_create_query_index_req(options->op_args); + req.bucket_name = bucket_name; + req.timeout = options->timeout_ms; + if (!scope_name.empty()) { + req.scope_name = scope_name; + } + if (!collection_name.empty()) { + req.collection_name = collection_name; + } + + res = + do_query_index_mgmt_op<couchbase::core::operations::management::query_index_create_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case QueryIndexManagementOperations::DROP_INDEX: { + auto req = get_drop_query_index_req(options->op_args); + req.bucket_name = bucket_name; + req.timeout = options->timeout_ms; + if (!scope_name.empty()) { + req.scope_name = scope_name; + } + if (!collection_name.empty()) { + req.collection_name = collection_name; + } + + res = + do_query_index_mgmt_op<couchbase::core::operations::management::query_index_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case QueryIndexManagementOperations::GET_ALL_INDEXES: { + couchbase::core::operations::management::query_index_get_all_request req{}; + req.bucket_name = bucket_name; + req.timeout = options->timeout_ms; + if (!scope_name.empty()) { + req.scope_name = scope_name; + } + if (!collection_name.empty()) { + req.collection_name = collection_name; + } + + res = do_query_index_mgmt_op< + couchbase::core::operations::management::query_index_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case QueryIndexManagementOperations::BUILD_DEFERRED_INDEXES: { + couchbase::core::operations::management::query_index_build_deferred_request req{}; + req.bucket_name = bucket_name; + req.timeout = options->timeout_ms; + if (!scope_name.empty()) { + req.scope_name = scope_name; + } + if (!collection_name.empty()) { + req.collection_name = collection_name; + } + res = do_query_index_mgmt_op< + couchbase::core::operations::management::query_index_build_deferred_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized query index mgmt operation passed in."); + barrier->set_value(nullptr); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = fut.get(); + Py_END_ALLOW_THREADS return ret; + } + return res; +} + +void +add_query_index_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = + PyUnicode_FromString(QueryIndexManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("QueryIndexManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "query_index_mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/query_index_management.hxx b/src/management/query_index_management.hxx new file mode 100644 index 000000000..d19d6db6a --- /dev/null +++ b/src/management/query_index_management.hxx @@ -0,0 +1,84 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class QueryIndexManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + CREATE_INDEX, + DROP_INDEX, + GET_ALL_INDEXES, + BUILD_DEFERRED_INDEXES + }; + + QueryIndexManagementOperations() + : QueryIndexManagementOperations{ UNKNOWN } + { + } + constexpr QueryIndexManagementOperations(QueryIndexManagementOperations::OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(QueryIndexManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(QueryIndexManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "CREATE_INDEX " + "DROP_INDEX " + "GET_ALL_INDEXES " + "BUILD_DEFERRED_INDEXES"; + + return ops; + } + +private: + OperationType operation; +}; + +struct query_index_mgmt_options { + PyObject* op_args; + QueryIndexManagementOperations::OperationType op_type = QueryIndexManagementOperations::UNKNOWN; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; +}; + +PyObject* +handle_query_index_mgmt_op(connection* conn, + struct query_index_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback); + +void +add_query_index_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/management/search_index_management.cxx b/src/management/search_index_management.cxx new file mode 100644 index 000000000..8b50d1363 --- /dev/null +++ b/src/management/search_index_management.cxx @@ -0,0 +1,926 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "search_index_management.hxx" + +#include <core/management/search_index.hxx> +#include <core/operations/management/search.hxx> + +#include "../exceptions.hxx" +#include "../result.hxx" + +PyObject* +build_search_index(couchbase::core::management::search::index index) +{ + PyObject* pyObj_index = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(index.uuid.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "uuid", pyObj_tmp)) { + Py_XDECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(index.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "name", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(index.type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "type", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (!index.params_json.empty()) { + pyObj_tmp = PyUnicode_FromString(index.params_json.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "params_json", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = PyUnicode_FromString(index.source_uuid.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "source_uuid", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(index.source_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "source_name", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(index.source_type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "source_type", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (!index.source_params_json.empty()) { + pyObj_tmp = PyUnicode_FromString(index.source_params_json.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "source_params_json", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (!index.plan_params_json.empty()) { + pyObj_tmp = PyUnicode_FromString(index.plan_params_json.c_str()); + if (-1 == PyDict_SetItemString(pyObj_index, "plan_params_json", pyObj_tmp)) { + Py_DECREF(pyObj_index); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + return pyObj_index; +} + +template<typename T> +result* +create_result_from_search_index_mgmt_response(const T& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.error.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "error", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + return res; +} + +template<> +result* +create_result_from_search_index_mgmt_response( + const couchbase::core::operations::management::search_index_get_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.error.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "error", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = build_search_index(resp.index); + if (pyObj_tmp == nullptr) { + Py_XDECREF(pyObj_result); + return nullptr; + } + if (-1 == PyDict_SetItemString(res->dict, "index", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + return res; +} + +template<> +result* +create_result_from_search_index_mgmt_response( + const couchbase::core::operations::management::search_index_get_all_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.impl_version.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "impl_version", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_indexes = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& index : resp.indexes) { + PyObject* pyObj_index = build_search_index(index); + if (pyObj_index == nullptr) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_indexes); + return nullptr; + } + PyList_Append(pyObj_indexes, pyObj_index); + Py_DECREF(pyObj_index); + } + + if (-1 == PyDict_SetItemString(res->dict, "indexes", pyObj_indexes)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_indexes); + return nullptr; + } + Py_DECREF(pyObj_indexes); + + return res; +} + +template<> +result* +create_result_from_search_index_mgmt_response( + const couchbase::core::operations::management::search_index_get_documents_count_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.error.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "error", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromLongLong(resp.count); + if (-1 == PyDict_SetItemString(res->dict, "count", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + return res; +} + +template<> +result* +create_result_from_search_index_mgmt_response( + const couchbase::core::operations::management::search_index_get_stats_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.error.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "error", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.stats.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "stats", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + return res; +} + +template<> +result* +create_result_from_search_index_mgmt_response( + const couchbase::core::operations::management::search_index_analyze_document_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.error.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "error", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.analysis.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "analysis", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + return res; +} + +template<> +result* +create_result_from_search_index_mgmt_response( + const couchbase::core::operations::management::search_get_stats_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_stats = PyUnicode_FromString(resp.stats.c_str()); + if (-1 == PyDict_SetItemString(res->dict, "stats", pyObj_stats)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_stats); + return nullptr; + } + Py_DECREF(pyObj_stats); + + return res; +} + +template<typename Response> +void +create_result_from_search_index_mgmt_op_response(const Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing search index mgmt operation.", "SearchIndexMgmt"); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + // pyObj_exc = build_exception_from_context(resp.ctx); + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_search_index_mgmt_response(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Search index mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +couchbase::core::management::search::index +get_search_index(PyObject* pyObj_index) +{ + couchbase::core::management::search::index index{}; + + PyObject* pyObj_uuid = PyDict_GetItemString(pyObj_index, "uuid"); + if (pyObj_uuid != nullptr) { + auto uuid = std::string(PyUnicode_AsUTF8(pyObj_uuid)); + index.uuid = uuid; + } + + PyObject* pyObj_name = PyDict_GetItemString(pyObj_index, "name"); + if (pyObj_name != nullptr) { + auto name = std::string(PyUnicode_AsUTF8(pyObj_name)); + index.name = name; + } + + PyObject* pyObj_type = PyDict_GetItemString(pyObj_index, "type"); + if (pyObj_type != nullptr) { + auto type = std::string(PyUnicode_AsUTF8(pyObj_type)); + index.type = type; + } + + PyObject* pyObj_params_json = PyDict_GetItemString(pyObj_index, "params_json"); + if (pyObj_params_json != nullptr) { + auto params_json = std::string(PyUnicode_AsUTF8(pyObj_params_json)); + index.params_json = params_json; + } + + PyObject* pyObj_source_uuid = PyDict_GetItemString(pyObj_index, "source_uuid"); + if (pyObj_source_uuid != nullptr) { + auto source_uuid = std::string(PyUnicode_AsUTF8(pyObj_source_uuid)); + index.source_uuid = source_uuid; + } + + PyObject* pyObj_source_name = PyDict_GetItemString(pyObj_index, "source_name"); + if (pyObj_source_name != nullptr) { + auto source_name = std::string(PyUnicode_AsUTF8(pyObj_source_name)); + index.source_name = source_name; + } + + PyObject* pyObj_source_type = PyDict_GetItemString(pyObj_index, "source_type"); + if (pyObj_source_type != nullptr) { + auto source_type = std::string(PyUnicode_AsUTF8(pyObj_source_type)); + index.source_type = source_type; + } + + PyObject* pyObj_source_params_json = PyDict_GetItemString(pyObj_index, "source_params_json"); + if (pyObj_source_params_json != nullptr) { + auto source_params_json = std::string(PyUnicode_AsUTF8(pyObj_source_params_json)); + index.source_params_json = source_params_json; + } + + PyObject* pyObj_plan_params_json = PyDict_GetItemString(pyObj_index, "plan_params_json"); + if (pyObj_plan_params_json != nullptr) { + auto plan_params_json = std::string(PyUnicode_AsUTF8(pyObj_plan_params_json)); + index.plan_params_json = plan_params_json; + } + + return index; +} + +couchbase::core::operations::management::search_index_control_ingest_request +get_search_index_control_ingest_req(PyObject* op_args) +{ + couchbase::core::operations::management::search_index_control_ingest_request req{}; + + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + + PyObject* pyObj_pause = PyDict_GetItemString(op_args, "pause"); + if (pyObj_pause != nullptr) { + if (pyObj_pause == Py_True) { + req.pause = true; + } else { + req.pause = false; + } + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + return req; +} + +couchbase::core::operations::management::search_index_control_plan_freeze_request +get_search_index_control_freeze_req(PyObject* op_args) +{ + couchbase::core::operations::management::search_index_control_plan_freeze_request req{}; + + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + + PyObject* pyObj_freeze = PyDict_GetItemString(op_args, "freeze"); + if (pyObj_freeze != nullptr) { + if (pyObj_freeze == Py_True) { + req.freeze = true; + } else { + req.freeze = false; + } + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + return req; +} + +couchbase::core::operations::management::search_index_control_query_request +get_search_index_control_query_req(PyObject* op_args) +{ + couchbase::core::operations::management::search_index_control_query_request req{}; + + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + + PyObject* pyObj_allow = PyDict_GetItemString(op_args, "allow"); + if (pyObj_allow != nullptr) { + if (pyObj_allow == Py_True) { + req.allow = true; + } else { + req.allow = false; + } + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + return req; +} + +couchbase::core::operations::management::search_index_analyze_document_request +get_search_index_analyze_doc_req(PyObject* op_args) +{ + couchbase::core::operations::management::search_index_analyze_document_request req{}; + + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + + PyObject* pyObj_encoded_document = PyDict_GetItemString(op_args, "encoded_document"); + auto encoded_document = std::string(PyUnicode_AsUTF8(pyObj_encoded_document)); + req.encoded_document = encoded_document; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + return req; +} + +template<typename T> +T +get_search_index_with_name_req(PyObject* op_args) +{ + T req{}; + + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + return req; +} + +template<typename T> +T +get_search_index_req(PyObject* op_args) +{ + T req{}; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + return req; +} + +couchbase::core::operations::management::search_index_upsert_request +get_search_index_upsert_req(PyObject* op_args) +{ + couchbase::core::operations::management::search_index_upsert_request req{}; + + PyObject* pyObj_index = PyDict_GetItemString(op_args, "index"); + Py_INCREF(pyObj_index); + req.index = get_search_index(pyObj_index); + Py_DECREF(pyObj_index); + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + return req; +} + +couchbase::core::operations::management::search_index_get_stats_request +get_search_index_stats_req(PyObject* op_args) +{ + couchbase::core::operations::management::search_index_get_stats_request req{}; + + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + req.index_name = index_name; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + return req; +} + +couchbase::core::operations::management::search_get_stats_request +get_search_all_stats_req(PyObject* op_args) +{ + couchbase::core::operations::management::search_get_stats_request req{}; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + return req; +} + +template<typename Request> +PyObject* +do_search_index_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_search_index_mgmt_op_response( + resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_search_index_mgmt_op(connection* conn, + struct search_index_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + PyObject* res = nullptr; + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + switch (options->op_type) { + case SearchIndexManagementOperations::UPSERT_INDEX: { + auto req = get_search_index_upsert_req(options->op_args); + req.timeout = options->timeout_ms; + + res = do_search_index_mgmt_op< + couchbase::core::operations::management::search_index_upsert_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::GET_INDEX: { + auto req = get_search_index_with_name_req< + couchbase::core::operations::management::search_index_get_request>(options->op_args); + req.timeout = options->timeout_ms; + + res = + do_search_index_mgmt_op<couchbase::core::operations::management::search_index_get_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::DROP_INDEX: { + auto req = get_search_index_with_name_req< + couchbase::core::operations::management::search_index_drop_request>(options->op_args); + req.timeout = options->timeout_ms; + + res = + do_search_index_mgmt_op<couchbase::core::operations::management::search_index_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::GET_INDEX_DOCUMENT_COUNT: { + auto req = get_search_index_with_name_req< + couchbase::core::operations::management::search_index_get_documents_count_request>( + options->op_args); + req.timeout = options->timeout_ms; + + res = do_search_index_mgmt_op< + couchbase::core::operations::management::search_index_get_documents_count_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::GET_ALL_INDEXES: { + auto req = + get_search_index_req<couchbase::core::operations::management::search_index_get_all_request>( + options->op_args); + req.timeout = options->timeout_ms; + + res = do_search_index_mgmt_op< + couchbase::core::operations::management::search_index_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::GET_INDEX_STATS: { + auto req = get_search_index_stats_req(options->op_args); + req.timeout = options->timeout_ms; + + res = do_search_index_mgmt_op< + couchbase::core::operations::management::search_index_get_stats_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::GET_ALL_STATS: { + auto req = get_search_all_stats_req(options->op_args); + req.timeout = options->timeout_ms; + + res = + do_search_index_mgmt_op<couchbase::core::operations::management::search_get_stats_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::FREEZE_PLAN: { + auto req = get_search_index_control_freeze_req(options->op_args); + req.timeout = options->timeout_ms; + + res = do_search_index_mgmt_op< + couchbase::core::operations::management::search_index_control_plan_freeze_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::CONTROL_INGEST: { + auto req = get_search_index_control_ingest_req(options->op_args); + req.timeout = options->timeout_ms; + + res = do_search_index_mgmt_op< + couchbase::core::operations::management::search_index_control_ingest_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::ANALYZE_DOCUMENT: { + auto req = get_search_index_analyze_doc_req(options->op_args); + req.timeout = options->timeout_ms; + + res = do_search_index_mgmt_op< + couchbase::core::operations::management::search_index_analyze_document_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case SearchIndexManagementOperations::CONTROL_QUERY: { + auto req = get_search_index_control_query_req(options->op_args); + req.timeout = options->timeout_ms; + + res = do_search_index_mgmt_op< + couchbase::core::operations::management::search_index_control_query_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized search index mgmt operation passed in."); + barrier->set_value(nullptr); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + return res; +} + +void +add_search_index_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = + PyUnicode_FromString(SearchIndexManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("SearchIndexManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "search_index_mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/search_index_management.hxx b/src/management/search_index_management.hxx new file mode 100644 index 000000000..e31e09f5b --- /dev/null +++ b/src/management/search_index_management.hxx @@ -0,0 +1,98 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class SearchIndexManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + UPSERT_INDEX, + GET_INDEX, + DROP_INDEX, + GET_INDEX_DOCUMENT_COUNT, + GET_ALL_INDEXES, + GET_INDEX_STATS, + GET_ALL_STATS, + FREEZE_PLAN, + CONTROL_INGEST, + ANALYZE_DOCUMENT, + CONTROL_QUERY + }; + + SearchIndexManagementOperations() + : SearchIndexManagementOperations{ UNKNOWN } + { + } + constexpr SearchIndexManagementOperations(SearchIndexManagementOperations::OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(SearchIndexManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(SearchIndexManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "UPSERT_INDEX " + "GET_INDEX " + "DROP_INDEX " + "GET_INDEX_DOCUMENT_COUNT " + "GET_ALL_INDEXES " + "GET_INDEX_STATS " + "GET_ALL_STATS " + "FREEZE_PLAN " + "CONTROL_INGEST " + "ANALYZE_DOCUMENT " + "CONTROL_QUERY"; + + return ops; + } + +private: + OperationType operation; +}; + +struct search_index_mgmt_options { + PyObject* op_args; + SearchIndexManagementOperations::OperationType op_type = SearchIndexManagementOperations::UNKNOWN; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; +}; + +PyObject* +handle_search_index_mgmt_op(connection* conn, + struct search_index_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback); + +void +add_search_index_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/management/user_management.cxx b/src/management/user_management.cxx new file mode 100644 index 000000000..503423ea5 --- /dev/null +++ b/src/management/user_management.cxx @@ -0,0 +1,1034 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "user_management.hxx" + +#include <core/management/rbac.hxx> +#include <core/operations/management/user.hxx> + +#include "../exceptions.hxx" +#include "../result.hxx" + +couchbase::core::management::rbac::auth_domain +str_to_auth_domain(std::string domain) +{ + if (domain.compare("external") == 0) { + return couchbase::core::management::rbac::auth_domain::external; + } + return couchbase::core::management::rbac::auth_domain::local; + ; +} + +PyObject* +auth_domain_to_str(couchbase::core::management::rbac::auth_domain domain) +{ + PyObject* pyObj_domain = nullptr; + + if (domain == couchbase::core::management::rbac::auth_domain::local) { + pyObj_domain = PyUnicode_FromString("local"); + } else if (domain == couchbase::core::management::rbac::auth_domain::external) { + pyObj_domain = PyUnicode_FromString("external"); + } else { + pyObj_domain = PyUnicode_FromString("unknown"); + } + + return pyObj_domain; +} + +couchbase::core::management::rbac::role +get_role(PyObject* pyObj_role) +{ + couchbase::core::management::rbac::role role{}; + + PyObject* role_name = PyDict_GetItemString(pyObj_role, "name"); + if (role_name) { + role.name = std::string(PyUnicode_AsUTF8(role_name)); + } + + PyObject* pyObj_role_bucket = PyDict_GetItemString(pyObj_role, "bucket"); + if (pyObj_role_bucket && pyObj_role_bucket != Py_None) { + role.bucket = std::string(PyUnicode_AsUTF8(pyObj_role_bucket)); + } + + PyObject* pyObj_role_scope = PyDict_GetItemString(pyObj_role, "scope"); + if (pyObj_role_scope && pyObj_role_scope != Py_None) { + role.scope = std::string(PyUnicode_AsUTF8(pyObj_role_scope)); + } + + PyObject* pyObj_role_collection = PyDict_GetItemString(pyObj_role, "collection"); + if (pyObj_role_collection && pyObj_role_collection != Py_None) { + role.collection = std::string(PyUnicode_AsUTF8(pyObj_role_collection)); + } + + return role; +} + +couchbase::core::management::rbac::user +get_user(PyObject* pyObj_user) +{ + couchbase::core::management::rbac::user user{}; + + PyObject* pyObj_username = PyDict_GetItemString(pyObj_user, "username"); + if (pyObj_username) { + user.username = std::string(PyUnicode_AsUTF8(pyObj_username)); + } + + PyObject* pyObj_name = PyDict_GetItemString(pyObj_user, "name"); + if (pyObj_name && pyObj_name != Py_None) { + user.display_name = std::string(PyUnicode_AsUTF8(pyObj_name)); + } + + PyObject* pyObj_password = PyDict_GetItemString(pyObj_user, "password"); + if (pyObj_password && pyObj_password != Py_None) { + user.password = std::string(PyUnicode_AsUTF8(pyObj_password)); + } + + PyObject* pyObj_roles = PyDict_GetItemString(pyObj_user, "roles"); + if (pyObj_roles) { + for (Py_ssize_t i = 0; i < PyList_Size(pyObj_roles); i++) { + PyObject* pyObj_role = PyList_GetItem(pyObj_roles, i); + auto role = get_role(pyObj_role); + user.roles.emplace_back(role); + } + } + + PyObject* pyObj_groups = PyDict_GetItemString(pyObj_user, "groups"); + if (pyObj_groups) { + for (Py_ssize_t i = 0; i < PyList_Size(pyObj_groups); i++) { + PyObject* pyObj_group = PyList_GetItem(pyObj_groups, i); + user.groups.emplace(std::string(PyUnicode_AsUTF8(pyObj_group))); + } + } + + return user; +} + +couchbase::core::management::rbac::group +get_group(PyObject* pyObj_group) +{ + couchbase::core::management::rbac::group group{}; + + PyObject* pyObj_name = PyDict_GetItemString(pyObj_group, "name"); + if (pyObj_name) { + group.name = std::string(PyUnicode_AsUTF8(pyObj_name)); + } + + PyObject* pyObj_description = PyDict_GetItemString(pyObj_group, "description"); + if (pyObj_description && pyObj_description != Py_None) { + group.description = std::string(PyUnicode_AsUTF8(pyObj_description)); + } + + PyObject* pyObj_roles = PyDict_GetItemString(pyObj_group, "roles"); + if (pyObj_roles) { + for (Py_ssize_t i = 0; i < PyList_Size(pyObj_roles); i++) { + PyObject* pyObj_role = PyList_GetItem(pyObj_roles, i); + auto role = get_role(pyObj_role); + group.roles.emplace_back(role); + } + } + + PyObject* pyObj_ldap_group_reference = PyDict_GetItemString(pyObj_group, "ldap_group_reference"); + if (pyObj_ldap_group_reference && pyObj_ldap_group_reference != Py_None) { + group.ldap_group_reference = std::string(PyUnicode_AsUTF8(pyObj_ldap_group_reference)); + } + + return group; +} + +template<typename T> +PyObject* +build_role(const T& role) +{ + PyObject* pyObj_role = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(role.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_role, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_role); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (role.bucket.has_value()) { + pyObj_tmp = PyUnicode_FromString(role.bucket.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_role, "bucket_name", pyObj_tmp)) { + Py_DECREF(pyObj_role); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (role.scope.has_value()) { + pyObj_tmp = PyUnicode_FromString(role.scope.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_role, "scope_name", pyObj_tmp)) { + Py_DECREF(pyObj_role); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (role.collection.has_value()) { + pyObj_tmp = PyUnicode_FromString(role.collection.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_role, "collection_name", pyObj_tmp)) { + Py_DECREF(pyObj_role); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + return pyObj_role; +} + +PyObject* +build_role_and_origins(couchbase::core::management::rbac::role_and_origins role) +{ + PyObject* pyObj_role_and_origin = PyDict_New(); + PyObject* pyObj_role = build_role(role); + if (pyObj_role == nullptr) { + Py_XDECREF(pyObj_role_and_origin); + return nullptr; + } + + if (-1 == PyDict_SetItemString(pyObj_role_and_origin, "role", pyObj_role)) { + Py_XDECREF(pyObj_role_and_origin); + Py_DECREF(pyObj_role); + return nullptr; + } + Py_DECREF(pyObj_role); + + PyObject* pyObj_origins = PyList_New(static_cast<Py_ssize_t>(0)); + PyObject* pyObj_tmp = nullptr; + for (auto const& origin : role.origins) { + PyObject* pyObj_origin = PyDict_New(); + pyObj_tmp = PyUnicode_FromString(origin.type.c_str()); + if (-1 == PyDict_SetItemString(pyObj_origin, "type", pyObj_tmp)) { + Py_XDECREF(pyObj_origin); + Py_XDECREF(pyObj_origins); + Py_DECREF(pyObj_role_and_origin); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (origin.name.has_value()) { + pyObj_tmp = PyUnicode_FromString(origin.name.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_origin, "name", pyObj_tmp)) { + Py_DECREF(pyObj_origin); + Py_XDECREF(pyObj_origins); + Py_DECREF(pyObj_role_and_origin); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyList_Append(pyObj_origins, pyObj_origin); + Py_DECREF(pyObj_origin); + } + + if (-1 == PyDict_SetItemString(pyObj_role_and_origin, "origins", pyObj_origins)) { + Py_DECREF(pyObj_origins); + Py_DECREF(pyObj_role_and_origin); + return nullptr; + } + Py_DECREF(pyObj_origins); + + return pyObj_role_and_origin; +} + +PyObject* +build_user(couchbase::core::management::rbac::user_and_metadata uam) +{ + PyObject* pyObj_user = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(uam.username.c_str()); + if (-1 == PyDict_SetItemString(pyObj_user, "username", pyObj_tmp)) { + Py_XDECREF(pyObj_user); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (uam.display_name.has_value()) { + pyObj_tmp = PyUnicode_FromString(uam.display_name.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_user, "display_name", pyObj_tmp)) { + Py_DECREF(pyObj_user); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyObject* pyObj_groups = PySet_New(nullptr); + for (auto const& group : uam.groups) { + pyObj_tmp = PyUnicode_FromString(group.c_str()); + if (-1 == PySet_Add(pyObj_groups, pyObj_tmp)) { + Py_DECREF(pyObj_user); + Py_XDECREF(pyObj_groups); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyDict_SetItemString(pyObj_user, "groups", pyObj_groups)) { + Py_DECREF(pyObj_user); + Py_XDECREF(pyObj_groups); + return nullptr; + } + Py_DECREF(pyObj_groups); + + PyObject* pyObj_roles = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& role : uam.roles) { + PyObject* pyObj_role = build_role(role); + if (pyObj_role == nullptr) { + Py_XDECREF(pyObj_roles); + Py_DECREF(pyObj_user); + return nullptr; + } + PyList_Append(pyObj_roles, pyObj_role); + Py_DECREF(pyObj_role); + } + + if (-1 == PyDict_SetItemString(pyObj_user, "roles", pyObj_roles)) { + Py_DECREF(pyObj_user); + Py_XDECREF(pyObj_roles); + return nullptr; + } + Py_DECREF(pyObj_roles); + + return pyObj_user; +} + +PyObject* +build_user_and_metadata(couchbase::core::management::rbac::user_and_metadata uam) +{ + PyObject* pyObj_uam = PyDict_New(); + + PyObject* pyObj_user = build_user(uam); + if (pyObj_user == nullptr) { + Py_XDECREF(pyObj_uam); + return nullptr; + } + + if (-1 == PyDict_SetItemString(pyObj_uam, "user", pyObj_user)) { + Py_DECREF(pyObj_user); + Py_XDECREF(pyObj_uam); + return nullptr; + } + Py_DECREF(pyObj_user); + + PyObject* pyObj_tmp = auth_domain_to_str(uam.domain); + if (-1 == PyDict_SetItemString(pyObj_uam, "domain", pyObj_tmp)) { + Py_DECREF(pyObj_uam); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_eff_roles = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& role : uam.effective_roles) { + PyObject* pyObj_role_and_origins = build_role_and_origins(role); + if (pyObj_role_and_origins == nullptr) { + Py_XDECREF(pyObj_eff_roles); + Py_DECREF(pyObj_uam); + return nullptr; + } + PyList_Append(pyObj_eff_roles, pyObj_role_and_origins); + Py_DECREF(pyObj_role_and_origins); + } + + if (-1 == PyDict_SetItemString(pyObj_uam, "effective_roles", pyObj_eff_roles)) { + Py_DECREF(pyObj_uam); + Py_DECREF(pyObj_eff_roles); + return nullptr; + } + Py_DECREF(pyObj_eff_roles); + + if (uam.password_changed.has_value()) { + pyObj_tmp = PyUnicode_FromString(uam.password_changed.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_uam, "password_changed", pyObj_tmp)) { + Py_DECREF(pyObj_uam); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyObject* pyObj_ext_groups = PySet_New(nullptr); + for (auto const& group : uam.external_groups) { + pyObj_tmp = PyUnicode_FromString(group.c_str()); + if (-1 == PySet_Add(pyObj_ext_groups, pyObj_tmp)) { + Py_DECREF(pyObj_uam); + Py_XDECREF(pyObj_ext_groups); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyDict_SetItemString(pyObj_uam, "external_groups", pyObj_ext_groups)) { + Py_DECREF(pyObj_uam); + Py_DECREF(pyObj_ext_groups); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_ext_groups); + + return pyObj_uam; +} + +PyObject* +build_group(couchbase::core::management::rbac::group group) +{ + PyObject* pyObj_group = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(group.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_group, "name", pyObj_tmp)) { + Py_XDECREF(pyObj_group); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (group.description.has_value()) { + pyObj_tmp = PyUnicode_FromString(group.description.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_group, "description", pyObj_tmp)) { + Py_DECREF(pyObj_group); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyObject* pyObj_roles = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& role : group.roles) { + PyObject* pyObj_role = build_role(role); + if (pyObj_role == nullptr) { + Py_XDECREF(pyObj_roles); + Py_DECREF(pyObj_group); + return nullptr; + } + PyList_Append(pyObj_roles, pyObj_role); + Py_DECREF(pyObj_role); + } + + if (-1 == PyDict_SetItemString(pyObj_group, "roles", pyObj_roles)) { + Py_DECREF(pyObj_group); + Py_XDECREF(pyObj_roles); + return nullptr; + } + Py_DECREF(pyObj_roles); + + if (group.ldap_group_reference.has_value()) { + pyObj_tmp = PyUnicode_FromString(group.ldap_group_reference.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_group, "ldap_group_reference", pyObj_tmp)) { + Py_DECREF(pyObj_group); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + return pyObj_group; +} + +PyObject* +get_error_messages(std::vector<std::string> messages) +{ + PyObject* pyObj_messages = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& message : messages) { + PyObject* pyObj_message = PyUnicode_FromString(message.c_str()); + PyList_Append(pyObj_messages, pyObj_message); + Py_DECREF(pyObj_message); + } + + return pyObj_messages; +} + +template<typename T> +result* +create_result_from_user_mgmt_response([[maybe_unused]] const T& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + return res; +} + +template<> +result* +create_result_from_user_mgmt_response<couchbase::core::operations::management::user_get_response>( + const couchbase::core::operations::management::user_get_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_uam = build_user_and_metadata(resp.user); + if (pyObj_uam == nullptr) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_uam); + return nullptr; + } + if (-1 == PyDict_SetItemString(res->dict, "user_and_metadata", pyObj_uam)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_uam); + return nullptr; + } + Py_DECREF(pyObj_uam); + return res; +} + +template<> +result* +create_result_from_user_mgmt_response< + couchbase::core::operations::management::user_get_all_response>( + const couchbase::core::operations::management::user_get_all_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_users = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& uam : resp.users) { + PyObject* pyObj_uam = build_user_and_metadata(uam); + if (pyObj_uam == nullptr) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_users); + Py_XDECREF(pyObj_uam); + return nullptr; + } + PyList_Append(pyObj_users, pyObj_uam); + Py_DECREF(pyObj_uam); + } + + if (-1 == PyDict_SetItemString(res->dict, "users", pyObj_users)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_users); + return nullptr; + } + Py_DECREF(pyObj_users); + + return res; +} + +template<> +result* +create_result_from_user_mgmt_response< + couchbase::core::operations::management::role_get_all_response>( + const couchbase::core::operations::management::role_get_all_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_roles = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& role : resp.roles) { + PyObject* pyObj_role = build_role(role); + if (pyObj_role == nullptr) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_roles); + return nullptr; + } + + PyObject* pyObj_tmp = PyUnicode_FromString(role.display_name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_role, "display_name", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + Py_XDECREF(pyObj_role); + Py_XDECREF(pyObj_roles); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(role.description.c_str()); + if (-1 == PyDict_SetItemString(pyObj_role, "description", pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + Py_DECREF(pyObj_role); + Py_XDECREF(pyObj_roles); + return nullptr; + } + Py_DECREF(pyObj_tmp); + PyList_Append(pyObj_roles, pyObj_role); + Py_DECREF(pyObj_role); + } + + if (-1 == PyDict_SetItemString(res->dict, "roles", pyObj_roles)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_roles); + return nullptr; + } + Py_DECREF(pyObj_roles); + + return res; +} + +template<> +result* +create_result_from_user_mgmt_response<couchbase::core::operations::management::group_get_response>( + const couchbase::core::operations::management::group_get_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_group = build_group(resp.group); + if (-1 == PyDict_SetItemString(res->dict, "group", pyObj_group)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_group); + return nullptr; + } + Py_DECREF(pyObj_group); + return res; +} + +template<> +result* +create_result_from_user_mgmt_response< + couchbase::core::operations::management::group_get_all_response>( + const couchbase::core::operations::management::group_get_all_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_groups = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& group : resp.groups) { + PyObject* pyObj_group = build_group(group); + PyList_Append(pyObj_groups, pyObj_group); + Py_DECREF(pyObj_group); + } + + if (-1 == PyDict_SetItemString(res->dict, "groups", pyObj_groups)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_groups); + return nullptr; + } + Py_DECREF(pyObj_groups); + + return res; +} + +template<typename Response> +void +create_result_from_user_mgmt_op_response(const Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing user mgmt operation.", "UserMgmt"); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_user_mgmt_response(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "User mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<> +void +create_result_from_user_mgmt_op_response< + couchbase::core::operations::management::user_upsert_response>( + const couchbase::core::operations::management::user_upsert_response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + PyObject* pyObj_err_msgs = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + // group might have error messages + pyObj_err_msgs = get_error_messages(resp.errors); + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing user mgmt upsert operation.", "UserMgmt"); + pycbc_add_exception_info(pyObj_exc, "error_msgs", pyObj_err_msgs); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_user_mgmt_response(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "User mgmt upsert operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<> +void +create_result_from_user_mgmt_op_response< + couchbase::core::operations::management::group_upsert_response>( + const couchbase::core::operations::management::group_upsert_response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + PyObject* pyObj_err_msgs = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + // group might have error messages + pyObj_err_msgs = get_error_messages(resp.errors); + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing user mgmt group upsert operation.", "UserMgmt"); + pycbc_add_exception_info(pyObj_exc, "error_msgs", pyObj_err_msgs); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_user_mgmt_response(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception(PycbcError::UnableToBuildResult, + __FILE__, + __LINE__, + "User mgmt group upsert operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<typename Request> +PyObject* +do_user_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_user_mgmt_op_response(resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_user_mgmt_op(connection* conn, + struct user_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + PyObject* res = nullptr; + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + + switch (options->op_type) { + case UserManagementOperations::UPSERT_USER: { + PyObject* pyObj_domain = PyDict_GetItemString(options->op_args, "domain"); + auto domain = str_to_auth_domain(std::string(PyUnicode_AsUTF8(pyObj_domain))); + PyObject* pyObj_user = PyDict_GetItemString(options->op_args, "user"); + auto user = get_user(pyObj_user); + + couchbase::core::operations::management::user_upsert_request req{}; + req.domain = domain; + req.user = user; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::user_upsert_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::GET_USER: { + PyObject* pyObj_domain = PyDict_GetItemString(options->op_args, "domain"); + auto domain = str_to_auth_domain(std::string(PyUnicode_AsUTF8(pyObj_domain))); + PyObject* pyObj_username = PyDict_GetItemString(options->op_args, "username"); + auto username = std::string(PyUnicode_AsUTF8(pyObj_username)); + + couchbase::core::operations::management::user_get_request req{}; + req.domain = domain; + req.username = username; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::user_get_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::GET_ALL_USERS: { + PyObject* pyObj_domain = PyDict_GetItemString(options->op_args, "domain"); + auto domain = str_to_auth_domain(std::string(PyUnicode_AsUTF8(pyObj_domain))); + + couchbase::core::operations::management::user_get_all_request req{}; + req.domain = domain; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::user_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::DROP_USER: { + PyObject* pyObj_domain = PyDict_GetItemString(options->op_args, "domain"); + auto domain = str_to_auth_domain(std::string(PyUnicode_AsUTF8(pyObj_domain))); + PyObject* pyObj_username = PyDict_GetItemString(options->op_args, "username"); + auto username = std::string(PyUnicode_AsUTF8(pyObj_username)); + + couchbase::core::operations::management::user_drop_request req{}; + req.domain = domain; + req.username = username; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::user_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::CHANGE_PASSWORD: { + PyObject* pyObj_newPassword = PyDict_GetItemString(options->op_args, "password"); + auto newPassword = std::string(PyUnicode_AsUTF8(pyObj_newPassword)); + + couchbase::core::operations::management::change_password_request req{}; + req.newPassword = newPassword; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::change_password_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::GET_ROLES: { + couchbase::core::operations::management::role_get_all_request req{}; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::role_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::UPSERT_GROUP: { + PyObject* pyObj_group = PyDict_GetItemString(options->op_args, "group"); + auto group = get_group(pyObj_group); + + couchbase::core::operations::management::group_upsert_request req{}; + req.group = group; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::group_upsert_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::GET_GROUP: { + PyObject* pyObj_name = PyDict_GetItemString(options->op_args, "name"); + auto name = std::string(PyUnicode_AsUTF8(pyObj_name)); + + couchbase::core::operations::management::group_get_request req{}; + req.name = name; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::group_get_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::GET_ALL_GROUPS: { + couchbase::core::operations::management::group_get_all_request req{}; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::group_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case UserManagementOperations::DROP_GROUP: { + PyObject* pyObj_name = PyDict_GetItemString(options->op_args, "name"); + auto name = std::string(PyUnicode_AsUTF8(pyObj_name)); + + couchbase::core::operations::management::group_drop_request req{}; + req.name = name; + req.timeout = options->timeout_ms; + + res = do_user_mgmt_op<couchbase::core::operations::management::group_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized user mgmt operation passed in."); + barrier->set_value(nullptr); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + } + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + return res; +} + +void +add_user_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = PyUnicode_FromString(UserManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("UserManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "user_mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/user_management.hxx b/src/management/user_management.hxx new file mode 100644 index 000000000..4073a5987 --- /dev/null +++ b/src/management/user_management.hxx @@ -0,0 +1,96 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class UserManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + UPSERT_USER, + GET_USER, + GET_ALL_USERS, + DROP_USER, + CHANGE_PASSWORD, + GET_ROLES, + UPSERT_GROUP, + GET_GROUP, + GET_ALL_GROUPS, + DROP_GROUP + }; + + UserManagementOperations() + : UserManagementOperations{ UNKNOWN } + { + } + constexpr UserManagementOperations(UserManagementOperations::OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(UserManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(UserManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "UPSERT_USER " + "GET_USER " + "GET_ALL_USERS " + "DROP_USER " + "CHANGE_PASSWORD " + "GET_ROLES " + "UPSERT_GROUP " + "GET_GROUP " + "GET_ALL_GROUPS " + "DROP_GROUP "; + + return ops; + } + +private: + OperationType operation; +}; + +struct user_mgmt_options { + PyObject* op_args; + UserManagementOperations::OperationType op_type = UserManagementOperations::UNKNOWN; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; +}; + +PyObject* +handle_user_mgmt_op(connection* conn, + struct user_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback); + +void +add_user_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/management/view_index_management.cxx b/src/management/view_index_management.cxx new file mode 100644 index 000000000..10f49ab56 --- /dev/null +++ b/src/management/view_index_management.cxx @@ -0,0 +1,501 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "view_index_management.hxx" + +#include <core/design_document_namespace.hxx> +#include <core/management/design_document.hxx> +#include <core/operations/management/view.hxx> + +#include "../exceptions.hxx" +#include "../result.hxx" + +PyObject* +build_design_doc(couchbase::core::management::views::design_document dd) +{ + PyObject* pyObj_dd = PyDict_New(); + + PyObject* pyObj_tmp = nullptr; + if (dd.rev.has_value()) { + pyObj_tmp = PyUnicode_FromString(dd.rev.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_dd, "rev", pyObj_tmp)) { + Py_XDECREF(pyObj_dd); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = PyUnicode_FromString(dd.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_dd, "name", pyObj_tmp)) { + Py_DECREF(pyObj_dd); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + std::string ns = "development"; + if (dd.ns == couchbase::core::design_document_namespace::production) { + ns = "production"; + } + + pyObj_tmp = PyUnicode_FromString(ns.c_str()); + if (-1 == PyDict_SetItemString(pyObj_dd, "namespace", pyObj_tmp)) { + Py_DECREF(pyObj_dd); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_views = PyDict_New(); + for (const auto [name, view] : dd.views) { + PyObject* pyObj_view = PyDict_New(); + + if (view.map.has_value()) { + pyObj_tmp = PyUnicode_FromString(view.map.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_view, "map", pyObj_tmp)) { + Py_DECREF(pyObj_dd); + Py_XDECREF(pyObj_view); + Py_XDECREF(pyObj_views); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (view.reduce.has_value()) { + pyObj_tmp = PyUnicode_FromString(view.reduce.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_view, "reduce", pyObj_tmp)) { + Py_DECREF(pyObj_dd); + Py_DECREF(pyObj_view); + Py_XDECREF(pyObj_views); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyDict_SetItemString(pyObj_views, name.c_str(), pyObj_view)) { + Py_DECREF(pyObj_dd); + Py_DECREF(pyObj_view); + Py_XDECREF(pyObj_views); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_view); + } + + if (-1 == PyDict_SetItemString(pyObj_dd, "views", pyObj_views)) { + Py_DECREF(pyObj_dd); + Py_XDECREF(pyObj_views); + return nullptr; + } + Py_DECREF(pyObj_views); + + return pyObj_dd; +} + +template<typename T> +result* +create_result_from_view_index_mgmt_response([[maybe_unused]] const T& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + return res; +} + +template<> +result* +create_result_from_view_index_mgmt_response< + couchbase::core::operations::management::view_index_get_all_response>( + const couchbase::core::operations::management::view_index_get_all_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_design_documents = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& design_document : resp.design_documents) { + PyObject* pyObj_design_doc = build_design_doc(design_document); + if (pyObj_design_doc == nullptr) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_design_documents); + return nullptr; + } + PyList_Append(pyObj_design_documents, pyObj_design_doc); + Py_DECREF(pyObj_design_doc); + } + + if (-1 == PyDict_SetItemString(res->dict, "design_documents", pyObj_design_documents)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_design_documents); + return nullptr; + } + Py_DECREF(pyObj_design_documents); + return res; +} + +template<> +result* +create_result_from_view_index_mgmt_response< + couchbase::core::operations::management::view_index_get_response>( + const couchbase::core::operations::management::view_index_get_response& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_design_doc = build_design_doc(resp.document); + if (-1 == PyDict_SetItemString(res->dict, "design_document", pyObj_design_doc)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_design_doc); + return nullptr; + } + Py_DECREF(pyObj_design_doc); + + return res; +} + +template<typename Response> +void +create_result_from_view_index_mgmt_op_response(const Response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyObject* pyObj_args = nullptr; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing view index mgmt operation.", "ViewIndexMgmt"); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_result_from_view_index_mgmt_response(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "View index mgmt operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +couchbase::core::management::views::design_document +get_design_doc(PyObject* pyObj_dd) +{ + PyObject* pyObj_name = PyDict_GetItemString(pyObj_dd, "name"); + auto name = std::string(PyUnicode_AsUTF8(pyObj_name)); + + PyObject* pyObj_namespace = PyDict_GetItemString(pyObj_dd, "namespace"); + auto namespace_ = std::string(PyUnicode_AsUTF8(pyObj_namespace)); + + auto ns = couchbase::core::design_document_namespace::development; + if (namespace_.compare("production") == 0) { + ns = couchbase::core::design_document_namespace::production; + } + + std::map<std::string, couchbase::core::management::views::design_document::view> views{}; + PyObject* pyObj_views = PyDict_GetItemString(pyObj_dd, "views"); + if (pyObj_views && PyDict_Check(pyObj_views)) { + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_views, &pos, &pyObj_key, &pyObj_value)) { + std::string k; + if (PyUnicode_Check(pyObj_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_key)); + } + if (PyDict_Check(pyObj_value) && !k.empty()) { + couchbase::core::management::views::design_document::view view{ k }; + + PyObject* pyObj_tmp = PyDict_GetItemString(pyObj_value, "map"); + if (pyObj_tmp != nullptr) { + auto map = std::string(PyUnicode_AsUTF8(pyObj_tmp)); + view.map = map; + } + + pyObj_tmp = PyDict_GetItemString(pyObj_value, "reduce"); + if (pyObj_tmp != nullptr) { + auto reduce = std::string(PyUnicode_AsUTF8(pyObj_tmp)); + view.reduce = reduce; + } + views.emplace(k, view); + } + } + } + couchbase::core::management::views::design_document dd{}; + dd.name = name; + dd.ns = ns; + + if (views.size() > 0) { + dd.views = views; + } + + PyObject* pyObj_rev = PyDict_GetItemString(pyObj_dd, "rev"); + if (pyObj_rev != nullptr) { + auto rev = std::string(PyUnicode_AsUTF8(pyObj_rev)); + dd.rev = rev; + } + + return dd; +} + +template<typename T> +T +get_view_mgmt_req_base(PyObject* op_args) +{ + T req{}; + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + return req; +} + +couchbase::core::operations::management::view_index_get_all_request +get_view_index_get_all_req(PyObject* op_args) +{ + auto req = + get_view_mgmt_req_base<couchbase::core::operations::management::view_index_get_all_request>( + op_args); + + PyObject* pyObj_namespace = PyDict_GetItemString(op_args, "namespace"); + auto namespace_ = std::string(PyUnicode_AsUTF8(pyObj_namespace)); + + auto ns = couchbase::core::design_document_namespace::development; + if (namespace_.compare("production") == 0) { + ns = couchbase::core::design_document_namespace::production; + } + req.ns = ns; + + return req; +} + +couchbase::core::operations::management::view_index_get_request +get_view_index_get_req(PyObject* op_args) +{ + auto req = + get_view_mgmt_req_base<couchbase::core::operations::management::view_index_get_request>( + op_args); + PyObject* pyObj_document_name = PyDict_GetItemString(op_args, "document_name"); + auto document_name = std::string(PyUnicode_AsUTF8(pyObj_document_name)); + req.document_name = document_name; + + PyObject* pyObj_namespace = PyDict_GetItemString(op_args, "namespace"); + auto namespace_ = std::string(PyUnicode_AsUTF8(pyObj_namespace)); + + auto ns = couchbase::core::design_document_namespace::development; + if (namespace_.compare("production") == 0) { + ns = couchbase::core::design_document_namespace::production; + } + req.ns = ns; + + return req; +} + +couchbase::core::operations::management::view_index_drop_request +get_view_index_drop_req(PyObject* op_args) +{ + auto req = + get_view_mgmt_req_base<couchbase::core::operations::management::view_index_drop_request>( + op_args); + PyObject* pyObj_document_name = PyDict_GetItemString(op_args, "document_name"); + auto document_name = std::string(PyUnicode_AsUTF8(pyObj_document_name)); + req.document_name = document_name; + + PyObject* pyObj_namespace = PyDict_GetItemString(op_args, "namespace"); + auto namespace_ = std::string(PyUnicode_AsUTF8(pyObj_namespace)); + + auto ns = couchbase::core::design_document_namespace::development; + if (namespace_.compare("production") == 0) { + ns = couchbase::core::design_document_namespace::production; + } + req.ns = ns; + + return req; +} + +couchbase::core::operations::management::view_index_upsert_request +get_view_index_upsert_req(PyObject* op_args) +{ + auto req = + get_view_mgmt_req_base<couchbase::core::operations::management::view_index_upsert_request>( + op_args); + PyObject* pyObj_design_doc = PyDict_GetItemString(op_args, "design_docucment"); + if (pyObj_design_doc != nullptr) { + auto design_doc = get_design_doc(pyObj_design_doc); + req.document = design_doc; + } + return req; +} + +template<typename Request> +PyObject* +do_view_index_mgmt_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_view_index_mgmt_op_response(resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +handle_view_index_mgmt_op(connection* conn, + struct view_index_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + PyObject* res = nullptr; + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto f = barrier->get_future(); + switch (options->op_type) { + case ViewIndexManagementOperations::UPSERT_INDEX: { + auto req = get_view_index_upsert_req(options->op_args); + req.timeout = options->timeout_ms; + + res = + do_view_index_mgmt_op<couchbase::core::operations::management::view_index_upsert_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case ViewIndexManagementOperations::GET_INDEX: { + auto req = get_view_index_get_req(options->op_args); + req.timeout = options->timeout_ms; + + res = do_view_index_mgmt_op<couchbase::core::operations::management::view_index_get_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case ViewIndexManagementOperations::DROP_INDEX: { + auto req = get_view_index_drop_req(options->op_args); + req.timeout = options->timeout_ms; + + res = do_view_index_mgmt_op<couchbase::core::operations::management::view_index_drop_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + case ViewIndexManagementOperations::GET_ALL_INDEXES: { + auto req = get_view_index_get_all_req(options->op_args); + req.timeout = options->timeout_ms; + + res = + do_view_index_mgmt_op<couchbase::core::operations::management::view_index_get_all_request>( + *conn, req, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized view index mgmt operation passed in."); + barrier->set_value(nullptr); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = f.get(); + Py_END_ALLOW_THREADS return ret; + } + return res; +} + +void +add_view_index_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class) +{ + PyObject* pyObj_enum_values = + PyUnicode_FromString(ViewIndexManagementOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("ViewIndexManagementOperations"); + // PyTuple_Pack returns new reference, need to Py_DECREF values provided + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* pyObj_mgmt_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "view_index_mgmt_operations", pyObj_mgmt_operations) < 0) { + // only need to Py_DECREF on failure to add when using PyModule_AddObject() + Py_XDECREF(pyObj_mgmt_operations); + return; + } +} diff --git a/src/management/view_index_management.hxx b/src/management/view_index_management.hxx new file mode 100644 index 000000000..a0592e87f --- /dev/null +++ b/src/management/view_index_management.hxx @@ -0,0 +1,84 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" + +class ViewIndexManagementOperations +{ +public: + enum OperationType { + UNKNOWN, + UPSERT_INDEX, + GET_INDEX, + DROP_INDEX, + GET_ALL_INDEXES + }; + + ViewIndexManagementOperations() + : ViewIndexManagementOperations{ UNKNOWN } + { + } + constexpr ViewIndexManagementOperations(ViewIndexManagementOperations::OperationType op) + : operation{ op } + { + } + + operator OperationType() const + { + return operation; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(ViewIndexManagementOperations op) const + { + return operation == op.operation; + } + constexpr bool operator!=(ViewIndexManagementOperations op) const + { + return operation != op.operation; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "UPSERT_INDEX " + "GET_INDEX " + "DROP_INDEX " + "GET_ALL_INDEXES"; + + return ops; + } + +private: + OperationType operation; +}; + +struct view_index_mgmt_options { + PyObject* op_args; + ViewIndexManagementOperations::OperationType op_type = ViewIndexManagementOperations::UNKNOWN; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::management_timeout; +}; + +PyObject* +handle_view_index_mgmt_op(connection* conn, + struct view_index_mgmt_options* options, + PyObject* pyObj_callback, + PyObject* pyObj_errback); + +void +add_view_index_mgmt_ops_enum(PyObject* pyObj_module, PyObject* pyObj_enum_class); diff --git a/src/metrics.hxx b/src/metrics.hxx new file mode 100644 index 000000000..929c14bed --- /dev/null +++ b/src/metrics.hxx @@ -0,0 +1,112 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <couchbase/metrics/meter.hxx> +// NOLINTNEXTLINE +#include "Python.h" // NOLINT + +namespace metrics = couchbase::metrics; + +namespace pycbc +{ +class value_recorder : public metrics::value_recorder +{ +public: + explicit value_recorder(PyObject* recorder) + : metrics::value_recorder() + , pyObj_recorder_(recorder) + { + // A value_recorder is only created from meter::get_value_recorder, which is responsible for + // obtaining the GIL + Py_INCREF(pyObj_recorder_); + pyObj_record_value_ = PyObject_GetAttrString(pyObj_recorder_, "record_value"); + CB_LOG_DEBUG("{}: created value_recorder", "PYCBC"); + } + + ~value_recorder() override + { + PyGILState_STATE state = PyGILState_Ensure(); + Py_DECREF(pyObj_recorder_); + Py_DECREF(pyObj_record_value_); + PyGILState_Release(state); + CB_LOG_DEBUG("{}: destroyed value_recorder", "PYCBC"); + } + + void record_value(std::int64_t value) override + { + PyGILState_STATE state = PyGILState_Ensure(); + auto pyObj_args = Py_BuildValue("(n)", static_cast<Py_ssize_t>(value)); + PyObject_CallObject(pyObj_record_value_, pyObj_args); + Py_DECREF(pyObj_args); + PyGILState_Release(state); + } + +private: + PyObject* pyObj_recorder_; + PyObject* pyObj_record_value_; +}; + +class meter : public metrics::meter +{ +public: + meter(PyObject* meter) + : pyObj_meter_(meter) + { + // Assume we have the GIL when creating a CouchbaseMeter + Py_INCREF(meter); + pyObj_value_recorder_ = PyObject_GetAttrString(meter, "value_recorder"); + assert(pyObj_value_recorder_); + } + + ~meter() override + { + PyGILState_STATE state = PyGILState_Ensure(); + Py_DECREF(pyObj_value_recorder_); + Py_DECREF(pyObj_meter_); + PyGILState_Release(state); + } + + std::shared_ptr<metrics::value_recorder> get_value_recorder( + const std::string& name, + const std::map<std::string, std::string>& tags) override + { + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* pyObj_name = PyUnicode_FromString(name.c_str()); + PyObject* pyObj_tags = PyDict_New(); + for (const auto& [key, value] : tags) { + PyObject* pyObj_value = PyUnicode_FromString(value.c_str()); + PyDict_SetItemString(pyObj_tags, key.c_str(), pyObj_value); + Py_DECREF(pyObj_value); + } + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_name, pyObj_tags); + auto pyObj_value_recorder = PyObject_CallObject(pyObj_value_recorder_, pyObj_args); + auto retval = std::make_shared<value_recorder>(pyObj_value_recorder); + Py_DECREF(pyObj_name); + Py_DECREF(pyObj_tags); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_value_recorder); + PyGILState_Release(state); + return retval; + } + +private: + PyObject* pyObj_meter_; + PyObject* pyObj_value_recorder_; +}; +} // namespace pycbc diff --git a/src/miscops.c b/src/miscops.c deleted file mode 100644 index 49f7f375b..000000000 --- a/src/miscops.c +++ /dev/null @@ -1,368 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "oputil.h" - -/** - * This file contains 'miscellaneous' operations. Functions contained here - * might move to other files if they become more complex. - * - * More specifically, this contains 'key-only' operations that don't - * require a value. - */ - - -/** - * This is called during each iteration of delete/unlock - */ -static int -handle_single_keyop(pycbc_Bucket *self, struct pycbc_common_vars *cv, int optype, - PyObject *curkey, PyObject *curval, PyObject *options, pycbc_Item *item, - void *arg) -{ - int rv; - char *key; - size_t nkey; - lcb_U64 cas = 0; - lcb_error_t err; - - union { - lcb_CMDBASE base; - lcb_CMDREMOVE rm; - lcb_CMDUNLOCK unl; - lcb_CMDENDURE endure; - } ucmd; - - (void)options; (void)arg; - - memset(&ucmd, 0, sizeof ucmd); - - if ( (optype == PYCBC_CMD_UNLOCK || optype == PYCBC_CMD_ENDURE) - && PYCBC_OPRES_CHECK(curkey)) { - curval = curkey; - curkey = ((pycbc_OperationResult*)curkey)->key; - } - - rv = pycbc_tc_encode_key(self, &curkey, (void**)&key, &nkey); - if (rv == -1) { - return -1; - } - - if (!nkey) { - PYCBC_EXCTHROW_EMPTYKEY(); - rv = -1; - goto GT_DONE; - } - - if (item) { - cas = item->cas; - - } else if (curval) { - if (PyDict_Check(curval)) { - PyObject *cas_o = PyDict_GetItemString(curval, "cas"); - if (!cas_o) { - PyErr_Clear(); - } - cas = pycbc_IntAsULL(cas_o); - - } else if (PYCBC_OPRES_CHECK(curval)) { - /* If we're passed a Result object, just extract its CAS */ - cas = ((pycbc_OperationResult*)curval)->cas; - - } else if (PyNumber_Check(curval)) { - cas = pycbc_IntAsULL(curval); - - } - - if (cas == (lcb_uint64_t)-1 && PyErr_Occurred()) { - PyErr_Clear(); - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Invalid CAS specified"); - return -1; - } - } - - LCB_CMD_SET_KEY(&ucmd.base, key, nkey); - ucmd.base.cas = cas; - - if (optype == PYCBC_CMD_UNLOCK) { - if (!cas) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "CAS must be specified for unlock"); - return -1; - } - err = lcb_unlock3(self->instance, cv->mres, &ucmd.unl); - - } else if (optype == PYCBC_CMD_ENDURE) { - err = cv->mctx->addcmd(cv->mctx, &ucmd.base); - - } else { - err = lcb_remove3(self->instance, cv->mres, &ucmd.rm); - } - if (err == LCB_SUCCESS) { - rv = 0; - } else { - rv = -1; - PYCBC_EXCTHROW_SCHED(err); - } - - GT_DONE: - Py_XDECREF(curkey); - return rv; -} - -static PyObject * -keyop_common(pycbc_Bucket *self, PyObject *args, PyObject *kwargs, int optype, - int argopts) -{ - int rv; - Py_ssize_t ncmds = 0; - pycbc_seqtype_t seqtype; - PyObject *casobj = NULL; - PyObject *is_quiet = NULL; - PyObject *kobj = NULL; - char persist_to = 0, replicate_to = 0; - struct pycbc_common_vars cv = PYCBC_COMMON_VARS_STATIC_INIT; - - static char *kwlist[] = { - "keys", "cas", "quiet", "persist_to", "replicate_to", NULL - }; - - rv = PyArg_ParseTupleAndKeywords(args, - kwargs, - "O|OOBB", - kwlist, - &kobj, - &casobj, - &is_quiet, - &persist_to, &replicate_to); - - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_check_sequence(kobj, 1, &ncmds, &seqtype); - if (rv < 0) { - return NULL; - } - - if (casobj && PyObject_IsTrue(casobj)) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "Can't pass CAS for multiple keys"); - } - - } else { - ncmds = 1; - } - - rv = pycbc_common_vars_init(&cv, self, argopts, ncmds, 0); - if (rv < 0) { - return NULL; - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_iter_multi(self, seqtype, kobj, &cv, optype, - handle_single_keyop, NULL); - } else { - rv = handle_single_keyop(self, &cv, optype, kobj, casobj, NULL, NULL, NULL); - } - - if (rv < 0) { - goto GT_DONE; - } - - if (optype == PYCBC_CMD_DELETE) { - rv = pycbc_handle_durability_args(self, &cv.mres->dur, - persist_to, replicate_to); - if (rv == 1) { - cv.mres->mropts |= PYCBC_MRES_F_DURABILITY; - - } else if (rv == -1) { - goto GT_DONE; - } - if (pycbc_maybe_set_quiet(cv.mres, is_quiet) == -1) { - goto GT_DONE; - } - } - - if (-1 == pycbc_common_vars_wait(&cv, self)) { - goto GT_DONE; - } - - GT_DONE: - pycbc_common_vars_finalize(&cv, self); - return cv.ret; -} - - -PyObject * -pycbc_Bucket_endure_multi(pycbc_Bucket *self, PyObject *args, PyObject *kwargs) -{ - int rv; - Py_ssize_t ncmds; - pycbc_seqtype_t seqtype; - char persist_to = 0, replicate_to = 0; - lcb_durability_opts_t dopts = { 0 }; - PyObject *keys; - PyObject *is_delete_O = Py_False; - lcb_error_t err; - float timeout = 0.0; - float interval = 0.0; - - struct pycbc_common_vars cv = PYCBC_COMMON_VARS_STATIC_INIT; - - static char *kwlist[] = { - "keys", - "persist_to", - "replicate_to", - "check_removed", - "timeout", - "interval", - NULL - }; - - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "OBB|Off", kwlist, - &keys, - &persist_to, &replicate_to, - &is_delete_O, &timeout, &interval); - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - rv = pycbc_oputil_check_sequence(keys, 1, &ncmds, &seqtype); - if (rv < 0) { - return NULL; - } - rv = pycbc_common_vars_init(&cv, self, PYCBC_ARGOPT_MULTI, ncmds, 0); - if (rv < 0) { - return NULL; - } - - dopts.v.v0.cap_max = persist_to < 0 || replicate_to < 0; - dopts.v.v0.check_delete = is_delete_O && PyObject_IsTrue(is_delete_O); - dopts.v.v0.timeout = (lcb_uint32_t)(timeout * 1000000.0); - dopts.v.v0.interval = (lcb_uint32_t)(interval * 1000000.0); - dopts.v.v0.persist_to = persist_to; - dopts.v.v0.replicate_to = replicate_to; - cv.mctx = lcb_endure3_ctxnew(self->instance, &dopts, &err); - if (cv.mctx == NULL) { - PYCBC_EXCTHROW_SCHED(err); - goto GT_DONE; - } - - rv = pycbc_oputil_iter_multi(self, seqtype, keys, &cv, PYCBC_CMD_ENDURE, - handle_single_keyop, NULL); - if (rv < 0) { - goto GT_DONE; - } - - if (-1 == pycbc_common_vars_wait(&cv, self)) { - goto GT_DONE; - } - - GT_DONE: - pycbc_common_vars_finalize(&cv, self); - return cv.ret; - -} - -#define DECLFUNC(name, operation, mode) \ - PyObject *pycbc_Bucket_##name(pycbc_Bucket *self, \ - PyObject *args, PyObject *kwargs) { \ - return keyop_common(self, args, kwargs, operation, mode); \ -} - -DECLFUNC(remove, PYCBC_CMD_DELETE, PYCBC_ARGOPT_SINGLE) -DECLFUNC(unlock, PYCBC_CMD_UNLOCK, PYCBC_ARGOPT_SINGLE) -DECLFUNC(remove_multi, PYCBC_CMD_DELETE, PYCBC_ARGOPT_MULTI) -DECLFUNC(unlock_multi, PYCBC_CMD_UNLOCK, PYCBC_ARGOPT_MULTI) - - -PyObject * -pycbc_Bucket__stats(pycbc_Bucket *self, PyObject *args, PyObject *kwargs) -{ - int rv; - int ii; - Py_ssize_t ncmds; - lcb_error_t err = LCB_ERROR; - PyObject *keys = NULL, *is_keystats = NULL; - struct pycbc_common_vars cv = PYCBC_COMMON_VARS_STATIC_INIT; - static char *kwlist[] = { "keys", "keystats", NULL }; - lcb_CMDSTATS cmd = { 0 }; - - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "|OO", kwlist, - &keys, &is_keystats); - - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - if (keys == NULL || PyObject_IsTrue(keys) == 0) { - keys = NULL; - ncmds = 1; - - } else { - if (!PySequence_Check(keys)) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "keys argument must be sequence"); - return NULL; - } - ncmds = PySequence_Size(keys); - } - - rv = pycbc_common_vars_init(&cv, self, PYCBC_ARGOPT_MULTI, ncmds, 0); - if (rv < 0) { - return NULL; - } - - if (keys) { - for (ii =0; ii < ncmds; ii++) { - char *key; - Py_ssize_t nkey; - PyObject *newkey = NULL; - - PyObject *curkey = PySequence_GetItem(keys, ii); - rv = pycbc_BufFromString(curkey, &key, &nkey, &newkey); - if (rv < 0) { - PYCBC_EXC_WRAP_KEY(PYCBC_EXC_ARGUMENTS, 0, "bad key type in stats", curkey); - goto GT_DONE; - } - - LCB_CMD_SET_KEY(&cmd, key, nkey); - if (is_keystats && PyObject_IsTrue(is_keystats)) { - cmd.cmdflags |= LCB_CMDSTATS_F_KV; - } - err = lcb_stats3(self->instance, cv.mres, &cmd); - Py_XDECREF(newkey); - } - - } else { - err = lcb_stats3(self->instance, cv.mres, &cmd); - } - - if (err != LCB_SUCCESS) { - PYCBC_EXCTHROW_SCHED(err); - goto GT_DONE; - } - - if (-1 == pycbc_common_vars_wait(&cv, self)) { - goto GT_DONE; - } - - GT_DONE: - pycbc_common_vars_finalize(&cv, self); - return cv.ret; -} diff --git a/src/mresdict.h b/src/mresdict.h deleted file mode 100644 index 379212ea7..000000000 --- a/src/mresdict.h +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Abstraction layer for interacting with the MultiResult dict. - */ - - -#define PYCBC_MULTIRESULT_BASE PyDictObject _dict_private - -/** - * Get a borrowed reference to the dictionary - */ -#define pycbc_multiresult_dict(mres) \ - (PyObject *)(&((mres)->_dict_private)) - -/** - * Set the base class for the MultiResult - */ -#define pycbc_multiresult_set_base(tobj) (tobj)->tp_base = &PyDict_Type - -/** - * __init__ for the dictionary - */ -#define pycbc_multiresult_init_dict(iobj, args, kwargs) \ - (PyDict_Type.tp_init((PyObject *)iobj, args, kwargs)) - -/** - * Destroy the dictionary - */ -#define pycbc_multiresult_destroy_dict(iobj) \ - PyDict_Type.tp_dealloc((PyObject *)iobj) - -#define pycbc_multiresult_check(obj) \ - (Py_TYPE(obj) == &pycbc_MultiResultType || \ - Py_TYPE(obj) == &pycbc_AsyncResultType) diff --git a/src/multiresult.c b/src/multiresult.c deleted file mode 100644 index 5f0d0b412..000000000 --- a/src/multiresult.c +++ /dev/null @@ -1,411 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "structmember.h" - - -static struct PyMemberDef MultiResult_TABLE_members[] = { - { "all_ok", - T_INT, offsetof(pycbc_MultiResult, all_ok), - READONLY, - PyDoc_STR("Whether all the items in this result are successful") - }, - { NULL } -}; - -PyTypeObject pycbc_MultiResultType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static PyMethodDef MultiResult_TABLE_methods[] = { - { NULL } -}; - -PyTypeObject pycbc_AsyncResultType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static struct PyMemberDef AsyncResult_TABLE_members[] = { - { "remaining", - T_UINT, offsetof(pycbc_AsyncResult, nops), - READONLY, - PyDoc_STR("Number of operations remaining for this 'AsyncResult") - }, - - { "callback", - T_OBJECT_EX, offsetof(pycbc_AsyncResult, callback), - 0, - PyDoc_STR("Callback to be invoked with this result") - }, - - { "errback", - T_OBJECT_EX, offsetof(pycbc_AsyncResult, errback), - 0, - PyDoc_STR("Callback to be invoked with any errors") - }, - - { NULL } -}; - -static PyObject * -AsyncResult_set_callbacks(pycbc_AsyncResult *self, PyObject *args) -{ - PyObject *errcb = NULL; - PyObject *okcb = NULL; - - if (!PyArg_ParseTuple(args, "OO", &okcb, &errcb)) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - Py_XINCREF(errcb); - Py_XINCREF(okcb); - - Py_XDECREF(self->callback); - Py_XDECREF(self->errback); - - self->callback = okcb; - self->errback = errcb; - Py_RETURN_NONE; -} - -static PyObject * -AsyncResult_set_single(pycbc_AsyncResult *self) -{ - if (self->nops != 1) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, - "Cannot set mode to single. " - "AsyncResult has more than one operation"); - return NULL; - } - - self->base.mropts |= PYCBC_MRES_F_SINGLE; - Py_RETURN_NONE; -} - -static PyObject * -AsyncResult_clear_callbacks(pycbc_AsyncResult *self, PyObject *args) -{ - (void)args; - Py_CLEAR(self->errback); - Py_CLEAR(self->callback); - Py_RETURN_NONE; -} - -static struct PyMethodDef AsyncResult_TABLE_methods[] = { - { "set_callbacks", (PyCFunction)AsyncResult_set_callbacks, - METH_VARARGS, - PyDoc_STR("Set the ok and error callbacks") - }, - { "clear_callbacks", (PyCFunction)AsyncResult_clear_callbacks, - METH_NOARGS, - PyDoc_STR("Convenience function to clear all callbacks. This\n" - "may be more performant than setting the fields manually") - }, - { "_set_single", (PyCFunction)AsyncResult_set_single, - METH_NOARGS, - PyDoc_STR("Indicate that this is a 'single' result to be " - "wrapped") - }, - - { NULL } -}; - -static int -MultiResultType__init__(pycbc_MultiResult *self, PyObject *args, PyObject *kwargs) -{ - if (pycbc_multiresult_init_dict(self, args, kwargs) < 0) { - PyErr_Print(); - abort(); - return -1; - } - - self->all_ok = 1; - self->exceptions = NULL; - self->errop = NULL; - self->mropts = 0; - - return 0; -} - -static void -MultiResult_dealloc(pycbc_MultiResult *self) -{ - Py_XDECREF(self->parent); - Py_XDECREF(self->exceptions); - Py_XDECREF(self->errop); - pycbc_multiresult_destroy_dict(self); -} - -int -pycbc_MultiResultType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_MultiResultType; - - *ptr = (PyObject*)p; - if (p->tp_name) { - return 0; - } - - pycbc_multiresult_set_base(p); - - p->tp_init = (initproc)MultiResultType__init__; - p->tp_dealloc = (destructor)MultiResult_dealloc; - - p->tp_name = "MultiResult"; - p->tp_doc = PyDoc_STR( - ":class:`dict` subclass to hold :class:`Result` objects\n" - "\n" - "This object also contains some of the heavy lifting, but this\n" - "is not currently exposed in python-space\n." - "\n" - "An additional method is :meth:`all_ok`, which allows to see\n" - "if all commands completed successfully\n" - "\n"); - - p->tp_basicsize = sizeof(pycbc_MultiResult); - p->tp_members = MultiResult_TABLE_members; - p->tp_methods = MultiResult_TABLE_methods; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - - return PyType_Ready(p); -} - -/** - * AsyncResult - */ -static int -AsyncResult__init__(pycbc_AsyncResult *self, PyObject *args, PyObject *kwargs) -{ - if (MultiResultType__init__((pycbc_MultiResult *)self, args, kwargs) != 0) { - return -1; - } - - self->nops = 0; - self->callback = NULL; - self->errback = NULL; - self->base.mropts |= PYCBC_MRES_F_ASYNC; - return 0; -} - -static void -AsyncResult_dealloc(pycbc_AsyncResult *self) -{ - Py_XDECREF(self->callback); - Py_XDECREF(self->errback); - MultiResult_dealloc(&self->base); -} - -int -pycbc_AsyncResultType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_AsyncResultType; - *ptr = (PyObject *)p; - if (p->tp_name) { - return 0; - } - - p->tp_base = &pycbc_MultiResultType; - p->tp_init = (initproc)AsyncResult__init__; - p->tp_dealloc = (destructor)AsyncResult_dealloc; - p->tp_members = AsyncResult_TABLE_members; - p->tp_basicsize = sizeof(pycbc_AsyncResult); - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_methods = AsyncResult_TABLE_methods; - p->tp_name = "AsyncResult"; - - return PyType_Ready(p); -} - -PyObject * -pycbc_multiresult_new(pycbc_Bucket *parent) -{ - PyTypeObject *initmeth; - pycbc_MultiResult *ret; - - if (parent->flags & PYCBC_CONN_F_ASYNC) { - initmeth = &pycbc_AsyncResultType; - - } else { - initmeth = &pycbc_MultiResultType; - } - - ret = (pycbc_MultiResult*) PyObject_CallFunction((PyObject*)initmeth, - NULL, - NULL); - if (!ret) { - PyErr_Print(); - abort(); - } - ret->parent = parent; - Py_INCREF(parent); - - if (parent->pipeline_queue) { - PyList_Append(parent->pipeline_queue, (PyObject *)ret); - } - - return (PyObject*)ret; -} - -void -pycbc_multiresult_adderr(pycbc_MultiResult* mres) -{ - PyObject *etuple; - mres->all_ok = 0; - if (!mres->exceptions) { - mres->exceptions = PyList_New(0); - } - - etuple = pycbc_exc_mktuple(); - PyList_Append(mres->exceptions, etuple); - Py_DECREF(etuple); -} - -/** - * This function raises exceptions from the MultiResult object, as required - */ -int -pycbc_multiresult_maybe_raise(pycbc_MultiResult *self) -{ - PyObject *type = NULL, *value = NULL, *traceback = NULL; - - if (self->errop == NULL && self->exceptions == NULL) { - return 0; - } - - if (self->exceptions) { - PyObject *tuple = PyList_GetItem(self->exceptions, 0); - - pycbc_assert(tuple); - pycbc_assert(PyTuple_Size(tuple) == 3); - - type = PyTuple_GetItem(tuple, 0); - value = PyTuple_GetItem(tuple, 1); - traceback = PyTuple_GetItem(tuple, 2); - PyErr_NormalizeException(&type, &value, &traceback); - Py_XINCREF(type); - Py_XINCREF(value); - Py_XINCREF(traceback); - } else { - pycbc_Result *res = (pycbc_Result*)self->errop; - - /** Craft an exception based on the operation */ - PYCBC_EXC_WRAP_KEY(PYCBC_EXC_LCBERR, res->rc, "Operational Error", res->key); - - /** Now we have an exception. Let's fetch it back */ - PyErr_Fetch(&type, &value, &traceback); - PyObject_SetAttrString(value, "result", (PyObject*)res); - } - - PyErr_Restore(type, value, traceback); - - /** - * This is needed since the exception object will later contain - * a reference to ourselves. If we don't free the original exception, - * then we'll be stuck with a circular reference - */ - - if (PyObject_IsInstance(value, pycbc_helpers.default_exception)) { - PyObject_SetAttrString(value, "all_results", (PyObject*)self); - Py_XDECREF(self->exceptions); - } - Py_XDECREF(self->errop); - self->exceptions = NULL; - self->errop = NULL; - - return 1; -} - -PyObject * -pycbc_multiresult_get_result(pycbc_MultiResult *self) -{ - int rv; - Py_ssize_t dictpos = 0; - PyObject *key, *value; - - if (!(self->mropts & PYCBC_MRES_F_SINGLE)) { - PyObject *res = (PyObject *)self; - Py_INCREF(res); - return res; - } - - rv = PyDict_Next(pycbc_multiresult_dict(self), &dictpos, &key, &value); - if (!rv) { - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "No objects in MultiResult"); - return NULL; - } - - Py_INCREF(value); - return value; -} - -void -pycbc_asyncresult_invoke(pycbc_AsyncResult *ares) -{ - PyObject *argtuple; - PyObject *cbmeth; - - if (!pycbc_multiresult_maybe_raise(&ares->base)) { - /** All OK */ - PyObject *eres = pycbc_multiresult_get_result(&ares->base); - cbmeth = ares->callback; - argtuple = PyTuple_New(1); - PyTuple_SET_ITEM(argtuple, 0, (PyObject *)eres); - - } else { - PyObject *ex_value, *ex_type, *ex_tb; - - PyErr_Fetch(&ex_type, &ex_value, &ex_tb); - - if (!ex_type) { - ex_type = Py_None; Py_INCREF(ex_type); - } - if (!ex_value) { - ex_value = Py_None; Py_INCREF(ex_value); - } - if (!ex_tb) { - ex_tb = Py_None; Py_INCREF(ex_tb); - } - - cbmeth = ares->errback; - argtuple = PyTuple_New(4); - - PyTuple_SET_ITEM(argtuple, 0, (PyObject *)ares); - Py_INCREF(ares); - - PyTuple_SET_ITEM(argtuple, 1, ex_type); - PyTuple_SET_ITEM(argtuple, 2, ex_value); - PyTuple_SET_ITEM(argtuple, 3, ex_tb); - } - - if (!cbmeth) { - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "No callbacks provided"); - } else { - PyObject *res = PyObject_CallObject(cbmeth, argtuple); - if (res) { - Py_XDECREF(res); - } else { - PyErr_Print(); - } - } - - Py_CLEAR(ares->base.parent); - Py_DECREF(argtuple); - Py_DECREF(ares); -} diff --git a/src/n1ql.c b/src/n1ql.c deleted file mode 100644 index 081743c03..000000000 --- a/src/n1ql.c +++ /dev/null @@ -1,99 +0,0 @@ -#include "pycbc.h" -#include "oputil.h" -#include "structmember.h" -#include <libcouchbase/n1ql.h> - -static void -n1ql_row_callback(lcb_t instance, int ign, const lcb_RESPN1QL *resp) -{ - pycbc_MultiResult *mres = (pycbc_MultiResult *)resp->cookie; - pycbc_Bucket *bucket = mres->parent; - pycbc_ViewResult *vres; - const char * const * hdrs = NULL; - short htcode = 0; - - PYCBC_CONN_THR_END(bucket); - vres = (pycbc_ViewResult *)PyDict_GetItem((PyObject*)mres, Py_None); - - if (resp->htresp) { - hdrs = resp->htresp->headers; - htcode = resp->htresp->htstatus; - } - - if (resp->rflags & LCB_RESP_F_FINAL) { - pycbc_httpresult_add_data(mres, &vres->base, resp->row, resp->nrow); - } else { - /* Like views, try to decode the row and invoke the callback; if we can */ - /* Assume success! */ - pycbc_viewresult_addrow(vres, mres, resp->row, resp->nrow); - } - - pycbc_viewresult_step(vres, mres, bucket, resp->rflags & LCB_RESP_F_FINAL); - - if (resp->rflags & LCB_RESP_F_FINAL) { - pycbc_httpresult_complete(&vres->base, mres, resp->rc, htcode, hdrs); - } else { - PYCBC_CONN_THR_BEGIN(bucket); - } -} - -PyObject * -pycbc_Bucket__n1ql_query(pycbc_Bucket *self, PyObject *args, PyObject *kwargs) -{ - int rv; - PyObject *ret = NULL; - pycbc_MultiResult *mres; - pycbc_ViewResult *vres; - lcb_error_t rc; - lcb_CMDN1QL cmd = { 0 }; - const char *params; - pycbc_strlen_t nparams; - int prepared = 0; - - static char *kwlist[] = { "params", "prepare", NULL }; - rv = PyArg_ParseTupleAndKeywords( - args, kwargs, "s#|i", kwlist, ¶ms, &nparams, &prepared); - - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - if (-1 == pycbc_oputil_conn_lock(self)) { - return NULL; - } - if (self->pipeline_queue) { - PYCBC_EXC_WRAP(PYCBC_EXC_PIPELINE, 0, - "N1QL queries cannot be executed in " - "pipeline context"); - } - - mres = (pycbc_MultiResult *)pycbc_multiresult_new(self); - vres = (pycbc_ViewResult *)PYCBC_TYPE_CTOR(&pycbc_ViewResultType); - pycbc_httpresult_init(&vres->base, mres); - vres->rows = PyList_New(0); - vres->base.format = PYCBC_FMT_JSON; - vres->base.htype = PYCBC_HTTP_HN1QL; - - cmd.content_type = "application/json"; - cmd.callback = n1ql_row_callback; - cmd.query = params; - cmd.nquery = nparams; - cmd.handle = &vres->base.u.nq; - if (prepared) { - cmd.cmdflags |= LCB_CMDN1QL_F_PREPCACHE; - } - rc = lcb_n1ql_query(self->instance, mres, &cmd); - - if (rc != LCB_SUCCESS) { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, rc, "Couldn't schedule n1ql query"); - goto GT_DONE; - } - - ret = (PyObject *)mres; - mres = NULL; - - GT_DONE: - Py_XDECREF(mres); - pycbc_oputil_conn_unlock(self); - return ret; -} diff --git a/src/n1ql.cxx b/src/n1ql.cxx new file mode 100644 index 000000000..b89b36cf0 --- /dev/null +++ b/src/n1ql.cxx @@ -0,0 +1,406 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "n1ql.hxx" +#include "exceptions.hxx" +#include "utils.hxx" + +std::string +scan_consistency_type_to_string(couchbase::query_scan_consistency consistency) +{ + switch (consistency) { + case couchbase::query_scan_consistency::not_bounded: + return "not_bounded"; + case couchbase::query_scan_consistency::request_plus: + return "request_plus"; + } + // should not be able to reach here, since this is an enum class + return "unknown"; +} + +std::string +profile_mode_to_str(couchbase::query_profile profile_mode) +{ + switch (profile_mode) { + case couchbase::query_profile::off: + return "off"; + case couchbase::query_profile::phases: + return "phases"; + case couchbase::query_profile::timings: + return "timings"; + } + return "unknown profile_mode"; +} + +PyObject* +get_result_metrics(couchbase::core::operations::query_response::query_metrics metrics) +{ + PyObject* pyObj_metrics = PyDict_New(); + std::chrono::duration<unsigned long long, std::nano> int_nsec = metrics.elapsed_time; + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(int_nsec.count()); + if (-1 == PyDict_SetItemString(pyObj_metrics, "elapsed_time", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + int_nsec = metrics.execution_time; + pyObj_tmp = PyLong_FromUnsignedLongLong(int_nsec.count()); + if (-1 == PyDict_SetItemString(pyObj_metrics, "execution_time", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.result_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "result_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.result_size); + if (-1 == PyDict_SetItemString(pyObj_metrics, "result_size", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.sort_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "sort_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.mutation_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "mutation_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.error_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "error_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.warning_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "warning_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + return pyObj_metrics; +} + +PyObject* +get_result_metadata(couchbase::core::operations::query_response::query_meta_data metadata, + bool include_metrics) +{ + PyObject* pyObj_metadata = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(metadata.request_id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "request_id", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(metadata.client_context_id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "client_context_id", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(metadata.status.c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "status", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (metadata.signature.has_value()) { + pyObj_tmp = PyUnicode_FromString(metadata.signature.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "signature", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + } + + if (metadata.profile.has_value()) { + pyObj_tmp = PyUnicode_FromString(metadata.profile.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "profile", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + } + + if (metadata.warnings.has_value()) { + PyObject* pyObj_warnings = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& warning : metadata.warnings.value()) { + PyObject* pyObj_warning = PyDict_New(); + + pyObj_tmp = PyLong_FromLong(warning.code); + if (-1 == PyDict_SetItemString(pyObj_warning, "code", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(warning.message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_warning, "message", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == PyList_Append(pyObj_warnings, pyObj_warning)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_warning); + } + + if (-1 == PyDict_SetItemString(pyObj_metadata, "warnings", pyObj_warnings)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_warnings); + } + + if (metadata.errors.has_value()) { + PyObject* pyObj_errors = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& error : metadata.errors.value()) { + PyObject* pyObj_error = PyDict_New(); + + pyObj_tmp = PyLong_FromLong(error.code); + if (-1 == PyDict_SetItemString(pyObj_error, "code", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(error.message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error, "message", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + if (-1 == PyList_Append(pyObj_errors, pyObj_error)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_error); + } + + if (-1 == PyDict_SetItemString(pyObj_metadata, "errors", pyObj_errors)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_errors); + } + + if (include_metrics && metadata.metrics.has_value()) { + PyObject* pyObject_metrics = get_result_metrics(metadata.metrics.value()); + + if (-1 == PyDict_SetItemString(pyObj_metadata, "metrics", pyObject_metrics)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObject_metrics); + } + + return pyObj_metadata; +} + +result* +create_result_from_query_response(couchbase::core::operations::query_response resp, + bool include_metrics) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_payload = PyDict_New(); + + PyObject* pyObject_metadata = get_result_metadata(resp.meta, include_metrics); + if (-1 == PyDict_SetItemString(pyObj_payload, "metadata", pyObject_metadata)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObject_metadata); + + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_payload)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_payload); + + return res; +} + +void +create_query_result(couchbase::core::operations::query_response resp, + bool include_metrics, + std::shared_ptr<rows_queue<PyObject*>> rows, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + + auto set_exception = false; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_args = NULL; + PyObject* pyObj_func = NULL; + PyObject* pyObj_callback_res = nullptr; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = + build_exception_from_context(resp.ctx, __FILE__, __LINE__, "Error doing N1QL operation."); + // lets clear any errors + PyErr_Clear(); + rows->put(pyObj_exc); + } else { + for (auto const& row : resp.rows) { + PyObject* pyObj_row = PyBytes_FromStringAndSize(row.c_str(), row.length()); + rows->put(pyObj_row); + } + + auto res = create_result_from_query_response(resp, include_metrics); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + // None indicates done (i.e. raise StopIteration) + Py_INCREF(Py_None); + rows->put(Py_None); + rows->put(reinterpret_cast<PyObject*>(res)); + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "N1QL operation error."); + rows->put(pyObj_exc); + } + + // This is for txcouchbase -- let it knows we're done w/ the query request + if (pyObj_callback != nullptr) { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, PyBool_FromLong(static_cast<long>(1))); + } + + if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_CallObject(pyObj_func, pyObj_args); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + pycbc_set_python_exception( + PycbcError::InternalSDKError, __FILE__, __LINE__, "N1QL complete callback failed."); + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +streamed_result* +handle_n1ql_query([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + PyObject* pyObj_query_args = nullptr; + std::uint64_t streaming_timeout_us = 0; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_row_callback = nullptr; + + static const char* kw_list[] = { "conn", "query_args", "streaming_timeout", + "callback", "errback", "row_callback", + nullptr }; + + const char* kw_format = "O!|OKOOO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &pyObj_query_args, + &streaming_timeout_us, + &pyObj_callback, + &pyObj_errback, + &pyObj_row_callback); + if (!ret) { + PyErr_SetString(PyExc_ValueError, "Unable to parse arguments"); + return nullptr; + } + + connection* conn = nullptr; + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + PyErr_SetString(PyExc_ValueError, "passed null connection"); + return nullptr; + } + PyErr_Clear(); + + auto req = build_query_request(pyObj_query_args); + if (PyErr_Occurred()) { + return nullptr; + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + Py_XINCREF(pyObj_errback); + Py_XINCREF(pyObj_callback); + + // timeout is always set either to default, or timeout provided in options + auto streaming_timeout = couchbase::core::timeout_defaults::query_timeout; + if (streaming_timeout_us > 0) { + streaming_timeout = std::chrono::milliseconds(streaming_timeout_us / 1000ULL); + } + streamed_result* streamed_res = create_streamed_result_obj(streaming_timeout); + + // TODO: let the couchbase++ streaming stabilize a bit more... + // req.row_callback = [rows = streamed_res->rows](std::string&& row) { + // PyGILState_STATE state = PyGILState_Ensure(); + // PyObject* pyObj_row = PyBytes_FromStringAndSize(row.c_str(), row.length()); + // rows->put(pyObj_row); + // PyGILState_Release(state); + // return couchbase::core::utils::json::stream_control::next_row; + // }; + + { + Py_BEGIN_ALLOW_THREADS conn->cluster_.execute( + req, + [rows = streamed_res->rows, include_metrics = req.metrics, pyObj_callback, pyObj_errback]( + couchbase::core::operations::query_response resp) { + create_query_result(resp, include_metrics, rows, pyObj_callback, pyObj_errback); + }); + Py_END_ALLOW_THREADS + } + return streamed_res; +} diff --git a/src/n1ql.hxx b/src/n1ql.hxx new file mode 100644 index 000000000..100f32aaa --- /dev/null +++ b/src/n1ql.hxx @@ -0,0 +1,30 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" +#include "result.hxx" + +streamed_result* +handle_n1ql_query(PyObject* self, PyObject* args, PyObject* kwargs); + +std::string +scan_consistency_type_to_string(couchbase::query_scan_consistency consistency); + +couchbase::query_profile +str_to_profile_mode(std::string profile_mode); diff --git a/src/observe.c b/src/observe.c deleted file mode 100644 index 9a59aad98..000000000 --- a/src/observe.c +++ /dev/null @@ -1,230 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ -#include "oputil.h" -#include "structmember.h" - -static struct PyMemberDef ObserveInfo_TABLE_members[] = { - { "flags", - T_UINT, offsetof(pycbc_ObserveInfo, flags), - READONLY, - PyDoc_STR("Server-side flags received from observe") - }, - { "from_master", - T_INT, offsetof(pycbc_ObserveInfo, from_master), - READONLY, PyDoc_STR( - "Whether this response is from the " - "master node. This evaluates to False if this " - "status is from a replica") - }, - { "cas", - T_ULONGLONG, offsetof(pycbc_ObserveInfo, cas), - READONLY, - PyDoc_STR( - "CAS as it exists on the given node. " - "It is possible (though not likely) that different " - "nodes will have a different CAS value for a given " - "key. In this case, the actual CAS being used should " - "be the one from the *master* (use :attr:`from_master`)") - }, - { NULL } -}; - -static PyTypeObject pycbc_ResultInfoType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static void -ObserveInfo_dealloc(pycbc_ObserveInfo *self) -{ - Py_TYPE(self)->tp_free((PyObject*)self); -} - -static PyObject * -ObserveInfo_repr(PyObject *self) -{ - return PyObject_CallFunction(pycbc_helpers.obsinfo_reprfunc, "O", self); -} - -int -pycbc_ObserveInfoType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_ResultInfoType; - *ptr = (PyObject*)p; - if (p->tp_name) { - return 0; - } - p->tp_name = "ObserveInfo"; - p->tp_doc = PyDoc_STR("Object containing information about " - "a key's OBSERVED state"); - p->tp_new = PyType_GenericNew; - p->tp_basicsize = sizeof(pycbc_ObserveInfo); - p->tp_members = ObserveInfo_TABLE_members; - p->tp_flags = Py_TPFLAGS_DEFAULT; - p->tp_dealloc = (destructor)ObserveInfo_dealloc; - p->tp_repr = ObserveInfo_repr; - return PyType_Ready(p); -} - -pycbc_ObserveInfo * -pycbc_observeinfo_new(pycbc_Bucket *parent) -{ - (void)parent; - return (pycbc_ObserveInfo*)PyObject_CallFunction((PyObject*)&pycbc_ResultInfoType, - NULL, NULL); -} - -static int -handle_single_observe(pycbc_Bucket *self, PyObject *curkey, int master_only, - struct pycbc_common_vars *cv) -{ - int rv; - char *key; - size_t nkey; - lcb_CMDOBSERVE cmd = { 0 }; - lcb_error_t err; - - rv = pycbc_tc_encode_key(self, &curkey, (void**)&key, &nkey); - if (rv < 0) { - return -1; - } - LCB_CMD_SET_KEY(&cmd, key, nkey); - - if (!nkey) { - PYCBC_EXCTHROW_EMPTYKEY(); - rv = -1; - goto GT_DONE; - } - - if (master_only) { - cmd.cmdflags |= LCB_CMDOBSERVE_F_MASTER_ONLY; - } - - err = cv->mctx->addcmd(cv->mctx, (lcb_CMDBASE*)&cmd); - if (err == LCB_SUCCESS) { - rv = 0; - } else { - PYCBC_EXCTHROW_SCHED(err); - rv = -1; - } - - GT_DONE: - Py_XDECREF(curkey); - return rv; -} - -static PyObject * -observe_common(pycbc_Bucket *self, PyObject *args, PyObject *kwargs, int argopts) -{ - int rv; - int ii; - Py_ssize_t ncmds; - PyObject *kobj = NULL; - pycbc_seqtype_t seqtype; - int master_only = 0; - PyObject *master_only_O = NULL; - - struct pycbc_common_vars cv = PYCBC_COMMON_VARS_STATIC_INIT; - - static char *kwlist[] = { "keys", "master_only", NULL }; - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "O|O", kwlist, - &kobj, &master_only_O); - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_check_sequence(kobj, 1, &ncmds, &seqtype); - if (rv < 0) { - return NULL; - } - } else { - ncmds = 1; - } - - master_only = master_only_O && PyObject_IsTrue(master_only_O); - - rv = pycbc_common_vars_init(&cv, self, argopts, ncmds, 0); - if (rv < 0) { - return NULL; - } - - cv.mctx = lcb_observe3_ctxnew(self->instance); - if (cv.mctx == NULL) { - PYCBC_EXCTHROW_SCHED(LCB_CLIENT_ENOMEM); - goto GT_DONE; - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - Py_ssize_t dictpos; - PyObject *curseq, *iter = NULL; - curseq = pycbc_oputil_iter_prepare(seqtype, kobj, &iter, &dictpos); - - if (!curseq) { - goto GT_DONE; - } - - for (ii = 0; ii < ncmds; ii++) { - PyObject *curkey = NULL, *curvalue = NULL; - - rv = pycbc_oputil_sequence_next(seqtype, curseq, &dictpos, ii, - &curkey, &curvalue); - if (rv < 0) { - goto GT_ITER_DONE; - } - - rv = handle_single_observe(self, curkey, master_only, &cv); - - GT_ITER_DONE: - Py_XDECREF(curkey); - Py_XDECREF(curvalue); - - if (rv < 0) { - goto GT_DONE; - } - } - - } else { - rv = handle_single_observe(self, kobj, master_only, &cv); - - if (rv < 0) { - goto GT_DONE; - } - } - - cv.is_seqcmd = 1; - if (-1 == pycbc_common_vars_wait(&cv, self)) { - goto GT_DONE; - } - - GT_DONE: - - pycbc_common_vars_finalize(&cv, self); - return cv.ret; -} - -PyObject * -pycbc_Bucket_observe(pycbc_Bucket *self, PyObject *args, PyObject *kw) -{ - return observe_common(self, args, kw, PYCBC_ARGOPT_SINGLE); -} - -PyObject * -pycbc_Bucket_observe_multi(pycbc_Bucket *self, PyObject *args, PyObject *kw) -{ - return observe_common(self, args, kw, PYCBC_ARGOPT_MULTI); -} diff --git a/src/opresult.c b/src/opresult.c deleted file mode 100644 index 03796f63f..000000000 --- a/src/opresult.c +++ /dev/null @@ -1,249 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "structmember.h" - - -static PyObject * -ValueResult_value(pycbc_ValueResult *self, void *closure) -{ - (void)closure; - - if (!self->value) { - Py_INCREF(Py_None); - return Py_None; - } - Py_INCREF(self->value); - return self->value; -} - -static void -OperationResult_dealloc(pycbc_OperationResult *self) -{ - Py_CLEAR(self->mutinfo); - pycbc_Result_dealloc((pycbc_Result*)self); -} - -static void -ValueResult_dealloc(pycbc_ValueResult *self) -{ - Py_XDECREF(self->value); - OperationResult_dealloc((pycbc_OperationResult*)self); -} - -static void -Item_dealloc(pycbc_Item *self) -{ - Py_XDECREF(self->vdict); - ValueResult_dealloc((pycbc_ValueResult*)self); -} - -static int -Item__init__(pycbc_Item *item, PyObject *args, PyObject *kwargs) -{ - if (pycbc_ValueResultType.tp_init((PyObject *)item, args, kwargs) != 0) { - return -1; - } - - if (!item->vdict) { - item->vdict = PyDict_New(); - } - return 0; -} - -static struct PyMemberDef OperationResult_TABLE_members[] = { - { "cas", - T_ULONGLONG, offsetof(pycbc_OperationResult, cas), - READONLY, PyDoc_STR("CAS For the key") - }, - { "_mutinfo", - T_OBJECT_EX, offsetof(pycbc_OperationResult, mutinfo), - READONLY, PyDoc_STR("Mutation info") - }, - { NULL } -}; - -static struct PyMemberDef ValueResult_TABLE_members[] = { - { "flags", - T_ULONG, offsetof(pycbc_ValueResult, flags), - READONLY, PyDoc_STR("Flags for the value") - }, - { NULL } -}; - -static PyGetSetDef ValueResult_TABLE_getset[] = { - { "value", - (getter)ValueResult_value, - NULL, - PyDoc_STR("Value for the operation") - }, - { NULL } -}; - -/** - * We need to re-define all these fields again and indicate their permissions - * as being writable - */ -static PyMemberDef Item_TABLE_members[] = { - { "__dict__", - T_OBJECT_EX, offsetof(pycbc_Item, vdict), READONLY - }, - - { "value", - T_OBJECT_EX, offsetof(pycbc_Item, value), 0, - PyDoc_STR("The value of the Item.\n\n" - "For storage operations, this value is read. For retrieval\n" - "operations, this field is set\n") - }, - - { "cas", - T_ULONGLONG, offsetof(pycbc_Item, cas), 0, - PyDoc_STR("The CAS of the Item.\n\n" - "This field is always updated. On storage operations,\n" - "this field (if not ``0``) is used as the CAS for the\n" - "current operation. If the CAS on the server does not\n" - "match the value in this property, the operation will\n" - "fail.\n" - "For retrieval operations, this field is simply\n" - "set with the current CAS of the Item\n") - }, - - { "flags", - T_ULONG, offsetof(pycbc_Item, flags), 0, - PyDoc_STR("The flags (format) of the Item.\n\n" - "This field is set\n" - "During a retrieval operation. It is not read for a \n" - "storage operation\n") - }, - - {"key", - T_OBJECT_EX, offsetof(pycbc_Item, key), 0, - PyDoc_STR("This is the key for the Item. It *must* be set\n" - "before passing this item along in any operation\n") - }, - - { NULL } -}; - -PyTypeObject pycbc_OperationResultType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -PyTypeObject pycbc_ValueResultType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -PyTypeObject pycbc_ItemType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -int -pycbc_ValueResultType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_ValueResultType; - *ptr = (PyObject*)p; - - - if (p->tp_name) { - return 0; - } - - p->tp_name = "ValueResult"; - p->tp_doc = PyDoc_STR( - "The result type returned for operations which retrieve a value\n"); - p->tp_new = PyType_GenericNew; - p->tp_basicsize = sizeof(pycbc_ValueResult); - p->tp_base = &pycbc_OperationResultType; - p->tp_getset = ValueResult_TABLE_getset; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_members = ValueResult_TABLE_members; - p->tp_dealloc = (destructor)ValueResult_dealloc; - return pycbc_ResultType_ready(p, PYCBC_VALRESULT_BASEFLDS); -} - -int -pycbc_OperationResultType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_OperationResultType; - - *ptr = (PyObject*)&pycbc_OperationResultType; - if (pycbc_OperationResultType.tp_name) { - return 0; - } - - p->tp_name = "OperationResult"; - p->tp_doc = PyDoc_STR( - "Result type returned for operations which do not fetch data\n"); - p->tp_basicsize = sizeof(pycbc_OperationResult); - p->tp_base = &pycbc_ResultType; - p->tp_members = OperationResult_TABLE_members; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_dealloc = (destructor)OperationResult_dealloc; - return pycbc_ResultType_ready(p, PYCBC_OPRESULT_BASEFLDS); -} - -int pycbc_ItemType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_ItemType; - *ptr = (PyObject *)p; - if (p->tp_name) { - return 0; - } - p->tp_name = "Item"; - p->tp_doc = PyDoc_STR( - "Subclass of a :class:`~couchbase.result.ValueResult`.\n" - "This can contain user-defined fields\n" - "This can also be used as an item in either a\n" - ":class:`ItemOptionDict` or a :class:`ItemSequence` object which\n" - "can then be passed along to one of the ``_multi`` operations\n" - "\n"); - p->tp_basicsize = sizeof(pycbc_Item); - p->tp_base = &pycbc_ValueResultType; - p->tp_members = Item_TABLE_members; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_init = (initproc)Item__init__; - p->tp_dealloc = (destructor)Item_dealloc; - p->tp_dictoffset = offsetof(pycbc_Item, vdict); - return pycbc_ResultType_ready(p, PYCBC_VALRESULT_BASEFLDS); -} - -pycbc_ValueResult * -pycbc_valresult_new(pycbc_Bucket *parent) -{ - (void)parent; - return (pycbc_ValueResult*) - PyObject_CallFunction((PyObject*)&pycbc_ValueResultType, NULL, NULL); -} - -pycbc_OperationResult * -pycbc_opresult_new(pycbc_Bucket *parent) -{ - (void)parent; - return (pycbc_OperationResult*) - PyObject_CallFunction((PyObject*)&pycbc_OperationResultType, NULL, NULL); -} - -pycbc_Item * -pycbc_item_new(pycbc_Bucket *parent) -{ - (void)parent; - return (pycbc_Item *) - PyObject_CallFunction((PyObject*)&pycbc_ItemType, NULL, NULL); -} diff --git a/src/oputil.c b/src/oputil.c deleted file mode 100644 index 8bb27d9e8..000000000 --- a/src/oputil.c +++ /dev/null @@ -1,496 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "oputil.h" - -void -pycbc_common_vars_finalize(struct pycbc_common_vars *cv, pycbc_Bucket *conn) -{ - if (cv->mctx) { - cv->mctx->fail(cv->mctx); - cv->mctx = NULL; - } - lcb_sched_fail(conn->instance); - Py_XDECREF(cv->mres); - - if (conn->lockmode) { - pycbc_oputil_conn_unlock(conn); - } -} - -int -pycbc_common_vars_wait(struct pycbc_common_vars *cv, pycbc_Bucket *self) -{ - Py_ssize_t nsched = cv->is_seqcmd ? 1 : cv->ncmds; - - if (cv->mctx) { - cv->mctx->done(cv->mctx, cv->mres); - cv->mctx = NULL; - } - lcb_sched_leave(self->instance); - self->nremaining += nsched; - - if (self->flags & PYCBC_CONN_F_ASYNC) { - /** For async, just do the right thing :) */ - cv->ret = (PyObject *)cv->mres; - ((pycbc_AsyncResult *)cv->mres)->nops = nsched; - - /** INCREF once more so it's alive in the event loop */ - Py_INCREF(cv->ret); - cv->mres = NULL; - return 0; - - } else if (self->pipeline_queue) { - cv->ret = Py_None; - Py_INCREF(Py_None); - return 0; - } - pycbc_oputil_wait_common(self); - - if (!pycbc_assert(self->nremaining == 0)) { - fprintf(stderr, "Remaining count != 0. Adjusting"); - self->nremaining = 0; - } - - if (pycbc_multiresult_maybe_raise(cv->mres)) { - return -1; - } - - cv->ret = pycbc_multiresult_get_result(cv->mres); - Py_DECREF(cv->mres); - cv->mres = NULL; - - if (cv->ret == NULL) { - return -1; - } - - return 0; -} - -int -pycbc_common_vars_init(struct pycbc_common_vars *cv, - pycbc_Bucket *self, - int argopts, - Py_ssize_t ncmds, - int want_vals) -{ - if (-1 == pycbc_oputil_conn_lock(self)) { - return -1; - } - - cv->ncmds = ncmds; - cv->mres = (pycbc_MultiResult*)pycbc_multiresult_new(self); - cv->argopts = argopts; - - if (argopts & PYCBC_ARGOPT_SINGLE) { - cv->mres->mropts |= PYCBC_MRES_F_SINGLE; - } - - if (!cv->mres) { - pycbc_oputil_conn_unlock(self); - return -1; - } - - lcb_sched_enter(self->instance); - return 0; -} - -/** - * Check that the object is not one of Python's typical string types - */ -#define _is_not_strtype(o) \ - (PyBytes_Check(o) == 0 && PyByteArray_Check(o) == 0 && PyUnicode_Check(o) == 0) - -int -pycbc_oputil_check_sequence(PyObject *sequence, - int allow_list, - Py_ssize_t *ncmds, - pycbc_seqtype_t *seqtype) -{ - int ret = 0; - pycbc_seqtype_t dummy; - if (!seqtype) { - seqtype = &dummy; - } - - *ncmds = 0; - - if (PyDict_Check(sequence)) { - *ncmds = PyDict_Size(sequence); - *seqtype = PYCBC_SEQTYPE_DICT; - ret = 0; - - } else if (allow_list == 0 && - PyObject_IsInstance(sequence, pycbc_helpers.itmcoll_base_type) == 0) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Keys must be a dictionary", - sequence); - ret = -1; - - } else if (PyList_Check(sequence)) { - *seqtype = PYCBC_SEQTYPE_LIST; - *ncmds = PyList_GET_SIZE(sequence); - - } else if (PyTuple_Check(sequence)) { - *seqtype = PYCBC_SEQTYPE_TUPLE; - *ncmds = PyTuple_GET_SIZE(sequence); - - } else if (PyObject_IsInstance(sequence, pycbc_helpers.itmcoll_base_type)) { - *ncmds = PyObject_Length(sequence); - if (*ncmds == -1) { - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, - "ItemCollection subclass did not return proper length"); - ret = -1; - } - *seqtype = PYCBC_SEQTYPE_GENERIC | PYCBC_SEQTYPE_F_ITM; - if (PyObject_IsInstance(sequence, pycbc_helpers.itmopts_dict_type)) { - *seqtype |= PYCBC_SEQTYPE_F_OPTS; - } - - - } else if (_is_not_strtype(sequence)) { - /** - * Previously we used PySequence_Check, but this failed on things - * which didn't have __getitem__ (they had a length, but the elements - * were not ordered, but we don't care about that here - */ - *seqtype = PYCBC_SEQTYPE_GENERIC; - *ncmds = PyObject_Length(sequence); - - if (*ncmds == -1) { - PyErr_Clear(); - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Keys must be iterable and have known length", - sequence); - ret = -1; - } - - } else { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Keys must be iterable and have known length", - sequence); - ret = -1; - } - - if (ret == 0 && *ncmds < 1) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "Key list is empty", sequence); - ret = -1; - } - - return ret; -} - -int -pycbc_maybe_set_quiet(pycbc_MultiResult *mres, PyObject *quiet) -{ - /** - * If quiet is 'None', then we default to Connection.quiet - */ - int tmp = 0; - if (quiet == NULL || quiet == Py_None) { - mres->mropts |= (mres->parent->quiet) ? PYCBC_MRES_F_QUIET : 0; - return 0; - } - - tmp |= PyObject_IsTrue(quiet); - - if (tmp == -1) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, - 0, "quiet must be True, False, or None'", quiet); - return -1; - } - - mres->mropts |= tmp ? PYCBC_MRES_F_QUIET : 0; - - return 0; -} - -PyObject * -pycbc_oputil_iter_prepare(pycbc_seqtype_t seqtype, - PyObject *sequence, - PyObject **iter, - Py_ssize_t *dictpos) -{ - if (seqtype & PYCBC_SEQTYPE_GENERIC) { - *iter = PyObject_GetIter(sequence); - if (!*iter) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Couldn't get iterator from object. Object " - "should implement __iter__", - sequence); - } - return *iter; - } else if (seqtype & PYCBC_SEQTYPE_DICT) { - *dictpos = 0; - } - *iter = NULL; - return sequence; -} - -/** - * I thought it better to make the function call a bit more complex, so as to - * have the iteration logic unified in a single place - */ -int -pycbc_oputil_sequence_next(pycbc_seqtype_t seqtype, - PyObject *seqobj, - Py_ssize_t *dictpos, - int ii, - PyObject **key, - PyObject **value) -{ - if (seqtype & PYCBC_SEQTYPE_DICT) { - int rv = PyDict_Next(seqobj, dictpos, key, value); - if (rv < 1) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_INTERNAL, - 0, "Couldn't iterate", seqobj); - return -1; - } - - Py_XINCREF(*key); - Py_XINCREF(*value); - return 0; - } - - *value = NULL; - if (seqtype & PYCBC_SEQTYPE_LIST) { - *key = PyList_GET_ITEM(seqobj, ii); - Py_INCREF(*key); - } else if (seqtype & PYCBC_SEQTYPE_TUPLE) { - *key = PyTuple_GET_ITEM(seqobj, ii); - Py_INCREF(*key); - } else { - *key = PyIter_Next(seqobj); - if (!*key) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "PyIter_Next returned NULL", seqobj); - return -1; - } - } - - return 0; -} - -static int -extract_item_params(struct pycbc_common_vars *cv, - PyObject *k, - pycbc_Item **itm, - PyObject **options) -{ - /** Key will always be an item */ - Py_ssize_t tsz; - - if (!PyTuple_Check(k)) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "Expected Tuple", k); - return -1; - } - - tsz = PyTuple_GET_SIZE(k); - if (tsz != 2 && tsz != 1) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Tuple from __iter__ must return 1 or 2 items", k); - return -1; - } - - *itm = (pycbc_Item *) PyTuple_GET_ITEM(k, 0); - if (!PyObject_IsInstance((PyObject*)*itm, (PyObject *)&pycbc_ItemType)) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Expected 'Item' instance", (PyObject*)*itm); - return -1; - } - - if (tsz == 2) { - *options = PyTuple_GET_ITEM(k, 1); - - if (*options == Py_None) { - *options = NULL; - - } else if (!PyDict_Check(*options)) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, - 0, "Options must be None or dict", *options); - return -1; - } - } - - if (! (*itm)->key) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "Item is missing key", (PyObject *)*itm); - return -1; - } - - /** Store the item inside the mres dictionary */ - PyDict_SetItem(pycbc_multiresult_dict(cv->mres), - (*itm)->key, (PyObject *)*itm); - cv->mres->mropts |= PYCBC_MRES_F_UALLOCED; - return 0; -} - -int -pycbc_oputil_iter_multi(pycbc_Bucket *self, - pycbc_seqtype_t seqtype, - PyObject *collection, - struct pycbc_common_vars *cv, - int optype, - pycbc_oputil_keyhandler handler, - void *arg) -{ - int rv = 0; - int ii; - PyObject *iterobj; - PyObject *seqobj; - Py_ssize_t dictpos = 0; - - seqobj = pycbc_oputil_iter_prepare(seqtype, collection, &iterobj, &dictpos); - if (seqobj == NULL) { - return -1; - } - - for (ii = 0; ii < cv->ncmds; ii++) { - PyObject *k, *v = NULL, *options = NULL; - PyObject *arg_k = NULL; - pycbc_Item *itm = NULL; - - rv = pycbc_oputil_sequence_next(seqtype, - seqobj, &dictpos, ii, &k, &v); - if (rv < 0) { - goto GT_ITER_DONE; - } - - if (seqtype & PYCBC_SEQTYPE_F_ITM) { - if ((rv = extract_item_params(cv, k, &itm, &options)) != 0) { - goto GT_ITER_DONE; - } - arg_k = itm->key; - - } else { - arg_k = k; - } - - rv = handler(self, cv, optype, arg_k, v, options, itm, arg); - - GT_ITER_DONE: - Py_XDECREF(k); - Py_XDECREF(v); - - if (rv == -1) { - break; - } - } - - Py_XDECREF(iterobj); - return rv; -} - -int -pycbc_oputil_conn_lock(pycbc_Bucket *self) -{ - int status; - int mode; - - if (!self->lockmode) { - return 0; - } - - mode = self->lockmode == PYCBC_LOCKMODE_WAIT ? WAIT_LOCK : NOWAIT_LOCK; - if (mode == WAIT_LOCK) { - /** - * We need to unlock the GIL here so that other objects can potentially - * access the Connection (and thus unlock it). - */ - Py_BEGIN_ALLOW_THREADS - status = PyThread_acquire_lock(self->lock, mode); - Py_END_ALLOW_THREADS - } else { - status = PyThread_acquire_lock(self->lock, mode); - } - - if (!status) { - PYCBC_EXC_WRAP(PYCBC_EXC_THREADING, - 0, - "Couldn't lock. If LOCKMODE_WAIT was passed, " - "then this means that something has gone wrong " - "internally. Otherwise, this means you are using " - "the Connection object from multiple threads. This " - "is not allowed (without an explicit " - "lockmode=LOCKMODE_WAIT constructor argument"); - return -1; - } - return 0; -} - -void -pycbc_oputil_conn_unlock(pycbc_Bucket *self) -{ - if (!self->lockmode) { - return; - } - PyThread_release_lock(self->lock); -} - -void -pycbc_oputil_wait_common(pycbc_Bucket *self) -{ - /** - * If we have a 'lockmode' specified, check to see that nothing else is - * using us. We lock in any event. - * - * We have two modes: - * - LOCKMODE_WAIT explicitly allows access from multiple threads. - * In this mode, we actually wait to acquire the lock. - * - * - LOCKMODE_EXC will raise an exception if it cannot lock immediately - * - * Note that LOCKMODE_EXC won't do strict checking - i.e. it's perfectly - * possible - */ - - PYCBC_CONN_THR_BEGIN(self); - lcb_wait3(self->instance, LCB_WAIT_NOCHECK); - PYCBC_CONN_THR_END(self); -} - -/** - * Returns 1 if durability was found, 0 if durability was not found, and -1 - * on error. - */ -int -pycbc_handle_durability_args(pycbc_Bucket *self, - pycbc_dur_params *params, - char persist_to, - char replicate_to) -{ - if (self->dur_global.persist_to || self->dur_global.replicate_to) { - if (persist_to == 0 && replicate_to == 0) { - persist_to = self->dur_global.persist_to; - replicate_to = self->dur_global.replicate_to; - } - } - - if (persist_to || replicate_to) { - int nreplicas = lcb_get_num_replicas(self->instance); - params->persist_to = persist_to; - params->replicate_to = replicate_to; - if (replicate_to > nreplicas || persist_to > (nreplicas + 1)) { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, LCB_DURABILITY_ETOOMANY, - "Durability requirements will never be satisfied"); - return -1; - } - - return 1; - } - - return 0; -} diff --git a/src/oputil.h b/src/oputil.h deleted file mode 100644 index 0cf2e0c78..000000000 --- a/src/oputil.h +++ /dev/null @@ -1,332 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#ifndef PYCBC_OPUTIL_H -#define PYCBC_OPUTIL_H - -/** - * This file contains various utilities needed by operation entry points. - */ - -#include "pycbc.h" - -/** - * Populated by pycbc_oputil_check_sequence, indicates the type of - * sequence being used. - * - * We optimize here as the tuple and list objects have highly efficient - * access methods. - */ -typedef enum { - /** Generic sequence. A PyObject_Iter is used to make an iterator */ - PYCBC_SEQTYPE_GENERIC = 1 << 0, - - /** Dictionary. We use PyDict_Next */ - PYCBC_SEQTYPE_DICT = 1 << 1, - - /** Tuple. PyTuple_GET_ITEM */ - PYCBC_SEQTYPE_TUPLE = 1 << 2, - - /** List, PyList_GET_ITEM */ - PYCBC_SEQTYPE_LIST = 1 << 3, - - /** Special sequence classes for Items */ - PYCBC_SEQTYPE_F_ITM = 1 << 4, - PYCBC_SEQTYPE_F_OPTS = 1 << 5 -} pycbc_seqtype_t; - - - - -/** - * Structure containing variables needed for commands. - * As a bonus, this also contains optimizations for single command situations. - */ -struct pycbc_common_vars { - /** - * Return value information. The MultiResult object is always allocated, - * however. If we need to return a single result, we extract it out of - * the MultiResult object and decref the latter. - * - * The MultiResult object can be considered as an async handle in the - * future as it allows us to maintain metadata about a result -- the - * actual info being populated by the callbacks which use the MultiResult - * as a generic 'context' object. - */ - int argopts; - - /** - * This is decref'd by finalize() - */ - pycbc_MultiResult *mres; - - /** - * When 'wait' is called successfuly, this is set to the mres (and the mres - * field itself is set to NULL. This way, finalize() doesn't decref it - */ - PyObject *ret; - - Py_ssize_t ncmds; - - /** - * Whether this command is expected to be received in sequence with a - * 'NUL' delimiter. If so, the remaining count will be modified by one - * only, with the callback decrementing it as needed - */ - char is_seqcmd; - - lcb_MULTICMD_CTX *mctx; -}; - -#define PYCBC_COMMON_VARS_STATIC_INIT { 0 } - -/** - * Handler for iterations - */ -typedef int (*pycbc_oputil_keyhandler) - (pycbc_Bucket *self, - struct pycbc_common_vars *cv, - int optype, - PyObject *key, - PyObject *value, - PyObject *options, - pycbc_Item *item, - void *arg); - -/** - * Examine the 'quiet' parameter and see if we should set the MultiResult's - * 'no_raise_enoent' flag. - */ -int pycbc_maybe_set_quiet(pycbc_MultiResult *mres, PyObject *quiet); - - -/** - * Verify the sequence passed to a multi_* method is valid. - * - * This function also weeds out strings (which are perfectly valid Python - * sequences) since passing a bare string to a 'multi_*' method is usually - * not what a user wants :) - * - * @param sequence the object the user passed as a sequence - * @param allow_list whether the sequence can be a 'list' (i.e. not a dict) - * @param ncmds populated with the actual size of the commands - * @param seqtype populated with the sequence type (see enum doc above) - * - * @return 0 on success, -1 on error. Error might be if: - * Object is not a dict, and 'allow_list' is false - * The object is a valid sequence, but is empty - * Other stuff. - */ -int pycbc_oputil_check_sequence(PyObject *sequence, - int allow_list, - Py_ssize_t *ncmds, - pycbc_seqtype_t *seqtype); - - -/** - * 'Prepares' the sequence object for iteration. This may happen - * if we needan actual Iterator object. Otherwise it doesn't do anything. - * - * I'm sorry that the functions here are a bit obtuse, but Python doesn't make - * it simple without sacrificing performance. The generic iterator classes - * and APIs are fairly slowish, and we should optimize if a user passes a simple - * object. - * - * @param seqtype The seqtype, as populated by a call to check_sequence - * @param sequence the sequence received from the user - * @param iter a pointer to an empty PyObject pointer. May be filled with an - * iterator object (if one is needed). Call Py_XDECREF when done iterating - * @param pointer to a dictionary position variable, if the item is a dictionary - * - * @return a sequence object which is to be fed to 'sequence_next' (see - * below). This may not be the same as the sequence object itself (i.e. it - * could be the iterator object). - * - * If an error is encountered, a CouchbaseError is set, and this function - * returns NULL. - */ - -PyObject *pycbc_oputil_iter_prepare(pycbc_seqtype_t seqtype, - PyObject *sequence, - PyObject **iter, - Py_ssize_t *dictpos); - - -/** - * Iterates over the sequence, getting the relevant keys and values for the - * current iteration - * @param seqtype the sequence type, as populated by check_sequence - * @param seqobj the object returned by 'iter_prepare' - * @param dictpos the dictionary position variable initialized by 'iter_prepare' - * @param ii the current position (should be incremented by used by one for - * each iteration) - * @param key the key for the current iteration. This is populated with a - * new reference. - * @param value the value for the current iteration. If the sequence is not - * a dictionary, this is always NULL. This is populated with a new referenced - * @return 0 on success, -1 on failure (with a CouchbaseError set) - */ -int pycbc_oputil_sequence_next(pycbc_seqtype_t seqtype, - PyObject *seqobj, - Py_ssize_t *dictpos, - int ii, - PyObject **key, - PyObject **value); - -/** - * Initialize the 'common_vars' structure. - * @param cv a pointer to a zero-populated common_vars struct - * @param ncmds the number of keys in the operation - * @param want_vals whether this operation will need to use values. This is - * used to determine if the 'encvals' field should be allocated - */ -int pycbc_common_vars_init(struct pycbc_common_vars *cv, - pycbc_Bucket *self, - int argopts, - Py_ssize_t ncmds, - int want_vals); - - -/** - * Iterate over a sequence of command objects - * @param self the connection object - * @param seqtype a Sequence Type. Initialized by seqtype_check - * @param collection the actual iterable of keys (verified by seqtype_check) - * @param cv Common vars structure - * @param optype the operation type. Passed to handler - * @param handler the actual handler to call for each key-value-item pair - * @param arg an opaque pointer passed to the handler - */ -int -pycbc_oputil_iter_multi(pycbc_Bucket *self, - pycbc_seqtype_t seqtype, - PyObject *collection, - struct pycbc_common_vars *cv, - int optype, - pycbc_oputil_keyhandler handler, - void *arg); - - -/** - * Clean up the 'common_vars' structure and free/decref any data. This - * automatically DECREFs any PyObject enckeys and encvaks. - */ -void pycbc_common_vars_finalize(struct pycbc_common_vars *cv, pycbc_Bucket *self); - -/** - * Wait for the operation to complete - * @return 0 on success, -1 on failure. - */ -int pycbc_common_vars_wait(struct pycbc_common_vars *cv, pycbc_Bucket *self); - - -/** - * Wrapper around lcb_wait(). This ensures threading contexts are properly - * initialized. - */ -void -pycbc_oputil_wait_common(pycbc_Bucket *self); - - -/** - * Lock the connection (if a lockmode has been set) - * @return 0 on success, nonzero on failure - */ -int pycbc_oputil_conn_lock(pycbc_Bucket *self); - -/** - * Unlock the connection previously acquired by conn_lock() - */ -void pycbc_oputil_conn_unlock(pycbc_Bucket *self); - -/** - * Returns 1 if durability was found, 0 if durability was not found, and -1 - * on error. - */ -int pycbc_handle_durability_args(pycbc_Bucket *self, - pycbc_dur_params *params, - char persist_to, - char replicate_to); - -/** - * Macro to declare prototypes for entry points. - * If the entry point is foo, then it is expected that there exist a C - * function called 'pycbc_Connection_foo'. - * - * We might want to expand this in the future. - */ -#define PYCBC_DECL_OP(name) \ - PyObject* pycbc_Bucket_##name(pycbc_Bucket*, PyObject*, PyObject*) - - -/* store.c */ -PYCBC_DECL_OP(upsert_multi); -PYCBC_DECL_OP(insert_multi); -PYCBC_DECL_OP(replace_multi); -PYCBC_DECL_OP(append_multi); -PYCBC_DECL_OP(prepend_multi); -PYCBC_DECL_OP(upsert); -PYCBC_DECL_OP(insert); -PYCBC_DECL_OP(replace); -PYCBC_DECL_OP(append); -PYCBC_DECL_OP(prepend); - -/* arithmetic.c */ -PYCBC_DECL_OP(counter); -PYCBC_DECL_OP(counter_multi); - -/* miscops.c */ -PYCBC_DECL_OP(remove); -PYCBC_DECL_OP(unlock); -PYCBC_DECL_OP(remove_multi); -PYCBC_DECL_OP(unlock_multi); - -PYCBC_DECL_OP(_stats); -PYCBC_DECL_OP(_keystats); - -PYCBC_DECL_OP(endure_multi); - - -/* get.c */ -PYCBC_DECL_OP(get); -PYCBC_DECL_OP(touch); -PYCBC_DECL_OP(lock); -PYCBC_DECL_OP(get_multi); -PYCBC_DECL_OP(touch_multi); -PYCBC_DECL_OP(lock_multi); - -/* get.c (replicas) */ -PYCBC_DECL_OP(_rget); -PYCBC_DECL_OP(_rget_multi); -PYCBC_DECL_OP(_rgetix); -PYCBC_DECL_OP(_rgetix_multi); -PYCBC_DECL_OP(_rgetall); -PYCBC_DECL_OP(_rgetall_multi); - -/* http.c */ -PYCBC_DECL_OP(_http_request); - -/* views.c */ -PYCBC_DECL_OP(_view_request); - -/* observe.c */ -PYCBC_DECL_OP(observe); -PYCBC_DECL_OP(observe_multi); - -/* n1ql.c */ -PYCBC_DECL_OP(_n1ql_query); - -#endif /* PYCBC_OPUTIL_H */ diff --git a/src/pipeline.c b/src/pipeline.c deleted file mode 100644 index 78b63aa20..000000000 --- a/src/pipeline.c +++ /dev/null @@ -1,91 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ -#include "pycbc.h" -#include "oputil.h" - -PyObject * -pycbc_Bucket__start_pipeline(pycbc_Bucket *self) -{ - if (self->pipeline_queue) { - PYCBC_EXC_WRAP(PYCBC_EXC_PIPELINE, 0, - "A pipeline is already in progress"); - return NULL; - } - - if (self->flags & PYCBC_CONN_F_ASYNC) { - PYCBC_EXC_WRAP(PYCBC_EXC_PIPELINE, 0, - "Pipeline mode not valid in async handle"); - return NULL; - } - - self->pipeline_queue = PyList_New(0); - Py_INCREF(self->pipeline_queue); - return self->pipeline_queue; -} - -PyObject * -pycbc_Bucket__end_pipeline(pycbc_Bucket *self) -{ - PyObject *rv; - int ii; - - if (!self->pipeline_queue) { - PYCBC_EXC_WRAP(PYCBC_EXC_PIPELINE, 0, - "No pipeline in progress"); - return NULL; - } - - rv = self->pipeline_queue; - - if (!self->nremaining) { - goto GT_DONE; - } - - pycbc_oputil_wait_common(self); - - pycbc_assert(self->nremaining == 0); - - for (ii = 0; ii < PyList_GET_SIZE(self->pipeline_queue); ii++) { - PyObject *retitem; - pycbc_MultiResult *mres = - (pycbc_MultiResult *)PyList_GET_ITEM(self->pipeline_queue, ii); - - if (pycbc_multiresult_maybe_raise(mres)) { - rv = NULL; - break; - } - - /** Returns new reference to something */ - retitem = pycbc_multiresult_get_result(mres); - if (retitem != (PyObject *)mres) { - PyList_SetItem(self->pipeline_queue, ii, retitem); - } else { - Py_DECREF(mres); - } - } - - - GT_DONE: - if (rv) { - Py_INCREF(rv); - pycbc_assert(rv == self->pipeline_queue); - } - - Py_XDECREF(self->pipeline_queue); - self->pipeline_queue = NULL; - - return rv; -} diff --git a/src/pycbc.h b/src/pycbc.h deleted file mode 100644 index e4528abd5..000000000 --- a/src/pycbc.h +++ /dev/null @@ -1,1098 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#ifndef PYCBC_H_ -#define PYCBC_H_ -/** - * This file contains the base header for the Python Couchbase Client - * @author Mark Nunberg - */ - -#include <Python.h> -#include <libcouchbase/couchbase.h> -#include <libcouchbase/api3.h> -#include <libcouchbase/views.h> -#include <libcouchbase/n1ql.h> - -#if LCB_VERSION < 0x020503 -#error "Couchbase Python SDK requires libcouchbase 2.5.3 or greater" -#endif - -#include <pythread.h> -#include "mresdict.h" - -#define PYCBC_REFCNT_ASSERT pycbc_assert - -/** - * See http://docs.python.org/2/c-api/arg.html for an explanation of this - * definition. - */ -#ifdef PY_SSIZE_T_CLEAN -typedef Py_ssize_t pycbc_strlen_t; -#else -typedef int pycbc_strlen_t; -#endif - -#define PYCBC_PACKAGE_NAME "couchbase" -#define PYCBC_MODULE_NAME "_libcouchbase" -#define PYCBC_FQNAME PYCBC_PACKAGE_NAME "." PYCBC_MODULE_NAME - -#define PYCBC_TCNAME_ENCODE_KEY "encode_key" -#define PYCBC_TCNAME_ENCODE_VALUE "encode_value" -#define PYCBC_TCNAME_DECODE_KEY "decode_key" -#define PYCBC_TCNAME_DECODE_VALUE "decode_value" - -/** - * Python 2.x and Python 3.x have different ideas of what a basic string - * and int types are. These blocks help us sort things out if we just want a - * "plain" integer or string - */ -#if PY_MAJOR_VERSION == 3 -#define PYCBC_POBJ_HEAD_INIT(t) { PyObject_HEAD_INIT(t) }, - -/** - * The IntFrom* macros get us a 'default' integer type from a long, etc. - * Implemented (if not a simple macro) in numutil.c - */ -#define pycbc_IntFromL PyLong_FromLong -#define pycbc_IntFromUL PyLong_FromUnsignedLong -#define pycbc_IntFromULL PyLong_FromUnsignedLongLong - -/** - * The IntAs* convert the integer type (long, int) into something we want - */ -#define pycbc_IntAsULL PyLong_AsUnsignedLongLong -#define pycbc_IntAsLL PyLong_AsLongLong -#define pycbc_IntAsUL PyLong_AsUnsignedLong -#define pycbc_IntAsL PyLong_AsLong - -/** - * The SimpleString macros generate strings for us. The 'Z' variant takes a - * NUL-terminated string, while the 'N' variant accepts a length specifier - */ -#define pycbc_SimpleStringZ(c) PyUnicode_FromString(c) -#define pycbc_SimpleStringN(c, n) PyUnicode_FromStringAndSize(c, n) - - -#else - -/** - * This defines the PyObject head for our types - */ -#define PYCBC_POBJ_HEAD_INIT(t) PyObject_HEAD_INIT(t) - -/** - * See above block for explanation of these macros - */ -#define pycbc_IntFromL PyInt_FromLong -#define pycbc_IntFromUL PyLong_FromUnsignedLong -#define pycbc_IntFromULL PyLong_FromUnsignedLongLong -#define pycbc_SimpleStringZ(c) PyString_FromString(c) -#define pycbc_SimpleStringN(c, n) PyString_FromStringAndSize(c, n) - -unsigned PY_LONG_LONG pycbc_IntAsULL(PyObject *o); -PY_LONG_LONG pycbc_IntAsLL(PyObject *o); -long pycbc_IntAsL(PyObject *o); -unsigned long pycbc_IntAsUL(PyObject *o); - - -#endif - -/** - * Fetches a valid TTL from the object - * @param obj an object to be parsed as the TTL - * @param ttl a pointer to the TTL itself - * @param nonzero whether to allow a value of 0 for the TTL - * @return 0 on success, nonzero on error. - */ -int pycbc_get_ttl(PyObject *obj, unsigned long *ttl, int nonzero); - -/** - * Fetches a valid 32 bit integer from the object. The object must be a long - * or int. - * @param obj the object containing the number - * @param out a pointer to a 32 bit integer to be populated - * @return 0 on success, -1 on failure. On failure, the error indicator is also - * set - */ -int pycbc_get_u32(PyObject *obj, lcb_uint32_t *out); - -/** - * Converts the object into an PyInt (2.x only) or PyLong (2.x or 3.x) - */ -PyObject *pycbc_maybe_convert_to_int(PyObject *o); - -/** - * Gives us a C buffer from a Python string. - * @param orig the original object containg a string thing. This is something - * we can convert into a byte buffer - * - * @param buf out, the C buffer, out, set to the new buffer - * @param nbuf, out, the length of the new buffer - * @param newkey, out, the new PyObject, which will back the buffer. - * This should not be DECREF'd until the @c buf is no longer needed - */ -int pycbc_BufFromString(PyObject *orig, - char **buf, - Py_ssize_t *nbuf, - PyObject **newkey); - - -/** - * These constants are used internally to figure out the high level - * operation being performed. - * - * Note that not all operations are defined here; it is only those operations - * where a single C function can handle multiple entry points. - */ -enum { - PYCBC_CMD_GET = 500, - PYCBC_CMD_LOCK, - PYCBC_CMD_TOUCH, - PYCBC_CMD_GAT, - PYCBC_CMD_COUNTER, - PYCBC_CMD_DELETE, - PYCBC_CMD_UNLOCK, - PYCBC_CMD_GETREPLICA, - /** "Extended" get replica, provides for more options */ - PYCBC_CMD_GETREPLICA_INDEX, - PYCBC_CMD_GETREPLICA_ALL, - PYCBC_CMD_ENDURE -}; - -/** - * Various exception types to be thrown - */ -enum { - /** Argument Error. User passed the wrong arguments */ - PYCBC_EXC_ARGUMENTS, - - /** Couldn't encode/decode something */ - PYCBC_EXC_ENCODING, - - /** Operational error returned from LCB */ - PYCBC_EXC_LCBERR, - - /** Internal error. There's something wrong with our code */ - PYCBC_EXC_INTERNAL, - - /** HTTP Error */ - PYCBC_EXC_HTTP, - - /** ObjectThreadError */ - PYCBC_EXC_THREADING, - - /** Object destroyed before it could connect */ - PYCBC_EXC_DESTROYED, - - /** Illegal operation in pipeline context */ - PYCBC_EXC_PIPELINE -}; - -/* Argument options */ -enum { - /** Entry point is a single key variant */ - PYCBC_ARGOPT_SINGLE = 0x1, - - /** Entry point is a multi key variant */ - PYCBC_ARGOPT_MULTI = 0x2 -}; - -/** - * Format flags - */ -enum { - PYCBC_FMT_LEGACY_JSON = 0x00, - PYCBC_FMT_LEGACY_PICKLE = 0x01, - PYCBC_FMT_LEGACY_BYTES = 0x02, - PYCBC_FMT_LEGACY_UTF8 = 0x04, - PYCBC_FMT_LEGACY_MASK = 0x07, - - PYCBC_FMT_COMMON_PICKLE = (0x01U << 24), - PYCBC_FMT_COMMON_JSON = (0x02U << 24), - PYCBC_FMT_COMMON_BYTES = (0x03U << 24), - PYCBC_FMT_COMMON_UTF8 = (0x04U << 24), - PYCBC_FMT_COMMON_MASK = (0xFFU << 24), - - PYCBC_FMT_JSON = PYCBC_FMT_LEGACY_JSON|PYCBC_FMT_COMMON_JSON, - PYCBC_FMT_PICKLE = PYCBC_FMT_LEGACY_PICKLE|PYCBC_FMT_COMMON_PICKLE, - PYCBC_FMT_BYTES = PYCBC_FMT_LEGACY_BYTES|PYCBC_FMT_COMMON_BYTES, - PYCBC_FMT_UTF8 = PYCBC_FMT_LEGACY_UTF8|PYCBC_FMT_COMMON_UTF8 -}; - -typedef enum { - PYCBC_LOCKMODE_NONE = 0, - PYCBC_LOCKMODE_EXC = 1, - PYCBC_LOCKMODE_WAIT = 2, - PYCBC_LOCKMODE_MAX -} pycbc_lockmode_t; - -enum { - PYCBC_CONN_F_WARNEXPLICIT = 1 << 0, - PYCBC_CONN_F_USEITEMRESULT = 1 << 1, - PYCBC_CONN_F_CLOSED = 1 << 2, - - /** - * For use with (but not limited to) Twisted. - * - * Deliver results asynchronously. This means: - * 1) Don't call lcb_wait() - * 2) Return an AsyncContainer (i.e. a MultiResult) - * 3) Invoke the MultiResult (AsyncContainer)'s callback as needed - */ - PYCBC_CONN_F_ASYNC = 1 << 3, - - /** Whether this instance has been connected */ - PYCBC_CONN_F_CONNECTED = 1 << 4, - - /** Schedule destruction of iops and lcb instance for later */ - PYCBC_CONN_F_ASYNC_DTOR = 1 << 5 -}; - -typedef struct { - char persist_to; - char replicate_to; -} pycbc_dur_params; - -typedef struct { - PyObject_HEAD - - /** LCB instance */ - lcb_t instance; - - /** Transcoder object */ - PyObject *tc; - - /** Default format, PyInt */ - PyObject *dfl_fmt; - - /** Callback to be invoked when connected */ - PyObject *conncb; - - /** - * Callback to be invoked upon destruction. Because we can fall out - * of scope in middle of an LCB function, this is required. - * - * The dtorcb is first called when the refcount of the connection - */ - PyObject *dtorcb; - - /** - * Test hook for reacting to durability/persistence settings from within - * mutator functions - */ - PyObject *dur_testhook; - - - /** String bucket */ - PyObject *bucket; - - /** Pipeline MultiResult container */ - PyObject *pipeline_queue; - - /** If using a custom IOPS, this contains it */ - PyObject *iopswrap; - - /** Thread state. Used to lock/unlock the GIL */ - PyThreadState *thrstate; - - PyThread_type_lock lock; - unsigned int lockmode; - - /** Whether to not raise any exceptions */ - unsigned int quiet; - - /** Whether GIL handling is in effect */ - unsigned int unlock_gil; - - /** Don't decode anything */ - unsigned int data_passthrough; - - /** whether __init__ has already been called */ - unsigned char init_called; - - /** How many operations are waiting for a reply */ - Py_ssize_t nremaining; - - unsigned int flags; - - pycbc_dur_params dur_global; - unsigned long dur_timeout; - -} pycbc_Bucket; - - -/***************** - * Result Objects. - ***************** - * - * These objects are returned to indicate the status/value of operations. - * The following defines a 'base' class and several 'subclasses'. - * - * See result.c and opresult.c - */ - -#define pycbc_Result_HEAD \ - PyObject_HEAD \ - lcb_error_t rc; \ - PyObject *key; - -#define pycbc_OpResult_HEAD \ - pycbc_Result_HEAD \ - lcb_uint64_t cas; \ - PyObject *mutinfo; - -typedef struct { - pycbc_Result_HEAD -} pycbc_Result; - -typedef struct { - pycbc_OpResult_HEAD -} pycbc_OperationResult; - - -#define pycbc_ValResult_HEAD \ - pycbc_OpResult_HEAD \ - PyObject *value; \ - lcb_uint32_t flags; - -typedef struct { - pycbc_ValResult_HEAD -} pycbc_ValueResult; - -/** - * Item or 'Document' object - */ -typedef struct { - pycbc_ValResult_HEAD - PyObject* vdict; -} pycbc_Item; - - -#define PYCBC_HTTP_HVIEW 1 -#define PYCBC_HTTP_HRAW 2 -#define PYCBC_HTTP_HN1QL 3 - -typedef struct { - pycbc_Result_HEAD - PyObject *http_data; - PyObject *headers; - pycbc_Bucket *parent; - union { - lcb_http_request_t htreq; - lcb_VIEWHANDLE vh; - lcb_N1QLHANDLE nq; - } u; - unsigned int format; - unsigned short htcode; - unsigned char done; - unsigned char htype; -} pycbc_HttpResult; - -typedef struct { - pycbc_HttpResult base; - PyObject *rows; - long rows_per_call; - char has_parse_error; -} pycbc_ViewResult; - - -enum { - /** 'quiet' boolean set */ - PYCBC_MRES_F_QUIET = 1 << 0, - - /** We're using a user-created Item; Don't create our own results */ - PYCBC_MRES_F_ITEMS = 1 << 1, - - /** Items are already allocated and present within the dictionary. */ - PYCBC_MRES_F_UALLOCED = 1 << 2, - - /** For GET (and possibly others), force FMT_BYTES */ - PYCBC_MRES_F_FORCEBYTES = 1 << 3, - - /** The commands have durability requirements */ - PYCBC_MRES_F_DURABILITY = 1 << 4, - - /** The command is an async subclass. Do we need this? */ - PYCBC_MRES_F_ASYNC = 1 << 5, - - /** This result is from a call to one of the single-item APIs */ - PYCBC_MRES_F_SINGLE = 1 << 6, - - /* Hint to dispatch to the view callback functions */ - PYCBC_MRES_F_VIEWS = 1 << 7 -}; -/** - * Object containing the result of a 'Multi' operation. It's the same as a - * normal dict, except we add an 'all_ok' field, so a user doesn't need to - * skim through all the pairs to determine if something failed. - * - * See multiresult.c - */ -typedef struct pycbc_MultiResult_st { - PYCBC_MULTIRESULT_BASE; - - /** parent Connection object */ - pycbc_Bucket *parent; - - /** - * A list of fatal exceptions, i.e. ones not resulting from a bad - * LCB error code - */ - PyObject *exceptions; - - /** A failed LCB operation, if any */ - PyObject *errop; - - pycbc_dur_params dur; - - /** Quick-check value to see if everything went well */ - int all_ok; - - /** Options for 'MultiResult' */ - int mropts; -} pycbc_MultiResult; - -typedef struct { - pycbc_MultiResult base; - - /* How many operations do we have remaining */ - unsigned int nops; - - /* Object for the callback */ - PyObject *callback; - - /* Object to be invoked with errors */ - PyObject *errback; -} pycbc_AsyncResult; - - -/** - * This structure is passed to our exception throwing function, it's - * usually wrapped by one of the macros below - */ -struct pycbc_exception_params { - /** C Source file at which the error was thrown (populated by macro */ - const char *file; - - /** C Source line, as above */ - int line; - - /** LCB Error code, if any */ - lcb_error_t err; - - /** Error message, if any */ - const char *msg; - - /** Key at which the error occurred. Not always present */ - PyObject *key; - - /** Single result which triggered the error, if present */ - PyObject *result; - - /** - * A MultiResult object. This contains other operations which may - * or may not have failed. This allows a user to check the status - * of multi operations in which one of the keys resulted in an - * exception - */ - PyObject *all_results; - - /** - * Extra info which caused the error. This is usually some kind of - * bad parameter. - */ - PyObject *objextra; -}; - -/** - * Initializes a pycbc_exception_params to contain the proper - * source context info - */ -#define PYCBC_EXC_STATIC_INIT { __FILE__, __LINE__ } - -/** - * Argument object, used for passing more information to the - * multi functions. This isn't documented API yest. - */ -typedef struct { - PyDictObject dict; - int dummy; /* avoid sizing issues */ -} pycbc_ArgumentObject; - - -/** - * Object used as the 'value' for observe responses - */ -typedef struct { - PyObject_HEAD - unsigned int flags; - int from_master; - unsigned PY_LONG_LONG cas; -} pycbc_ObserveInfo; - -/** - * Flags to use for each type to indicate which subfields are relevant to - * print out. - */ -enum { - PYCBC_RESFLD_RC = 1 << 0, - PYCBC_RESFLD_CAS = 1 << 1, - PYCBC_RESFLD_KEY = 1 << 2, - PYCBC_RESFLD_FLAGS = 1 << 3, - PYCBC_RESFLD_HTCODE = 1 << 4, - PYCBC_RESFLD_VALUE = 1 << 5, - PYCBC_RESFLD_URL = 1 << 6 -}; - -#define PYCBC_RESULT_BASEFLDS (PYCBC_RESFLD_RC) -#define PYCBC_OPRESULT_BASEFLDS \ - (PYCBC_RESULT_BASEFLDS| \ - PYCBC_RESFLD_CAS| \ - PYCBC_RESFLD_KEY) - -#define PYCBC_VALRESULT_BASEFLDS (PYCBC_OPRESULT_BASEFLDS| \ - PYCBC_RESFLD_VALUE| \ - PYCBC_RESFLD_FLAGS) - -#define PYCBC_HTRESULT_BASEFLDS \ - ( PYCBC_RESULT_BASEFLDS | \ - PYCBC_RESFLD_HTCODE | \ - PYCBC_RESFLD_URL | \ - PYCBC_RESFLD_VALUE) - -#define PYCBC_RESPROPS_NAME "_fldprops" -/** - * Wrapper around PyType_Ready which also injects the common flags properties - */ -int pycbc_ResultType_ready(PyTypeObject *p, int flags); - - -/** - * Extern PyTypeObject declaraions. - */ - -/* multiresult.c */ -extern PyTypeObject pycbc_MultiResultType; -extern PyTypeObject pycbc_AsyncResultType; - -/* result.c */ -extern PyTypeObject pycbc_ResultType; - -/* opresult.c */ -extern PyTypeObject pycbc_OperationResultType; -extern PyTypeObject pycbc_ValueResultType; -extern PyTypeObject pycbc_HttpResultType; -extern PyTypeObject pycbc_ItemType; - -/* views.c */ -extern PyTypeObject pycbc_ViewResultType; - -/** - * Result type check macros - */ -#define PYCBC_VALRES_CHECK(o) \ - PyObject_IsInstance(o, &pycbc_ValueResultType) - -#define PYCBC_OPRES_CHECK(o) \ - PyObject_IsInstance(o, (PyObject*)&pycbc_OperationResultType) - -extern PyTypeObject pycbc_ArgumentType; - -/** - * XXX: This isn't used. - */ -extern PyObject *pycbc_ExceptionType; - -/** - * X-macro to define the helpers we pass from _bootstrap.py along to - * the module's '_init_helpers' function. We use an xmacro here because - * the parameters may change and the argument handling is rather complex. - * See below (in the pycbc_helpers structure) and in ext.c for more usages. - */ -#define PYCBC_XHELPERS(X) \ - X(result_reprfunc) \ - X(fmt_utf8_flags) \ - X(fmt_bytes_flags) \ - X(fmt_json_flags) \ - X(fmt_pickle_flags) \ - X(pickle_encode) \ - X(pickle_decode) \ - X(json_encode) \ - X(json_decode) \ - X(lcb_errno_map) \ - X(misc_errno_map) \ - X(default_exception) \ - X(obsinfo_reprfunc) \ - X(itmcoll_base_type) \ - X(itmopts_dict_type) \ - X(itmopts_seq_type) \ - X(fmt_auto) \ - X(view_path_helper) - -#define PYCBC_XHELPERS_STRS(X) \ - X(tcname_encode_key, PYCBC_TCNAME_ENCODE_KEY) \ - X(tcname_encode_value, PYCBC_TCNAME_ENCODE_VALUE) \ - X(tcname_decode_key, PYCBC_TCNAME_DECODE_KEY) \ - X(tcname_decode_value, PYCBC_TCNAME_DECODE_VALUE) \ - X(ioname_modevent, "update_event") \ - X(ioname_modtimer, "update_timer") \ - X(ioname_startwatch, "start_watching") \ - X(ioname_stopwatch, "stop_watching") \ - X(ioname_mkevent, "io_event_factory") \ - X(ioname_mktimer, "timer_event_factory") \ - X(vkey_id, "id") \ - X(vkey_key, "key") \ - X(vkey_value, "value") \ - X(vkey_geo, "geometry") \ - X(vkey_docresp, "__DOCRESULT__") - -/** - * Definition of global helpers. This is only instantiated once as - * pycbc_helpers. - */ -struct pycbc_helpers_ST { - #define X(n) PyObject *n; - PYCBC_XHELPERS(X) - #undef X - - #define X(n, s) PyObject *n; - PYCBC_XHELPERS_STRS(X) - #undef X -}; - -/** - * We use this one a lot. This is defined in ext.c - */ -extern struct pycbc_helpers_ST pycbc_helpers; - -/** - * Threading macros - */ -#define PYCBC_USE_THREADS - -#ifdef PYCBC_USE_THREADS -#define PYCBC_CONN_THR_BEGIN(conn) \ - if ((conn)->unlock_gil) { \ - pycbc_assert((conn)->thrstate == NULL); \ - (conn)->thrstate = PyEval_SaveThread(); \ - } - -#define PYCBC_CONN_THR_END(conn) \ - if ((conn)->unlock_gil) { \ - pycbc_assert((conn)->thrstate); \ - PyEval_RestoreThread((conn)->thrstate); \ - (conn)->thrstate = NULL; \ - } - -#else -#define PYCBC_CONN_THR_BEGIN(X) -#define PYCBC_CONN_THR_END(X) -#endif - -/******************************* - * Type Initialization Functions - ******************************* - * - * These functions are called once from the extension's import method. - * See ext.c - * - * They basically initialize the corresponding Python type so that - * we can use them further on. - */ - -/** Initializes the constants, constants. */ -void pycbc_init_pyconstants(PyObject *module); -PyObject *pycbc_lcb_errstr(lcb_t instance, lcb_error_t err); -PyObject *pycbc_print_constants(PyObject *mod, PyObject *args); - -int pycbc_ResultType_init(PyObject **ptr); -int pycbc_BucketType_init(PyObject **ptr); -int pycbc_MultiResultType_init(PyObject **ptr); -int pycbc_ValueResultType_init(PyObject **ptr); -int pycbc_OperationResultType_init(PyObject **ptr); -int pycbc_HttpResultType_init(PyObject **ptr); -int pycbc_TranscoderType_init(PyObject **ptr); -int pycbc_ObserveInfoType_init(PyObject **ptr); -int pycbc_ItemType_init(PyObject **ptr); -int pycbc_EventType_init(PyObject **ptr); -int pycbc_TimerEventType_init(PyObject **ptr); -int pycbc_IOEventType_init(PyObject **ptr); -int pycbc_AsyncResultType_init(PyObject **ptr); -int pycbc_IOPSWrapperType_init(PyObject **ptr); -int pycbc_ViewResultType_init(PyObject **ptr); - -/** - * Calls the type's constructor with no arguments: - */ -#define PYCBC_TYPE_CTOR(t) PyObject_CallFunction((PyObject*)t, NULL, NULL) - - -/** - * Allocators for result functions. See callbacks.c:get_common - */ -PyObject *pycbc_result_new(pycbc_Bucket *parent); -PyObject *pycbc_multiresult_new(pycbc_Bucket *parent); -pycbc_ValueResult *pycbc_valresult_new(pycbc_Bucket *parent); -pycbc_OperationResult *pycbc_opresult_new(pycbc_Bucket *parent); -pycbc_Item *pycbc_item_new(pycbc_Bucket *parent); - -/* Not an allocator per-se, but rather an initializer */ -void pycbc_httpresult_init(pycbc_HttpResult *self, pycbc_MultiResult *parent); - -/* For observe info */ -pycbc_ObserveInfo * pycbc_observeinfo_new(pycbc_Bucket *parent); - -/** - * If an HTTP result was successful or not - */ -int pycbc_httpresult_ok(pycbc_HttpResult *self); - - -/** - * Append data to the HTTP result - * @param mres The multi result - * @param htres The HTTP result - * @param bytes Data to append - * @param nbytes Length of data - */ -void -pycbc_httpresult_add_data(pycbc_MultiResult *mres, pycbc_HttpResult *htres, - const void *bytes, size_t nbytes); - -/** - * Signal completion of an HTTP result. - * - * @param htres The HTTP result (Python) - * @param mres The MultiResult object - * @param err Error code (for the HTTP operation) - * @param status The status code - * @param headers The headers - */ -void -pycbc_httpresult_complete(pycbc_HttpResult *htres, pycbc_MultiResult *mres, - lcb_error_t err, short status, - const char * const *headers); - -/** - * Add more data to the view's row list. - * - * This function will attempt to parse the data as JSON, and store an appropriate - * error code otherwise. - * - * @param vres The ViewResult object - * @param mres The MultiResult object - * @param data Buffer - * @param n Length of buffer - */ -void -pycbc_viewresult_addrow(pycbc_ViewResult *vres, pycbc_MultiResult *mres, - const void *data, size_t n); - -/** - * Attempt to notify the relevant callbacks for new data, if the constraints - * allow it. - * - * This will invoke the callback in asynchronous mode, and will break - * the event loop - * - * @param vres The ViewResult - * @param mres The MultiResult - * @param bucket The Bucket - * @param force_callback whether the async callback should be forcefully invoked, - * ignoring the rows_per_call setting (usually only required on error or when - * there are no more rows). - */ -void -pycbc_viewresult_step(pycbc_ViewResult *vres, pycbc_MultiResult *mres, - pycbc_Bucket *bucket, int force_callback); - -/** - * Simple function, here because it's defined in result.c but needed in - * opresult.c - */ -void pycbc_Result_dealloc(pycbc_Result *self); - -/** - * Traps the current exception and adds it to the current MultiResult - * context. - * @param mres The MultiResult object. - * - * This calls pycbc_exc_mktuple(), so the constrains there apply to this - * function as well. - */ -void pycbc_multiresult_adderr(pycbc_MultiResult* mres); - -/** - * Raise an exception from a multi result. This will raise an exception if: - * 1) There is a 'fatal' error in the 'exceptions' list - * 2) There is an 'operr'. 'operr' can be a failed LCB code (if no_raise_enoent - * is on, this is not present if the failed code was LCB_KEY_ENOENT) - */ -int pycbc_multiresult_maybe_raise(pycbc_MultiResult *self); - -/** - * Return the effective user-facing value from this MultiResult object. - * This should only be called if 'maybe_raise' returns false. - * @param self the object - * @return a new reference to the final result, or NULL on error. - */ -PyObject* pycbc_multiresult_get_result(pycbc_MultiResult *self); - -/** - * Invokes a callback when an operation has been completed. This will either - * invoke the operation's "error callback" or the operation's "result callback" - * depending on the state. - */ -void pycbc_asyncresult_invoke(pycbc_AsyncResult *mres); - -/** - * Initialize the callbacks for the lcb_t - */ -void pycbc_callbacks_init(lcb_t instance); -void pycbc_http_callbacks_init(lcb_t instance); -void pycbc_views_callbacks_init(lcb_t instance); - -/** - * "Real" exception handler. - * @param mode one of the PYCBC_EXC_* constants - * @param p a struct of exception parameters - */ -void pycbc_exc_wrap_REAL(int mode, struct pycbc_exception_params *p); - -/** - * Get the appropriate Couchbase exception object. - * @param mode one of the PYCBC_EXC_* constants - * @param err the libcouchbase error, if any - * @return a borrowed reference to the appropriate exception class - */ -PyObject* pycbc_exc_map(int mode, lcb_error_t err); - -/** - * Creates a simple exception with a given message. The exception - * is not thrown. - */ -PyObject* pycbc_exc_message(int mode, lcb_error_t err, const char *msg); - -/** - * Gets the error classifier categories (as a set of bit flags) for a given - * error code. - */ -PyObject* pycbc_exc_get_categories(PyObject *self, PyObject *arg); -/** - * Throws an exception. If an exception is pending, it is caught and wrapped, - * delivered into the CouchbaseError's 'inner_cause' field - * - * @param e_mode one of the PYCBC_EXC_* constants - * @param e_err the LCB error code (use 0 if none) - * @param e_msg a string message, if any - * @param e_key the key during the handling of which the error occurred - * @param e_objextra the problematic object which actually caused the errror - */ -#define PYCBC_EXC_WRAP_EX(e_mode, e_err, e_msg, e_key, e_objextra) { \ - struct pycbc_exception_params __pycbc_ep = {0}; \ - __pycbc_ep.file = __FILE__; \ - __pycbc_ep.line = __LINE__; \ - __pycbc_ep.err = e_err; \ - __pycbc_ep.msg = e_msg; \ - __pycbc_ep.key = e_key; \ - __pycbc_ep.objextra = e_objextra; \ - pycbc_exc_wrap_REAL(e_mode, &__pycbc_ep); \ -} - -#define PYCBC_EXC_WRAP(mode, err, msg) \ - PYCBC_EXC_WRAP_EX(mode, err, msg, NULL, NULL) - -#define PYCBC_EXC_WRAP_OBJ(mode, err, msg, obj) \ - PYCBC_EXC_WRAP_EX(mode, err, msg, NULL, obj) - -#define PYCBC_EXC_WRAP_KEY(mode, err, msg, key) \ - PYCBC_EXC_WRAP_EX(mode, err, msg, key, NULL) - -#define PYCBC_EXC_WRAP_VALUE PYCBC_EXC_WRAP_KEY - -int pycbc_handle_assert(const char *msg, const char* file, int line); - -/** - * Creates a tuple of (class, object, traceback), similar to what would be - * returned from sys.exc_info() - * @return The error tuple. - * - * Calling this function will also clear the error - * state. This must be called only if PyErr_Occurred() is true. - */ -PyObject *pycbc_exc_mktuple(void); - -/** - * This macro can be used as an 'if' structure. It returns false if the - * condition fails and try otherwise - */ -#define pycbc_assert(e) ((e) ? 1 : pycbc_handle_assert(#e, __FILE__, __LINE__)) - -/** - * EXCTHROW macros. These provide error messages for common stages. - */ -#define PYCBC_EXCTHROW_WAIT(err) PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, err, \ - "There was a problem while trying to send/receive " \ - "your request over the network. This may be a result of a " \ - "bad network or a misconfigured client or server") - -#define PYCBC_EXCTHROW_SCHED(err) PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, err, \ - "There was a problem scheduling your request, or determining " \ - "the appropriate server or vBucket for the key(s) requested. "\ - "This may also be a bug in the SDK if there are no network issues") - -#define PYCBC_EXCTHROW_ARGS() PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, \ - "Bad/insufficient arguments provided") - -#define PYCBC_EXCTHROW_EMPTYKEY() PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, \ - "Empty key (i.e. '', empty string) passed") - -/** - * Encodes a key into a buffer. - * @param conn the connection object - * @param key. in-out. Input should be the Python key. Output should be the - * new python object which contains the underlying buffer for the key. - * @param buf a pointer to a buffer pointer - * @param nbuf pointer to the length of the buffer - * - * The buf parameter will likely be tied to the key parameter, so be sure not - * to decrement its refcount until buf is no longer needed - * - * @return - * 0 on success, nonzero on error - */ -int pycbc_tc_encode_key(pycbc_Bucket *conn, - PyObject **key, - void **buf, - size_t *nbuf); - -/** - * Decodes a key buffer into a python object. - * @param conn the connection object - * @param key the key buffer - * @param nkey the size of the key - * @param pobj a pointer to a PyObject*, will be set with a newly-created python - * object which represents the converted key - * - * @return - * 0 on success, nonzero on error - */ -int pycbc_tc_decode_key(pycbc_Bucket *conn, - const void *key, - size_t nkey, - PyObject **pobj); - -/** - * Encode a value with flags - * @param value. in-out. Input should be the Python value, Output should be the - * new python object which contains the converted value. - * @param flag_v. Python object representing 'flags'. This is used for efficiency - * in order to pass a pythonized version of the flags - * @param buf a pointer to a buffer, likely tied to 'buf' - * @param nbuf pointer to buffer length - * @param flags pointer to a flags variable, will be set with the appropriate - * flags - */ -int pycbc_tc_encode_value(pycbc_Bucket *conn, - PyObject **value, - PyObject *flag_v, - void **buf, - size_t *nbuf, - lcb_uint32_t *flags); - -/** - * Decode a value with flags - * @param conn the connection object - * @param value as received from the server - * @param nvalue length of value - * @param flags flags as received from the server - * @param pobj the pythonized value - */ -int pycbc_tc_decode_value(pycbc_Bucket *conn, - const void *value, - size_t nvalue, - lcb_uint32_t flags, - PyObject **pobj); - - - -/** - * Like encode_value, but only uses built-in encoders - */ -int pycbc_tc_simple_encode(PyObject **p, - void *buf, - size_t *nbuf, - lcb_uint32_t flags); - -/** - * Like decode_value, but only uses built-in decoders - */ -int pycbc_tc_simple_decode(PyObject **vp, - const char *buf, - size_t nbuf, - lcb_uint32_t flags); - -/** - * Automatically determine the format for the object. - */ -PyObject * -pycbc_tc_determine_format(PyObject *value); - -PyObject * -pycbc_iowrap_new(pycbc_Bucket *conn, PyObject *pyio); - -lcb_io_opt_t -pycbc_iowrap_getiops(PyObject *iowrap); - -/** - * Event callback handling - */ -void pycbc_invoke_connected_event(pycbc_Bucket *conn, lcb_error_t err); - -/** - * Schedule the dtor event - */ -void pycbc_schedule_dtor_event(pycbc_Bucket *self); - -/** - * Pipeline handlers - */ -PyObject* pycbc_Bucket__start_pipeline(pycbc_Bucket *); -PyObject* pycbc_Bucket__end_pipeline(pycbc_Bucket *); - -/** - * Control methods - */ -PyObject* pycbc_Bucket__cntl(pycbc_Bucket *, PyObject *, PyObject *); -PyObject* pycbc_Bucket__vbmap(pycbc_Bucket *, PyObject *); -PyObject* pycbc_Bucket__cntlstr(pycbc_Bucket *conn, PyObject *args, PyObject *kw); - -/** - * Flag to check if logging is enabled for the library via Python's logging - */ -extern PyObject* pycbc_log_handler; -extern struct lcb_logprocs_st pycbc_lcb_logprocs; - -/** - * Dummy tuple/keywords, used for PyArg_ParseTupleAndKeywordArgs, which dies - * if one of the arguments is NULL, so these contain empty tuples and dicts, - * respectively. - */ -extern PyObject *pycbc_DummyTuple; -extern PyObject *pycbc_DummyKeywords; - -#endif /* PYCBC_H_ */ diff --git a/src/result.c b/src/result.c deleted file mode 100644 index 7e9677ec5..000000000 --- a/src/result.c +++ /dev/null @@ -1,170 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "pycbc.h" -#include "structmember.h" - -static PyObject * -Result_success(pycbc_Result *self, void *closure) -{ - (void)closure; - return PyBool_FromLong(self->rc == LCB_SUCCESS); -} - -static PyObject * -Result_repr(pycbc_Result *self) -{ - return PyObject_CallFunction(pycbc_helpers.result_reprfunc, "O", self); -} - - -static PyObject * -Result_retnone(pycbc_Result *self, void *closure) -{ - (void)closure; - (void)self; - Py_INCREF(Py_None); - return Py_None; -} - -static PyObject * -Result_int0(pycbc_Result *self, void *closure) -{ - (void)closure; - (void)self; - return pycbc_IntFromL(0); -} - -static PyObject * -Result_errstr(pycbc_Result *self, void *closure) -{ - (void)closure; - return pycbc_lcb_errstr(NULL, self->rc); -} - -static struct PyMemberDef Result_TABLE_members[] = { - { "rc", - T_INT, offsetof(pycbc_Result, rc), - READONLY, - PyDoc_STR("libcouchbase error code") - }, - { "key", T_OBJECT_EX, offsetof(pycbc_Result, key), - READONLY, - PyDoc_STR("Key for the operation") - }, - - { NULL } -}; - -static struct PyGetSetDef Result_TABLE_getset[] = { - { "success", - (getter)Result_success, - NULL, - PyDoc_STR("Determine whether operation succeeded or not") - }, - { "value", - (getter)Result_retnone, - NULL, NULL, - }, - { "errstr", - (getter)Result_errstr, - NULL, - PyDoc_STR("Returns a textual representation of the error") - }, - { "cas", (getter)Result_int0, - NULL, NULL - }, - { NULL } -}; - -PyTypeObject pycbc_ResultType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static PyMethodDef Result_TABLE_methods[] = { - { NULL } -}; - -static void -Result_dealloc(pycbc_Result *self) -{ - Py_XDECREF(self->key); - Py_TYPE(self)->tp_free((PyObject*)self); -} - -void -pycbc_Result_dealloc(pycbc_Result *self) -{ - Result_dealloc(self); -} - -int -pycbc_ResultType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_ResultType; - *ptr = (PyObject*)p; - - if (p->tp_name) { - return 0; - } - - p->tp_name = "Result"; - p->tp_doc = PyDoc_STR( - "The standard return type for Couchbase operations.\n" - "\n" - "This is a lightweight object and may be subclassed by other\n" - "operations which may required additional fields."); - - p->tp_new = PyType_GenericNew; - p->tp_dealloc = (destructor)Result_dealloc; - p->tp_basicsize = sizeof(pycbc_Result); - p->tp_members = Result_TABLE_members; - p->tp_methods = Result_TABLE_methods; - p->tp_getset = Result_TABLE_getset; - p->tp_repr = (reprfunc)Result_repr; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - - return pycbc_ResultType_ready(p, PYCBC_RESULT_BASEFLDS); -} - -PyObject * -pycbc_result_new(pycbc_Bucket *parent) -{ - PyObject *obj = PyObject_CallFunction((PyObject*) &pycbc_ResultType, - NULL, - NULL); - (void)parent; - return obj; -} - -int -pycbc_ResultType_ready(PyTypeObject *p, int flags) -{ - int rv; - PyObject *flags_o; - - rv = PyType_Ready(p); - if (rv) { - return rv; - } - - flags_o = pycbc_IntFromUL(flags); - PyDict_SetItemString(p->tp_dict, PYCBC_RESPROPS_NAME, flags_o); - Py_DECREF(flags_o); - - return rv; -} diff --git a/src/result.cxx b/src/result.cxx new file mode 100644 index 000000000..15d3fbbef --- /dev/null +++ b/src/result.cxx @@ -0,0 +1,544 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "result.hxx" +#include "client.hxx" +#include "exceptions.hxx" + +/* result type methods */ + +static void +result_dealloc([[maybe_unused]] result* self) +{ + if (self->dict) { + PyDict_Clear(self->dict); + Py_DECREF(self->dict); + } + // CB_LOG_DEBUG("pycbc - dealloc result: result->refcnt: {}, result->dict->refcnt: {}", + // Py_REFCNT(self), Py_REFCNT(self->dict)); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject* +result__get__(result* self, PyObject* args) +{ + const char* field_name = nullptr; + PyObject* default_value = nullptr; + + if (!PyArg_ParseTuple(args, "s|O", &field_name, &default_value)) { + PyErr_Print(); + PyErr_Clear(); + Py_RETURN_NONE; + } + // PyDict_GetItem will return NULL if key doesn't exist; also suppresses errors + PyObject* val = PyDict_GetItemString(self->dict, field_name); + + if (val == nullptr && default_value == nullptr) { + Py_RETURN_NONE; + } + if (val == nullptr) { + val = default_value; + } + Py_INCREF(val); + if (default_value != nullptr) { + Py_XDECREF(default_value); + } + + return val; +} + +static PyObject* +result__str__(result* self) +{ + const char* format_string = "result:{value=%S}"; + return PyUnicode_FromFormat(format_string, self->dict); +} + +static PyMethodDef result_methods[] = { + { "get", (PyCFunction)result__get__, METH_VARARGS, PyDoc_STR("get field in result object") }, + { NULL, NULL, 0, NULL } +}; + +static struct PyMemberDef result_members[] = { { "raw_result", + T_OBJECT_EX, + offsetof(result, dict), + 0, + PyDoc_STR("Object for the raw result data.\n") }, + { NULL } }; + +static PyObject* +result__new__(PyTypeObject* type, PyObject*, PyObject*) +{ + result* self = reinterpret_cast<result*>(type->tp_alloc(type, 0)); + self->dict = PyDict_New(); + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_result_type() +{ + PyTypeObject obj = {}; + obj.ob_base = PyVarObject_HEAD_INIT(NULL, 0) obj.tp_name = "pycbc_core.result"; + obj.tp_doc = PyDoc_STR("Result of operation on client"); + obj.tp_basicsize = sizeof(result); + obj.tp_itemsize = 0; + obj.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + obj.tp_new = result__new__; + obj.tp_dealloc = (destructor)result_dealloc; + obj.tp_methods = result_methods; + obj.tp_members = result_members; + obj.tp_repr = (reprfunc)result__str__; + return obj; +} + +static PyTypeObject result_type = init_result_type(); + +PyObject* +create_result_obj() +{ + return PyObject_CallObject(reinterpret_cast<PyObject*>(&result_type), nullptr); +} + +/* mutation_token type methods */ + +static void +mutation_token_dealloc([[maybe_unused]] mutation_token* self) +{ + delete self->token; + // CB_LOG_DEBUG("pycbc - dealloc mutation_token: token->refcnt: {}", Py_REFCNT(self)); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject* +mutation_token__get__(mutation_token* self, [[maybe_unused]] PyObject* args) +{ + PyObject* pyObj_mutation_token = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(self->token->bucket_name().c_str()); + if (-1 == PyDict_SetItemString(pyObj_mutation_token, "bucket_name", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(self->token->partition_uuid()); + if (-1 == PyDict_SetItemString(pyObj_mutation_token, "partition_uuid", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(self->token->sequence_number()); + if (-1 == PyDict_SetItemString(pyObj_mutation_token, "sequence_number", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(self->token->partition_id()); + if (-1 == PyDict_SetItemString(pyObj_mutation_token, "partition_id", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + return pyObj_mutation_token; +} + +static PyMethodDef mutation_token_methods[] = { { "get", + (PyCFunction)mutation_token__get__, + METH_NOARGS, + PyDoc_STR("get mutation token as dict") }, + { NULL } }; + +static PyObject* +mutation_token__new__(PyTypeObject* type, PyObject*, PyObject*) +{ + mutation_token* self = reinterpret_cast<mutation_token*>(type->tp_alloc(type, 0)); + self->token = new couchbase::mutation_token(); + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_mutation_token_type() +{ + PyTypeObject obj = {}; + obj.ob_base = PyVarObject_HEAD_INIT(NULL, 0) obj.tp_name = "pycbc_core.mutation_token"; + obj.tp_doc = PyDoc_STR("Object for c++ client mutation token"); + obj.tp_basicsize = sizeof(mutation_token); + obj.tp_itemsize = 0; + obj.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + obj.tp_new = mutation_token__new__; + obj.tp_dealloc = (destructor)mutation_token_dealloc; + obj.tp_methods = mutation_token_methods; + return obj; +} + +static PyTypeObject mutation_token_type = init_mutation_token_type(); + +PyObject* +create_mutation_token_obj(couchbase::mutation_token mt) +{ + PyObject* pyObj_mut = + PyObject_CallObject(reinterpret_cast<PyObject*>(&mutation_token_type), nullptr); + mutation_token* mut_token = reinterpret_cast<mutation_token*>(pyObj_mut); + auto token = couchbase::mutation_token{ + mt.partition_uuid(), mt.sequence_number(), mt.partition_id(), mt.bucket_name() + }; + *mut_token->token = token; + return reinterpret_cast<PyObject*>(mut_token); +} + +/* streamed_result type methods */ + +static void +streamed_result_dealloc([[maybe_unused]] streamed_result* self) +{ + // CB_LOG_DEBUG("pycbc - dealloc streamed_result: result->refcnt: {}", Py_REFCNT(self)); + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyMethodDef streamed_result_TABLE_methods[] = { { NULL } }; + +PyObject* +streamed_result_iter(PyObject* self) +{ + Py_INCREF(self); + return self; +} + +PyObject* +streamed_result_iternext(PyObject* self) +{ + streamed_result* s_res = reinterpret_cast<streamed_result*>(self); + PyObject* row; + { + Py_BEGIN_ALLOW_THREADS row = s_res->rows->get(s_res->timeout_ms); + Py_END_ALLOW_THREADS + } + + if (row != nullptr) { + return row; + } else { + PyErr_SetString(PyExc_StopIteration, "Timeout occurred waiting for next item in queue."); + return nullptr; + } +} + +static PyObject* +streamed_result__new__(PyTypeObject* type, PyObject*, PyObject*) +{ + streamed_result* self = reinterpret_cast<streamed_result*>(type->tp_alloc(type, 0)); + self->ec = std::error_code(); + self->rows = std::make_shared<rows_queue<PyObject*>>(); + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_streamed_result_type() +{ + PyTypeObject obj = {}; + obj.ob_base = PyVarObject_HEAD_INIT(NULL, 0) obj.tp_name = "pycbc_core.streamed_result"; + obj.tp_doc = PyDoc_STR("Result of streaming operation on client"); + obj.tp_basicsize = sizeof(streamed_result); + obj.tp_itemsize = 0; + obj.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + obj.tp_new = streamed_result__new__; + obj.tp_dealloc = (destructor)streamed_result_dealloc; + obj.tp_methods = streamed_result_TABLE_methods; + obj.tp_iter = streamed_result_iter; + obj.tp_iternext = streamed_result_iternext; + return obj; +} + +static PyTypeObject streamed_result_type = init_streamed_result_type(); + +streamed_result* +create_streamed_result_obj(std::chrono::milliseconds timeout_ms) +{ + PyObject* pyObj_res = + PyObject_CallObject(reinterpret_cast<PyObject*>(&streamed_result_type), nullptr); + streamed_result* streamed_res = reinterpret_cast<streamed_result*>(pyObj_res); + streamed_res->timeout_ms = timeout_ms; + return streamed_res; +} + +/* scan_iterator type methods */ + +static void +scan_iterator_dealloc(scan_iterator* self) +{ + Py_TYPE(self)->tp_free((PyObject*)self); +} + +static PyObject* +scan_iterator__cancel_scan__(scan_iterator* self) +{ + scan_iterator* scan_iter = reinterpret_cast<scan_iterator*>(self); + scan_iter->scan_result->cancel(); + Py_RETURN_NONE; +} + +static PyObject* +scan_iterator__is_cancelled__(scan_iterator* self) +{ + scan_iterator* scan_iter = reinterpret_cast<scan_iterator*>(self); + if (scan_iter->scan_result->is_cancelled()) { + Py_INCREF(Py_True); + return Py_True; + } else { + Py_INCREF(Py_False); + return Py_False; + } +} + +static PyMethodDef scan_iterator_TABLE_methods[] = { { "cancel_scan", + (PyCFunction)scan_iterator__cancel_scan__, + METH_NOARGS, + PyDoc_STR("Cancel range scan streaming.") }, + { "is_cancelled", + (PyCFunction)scan_iterator__is_cancelled__, + METH_NOARGS, + PyDoc_STR("Get mutation token as dict") }, + { NULL } }; + +PyObject* +scan_iterator_iter(PyObject* self) +{ + Py_INCREF(self); + return self; +} + +PyObject* +build_scan_item(couchbase::core::range_scan_item item) +{ + // Should already have the GIL + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_tmp = nullptr; + + try { + pyObj_tmp = PyUnicode_FromString(item.key.c_str()); + } catch (const std::exception& e) { + Py_XDECREF(pyObj_result); + pyObj_tmp = + pycbc_build_exception(PycbcError::UnsuccessfulOperation, __FILE__, __LINE__, e.what()); + return pyObj_tmp; + } + + if (-1 == PyDict_SetItemString(res->dict, RESULT_KEY, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + pyObj_tmp = pycbc_build_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to add KV range scan item key to result."); + return pyObj_tmp; + } + + if (item.body.has_value()) { + pyObj_tmp = PyLong_FromUnsignedLong(item.body.value().flags); + if (-1 == PyDict_SetItemString(res->dict, RESULT_FLAGS, pyObj_tmp)) { + Py_DECREF(pyObj_result); + pyObj_tmp = pycbc_build_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to add KV range scan item flags to result."); + return pyObj_tmp; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(item.body.value().expiry); + if (-1 == PyDict_SetItemString(res->dict, RESULT_EXPIRY, pyObj_tmp)) { + Py_DECREF(pyObj_result); + pyObj_tmp = pycbc_build_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to add KV range scan item expiry to result."); + return pyObj_tmp; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(item.body.value().cas.value()); + if (-1 == PyDict_SetItemString(res->dict, RESULT_CAS, pyObj_tmp)) { + Py_DECREF(pyObj_result); + pyObj_tmp = pycbc_build_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to add KV range scan item cas to result."); + return pyObj_tmp; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(item.body.value().sequence_number); + if (-1 == PyDict_SetItemString(res->dict, "sequence_number", pyObj_tmp)) { + Py_DECREF(pyObj_result); + pyObj_tmp = + pycbc_build_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to add KV range scan item sequence_number to result."); + return pyObj_tmp; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(std::to_integer<std::uint8_t>(item.body.value().datatype)); + if (-1 == PyDict_SetItemString(res->dict, "datatype", pyObj_tmp)) { + Py_DECREF(pyObj_result); + pyObj_tmp = pycbc_build_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to add KV range scan item datatype to result."); + return pyObj_tmp; + } + Py_DECREF(pyObj_tmp); + + try { + pyObj_tmp = binary_to_PyObject(item.body.value().value); + } catch (const std::exception& e) { + Py_DECREF(pyObj_result); + pyObj_tmp = + pycbc_build_exception(PycbcError::UnsuccessfulOperation, __FILE__, __LINE__, e.what()); + return pyObj_tmp; + } + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_tmp)) { + Py_DECREF(pyObj_result); + pyObj_tmp = pycbc_build_exception(PycbcError::UnsuccessfulOperation, + __FILE__, + __LINE__, + "Unable to add KV range scan item key to result."); + return pyObj_tmp; + } + Py_DECREF(pyObj_tmp); + } + return reinterpret_cast<PyObject*>(res); +} + +PyObject* +scan_iterator_iternext(PyObject* self) +{ + scan_iterator* scan_iter = reinterpret_cast<scan_iterator*>(self); + tl::expected<couchbase::core::range_scan_item, std::error_code> result; + { + Py_BEGIN_ALLOW_THREADS result = scan_iter->scan_result->next(); + Py_END_ALLOW_THREADS + } + + if (!result.has_value()) { + PyObject* pyObj_exc = pycbc_build_exception( + result.error(), __FILE__, __LINE__, "Error retrieving next scan result item."); + return pyObj_exc; + } + + return build_scan_item(result.value()); +} + +static PyObject* +scan_iterator__new__(PyTypeObject* type, PyObject*, PyObject*) +{ + scan_iterator* self = reinterpret_cast<scan_iterator*>(type->tp_alloc(type, 0)); + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_scan_iterator_type() +{ + PyTypeObject obj = {}; + obj.ob_base = PyVarObject_HEAD_INIT(NULL, 0) obj.tp_name = "pycbc_core.scan_iterator"; + obj.tp_doc = PyDoc_STR("Result of range scan operation on client"); + obj.tp_basicsize = sizeof(scan_iterator); + obj.tp_itemsize = 0; + obj.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + obj.tp_new = scan_iterator__new__; + obj.tp_dealloc = (destructor)scan_iterator_dealloc; + obj.tp_methods = scan_iterator_TABLE_methods; + obj.tp_iter = scan_iterator_iter; + obj.tp_iternext = scan_iterator_iternext; + return obj; +} + +static PyTypeObject scan_iterator_type = init_scan_iterator_type(); + +scan_iterator* +create_scan_iterator_obj(couchbase::core::scan_result result) +{ + PyObject* pyObj_res = + PyObject_CallObject(reinterpret_cast<PyObject*>(&scan_iterator_type), nullptr); + scan_iterator* scan_iter = reinterpret_cast<scan_iterator*>(pyObj_res); + scan_iter->scan_result = std::make_shared<couchbase::core::scan_result>(result); + return scan_iter; +} + +PyObject* +add_result_objects(PyObject* pyObj_module) +{ + // mutation_token_type + if (PyType_Ready(&mutation_token_type) < 0) { + return nullptr; + } + Py_INCREF(&mutation_token_type); + if (PyModule_AddObject( + pyObj_module, "mutation_token", reinterpret_cast<PyObject*>(&mutation_token_type)) < 0) { + Py_DECREF(&mutation_token_type); + return nullptr; + } + + // result_type, need to DECREF previous types on failure + if (PyType_Ready(&result_type) < 0) { + Py_DECREF(&mutation_token_type); + return nullptr; + } + Py_INCREF(&result_type); + if (PyModule_AddObject(pyObj_module, "result", reinterpret_cast<PyObject*>(&result_type)) < 0) { + Py_DECREF(&mutation_token_type); + Py_DECREF(&result_type); + return nullptr; + } + + // scan_iterator_type, need to DECREF previous types on failure + if (PyType_Ready(&scan_iterator_type) < 0) { + Py_DECREF(&mutation_token_type); + Py_DECREF(&result_type); + return nullptr; + } + Py_INCREF(&scan_iterator_type); + if (PyModule_AddObject( + pyObj_module, "scan_iterator", reinterpret_cast<PyObject*>(&scan_iterator_type)) < 0) { + Py_DECREF(&mutation_token_type); + Py_DECREF(&result_type); + Py_DECREF(&scan_iterator_type); + return nullptr; + } + + // streamed_result_type, need to DECREF previous types on failure + if (PyType_Ready(&streamed_result_type) < 0) { + Py_DECREF(&mutation_token_type); + Py_DECREF(&result_type); + Py_DECREF(&scan_iterator_type); + return nullptr; + } + Py_INCREF(&streamed_result_type); + if (PyModule_AddObject( + pyObj_module, "streamed_result", reinterpret_cast<PyObject*>(&streamed_result_type)) < 0) { + Py_DECREF(&mutation_token_type); + Py_DECREF(&result_type); + Py_DECREF(&scan_iterator_type); + Py_DECREF(&streamed_result_type); + return nullptr; + } + + return pyObj_module; +} diff --git a/src/result.hxx b/src/result.hxx new file mode 100644 index 000000000..c738d17c8 --- /dev/null +++ b/src/result.hxx @@ -0,0 +1,108 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" +#include "utils.hxx" +#include <core/scan_result.hxx> +#include <queue> + +template<class T> +class rows_queue +{ +public: + rows_queue() + : rows_() + , mut_() + , cv_() + { + } + + ~rows_queue() + { + } + + void put(T row) + { + std::lock_guard<std::mutex> lock(mut_); + rows_.push(row); + cv_.notify_one(); + } + + T get(std::chrono::milliseconds timeout_ms) + { + std::unique_lock<std::mutex> lock(mut_); + + while (rows_.empty()) { + auto now = std::chrono::system_clock::now(); + if (cv_.wait_until(lock, now + timeout_ms) == std::cv_status::timeout) { + // this will cause iternext to return nullptr, which stops iteration + return nullptr; + } + } + + auto row = rows_.front(); + rows_.pop(); + return row; + } + + int size() + { + std::lock_guard<std::mutex> lock(mut_); + return rows_.size(); + } + +private: + std::queue<T> rows_; + std::mutex mut_; + bool cancel_streaming_{ false }; + std::condition_variable cv_; +}; + +struct result { + PyObject_HEAD PyObject* dict; +}; + +PyObject* +create_result_obj(); + +struct mutation_token { + PyObject_HEAD couchbase::mutation_token* token; +}; + +PyObject* +create_mutation_token_obj(struct couchbase::mutation_token mt); + +struct streamed_result { + PyObject_HEAD std::error_code ec; + std::shared_ptr<rows_queue<PyObject*>> rows; + std::chrono::milliseconds timeout_ms{}; +}; + +streamed_result* +create_streamed_result_obj(std::chrono::milliseconds timeout_ms); + +struct scan_iterator { + PyObject_HEAD std::shared_ptr<couchbase::core::scan_result> scan_result; +}; + +scan_iterator* +create_scan_iterator_obj(couchbase::core::scan_result result); + +PyObject* +add_result_objects(PyObject* pyObj_module); diff --git a/src/search.cxx b/src/search.cxx new file mode 100644 index 000000000..a13ab4a73 --- /dev/null +++ b/src/search.cxx @@ -0,0 +1,942 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <core/operations/document_search.hxx> +#include <core/search_highlight_style.hxx> +#include <core/search_scan_consistency.hxx> + +#include "exceptions.hxx" +#include "search.hxx" +#include "tracing.hxx" +#include "utils.hxx" + +PyObject* +get_result_row_fragments(std::map<std::string, std::vector<std::string>> fragments) +{ + PyObject* pyObj_row_fragments = PyDict_New(); + for (auto const& fragment : fragments) { + + PyObject* pyObj_fragments = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& f : fragment.second) { + PyObject* pyObj_fragment = PyUnicode_FromString(f.c_str()); + if (-1 == PyList_Append(pyObj_fragments, pyObj_fragment)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_fragment); + } + if (-1 == PyDict_SetItemString(pyObj_row_fragments, fragment.first.c_str(), pyObj_fragments)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_fragments); + } + + return pyObj_row_fragments; +} + +PyObject* +get_result_row_locations( + std::vector<couchbase::core::operations::search_response::search_location> locations) +{ + PyObject* pyObj_row_locations = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& location : locations) { + PyObject* pyObj_row_location = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(location.field.c_str()); + if (-1 == PyDict_SetItemString(pyObj_row_location, "field", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(location.term.c_str()); + if (-1 == PyDict_SetItemString(pyObj_row_location, "term", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(location.position); + if (-1 == PyDict_SetItemString(pyObj_row_location, "position", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(location.start_offset); + if (-1 == PyDict_SetItemString(pyObj_row_location, "start", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(location.end_offset); + if (-1 == PyDict_SetItemString(pyObj_row_location, "end", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (location.array_positions.has_value()) { + PyObject* pyObj_array_positions = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& array_position : location.array_positions.value()) { + PyObject* pyObj_array_position = PyLong_FromUnsignedLongLong(array_position); + if (-1 == PyList_Append(pyObj_array_positions, pyObj_array_position)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_array_position); + } + + if (-1 == + PyDict_SetItemString(pyObj_row_location, "array_positions", pyObj_array_positions)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_array_positions); + } + + if (-1 == PyList_Append(pyObj_row_locations, pyObj_row_location)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_row_location); + } + return pyObj_row_locations; +} + +PyObject* +get_result_row(couchbase::core::operations::search_response::search_row row) +{ + PyObject* pyObj_row = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(row.index.c_str()); + if (-1 == PyDict_SetItemString(pyObj_row, "index", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(row.id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_row, "id", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyFloat_FromDouble(row.score); + if (-1 == PyDict_SetItemString(pyObj_row, "score", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (row.locations.size() > 0) { + pyObj_tmp = get_result_row_locations(row.locations); + if (-1 == PyDict_SetItemString(pyObj_row, "locations", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (row.fragments.size() > 0) { + pyObj_tmp = get_result_row_fragments(row.fragments); + if (-1 == PyDict_SetItemString(pyObj_row, "fragments", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = PyUnicode_FromString(row.fields.c_str()); + if (-1 == PyDict_SetItemString(pyObj_row, "fields", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(row.explanation.c_str()); + if (-1 == PyDict_SetItemString(pyObj_row, "explanation", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + return pyObj_row; +} + +PyObject* +get_result_numeric_range_facets( + std::vector<couchbase::core::operations::search_response::search_facet::numeric_range_facet> + numeric_range_facets) +{ + PyObject* pyObj_numeric_range_facets = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& numeric_range_facet : numeric_range_facets) { + PyObject* pyObj_numeric_range_facet = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(numeric_range_facet.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_numeric_range_facet, "name", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(numeric_range_facet.count); + if (-1 == PyDict_SetItemString(pyObj_numeric_range_facet, "count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (std::holds_alternative<std::uint64_t>(numeric_range_facet.min)) { + pyObj_tmp = PyLong_FromUnsignedLongLong(std::get<std::uint64_t>(numeric_range_facet.min)); + if (-1 == PyDict_SetItemString(pyObj_numeric_range_facet, "min", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } else if (std::holds_alternative<double>(numeric_range_facet.min)) { + pyObj_tmp = PyFloat_FromDouble(std::get<double>(numeric_range_facet.min)); + if (-1 == PyDict_SetItemString(pyObj_numeric_range_facet, "min", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (std::holds_alternative<std::uint64_t>(numeric_range_facet.max)) { + pyObj_tmp = PyLong_FromUnsignedLongLong(std::get<std::uint64_t>(numeric_range_facet.max)); + if (-1 == PyDict_SetItemString(pyObj_numeric_range_facet, "max", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } else if (std::holds_alternative<double>(numeric_range_facet.max)) { + pyObj_tmp = PyFloat_FromDouble(std::get<double>(numeric_range_facet.max)); + if (-1 == PyDict_SetItemString(pyObj_numeric_range_facet, "max", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyList_Append(pyObj_numeric_range_facets, pyObj_numeric_range_facet)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_numeric_range_facet); + } + + return pyObj_numeric_range_facets; +} + +PyObject* +get_result_date_range_facets( + std::vector<couchbase::core::operations::search_response::search_facet::date_range_facet> + date_range_facets) +{ + PyObject* pyObj_date_range_facets = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& date_range_facet : date_range_facets) { + PyObject* pyObj_date_range_facet = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(date_range_facet.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_date_range_facet, "name", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(date_range_facet.count); + if (-1 == PyDict_SetItemString(pyObj_date_range_facet, "count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (date_range_facet.start.has_value()) { + pyObj_tmp = PyUnicode_FromString(date_range_facet.start.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_date_range_facet, "start", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (date_range_facet.end.has_value()) { + pyObj_tmp = PyUnicode_FromString(date_range_facet.end.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_date_range_facet, "end", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyList_Append(pyObj_date_range_facets, pyObj_date_range_facet)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_date_range_facet); + } + + return pyObj_date_range_facets; +} + +PyObject* +get_result_term_facets( + std::vector<couchbase::core::operations::search_response::search_facet::term_facet> term_facets) +{ + PyObject* pyObj_term_facets = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& term_facet : term_facets) { + PyObject* pyObj_term_facet = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(term_facet.term.c_str()); + if (-1 == PyDict_SetItemString(pyObj_term_facet, "term", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(term_facet.count); + if (-1 == PyDict_SetItemString(pyObj_term_facet, "count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (-1 == PyList_Append(pyObj_term_facets, pyObj_term_facet)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_term_facet); + } + + return pyObj_term_facets; +} + +PyObject* +get_result_facets(std::vector<couchbase::core::operations::search_response::search_facet> facets) +{ + PyObject* pyObj_facets = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto const& facet : facets) { + PyObject* pyObj_facet = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(facet.name.c_str()); + if (-1 == PyDict_SetItemString(pyObj_facet, "name", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(facet.field.c_str()); + if (-1 == PyDict_SetItemString(pyObj_facet, "field", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(facet.total); + if (-1 == PyDict_SetItemString(pyObj_facet, "total", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(facet.missing); + if (-1 == PyDict_SetItemString(pyObj_facet, "missing", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(facet.other); + if (-1 == PyDict_SetItemString(pyObj_facet, "other", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (facet.terms.size() > 0) { + pyObj_tmp = get_result_term_facets(facet.terms); + if (-1 == PyDict_SetItemString(pyObj_facet, "terms", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (facet.date_ranges.size() > 0) { + pyObj_tmp = get_result_date_range_facets(facet.date_ranges); + if (-1 == PyDict_SetItemString(pyObj_facet, "date_ranges", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (facet.numeric_ranges.size() > 0) { + pyObj_tmp = get_result_numeric_range_facets(facet.numeric_ranges); + if (-1 == PyDict_SetItemString(pyObj_facet, "numeric_ranges", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyList_Append(pyObj_facets, pyObj_facet)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_facet); + } + + return pyObj_facets; +} + +PyObject* +get_result_metrics(couchbase::core::operations::search_response::search_metrics metrics) +{ + PyObject* pyObj_metrics = PyDict_New(); + std::chrono::duration<unsigned long long, std::nano> int_nsec = metrics.took; + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(int_nsec.count()); + if (-1 == PyDict_SetItemString(pyObj_metrics, "took", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.total_rows); + if (-1 == PyDict_SetItemString(pyObj_metrics, "total_rows", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyFloat_FromDouble(metrics.max_score); + if (-1 == PyDict_SetItemString(pyObj_metrics, "max_score", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.success_partition_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "success_partition_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLongLong(metrics.error_partition_count); + if (-1 == PyDict_SetItemString(pyObj_metrics, "error_partition_count", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_tmp); + + return pyObj_metrics; +} + +PyObject* +get_result_metadata(couchbase::core::operations::search_response::search_meta_data metadata, + bool include_metrics) +{ + PyObject* pyObj_metadata = PyDict_New(); + PyObject* pyObj_tmp = PyUnicode_FromString(metadata.client_context_id.c_str()); + if (-1 == PyDict_SetItemString(pyObj_metadata, "client_context_id", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (include_metrics) { + PyObject* pyObject_metrics = get_result_metrics(metadata.metrics); + if (-1 == PyDict_SetItemString(pyObj_metadata, "metrics", pyObject_metrics)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObject_metrics); + } + + PyObject* pyObj_errors = PyDict_New(); + for (auto const& error : metadata.errors) { + PyObject* pyObj_value = PyUnicode_FromString(error.second.c_str()); + if (-1 == PyDict_SetItemString(pyObj_errors, error.first.c_str(), pyObj_value)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_value); + } + if (-1 == PyDict_SetItemString(pyObj_metadata, "errors", pyObj_errors)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_errors); + + return pyObj_metadata; +} + +result* +create_result_from_search_response(couchbase::core::operations::search_response resp, + bool include_metrics) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_payload = PyDict_New(); + + PyObject* pyObj_tmp = PyUnicode_FromString(resp.status.c_str()); + if (-1 == PyDict_SetItemString(pyObj_payload, "status", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.error.c_str()); + if (-1 == PyDict_SetItemString(pyObj_payload, "error", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObject_metadata = get_result_metadata(resp.meta, include_metrics); + if (-1 == PyDict_SetItemString(pyObj_payload, "metadata", pyObject_metadata)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObject_metadata); + + if (resp.facets.size() > 0) { + pyObj_tmp = get_result_facets(resp.facets); + if (-1 == PyDict_SetItemString(pyObj_payload, "facets", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + // if (resp.rows.size() > 0) { + // pyObj_tmp = get_result_rows(resp.rows); + // if (-1 == PyDict_SetItemString(pyObj_payload, "rows", pyObj_tmp)) { + // PyErr_Print(); + // PyErr_Clear(); + // } + // Py_DECREF(pyObj_tmp); + // } + + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_payload)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_payload); + + return res; +} + +void +create_search_result(couchbase::core::operations::search_response resp, + std::shared_ptr<rows_queue<PyObject*>> rows, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + bool include_metrics) +{ + auto set_exception = false; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_args = NULL; + PyObject* pyObj_func = NULL; + PyObject* pyObj_callback_res = nullptr; + + PyGILState_STATE state = PyGILState_Ensure(); + if (resp.ctx.ec.value()) { + pyObj_exc = build_exception_from_context( + resp.ctx, __FILE__, __LINE__, "Error doing full text search operation."); + // lets clear any errors + PyErr_Clear(); + rows->put(pyObj_exc); + } else { + for (auto const& row : resp.rows) { + PyObject* pyObj_row = get_result_row(row); + rows->put(pyObj_row); + } + + auto res = create_result_from_search_response(resp, include_metrics); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + // None indicates done (i.e. raise StopIteration) + Py_INCREF(Py_None); + rows->put(Py_None); + rows->put(reinterpret_cast<PyObject*>(res)); + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Full text search operation error."); + rows->put(pyObj_exc); + } + + // This is for txcouchbase -- let it knows we're done w/ the FTS request + if (pyObj_callback != nullptr) { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, PyBool_FromLong(static_cast<long>(1))); + } + + if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_CallObject(pyObj_func, pyObj_args); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + pycbc_set_python_exception(PycbcError::InternalSDKError, + __FILE__, + __LINE__, + "Full text search complete callback failed."); + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +std::map<std::string, std::string> +get_facets(PyObject* pyObj_facets) +{ + std::map<std::string, std::string> facets{}; + if (pyObj_facets && PyDict_Check(pyObj_facets)) { + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_facets, &pos, &pyObj_key, &pyObj_value)) { + std::string k; + if (PyUnicode_Check(pyObj_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_key)); + } + if (PyUnicode_Check(pyObj_value) && !k.empty()) { + auto res = std::string(PyUnicode_AsUTF8(pyObj_value)); + facets.emplace(k, res); + } + } + } + return facets; +} + +std::map<std::string, couchbase::core::json_string> +get_raw_options(PyObject* pyObj_raw) +{ + std::map<std::string, couchbase::core::json_string> raw_options{}; + if (pyObj_raw && PyDict_Check(pyObj_raw)) { + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_raw, &pos, &pyObj_key, &pyObj_value)) { + std::string k; + if (PyUnicode_Check(pyObj_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_key)); + } + if (PyUnicode_Check(pyObj_value) && !k.empty()) { + auto res = std::string(PyUnicode_AsUTF8(pyObj_value)); + raw_options.emplace(k, couchbase::core::json_string{ std::move(res) }); + } + } + } + return raw_options; +} + +couchbase::core::operations::search_request +get_search_request(PyObject* op_args) +{ + PyObject* pyObj_index_name = PyDict_GetItemString(op_args, "index_name"); + auto index_name = std::string(PyUnicode_AsUTF8(pyObj_index_name)); + + PyObject* pyObj_query = PyDict_GetItemString(op_args, "query"); + auto query = std::string(PyUnicode_AsUTF8(pyObj_query)); + + couchbase::core::operations::search_request req{ + index_name, couchbase::core::json_string{ std::move(query) } + }; + + PyObject* pyObj_vector_search = PyDict_GetItemString(op_args, "vector_search"); + if (pyObj_vector_search != nullptr) { + auto vector_search = std::string(PyUnicode_AsUTF8(pyObj_vector_search)); + req.vector_search = couchbase::core::json_string{ std::move(vector_search) }; + } + + PyObject* pyObj_vector_combo = PyDict_GetItemString(op_args, "vector_query_combination"); + if (pyObj_vector_combo != nullptr) { + auto vector_combo = std::string(PyUnicode_AsUTF8(pyObj_vector_combo)); + if (vector_combo.compare("and") == 0) { + req.vector_query_combination = couchbase::core::vector_query_combination::combination_and; + } + if (vector_combo.compare("or") == 0) { + req.vector_query_combination = couchbase::core::vector_query_combination::combination_or; + } + } + + PyObject* pyObj_show_request = PyDict_GetItemString(op_args, "show_request"); + if (pyObj_show_request != nullptr) { + req.show_request = pyObj_show_request == Py_True ? true : false; + } + + PyObject* pyObj_limit = PyDict_GetItemString(op_args, "limit"); + if (pyObj_limit != nullptr) { + auto limit = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_limit)); + req.limit = limit; + } + + PyObject* pyObj_skip = PyDict_GetItemString(op_args, "skip"); + if (pyObj_skip != nullptr) { + auto skip = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_skip)); + req.skip = skip; + } + + PyObject* pyObj_explain = PyDict_GetItemString(op_args, "explain"); + if (pyObj_explain != nullptr && pyObj_explain == Py_True) { + req.explain = true; + } + + PyObject* pyObj_disable_scoring = PyDict_GetItemString(op_args, "disable_scoring"); + if (pyObj_disable_scoring != nullptr && pyObj_disable_scoring == Py_True) { + req.disable_scoring = true; + } + + PyObject* pyObj_include_locations = PyDict_GetItemString(op_args, "include_locations"); + if (pyObj_include_locations != nullptr && pyObj_include_locations == Py_True) { + req.include_locations = true; + } + + PyObject* pyObj_highlight_style = PyDict_GetItemString(op_args, "highlight_style"); + if (pyObj_highlight_style != nullptr) { + auto highlight_style = std::string(PyUnicode_AsUTF8(pyObj_highlight_style)); + if (highlight_style.compare("html") == 0) { + req.highlight_style = couchbase::core::search_highlight_style::html; + } else if (highlight_style.compare("ansi") == 0) { + req.highlight_style = couchbase::core::search_highlight_style::ansi; + } + } + + PyObject* pyObj_highlight_fields = PyDict_GetItemString(op_args, "highlight_fields"); + if (pyObj_highlight_fields != nullptr && PyList_Check(pyObj_highlight_fields)) { + size_t nfields = static_cast<size_t>(PyList_GET_SIZE(pyObj_highlight_fields)); + std::vector<std::string> fields{}; + size_t ii; + for (ii = 0; ii < nfields; ++ii) { + PyObject* pyObj_field = PyList_GetItem(pyObj_highlight_fields, ii); + auto field = std::string(PyUnicode_AsUTF8(pyObj_field)); + fields.push_back(field); + } + + if (fields.size() > 0) { + req.highlight_fields = fields; + } + } + + PyObject* pyObj_fields = PyDict_GetItemString(op_args, "fields"); + if (pyObj_fields != nullptr && PyList_Check(pyObj_fields)) { + size_t nfields = static_cast<size_t>(PyList_GET_SIZE(pyObj_fields)); + std::vector<std::string> fields{}; + size_t ii; + for (ii = 0; ii < nfields; ++ii) { + PyObject* pyObj_field = PyList_GetItem(pyObj_fields, ii); + auto field = std::string(PyUnicode_AsUTF8(pyObj_field)); + fields.push_back(field); + } + + if (fields.size() > 0) { + req.fields = fields; + } + } + + PyObject* pyObj_collections = PyDict_GetItemString(op_args, "collections"); + if (pyObj_collections != nullptr && PyList_Check(pyObj_collections)) { + size_t ncollections = static_cast<size_t>(PyList_GET_SIZE(pyObj_collections)); + std::vector<std::string> collections{}; + size_t ii; + for (ii = 0; ii < ncollections; ++ii) { + PyObject* pyObj_collection = PyList_GetItem(pyObj_collections, ii); + auto collection = std::string(PyUnicode_AsUTF8(pyObj_collection)); + collections.push_back(collection); + } + + if (collections.size() > 0) { + req.collections = collections; + } + } + + PyObject* pyObj_scan_consistency = PyDict_GetItemString(op_args, "scan_consistency"); + if (pyObj_scan_consistency != nullptr) { + auto scan_consistency = std::string(PyUnicode_AsUTF8(pyObj_scan_consistency)); + if (scan_consistency.compare("not_bounded") == 0) { + req.scan_consistency = couchbase::core::search_scan_consistency::not_bounded; + } + } + + PyObject* pyObj_mutation_state = PyDict_GetItemString(op_args, "mutation_state"); + if (pyObj_mutation_state != nullptr && PyList_Check(pyObj_mutation_state)) { + req.mutation_state = get_mutation_state(pyObj_mutation_state); + } + + PyObject* pyObj_sort_specs = PyDict_GetItemString(op_args, "sort_specs"); + if (pyObj_sort_specs != nullptr && PyList_Check(pyObj_sort_specs)) { + size_t nspecs = static_cast<size_t>(PyList_GET_SIZE(pyObj_sort_specs)); + std::vector<std::string> sort_specs{}; + size_t ii; + for (ii = 0; ii < nspecs; ++ii) { + PyObject* pyObj_spec = PyList_GetItem(pyObj_sort_specs, ii); + auto spec = std::string(PyUnicode_AsUTF8(pyObj_spec)); + sort_specs.push_back(spec); + } + + if (sort_specs.size() > 0) { + req.sort_specs = sort_specs; + } + } + + PyObject* pyObj_facets = PyDict_GetItemString(op_args, "facets"); + if (pyObj_facets != nullptr) { + auto facets = get_facets(pyObj_facets); + if (facets.size() > 0) { + req.facets = facets; + } + } + + PyObject* pyObj_raw = PyDict_GetItemString(op_args, "raw"); + if (pyObj_raw != nullptr) { + auto raw_options = get_raw_options(pyObj_raw); + if (raw_options.size() > 0) { + req.raw = raw_options; + } + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (nullptr != pyObj_timeout) { + // comes in as microseconds + req.timeout = std::chrono::milliseconds(PyLong_AsUnsignedLongLong(pyObj_timeout) / 1000ULL); + } + + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + if (pyObj_bucket_name != nullptr) { + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + req.bucket_name = bucket_name; + } + + PyObject* pyObj_scope_name = PyDict_GetItemString(op_args, "scope_name"); + if (pyObj_scope_name != nullptr) { + auto scope_name = std::string(PyUnicode_AsUTF8(pyObj_scope_name)); + req.scope_name = scope_name; + } + + PyObject* pyObj_log_request = PyDict_GetItemString(op_args, "log_request"); + if (pyObj_log_request != nullptr) { + req.log_request = pyObj_log_request == Py_True ? true : false; + } + + PyObject* pyObj_log_response = PyDict_GetItemString(op_args, "log_response"); + if (pyObj_log_response != nullptr) { + req.log_response = pyObj_log_response == Py_True ? true : false; + } + + return req; +} + +streamed_result* +handle_search_query([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + // optional + PyObject* pyObj_op_args = nullptr; + std::uint64_t streaming_timeout_us = 0; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_row_callback = nullptr; + PyObject* pyObj_span = nullptr; + + static const char* kw_list[] = { "conn", "op_args", "streaming_timeout", + "callback", "errback", "row_callback", + "span", nullptr }; + + const char* kw_format = "O!|OKOOOO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &pyObj_op_args, + &streaming_timeout_us, + &pyObj_callback, + &pyObj_errback, + &pyObj_row_callback, + &pyObj_span); + if (!ret) { + PyErr_Print(); + PyErr_SetString(PyExc_ValueError, "Unable to parse arguments"); + return nullptr; + } + + connection* conn = nullptr; + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + PyErr_SetString(PyExc_ValueError, "passed null connection"); + return nullptr; + } + PyErr_Clear(); + + auto req = get_search_request(pyObj_op_args); + bool include_metrics = true; + PyObject* pyObj_metrics = PyDict_GetItemString(pyObj_op_args, "metrics"); + if (pyObj_metrics != nullptr && pyObj_metrics == Py_False) { + include_metrics = false; + } + if (nullptr != pyObj_span) { + req.parent_span = std::make_shared<pycbc::request_span>(pyObj_span); + } + + // timeout is always set either to default, or timeout provided in options + auto streaming_timeout = couchbase::core::timeout_defaults::search_timeout; + if (streaming_timeout_us > 0) { + streaming_timeout = std::chrono::milliseconds(streaming_timeout_us / 1000ULL); + } + streamed_result* streamed_res = create_streamed_result_obj(streaming_timeout); + + // TODO: let the couchbase++ streaming stabilize a bit more... + // req.row_callback = [rows = streamed_res->rows](std::string&& row) { + // PyGILState_STATE state = PyGILState_Ensure(); + // PyObject* pyObj_row = PyBytes_FromStringAndSize(row.c_str(), row.length()); + // rows->put(pyObj_row); + // PyGILState_Release(state); + // return couchbase::core::utils::json::stream_control::next_row; + // }; + + // we need the callback, errback, and logic to all stick around, so... + // use XINCREF b/c they _could_ be NULL + Py_XINCREF(pyObj_errback); + Py_XINCREF(pyObj_callback); + + Py_BEGIN_ALLOW_THREADS conn->cluster_.execute( + req, + [rows = streamed_res->rows, pyObj_callback, pyObj_errback, include_metrics]( + couchbase::core::operations::search_response resp) { + create_search_result(resp, rows, pyObj_callback, pyObj_errback, include_metrics); + }); + Py_END_ALLOW_THREADS return streamed_res; +} diff --git a/src/search.hxx b/src/search.hxx new file mode 100644 index 000000000..37f5bfc45 --- /dev/null +++ b/src/search.hxx @@ -0,0 +1,24 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" +#include "result.hxx" + +streamed_result* +handle_search_query(PyObject* self, PyObject* args, PyObject* kwargs); diff --git a/src/store.c b/src/store.c deleted file mode 100644 index 643e109aa..000000000 --- a/src/store.c +++ /dev/null @@ -1,340 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -#include "oputil.h" - -struct storecmd_vars { - int operation; - unsigned long ttl; - PyObject *flagsobj; - lcb_U64 single_cas; -}; - -struct single_key_context { - PyObject *value; - PyObject *flagsobj; - lcb_uint64_t cas; - unsigned long ttl; - -}; - -static int -handle_item_kv(pycbc_Item *itm, PyObject *options, const struct storecmd_vars *scv, - struct single_key_context *skc) -{ - int rv; - PyObject *ttl_O = NULL, *flagsobj_Oalt = NULL, *igncas_O = NULL; - PyObject *frag_O = NULL; - static char *itm_optlist[] = { - "ttl", "format", "ignore_cas", "fragment", NULL }; - - lcb_cas_t itmcas = itm->cas; - skc->value = itm->value; - - if (options) { - rv = PyArg_ParseTupleAndKeywords(pycbc_DummyTuple, options, "|OOOO", - itm_optlist, &ttl_O, &flagsobj_Oalt, &igncas_O, &frag_O); - if (!rv) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, - "Couldn't parse item options"); - return -1; - } - - if (ttl_O) { - if (-1 == pycbc_get_ttl(ttl_O, &skc->ttl, 1)) { - return -1; - } - - if (!skc->ttl) { - skc->ttl = scv->ttl; - } - } - - if (flagsobj_Oalt && flagsobj_Oalt != Py_None) { - skc->flagsobj = flagsobj_Oalt; - } - - if (igncas_O && PyObject_IsTrue(igncas_O)) { - itmcas = 0; - } - - if (frag_O == NULL) { - if (scv->operation == LCB_APPEND || scv->operation == LCB_PREPEND) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "append/prepend must provide options with 'fragment' specifier"); - return -1; - } - - } else { - if (scv->operation != LCB_APPEND && scv->operation != LCB_PREPEND) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "'fragment' only valid for append/prepend"); - return -1; - } - - skc->value = frag_O; - } - - } else { - if (scv->operation == LCB_APPEND || scv->operation == LCB_PREPEND) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "append/prepend must provide options with 'fragment' specifier"); - return -1; - } - } - - if (!skc->value) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "Value is empty", skc->value); - return -1; - } - - skc->cas = itmcas; - return 0; -} - -static int -handle_single_kv(pycbc_Bucket *self, struct pycbc_common_vars *cv, int optype, - PyObject *curkey, PyObject *curvalue, PyObject *options, pycbc_Item *itm, - void *arg) -{ - int rv; - const struct storecmd_vars *scv = (struct storecmd_vars *)arg; - struct single_key_context skc = { NULL }; - const void *key, *value; - /* backing key/values for to-bytes conversion. Freed at the end */ - PyObject *encvalue = NULL, *enckey = NULL; - size_t nkey, nvalue; - lcb_error_t err; - lcb_CMDSTORE cmd = { 0 }; - - skc.ttl = scv->ttl; - skc.flagsobj = scv->flagsobj; - skc.value = curvalue; - skc.cas = scv->single_cas; - - rv = pycbc_tc_encode_key(self, &curkey, (void**)&key, &nkey); - if (rv < 0) { - return -1; - } - enckey = curkey; - - if (!nkey) { - PYCBC_EXCTHROW_EMPTYKEY(); - rv = -1; - goto GT_DONE; - } - - if (itm) { - rv = handle_item_kv(itm, options, scv, &skc); - if (rv < 0) { - rv = -1; - goto GT_DONE; - } - } - - rv = pycbc_tc_encode_value(self, &skc.value, skc.flagsobj, - (void**)&value, &nvalue, &cmd.flags); - if (rv < 0) { - rv = -1; - goto GT_DONE; - } - /* Set the encoded value */ - encvalue = skc.value; - - if (scv->operation == LCB_APPEND || scv->operation == LCB_PREPEND) { - /* The server ignores these flags and libcouchbase will throw an error - * if the flags are present. We check elsewhere here to ensure that - * only UTF8/BYTES are accepted for append/prepend anyway */ - cmd.flags = 0; - } - - LCB_CMD_SET_KEY(&cmd, key, nkey); - LCB_CMD_SET_VALUE(&cmd, value, nvalue); - cmd.cas = skc.cas; - cmd.operation = scv->operation; - cmd.exptime = skc.ttl; - err = lcb_store3(self->instance, cv->mres, &cmd); - if (err == LCB_SUCCESS) { - rv = 0; - } else { - rv = -1; - PYCBC_EXCTHROW_SCHED(err); - } - - - GT_DONE: - /* Clean up our encoded keys and values */ - Py_XDECREF(enckey); - Py_XDECREF(encvalue); - return rv; -} - - -static int -handle_append_flags(pycbc_Bucket *self, PyObject **flagsobj) -{ - unsigned long val = 0; - - if (*flagsobj == NULL || *flagsobj == Py_None) { - *flagsobj = pycbc_helpers.fmt_utf8_flags; - return 0; - } - - if (self->tc) { - return 0; /* let the transcoder handle it */ - } - - val = pycbc_IntAsUL(*flagsobj); - if (val == (unsigned long)-1) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "invalid flags", *flagsobj); - return -1; - } - - if ((val & PYCBC_FMT_BYTES) == PYCBC_FMT_BYTES) { - return 0; - } else if ((val & PYCBC_FMT_UTF8) == PYCBC_FMT_UTF8) { - return 0; - } - - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "Only FMT_BYTES and FMT_UTF8 are supported for append/prepend", *flagsobj); - return -1; - -} - -static PyObject * -set_common(pycbc_Bucket *self, PyObject *args, PyObject *kwargs, - const lcb_storage_t operation, int argopts) -{ - int rv; - Py_ssize_t ncmds = 0; - PyObject *ttl_O = NULL; - PyObject *dict = NULL; - PyObject *key; - PyObject *value; - pycbc_seqtype_t seqtype; - struct pycbc_common_vars cv = PYCBC_COMMON_VARS_STATIC_INIT; - struct storecmd_vars scv = { 0 }; - char persist_to = 0, replicate_to = 0; - - - static char *kwlist_multi[] = { - "kv", "ttl", "format", - "persist_to", "replicate_to", - NULL - }; - - static char *kwlist_single[] = { - "key", "value", "cas", "ttl", "format", - "persist_to", "replicate_to", - NULL - }; - - scv.operation = operation; - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "O|OOBB", kwlist_multi, - &dict, - &ttl_O, &scv.flagsobj, - &persist_to, &replicate_to); - - } else { - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "OO|KOOBB", kwlist_single, - &key, &value, - &scv.single_cas, &ttl_O, &scv.flagsobj, - &persist_to, &replicate_to); - } - - if (!rv) { - PYCBC_EXC_WRAP(PYCBC_EXC_ARGUMENTS, 0, "couldn't parse arguments"); - return NULL; - } - - rv = pycbc_get_ttl(ttl_O, &scv.ttl, 1); - if (rv < 0) { - return NULL; - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_check_sequence(dict, 0, &ncmds, &seqtype); - if (rv < 0) { - return NULL; - } - - } else { - ncmds = 1; - } - - if (operation == LCB_APPEND || operation == LCB_PREPEND) { - rv = handle_append_flags(self, &scv.flagsobj); - if (rv < 0) { - return NULL; - } - - } else if (scv.flagsobj == NULL || scv.flagsobj == Py_None) { - scv.flagsobj = self->dfl_fmt; - } - - rv = pycbc_common_vars_init(&cv, self, argopts, ncmds, 1); - if (rv < 0) { - return NULL; - } - - rv = pycbc_handle_durability_args(self, &cv.mres->dur, - persist_to, replicate_to); - - if (rv == 1) { - cv.mres->mropts |= PYCBC_MRES_F_DURABILITY; - - } else if (rv == -1) { - goto GT_DONE; - } - - if (argopts & PYCBC_ARGOPT_MULTI) { - rv = pycbc_oputil_iter_multi(self, seqtype, dict, &cv, 0, handle_single_kv, &scv); - - } else { - rv = handle_single_kv(self, &cv, 0, key, value, NULL, NULL, &scv); - } - - if (rv < 0) { - goto GT_DONE; - } - - if (-1 == pycbc_common_vars_wait(&cv, self)) { - goto GT_DONE; - } - -GT_DONE: - pycbc_common_vars_finalize(&cv, self); - return cv.ret; -} - -#define DECLFUNC(name, operation, mode) \ - PyObject *pycbc_Bucket_##name(pycbc_Bucket *self, \ - PyObject *args, PyObject *kwargs) { \ - return set_common(self, args, kwargs, operation, mode); \ -} - -DECLFUNC(upsert_multi, LCB_SET, PYCBC_ARGOPT_MULTI) -DECLFUNC(insert_multi, LCB_ADD, PYCBC_ARGOPT_MULTI) -DECLFUNC(replace_multi, LCB_REPLACE, PYCBC_ARGOPT_MULTI) - -DECLFUNC(append_multi, LCB_APPEND, PYCBC_ARGOPT_MULTI) -DECLFUNC(prepend_multi, LCB_PREPEND, PYCBC_ARGOPT_MULTI) - -DECLFUNC(upsert, LCB_SET, PYCBC_ARGOPT_SINGLE) -DECLFUNC(insert, LCB_ADD, PYCBC_ARGOPT_SINGLE) -DECLFUNC(replace, LCB_REPLACE, PYCBC_ARGOPT_SINGLE) - -DECLFUNC(append, LCB_APPEND, PYCBC_ARGOPT_SINGLE) -DECLFUNC(prepend, LCB_PREPEND, PYCBC_ARGOPT_SINGLE) diff --git a/src/subdoc_ops.cxx b/src/subdoc_ops.cxx new file mode 100644 index 000000000..67798e5b5 --- /dev/null +++ b/src/subdoc_ops.cxx @@ -0,0 +1,1277 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "subdoc_ops.hxx" + +#include <couchbase/cas.hxx> + +#include "exceptions.hxx" +#include "result.hxx" +#include "tracing.hxx" +#include "utils.hxx" + +couchbase::core::impl::subdoc::opcode +to_subdoc_opcode(std::uint8_t opcode) +{ + if (opcode == 0x00) { + return couchbase::core::impl::subdoc::opcode::get_doc; + } else if (opcode == 0x01) { + return couchbase::core::impl::subdoc::opcode::set_doc; + } else if (opcode == 0x04) { + return couchbase::core::impl::subdoc::opcode::remove_doc; + } else if (opcode == 0xc5) { + return couchbase::core::impl::subdoc::opcode::get; + } else if (opcode == 0xc6) { + return couchbase::core::impl::subdoc::opcode::exists; + } else if (opcode == 0xc7) { + return couchbase::core::impl::subdoc::opcode::dict_add; + } else if (opcode == 0xc8) { + return couchbase::core::impl::subdoc::opcode::dict_upsert; + } else if (opcode == 0xc9) { + return couchbase::core::impl::subdoc::opcode::remove; + } else if (opcode == 0xca) { + return couchbase::core::impl::subdoc::opcode::replace; + } else if (opcode == 0xcb) { + return couchbase::core::impl::subdoc::opcode::array_push_last; + } else if (opcode == 0xcc) { + return couchbase::core::impl::subdoc::opcode::array_push_first; + } else if (opcode == 0xcd) { + return couchbase::core::impl::subdoc::opcode::array_insert; + } else if (opcode == 0xce) { + return couchbase::core::impl::subdoc::opcode::array_add_unique; + } else if (opcode == 0xcf) { + return couchbase::core::impl::subdoc::opcode::counter; + } else if (opcode == 0xd2) { + return couchbase::core::impl::subdoc::opcode::get_count; + } else if (opcode == 0xd3) { + return couchbase::core::impl::subdoc::opcode::replace_body_with_xattr; + } + + throw std::invalid_argument(fmt::format("Unknown subdoc op code: {}", opcode)); +} + +template<typename T> +result* +add_extras_to_result([[maybe_unused]] const T& t, result* res) +{ + return res; +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::lookup_in_any_replica_response>( + const couchbase::core::operations::lookup_in_any_replica_response& resp, + result* res) +{ + PyObject* pyObj_tmp = PyBool_FromLong(static_cast<long>(resp.is_replica)); + if (-1 == PyDict_SetItemString(res->dict, "is_replica", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_fields = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto f : resp.fields) { + PyObject* pyObj_field = PyDict_New(); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.opcode)); + if (-1 == PyDict_SetItemString(pyObj_field, "opcode", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyBool_FromLong(static_cast<long>(f.exists)); + if (-1 == PyDict_SetItemString(pyObj_field, "exists", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.status)); + if (-1 == PyDict_SetItemString(pyObj_field, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_DecodeUTF8(f.path.c_str(), f.path.length(), "strict"); + if (-1 == PyDict_SetItemString(pyObj_field, "path", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.original_index)); + if (-1 == PyDict_SetItemString(pyObj_field, "original_index", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (f.value.size() > 0) { + try { + pyObj_tmp = binary_to_PyObject(f.value); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_field, RESULT_VALUE, pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyList_Append(pyObj_fields, pyObj_field); + Py_DECREF(pyObj_field); + } + + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_fields)) { + Py_XDECREF(pyObj_fields); + return nullptr; + } + Py_DECREF(pyObj_fields); + return res; +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::lookup_in_all_replicas_response::entry>( + const couchbase::core::operations::lookup_in_all_replicas_response::entry& resp, + result* res) +{ + PyObject* pyObj_tmp = PyBool_FromLong(static_cast<long>(resp.is_replica)); + if (-1 == PyDict_SetItemString(res->dict, "is_replica", pyObj_tmp)) { + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + PyObject* pyObj_fields = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto f : resp.fields) { + PyObject* pyObj_field = PyDict_New(); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.opcode)); + if (-1 == PyDict_SetItemString(pyObj_field, "opcode", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyBool_FromLong(static_cast<long>(f.exists)); + if (-1 == PyDict_SetItemString(pyObj_field, "exists", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.status)); + if (-1 == PyDict_SetItemString(pyObj_field, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_DecodeUTF8(f.path.c_str(), f.path.length(), "strict"); + if (-1 == PyDict_SetItemString(pyObj_field, "path", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.original_index)); + if (-1 == PyDict_SetItemString(pyObj_field, "original_index", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (f.value.size() > 0) { + try { + pyObj_tmp = binary_to_PyObject(f.value); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_field, RESULT_VALUE, pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyList_Append(pyObj_fields, pyObj_field); + Py_DECREF(pyObj_field); + } + + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_fields)) { + Py_XDECREF(pyObj_fields); + return nullptr; + } + Py_DECREF(pyObj_fields); + return res; +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::lookup_in_response>( + const couchbase::core::operations::lookup_in_response& resp, + result* res) +{ + PyObject* pyObj_fields = PyList_New(static_cast<Py_ssize_t>(0)); + for (auto f : resp.fields) { + PyObject* pyObj_field = PyDict_New(); + + PyObject* pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.opcode)); + if (-1 == PyDict_SetItemString(pyObj_field, "opcode", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyBool_FromLong(static_cast<long>(f.exists)); + if (-1 == PyDict_SetItemString(pyObj_field, "exists", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.status)); + if (-1 == PyDict_SetItemString(pyObj_field, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_DecodeUTF8(f.path.c_str(), f.path.length(), "strict"); + if (-1 == PyDict_SetItemString(pyObj_field, "path", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(f.original_index)); + if (-1 == PyDict_SetItemString(pyObj_field, "original_index", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (f.value.size() > 0) { + try { + pyObj_tmp = binary_to_PyObject(f.value); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_field, RESULT_VALUE, pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + + PyList_Append(pyObj_fields, pyObj_field); + Py_DECREF(pyObj_field); + } + + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_fields)) { + Py_XDECREF(pyObj_fields); + return nullptr; + } + Py_DECREF(pyObj_fields); + return res; +} + +template<> +result* +add_extras_to_result<couchbase::core::operations::mutate_in_response>( + const couchbase::core::operations::mutate_in_response& resp, + result* res) +{ + PyObject* pyObj_mutation_token = create_mutation_token_obj(resp.token); + if (-1 == PyDict_SetItemString(res->dict, RESULT_MUTATION_TOKEN, pyObj_mutation_token)) { + Py_XDECREF(pyObj_mutation_token); + return nullptr; + } + Py_DECREF(pyObj_mutation_token); + + PyObject* pyObj_fields = PyList_New(static_cast<Py_ssize_t>(0)); + for (int i = 0; i < resp.fields.size(); i++) { + PyObject* pyObj_field = PyDict_New(); + PyObject* pyObj_tmp = + PyLong_FromUnsignedLong(static_cast<unsigned long>(resp.fields[i].opcode)); + if (-1 == PyDict_SetItemString(pyObj_field, "opcode", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(resp.fields[i].status)); + if (-1 == PyDict_SetItemString(pyObj_field, "status", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = + PyUnicode_DecodeUTF8(resp.fields[i].path.c_str(), resp.fields[i].path.length(), "strict"); + if (-1 == PyDict_SetItemString(pyObj_field, "path", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyLong_FromUnsignedLong(static_cast<unsigned long>(resp.fields[i].original_index)); + if (-1 == PyDict_SetItemString(pyObj_field, "original_index", pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (resp.fields[i].value.size()) { + try { + pyObj_tmp = binary_to_PyObject(resp.fields[i].value); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + if (-1 == PyDict_SetItemString(pyObj_field, RESULT_VALUE, pyObj_tmp)) { + Py_XDECREF(pyObj_fields); + Py_XDECREF(pyObj_field); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + PyList_Append(pyObj_fields, pyObj_field); + Py_DECREF(pyObj_field); + } + + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_fields)) { + Py_XDECREF(pyObj_fields); + return nullptr; + } + Py_DECREF(pyObj_fields); + return res; +} + +template<typename T> +result* +create_base_result_from_subdoc_op_response(const char* key, const T& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(resp.cas.value()); + if (-1 == PyDict_SetItemString(res->dict, RESULT_CAS, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString(res->dict, RESULT_FLAGS, Py_None)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + + if (nullptr != key) { + pyObj_tmp = PyUnicode_FromString(key); + if (-1 == PyDict_SetItemString(res->dict, RESULT_KEY, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + return res; +} + +template<> +result* +create_base_result_from_subdoc_op_response( + const char* key, + const couchbase::core::operations::lookup_in_all_replicas_response::entry& resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + PyObject* pyObj_tmp = PyLong_FromUnsignedLongLong(resp.cas.value()); + if (-1 == PyDict_SetItemString(res->dict, RESULT_CAS, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString(res->dict, RESULT_FLAGS, Py_None)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + + if (nullptr != key) { + pyObj_tmp = PyUnicode_FromString(key); + if (-1 == PyDict_SetItemString(res->dict, RESULT_KEY, pyObj_tmp)) { + Py_XDECREF(pyObj_result); + Py_XDECREF(pyObj_tmp); + return nullptr; + } + Py_DECREF(pyObj_tmp); + } + return res; +} + +template<typename T> +void +create_result_from_subdoc_op_response(const char* key, + const T& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* pyObj_args = NULL; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + if (resp.ctx.ec().value()) { + pyObj_exc = + build_exception_from_context(resp.ctx, __FILE__, __LINE__, "Subdoc operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + auto res = create_base_result_from_subdoc_op_response(key, resp); + if (res != nullptr) { + res = add_extras_to_result(resp, res); + } + + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Subdoc operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } + + if (!set_exception && pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<> +void +create_result_from_subdoc_op_response( + const char* key, + const couchbase::core::operations::lookup_in_all_replicas_response& resp, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* pyObj_args = NULL; + PyObject* pyObj_kwargs = nullptr; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_callback_res = nullptr; + auto set_exception = false; + + auto streamed_res = + create_streamed_result_obj(couchbase::core::timeout_defaults::key_value_durable_timeout); + + if (resp.ctx.ec().value()) { + pyObj_exc = + build_exception_from_context(resp.ctx, __FILE__, __LINE__, "Subdoc operation error."); + if (pyObj_errback == nullptr) { + barrier->set_value(pyObj_exc); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + // lets clear any errors + PyErr_Clear(); + } else { + for (auto const& entry : resp.entries) { + auto res = create_base_result_from_subdoc_op_response(key, entry); + if (res == nullptr) { + set_exception = true; + break; + } + res = add_extras_to_result(entry, res); + streamed_res->rows->put(reinterpret_cast<PyObject*>(res)); + } + + if (PyErr_Occurred() != nullptr) { + set_exception = true; + } else if (!set_exception) { + Py_INCREF(Py_None); + streamed_res->rows->put(Py_None); + if (pyObj_callback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(streamed_res)); + } else { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, reinterpret_cast<PyObject*>(streamed_res)); + } + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Subdoc operation error."); + streamed_res->rows->put(pyObj_exc); + if (pyObj_errback == nullptr) { + barrier->set_value(reinterpret_cast<PyObject*>(streamed_res)); + } else { + pyObj_func = pyObj_errback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_exc); + } + } else if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_Call(pyObj_func, pyObj_args, pyObj_kwargs); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + PyErr_Print(); + // @TODO: how to handle this situation? + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_kwargs); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + PyGILState_Release(state); +} + +template<typename Request> +void +do_subdoc_op(connection& conn, + Request& req, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + using response_type = typename Request::response_type; + Py_BEGIN_ALLOW_THREADS conn.cluster_.execute( + req, [key = req.id.key(), pyObj_callback, pyObj_errback, barrier](response_type resp) { + create_result_from_subdoc_op_response( + key.c_str(), resp, pyObj_callback, pyObj_errback, barrier); + }); + Py_END_ALLOW_THREADS +} + +PyObject* +prepare_and_execute_lookup_in_op(struct lookup_in_options* options, + size_t nspecs, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + size_t ii; + auto specs = std::vector<couchbase::core::impl::subdoc::command>{}; + for (ii = 0; ii < nspecs; ++ii) { + + struct lookup_in_spec new_spec = {}; + PyObject* pyObj_spec = nullptr; + if (PyTuple_Check(options->specs)) { + pyObj_spec = PyTuple_GetItem(options->specs, ii); + } else { + pyObj_spec = PyList_GetItem(options->specs, ii); + } + + if (!pyObj_spec) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unable to parse spec."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + if (!PyArg_ParseTuple(pyObj_spec, "bsp", &new_spec.op, &new_spec.path, &new_spec.xattr)) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unable to parse spec."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + try { + auto opcode = to_subdoc_opcode(new_spec.op); + specs.emplace_back(couchbase::core::impl::subdoc::command{ + opcode, + new_spec.path, + {}, + couchbase::core::impl::subdoc::build_lookup_in_path_flags(new_spec.xattr, false) }); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_ValueError, + fmt::format("Invalid subdocument opcode {}", new_spec.op).c_str()); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + + couchbase::core::operations::lookup_in_request req{ options->id }; + req.timeout = options->timeout_ms; + req.access_deleted = options->access_deleted; + req.specs = specs; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_subdoc_op(*(options->conn), req, pyObj_callback, pyObj_errback, barrier); + Py_RETURN_NONE; +} + +PyObject* +prepare_and_execute_lookup_in_any_replica_op(struct lookup_in_replica_options* options, + size_t nspecs, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + size_t ii; + auto specs = std::vector<couchbase::core::impl::subdoc::command>{}; + for (ii = 0; ii < nspecs; ++ii) { + + struct lookup_in_spec new_spec = {}; + PyObject* pyObj_spec = nullptr; + if (PyTuple_Check(options->specs)) { + pyObj_spec = PyTuple_GetItem(options->specs, ii); + } else { + pyObj_spec = PyList_GetItem(options->specs, ii); + } + + if (!pyObj_spec) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unable to parse spec."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + if (!PyArg_ParseTuple(pyObj_spec, "bsp", &new_spec.op, &new_spec.path, &new_spec.xattr)) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unable to parse spec."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + try { + auto opcode = to_subdoc_opcode(new_spec.op); + specs.emplace_back(couchbase::core::impl::subdoc::command{ + opcode, + new_spec.path, + {}, + couchbase::core::impl::subdoc::build_lookup_in_path_flags(new_spec.xattr, false) }); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_ValueError, + fmt::format("Invalid subdocument opcode {}", new_spec.op).c_str()); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + + couchbase::core::operations::lookup_in_any_replica_request req{ options->id }; + req.timeout = options->timeout_ms; + req.specs = specs; + req.read_preference = options->read_preference; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_subdoc_op(*(options->conn), req, pyObj_callback, pyObj_errback, barrier); + Py_RETURN_NONE; +} + +PyObject* +prepare_and_execute_lookup_in_all_replicas_op(struct lookup_in_replica_options* options, + size_t nspecs, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + size_t ii; + auto specs = std::vector<couchbase::core::impl::subdoc::command>{}; + for (ii = 0; ii < nspecs; ++ii) { + + struct lookup_in_spec new_spec = {}; + PyObject* pyObj_spec = nullptr; + if (PyTuple_Check(options->specs)) { + pyObj_spec = PyTuple_GetItem(options->specs, ii); + } else { + pyObj_spec = PyList_GetItem(options->specs, ii); + } + + if (!pyObj_spec) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unable to parse spec."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + if (!PyArg_ParseTuple(pyObj_spec, "bsp", &new_spec.op, &new_spec.path, &new_spec.xattr)) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unable to parse spec."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + try { + auto opcode = to_subdoc_opcode(new_spec.op); + specs.emplace_back(couchbase::core::impl::subdoc::command{ + opcode, + new_spec.path, + {}, + couchbase::core::impl::subdoc::build_lookup_in_path_flags(new_spec.xattr, false) }); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_ValueError, + fmt::format("Invalid subdocument opcode {}", new_spec.op).c_str()); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + + couchbase::core::operations::lookup_in_all_replicas_request req{ options->id }; + req.timeout = options->timeout_ms; + req.specs = specs; + req.read_preference = options->read_preference; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + do_subdoc_op(*(options->conn), req, pyObj_callback, pyObj_errback, barrier); + Py_RETURN_NONE; +} + +PyObject* +prepare_and_execute_mutate_in_op(struct mutate_in_options* options, + size_t nspecs, + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier) +{ + size_t ii; + auto specs = std::vector<couchbase::core::impl::subdoc::command>{}; + for (ii = 0; ii < nspecs; ++ii) { + + struct mutate_in_spec new_spec = {}; + PyObject* pyObj_spec = nullptr; + if (PyTuple_Check(options->specs)) { + pyObj_spec = PyTuple_GetItem(options->specs, ii); + } else { + pyObj_spec = PyList_GetItem(options->specs, ii); + } + + if (!pyObj_spec) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unable to parse spec."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + if (!PyArg_ParseTuple(pyObj_spec, + "bsppp|O", + &new_spec.op, + &new_spec.path, + &new_spec.create_parents, + &new_spec.xattr, + &new_spec.expand_macros, + &new_spec.pyObj_value)) { + pycbc_set_python_exception( + PycbcError::InvalidArgument, __FILE__, __LINE__, "Unable to parse spec."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + + if (new_spec.pyObj_value) { + try { + new_spec.value = PyObject_to_binary(new_spec.pyObj_value); + } catch (const std::exception& e) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, e.what()); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + + try { + auto opcode = to_subdoc_opcode(new_spec.op); + specs.emplace_back(couchbase::core::impl::subdoc::command{ + opcode, + new_spec.path, + new_spec.value, + couchbase::core::impl::subdoc::build_mutate_in_path_flags( + new_spec.xattr, new_spec.create_parents, new_spec.expand_macros, false) }); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_ValueError, + fmt::format("Invalid subdocument opcode {}", new_spec.op).c_str()); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + return nullptr; + } + } + + couchbase::core::operations::mutate_in_request req{ options->id }; + req.cas = options->cas; + req.specs = specs; + req.timeout = options->timeout_ms; + if (0 < options->expiry) { + req.expiry = options->expiry; + } + req.store_semantics = options->store_semantics; + req.access_deleted = options->access_deleted; + req.create_as_deleted = options->create_as_deleted; + req.preserve_expiry = options->preserve_expiry; + if (nullptr != options->span) { + req.parent_span = std::make_shared<pycbc::request_span>(options->span); + } + if (options->use_legacy_durability) { + auto req_legacy_durability = + couchbase::core::operations::mutate_in_request_with_legacy_durability{ + req, options->persist_to, options->replicate_to + }; + do_subdoc_op(*(options->conn), req_legacy_durability, pyObj_callback, pyObj_errback, barrier); + Py_RETURN_NONE; + } + req.durability_level = options->durability_level; + do_subdoc_op(*(options->conn), req, pyObj_callback, pyObj_errback, barrier); + Py_RETURN_NONE; +} + +struct lookup_in_options +get_lookup_in_options(PyObject* op_args) +{ + struct lookup_in_options opts { + }; + + PyObject* pyObj_span = PyDict_GetItemString(op_args, "span"); + if (pyObj_span != nullptr) { + opts.span = pyObj_span; + } + + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (pyObj_timeout != nullptr) { + auto timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_timeout)); + timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + if (0 < timeout) { + opts.timeout_ms = timeout_ms; + } + } + + PyObject* pyObj_access_deleted = PyDict_GetItemString(op_args, "access_deleted"); + opts.access_deleted = + pyObj_access_deleted != nullptr && pyObj_access_deleted == Py_True ? true : false; + + return opts; +} + +lookup_in_replica_options +get_lookup_in_replica_options(PyObject* op_args) +{ + struct lookup_in_replica_options opts; + + PyObject* pyObj_span = PyDict_GetItemString(op_args, "span"); + if (pyObj_span != nullptr) { + opts.span = pyObj_span; + } + + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (pyObj_timeout != nullptr) { + auto timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_timeout)); + timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + if (0 < timeout) { + opts.timeout_ms = timeout_ms; + } + } + + PyObject* pyObj_read_preference = PyDict_GetItemString(op_args, "read_preference"); + if (pyObj_read_preference != nullptr) { + opts.read_preference = PyObject_to_read_preference(pyObj_read_preference); + } + + return opts; +} + +mutate_in_options +get_mutate_in_options(PyObject* op_args) +{ + struct mutate_in_options opts; + + PyObject* pyObj_span = PyDict_GetItemString(op_args, "span"); + if (pyObj_span != nullptr) { + opts.span = pyObj_span; + } + + PyObject* pyObj_expiry = PyDict_GetItemString(op_args, "expiry"); + if (pyObj_expiry != nullptr) { + opts.expiry = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_expiry)); + } + + PyObject* pyObj_cas = PyDict_GetItemString(op_args, "cas"); + couchbase::cas cas = couchbase::cas{ 0 }; + if (pyObj_cas != nullptr) { + auto cas_int = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_cas)); + if (cas_int != 0) { + cas = couchbase::cas{ cas_int }; + } + } + opts.cas = cas; + + PyObject* pyObj_preserve_expiry = PyDict_GetItemString(op_args, "preserve_expiry"); + opts.preserve_expiry = + pyObj_preserve_expiry != nullptr && pyObj_preserve_expiry == Py_True ? true : false; + + PyObject* pyObj_access_deleted = PyDict_GetItemString(op_args, "access_deleted"); + opts.access_deleted = + pyObj_access_deleted != nullptr && pyObj_access_deleted == Py_True ? true : false; + + PyObject* pyObj_create_as_deleted = PyDict_GetItemString(op_args, "create_as_deleted"); + opts.create_as_deleted = + pyObj_create_as_deleted != nullptr && pyObj_create_as_deleted == Py_True ? true : false; + + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (pyObj_timeout != nullptr) { + auto timeout = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_timeout)); + timeout_ms = std::chrono::milliseconds(std::max(0ULL, timeout / 1000ULL)); + if (0 < timeout) { + opts.timeout_ms = timeout_ms; + } + } + + PyObject* pyObj_semantics = PyDict_GetItemString(op_args, "store_semantics"); + if (pyObj_semantics) { + auto semantics = static_cast<uint8_t>(PyLong_AsUnsignedLong(pyObj_semantics)); + switch (semantics) { + case 1: { + opts.store_semantics = couchbase::store_semantics::upsert; + break; + } + case 2: { + opts.store_semantics = couchbase::store_semantics::insert; + break; + } + default: { + opts.store_semantics = couchbase::store_semantics::replace; + break; + } + }; + } + + PyObject* pyObj_durability = PyDict_GetItemString(op_args, "durability"); + if (pyObj_durability) { + if (PyDict_Check(pyObj_durability)) { + auto durability = PyObject_to_durability(pyObj_durability); + opts.use_legacy_durability = true; + opts.persist_to = durability.first; + opts.replicate_to = durability.second; + } else if (PyLong_Check(pyObj_durability)) { + opts.durability_level = PyObject_to_durability_level(pyObj_durability); + } + } + + return opts; +} + +PyObject* +handle_subdoc_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + char* bucket = nullptr; + char* scope = nullptr; + char* collection = nullptr; + char* key = nullptr; + Operations::OperationType op_type = Operations::UNKNOWN; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_op_args = nullptr; + PyObject* pyObj_span = nullptr; + PyObject* pyObj_spec = nullptr; + + static const char* kw_list[] = { "conn", "bucket", "scope", "collection_name", "key", "op_type", + "spec", "op_args", nullptr }; + + const char* kw_format = "O!ssssI|OO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &bucket, + &scope, + &collection, + &key, + &op_type, + &pyObj_spec, + &pyObj_op_args); + + if (!ret) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform subdoc operation. Unable to parse args/kwargs."); + return nullptr; + } + + if (!PyTuple_Check(pyObj_spec) && !PyList_Check(pyObj_spec)) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform subdoc operation. Value must be a tuple or list."); + return nullptr; + } + + size_t nspecs; + if (PyTuple_Check(pyObj_spec)) { + nspecs = static_cast<size_t>(PyTuple_GET_SIZE(pyObj_spec)); + } else { + nspecs = static_cast<size_t>(PyList_GET_SIZE(pyObj_spec)); + } + + if (nspecs == 0) { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Cannot perform subdoc operation. Need at least one command."); + return nullptr; + } + + connection* conn = nullptr; + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, NULL_CONN_OBJECT); + return nullptr; + } + + // PyObjects that need to be around for the cxx client lambda + // have their increment/decrement handled w/in the callback_context struct + // struct callback_context callback_ctx = { pyObj_callback, pyObj_errback }; + pyObj_callback = PyDict_GetItemString(pyObj_op_args, "callback"); + pyObj_errback = PyDict_GetItemString(pyObj_op_args, "errback"); + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + + std::shared_ptr<std::promise<PyObject*>> barrier = nullptr; + std::future<PyObject*> fut; + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + barrier = std::make_shared<std::promise<PyObject*>>(); + fut = barrier->get_future(); + } + + switch (op_type) { + case Operations::LOOKUP_IN: { + auto opts = get_lookup_in_options(pyObj_op_args); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, key }; + opts.op_type = op_type; + opts.specs = pyObj_spec; + prepare_and_execute_lookup_in_op(&opts, nspecs, pyObj_callback, pyObj_errback, barrier); + break; + } + case Operations::LOOKUP_IN_ALL_REPLICAS: { + auto opts = get_lookup_in_replica_options(pyObj_op_args); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, key }; + opts.op_type = op_type; + opts.specs = pyObj_spec; + prepare_and_execute_lookup_in_all_replicas_op( + &opts, nspecs, pyObj_callback, pyObj_errback, barrier); + break; + } + case Operations::LOOKUP_IN_ANY_REPLICA: { + auto opts = get_lookup_in_replica_options(pyObj_op_args); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, key }; + opts.op_type = op_type; + opts.specs = pyObj_spec; + prepare_and_execute_lookup_in_any_replica_op( + &opts, nspecs, pyObj_callback, pyObj_errback, barrier); + break; + } + case Operations::MUTATE_IN: { + auto opts = get_mutate_in_options(pyObj_op_args); + opts.conn = conn; + opts.id = couchbase::core::document_id{ bucket, scope, collection, key }; + opts.op_type = op_type; + opts.specs = pyObj_spec; + prepare_and_execute_mutate_in_op(&opts, nspecs, pyObj_callback, pyObj_errback, barrier); + break; + } + default: { + pycbc_set_python_exception(PycbcError::InvalidArgument, + __FILE__, + __LINE__, + "Unrecognized subdoc operation passed in."); + if (barrier) { + barrier->set_value(nullptr); + } + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + break; + } + }; + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = fut.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} diff --git a/src/subdoc_ops.hxx b/src/subdoc_ops.hxx new file mode 100644 index 000000000..4e3c3aee1 --- /dev/null +++ b/src/subdoc_ops.hxx @@ -0,0 +1,102 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" +#include <core/impl/subdoc/opcode.hxx> +#include <core/impl/subdoc/path_flags.hxx> +#include <couchbase/read_preference.hxx> +#include <couchbase/store_semantics.hxx> + +struct mutate_in_spec { + uint8_t op; + uint8_t flags; + char* path; + std::vector<std::byte> value; + + PyObject* pyObj_value; + bool create_parents; + bool xattr; + bool expand_macros; +}; + +struct lookup_in_spec { + uint8_t op; + uint8_t flags; + char* path; + bool xattr; +}; + +struct lookup_in_options { + // required + connection* conn; + couchbase::core::document_id id; + Operations::OperationType op_type{ Operations::LOOKUP_IN }; + + // optional + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + bool access_deleted{ false }; + PyObject* span{ nullptr }; + PyObject* specs{ nullptr }; + + // TODO: + // retries? + // partition? +}; + +struct lookup_in_replica_options { + // required + connection* conn; + couchbase::core::document_id id; + Operations::OperationType op_type; + + // optional + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + couchbase::read_preference read_preference{ couchbase::read_preference::no_preference }; + PyObject* span{ nullptr }; + PyObject* specs{ nullptr }; +}; + +struct mutate_in_options { + // required + connection* conn; + couchbase::core::document_id id; + Operations::OperationType op_type{ Operations::MUTATE_IN }; + + // optional + couchbase::durability_level durability_level{ couchbase::durability_level::none }; + bool use_legacy_durability{ false }; + couchbase::replicate_to replicate_to{ couchbase::replicate_to::none }; + couchbase::persist_to persist_to{ couchbase::persist_to::none }; + couchbase::store_semantics store_semantics{ couchbase::store_semantics::replace }; + uint32_t expiry{ 0 }; + couchbase::cas cas; + std::chrono::milliseconds timeout_ms = couchbase::core::timeout_defaults::key_value_timeout; + bool preserve_expiry{ false }; + bool access_deleted{ false }; + bool create_as_deleted{ false }; + PyObject* span{ nullptr }; + PyObject* specs{ nullptr }; + + // TODO: + // retries? + // partition? +}; + +PyObject* +handle_subdoc_op(PyObject* self, PyObject* args, PyObject* kwargs); diff --git a/src/tracing.hxx b/src/tracing.hxx new file mode 100644 index 000000000..c4f9fe681 --- /dev/null +++ b/src/tracing.hxx @@ -0,0 +1,138 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include <couchbase/tracing/request_tracer.hxx> +// NOLINTNEXTLINE +#include "Python.h" // NOLINT +#include <iostream> +// convenient aliasing... +namespace tracing = couchbase::tracing; + +namespace pycbc +{ + +class request_span : public tracing::request_span +{ +public: + explicit request_span(PyObject* span, std::shared_ptr<tracing::request_span> parent = nullptr) + : tracing::request_span("", parent) // name doesn't matter - it is in the underlying python span + , pyObj_span_(span) + { + // only called by start_span, which has the GIL, so... + Py_INCREF(span); + pyObj_set_attribute_ = PyObject_GetAttrString(pyObj_span_, "set_attribute"); + } + + ~request_span() override + { + PyGILState_STATE state = PyGILState_Ensure(); + Py_DECREF(pyObj_set_attribute_); + Py_DECREF(pyObj_span_); + PyGILState_Release(state); + } + + void add_tag(const std::string& name, std::uint64_t value) override + { + PyGILState_STATE state = PyGILState_Ensure(); + auto pyObj_args = Py_BuildValue("(sn)", name.c_str(), static_cast<Py_ssize_t>(value)); + PyObject_Call(pyObj_set_attribute_, pyObj_args, nullptr); + Py_DECREF(pyObj_args); + PyGILState_Release(state); + } + void add_tag(const std::string& name, const std::string& value) override + { + PyGILState_STATE state = PyGILState_Ensure(); + auto pyObj_args = Py_BuildValue("(ss)", name.c_str(), value.c_str()); + PyObject_Call(pyObj_set_attribute_, pyObj_args, nullptr); + Py_DECREF(pyObj_args); + PyGILState_Release(state); + } + void end() override + { + PyGILState_STATE state = PyGILState_Ensure(); + auto pyObj_end = PyObject_GetAttrString(pyObj_span_, "finish"); + PyObject_CallObject(pyObj_end, nullptr); + Py_DECREF(pyObj_end); + PyGILState_Release(state); + } + + PyObject* py_span() + { + return pyObj_span_; + } + +private: + PyObject* pyObj_span_; + PyObject* pyObj_set_attribute_; + PyObject* pyObj_get_context_; +}; + +class request_tracer : public tracing::request_tracer +{ +public: + request_tracer(PyObject* tracer) + : pyObj_tracer_(tracer) + { + // Assumption here is we have the GIL when we wrap the python tracer here + Py_INCREF(tracer); + pyObj_start_span_ = PyObject_GetAttrString(tracer, "start_span"); + assert(pyObj_start_span_); + } + + ~request_tracer() + { + PyGILState_STATE state = PyGILState_Ensure(); + Py_DECREF(pyObj_start_span_); + Py_DECREF(pyObj_tracer_); + PyGILState_Release(state); + } + + std::shared_ptr<tracing::request_span> start_span( + std::string name, + std::shared_ptr<tracing::request_span> parent = {}) override + { + // defer to the pyObj_tracer_, and wrap the result in a pycbc span. Note: Taking the GIL here, + // and elsewhere (like in the request_span) isn't perhaps the most efficient strategy. We could + // cache spans and periodically (or just when asked) grab the GIL and create them. However, + // lets do this first, then think about optimizations + PyGILState_STATE state = PyGILState_Ensure(); + PyObject* pyObj_name = PyUnicode_FromString(name.c_str()); + PyObject* pyObj_args = PyTuple_New(0); + PyObject* pyObj_kwargs = PyDict_New(); + PyDict_SetItemString(pyObj_kwargs, "name", pyObj_name); + if (parent) { + auto pyObj_parent = std::dynamic_pointer_cast<pycbc::request_span>(parent)->py_span(); + PyDict_SetItemString(pyObj_kwargs, "parent", pyObj_parent); + } + auto pyObj_span = PyObject_Call(pyObj_start_span_, pyObj_args, pyObj_kwargs); + auto retval = std::make_shared<request_span>(pyObj_span, parent); + Py_DECREF(pyObj_name); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + Py_DECREF(pyObj_span); + PyGILState_Release(state); + return retval; + } + +private: + PyObject* pyObj_tracer_; + PyObject* pyObj_start_span_; +}; + +} // namespace pycbc diff --git a/src/transactions/transactions.cxx b/src/transactions/transactions.cxx new file mode 100644 index 000000000..e01f1d1a1 --- /dev/null +++ b/src/transactions/transactions.cxx @@ -0,0 +1,1570 @@ + +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "transactions.hxx" +#include "../exceptions.hxx" +#include "../n1ql.hxx" +#include "../utils.hxx" +#include <core/cluster.hxx> +#include <core/operations.hxx> +#include <core/transactions/durability_level.hxx> +#include <core/transactions/internal/exceptions_internal.hxx> +#include <core/transactions/transaction_get_result.hxx> +#include <couchbase/query_scan_consistency.hxx> +#include <sstream> + +void +add_to_dict(PyObject* dict, std::string key, std::string value) +{ + PyObject* pyObj_value = PyUnicode_FromString(value.c_str()); + PyDict_SetItemString(dict, key.c_str(), pyObj_value); + Py_DECREF(pyObj_value); +} + +void +add_to_dict(PyObject* dict, std::string key, int64_t value) +{ + PyObject* pyObj_val = PyLong_FromLongLong(value); + PyDict_SetItemString(dict, key.c_str(), pyObj_val); + Py_DECREF(pyObj_val); +} + +void +add_to_dict(PyObject* dict, std::string key, bool value) +{ + PyDict_SetItemString(dict, key.c_str(), value ? Py_True : Py_False); +} + +void +pycbc_txns::dealloc_transactions(PyObject* obj) +{ + auto txns = reinterpret_cast<pycbc_txns::transactions*>(PyCapsule_GetPointer(obj, "txns_")); + txns->txns->close(); + txns->txns.reset(); + CB_LOG_DEBUG("dealloc transactions"); +} + +void +pycbc_txns::dealloc_transaction_context(PyObject* obj) +{ + auto ctx = reinterpret_cast<pycbc_txns::transaction_context*>(PyCapsule_GetPointer(obj, "ctx_")); + delete ctx; + CB_LOG_DEBUG("dealloc transaction_context"); +} + +/* pycbc_txns::transaction_config type methods */ + +void +pycbc_txns::transaction_config__dealloc__(pycbc_txns::transaction_config* cfg) +{ + delete cfg->cfg; + Py_TYPE(cfg)->tp_free((PyObject*)cfg); + CB_LOG_DEBUG("dealloc transaction_config"); +} + +PyObject* +pycbc_txns::transaction_config__to_dict__(PyObject* self) +{ + auto conf = reinterpret_cast<pycbc_txns::transaction_config*>(self); + PyObject* retval = PyDict_New(); + add_to_dict(retval, "durability_level", static_cast<int64_t>(conf->cfg->durability_level())); + add_to_dict(retval, + "cleanup_window", + static_cast<int64_t>(conf->cfg->cleanup_config().cleanup_window().count())); + add_to_dict(retval, "timeout", static_cast<int64_t>(conf->cfg->timeout().count())); + add_to_dict(retval, "cleanup_lost_attempts", conf->cfg->cleanup_config().cleanup_lost_attempts()); + add_to_dict( + retval, "cleanup_client_attempts", conf->cfg->cleanup_config().cleanup_client_attempts()); + add_to_dict(retval, + "scan_consistency", + scan_consistency_type_to_string(conf->cfg->query_config().scan_consistency())); + if (conf->cfg->metadata_collection()) { + std::string meta = fmt::format("{}.{}.{}", + conf->cfg->metadata_collection()->bucket, + conf->cfg->metadata_collection()->scope, + conf->cfg->metadata_collection()->collection); + add_to_dict(retval, "metadata_collection", meta); + } + return retval; +} + +static PyMethodDef transaction_config_methods[] = { + { "to_dict", + (PyCFunction)pycbc_txns::transaction_config__to_dict__, + METH_NOARGS, + PyDoc_STR("transaction_config as a dict") }, + { NULL, NULL, 0, NULL } +}; + +PyObject* +pycbc_txns::transaction_config__new__(PyTypeObject* type, PyObject* args, PyObject* kwargs) +{ + PyObject* durability_level = nullptr; + PyObject* cleanup_window = nullptr; + PyObject* timeout = nullptr; + char* scan_consistency = nullptr; + PyObject* cleanup_lost_attempts = nullptr; + PyObject* cleanup_client_attempts = nullptr; + char* metadata_bucket = nullptr; + char* metadata_scope = nullptr; + char* metadata_collection = nullptr; + + const char* kw_list[] = { "durability_level", + "cleanup_window", + "timeout", + "cleanup_lost_attempts", + "cleanup_client_attempts", + "metadata_bucket", + "metadata_scope", + "metadata_collection", + "scan_consistency", + nullptr }; + const char* kw_format = "|OOOOOssss"; + auto self = reinterpret_cast<pycbc_txns::transaction_config*>(type->tp_alloc(type, 0)); + + self->cfg = new tx::transactions_config(); + + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &durability_level, + &cleanup_window, + &timeout, + &cleanup_lost_attempts, + &cleanup_client_attempts, + &metadata_bucket, + &metadata_scope, + &metadata_collection, + &scan_consistency)) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + Py_RETURN_NONE; + } + if (nullptr != durability_level) { + self->cfg->durability_level( + static_cast<couchbase::durability_level>(PyLong_AsUnsignedLong(durability_level))); + } + if (nullptr != cleanup_window) { + self->cfg->cleanup_config().cleanup_window( + std::chrono::microseconds(PyLong_AsUnsignedLongLong(cleanup_window))); + } + if (nullptr != timeout) { + self->cfg->timeout(std::chrono::microseconds(PyLong_AsUnsignedLongLong(timeout))); + } + if (nullptr != cleanup_lost_attempts) { + self->cfg->cleanup_config().cleanup_lost_attempts(!!PyObject_IsTrue(cleanup_lost_attempts)); + } + if (nullptr != cleanup_client_attempts) { + self->cfg->cleanup_config().cleanup_client_attempts(!!PyObject_IsTrue(cleanup_client_attempts)); + } + if (nullptr != metadata_bucket && nullptr != metadata_scope && nullptr != metadata_collection) { + auto keyspace = + tx::transaction_keyspace{ metadata_bucket, metadata_scope, metadata_collection }; + self->cfg->metadata_collection(keyspace); + } + if (nullptr != scan_consistency) { + self->cfg->query_config().scan_consistency( + str_to_scan_consistency_type<couchbase::query_scan_consistency>(scan_consistency)); + } + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_transaction_config_type() +{ + PyTypeObject r = {}; + r.tp_name = "pycbc_core.transaction_config"; + r.tp_doc = "Transaction configuration"; + r.tp_basicsize = sizeof(pycbc_txns::transaction_config); + r.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + r.tp_new = pycbc_txns::transaction_config__new__; + r.tp_dealloc = (destructor)pycbc_txns::transaction_config__dealloc__; + r.tp_methods = transaction_config_methods; + return r; +} + +static PyTypeObject transaction_config_type = init_transaction_config_type(); + +/* pycbc_txns::transaction_options type methods */ + +void +pycbc_txns::transaction_options__dealloc__(pycbc_txns::transaction_options* opts) +{ + delete opts->opts; + Py_TYPE(opts)->tp_free((PyObject*)opts); + CB_LOG_DEBUG("dealloc transaction_options"); +} + +PyObject* +pycbc_txns::transaction_options__to_dict__(PyObject* self) +{ + auto opts = reinterpret_cast<pycbc_txns::transaction_options*>(self); + PyObject* retval = PyDict_New(); + if (opts->opts->timeout()) { + add_to_dict(retval, "timeout", static_cast<int64_t>(opts->opts->timeout()->count())); + } + if (opts->opts->durability_level()) { + add_to_dict( + retval, "durability_level", static_cast<int64_t>(opts->opts->durability_level().value())); + } + if (opts->opts->scan_consistency()) { + add_to_dict( + retval, "scan_consistency", scan_consistency_type_to_string(*opts->opts->scan_consistency())); + } + if (opts->opts->metadata_collection()) { + std::string meta = fmt::format("{}.{}.{}", + opts->opts->metadata_collection()->bucket, + opts->opts->metadata_collection()->scope, + opts->opts->metadata_collection()->collection); + add_to_dict(retval, "metadata_collection", meta); + } + return retval; +} + +PyObject* +pycbc_txns::transaction_options__str__(PyObject* self) +{ + auto opts = reinterpret_cast<pycbc_txns::transaction_options*>(self)->opts; + std::stringstream stream; + stream << "transaction_options{"; + if (nullptr != opts) { + if (opts->durability_level()) { + stream << "durability: " << tx_core::durability_level_to_string(*opts->durability_level()) + << ", "; + } + if (opts->timeout()) { + stream << "timeout: " << opts->timeout()->count() << "ns, "; + } + if (opts->scan_consistency()) { + stream << "scan_consistency: " << scan_consistency_type_to_string(*opts->scan_consistency()); + } + } + stream << "}"; + return PyUnicode_FromString(stream.str().c_str()); +} + +static PyMethodDef transaction_options_methods[] = { + { "to_dict", + (PyCFunction)pycbc_txns::transaction_options__to_dict__, + METH_NOARGS, + PyDoc_STR("transaction_options as a dict") }, + { NULL, NULL, 0, NULL } +}; + +PyObject* +pycbc_txns::transaction_options__new__(PyTypeObject* type, PyObject* args, PyObject* kwargs) +{ + PyObject* durability_level = nullptr; + PyObject* timeout = nullptr; + char* scan_consistency = nullptr; + char* metadata_bucket = nullptr; + char* metadata_scope = nullptr; + char* metadata_collection = nullptr; + + const char* kw_list[] = { + "durability_level", "timeout", "scan_consistency", "metadata_bucket", "metadata_scope", + "metadata_collection", nullptr + }; + const char* kw_format = "|OOssss"; + auto self = reinterpret_cast<pycbc_txns::transaction_options*>(type->tp_alloc(type, 0)); + + self->opts = new tx::transaction_options(); + CB_LOG_DEBUG("transaction_options__new__ called"); + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &durability_level, + &timeout, + &scan_consistency, + &metadata_bucket, + &metadata_scope, + &metadata_collection)) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + Py_RETURN_NONE; + } + if (nullptr != durability_level) { + self->opts->durability_level( + static_cast<couchbase::durability_level>(PyLong_AsUnsignedLong(durability_level))); + } + if (nullptr != timeout) { + self->opts->timeout(std::chrono::microseconds(PyLong_AsUnsignedLongLong(timeout))); + } + if (nullptr != scan_consistency) { + self->opts->scan_consistency( + str_to_scan_consistency_type<couchbase::query_scan_consistency>(scan_consistency)); + } + if (nullptr != metadata_bucket && nullptr != metadata_scope && nullptr != metadata_collection) { + auto keyspace = + tx::transaction_keyspace{ metadata_bucket, metadata_scope, metadata_collection }; + self->opts->metadata_collection(keyspace); + } + + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_transaction_options_type() +{ + PyTypeObject r = {}; + r.tp_name = "pycbc_core.transaction_options"; + r.tp_doc = "Transaction options"; + r.tp_basicsize = sizeof(pycbc_txns::transaction_options); + r.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + r.tp_new = pycbc_txns::transaction_options__new__; + r.tp_str = (reprfunc)pycbc_txns::transaction_options__str__; + r.tp_dealloc = (destructor)pycbc_txns::transaction_options__dealloc__; + r.tp_methods = transaction_options_methods; + return r; +} + +static PyTypeObject transaction_options_type = init_transaction_options_type(); + +/* pycbc_txns::transaction_query_options type methods */ + +void +pycbc_txns::transaction_query_options__dealloc__(pycbc_txns::transaction_query_options* opts) +{ + delete opts->opts; + Py_TYPE(opts)->tp_free((PyObject*)opts); + CB_LOG_DEBUG("dealloc transaction_query_options"); +} + +PyObject* +pycbc_txns::transaction_query_options__to_dict__(PyObject* self) +{ + auto opts = reinterpret_cast<pycbc_txns::transaction_query_options*>(self); + PyObject* retval = PyDict_New(); + auto query_opts = opts->opts->get_query_options().build(); + add_to_dict(retval, "adhoc", query_opts.adhoc); + add_to_dict(retval, "metrics", query_opts.metrics); + add_to_dict(retval, "read_only", query_opts.readonly); + add_to_dict(retval, "flex_index", query_opts.flex_index); + add_to_dict(retval, "preserve_expiry", query_opts.preserve_expiry); + if (query_opts.max_parallelism.has_value()) { + add_to_dict( + retval, "max_parallelism", static_cast<int64_t>(query_opts.max_parallelism.value())); + } + if (query_opts.scan_cap.has_value()) { + add_to_dict(retval, "scan_cap", static_cast<int64_t>(query_opts.scan_cap.value())); + } + if (query_opts.scan_wait) { + add_to_dict(retval, "scan_wait", static_cast<int64_t>(query_opts.scan_wait->count())); + } + if (query_opts.pipeline_batch.has_value()) { + add_to_dict(retval, "pipeline_batch", static_cast<int64_t>(query_opts.pipeline_batch.value())); + } + if (query_opts.pipeline_cap.has_value()) { + add_to_dict(retval, "pipeline_cap", static_cast<int64_t>(query_opts.pipeline_cap.value())); + } + if (query_opts.client_context_id.has_value()) { + add_to_dict(retval, "client_context_id", query_opts.client_context_id.value()); + } + if (query_opts.scan_consistency.has_value()) { + add_to_dict(retval, + "scan_consistency", + scan_consistency_type_to_string(query_opts.scan_consistency.value())); + } + if (query_opts.profile.has_value()) { + add_to_dict(retval, "profile", profile_mode_to_str(query_opts.profile.value())); + } + + if (!query_opts.raw.empty()) { + PyObject* raw = PyDict_New(); + for (auto const& [key, val] : query_opts.raw) { + auto val_str = binary_to_string(val); + add_to_dict(raw, key, val_str); + } + PyDict_SetItemString(retval, "raw", raw); + Py_DECREF(raw); + } + + if (!query_opts.positional_parameters.empty()) { + PyObject* pyObj_pos = PyList_New(0); + for (auto& val : query_opts.positional_parameters) { + auto val_str = binary_to_string(val); + PyObject* pyObj_val = PyUnicode_FromString(val_str.c_str()); + PyList_Append(pyObj_pos, pyObj_val); + Py_DECREF(pyObj_val); + } + PyDict_SetItemString(retval, "positional_parameters", pyObj_pos); + Py_DECREF(pyObj_pos); + } + if (!query_opts.named_parameters.empty()) { + PyObject* pyObj_named = PyDict_New(); + for (auto& [key, value] : query_opts.named_parameters) { + auto val_str = binary_to_string(value); + add_to_dict(pyObj_named, key, val_str); + } + PyDict_SetItemString(retval, "named_parameters", pyObj_named); + Py_DECREF(pyObj_named); + } + return retval; +} + +static PyMethodDef transaction_query_options_methods[] = { + { "to_dict", + (PyCFunction)pycbc_txns::transaction_query_options__to_dict__, + METH_NOARGS, + PyDoc_STR("transaction_query_options as a dict") }, + { NULL, NULL, 0, NULL } +}; + +PyObject* +pycbc_txns::transaction_query_options__new__(PyTypeObject* type, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_query_args = nullptr; + const char* kw_list[] = { "query_args", nullptr }; + const char* kw_format = "|O"; + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, kw_format, const_cast<char**>(kw_list), &pyObj_query_args)) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + Py_RETURN_NONE; + } + auto self = reinterpret_cast<pycbc_txns::transaction_query_options*>(type->tp_alloc(type, 0)); + auto req = build_query_request(pyObj_query_args); + if (PyErr_Occurred()) { + return nullptr; + } + self->opts = new tx::transaction_query_options(); + self->opts->ad_hoc(req.adhoc); + self->opts->metrics(req.metrics); + self->opts->readonly(req.readonly); + // @TODO: add flex index to txn QueryOptions eventually? + // self->opts->flex_index(req.flex_index); + if (req.max_parallelism.has_value()) { + self->opts->max_parallelism(req.max_parallelism.value()); + } + if (req.scan_cap.has_value()) { + self->opts->scan_cap(req.scan_cap.value()); + } + if (req.scan_wait.has_value()) { + self->opts->scan_wait(req.scan_wait.value()); + } + if (req.scan_cap.has_value()) { + self->opts->scan_cap(req.scan_cap.value()); + } + if (req.pipeline_batch.has_value()) { + self->opts->pipeline_batch(req.pipeline_batch.value()); + } + if (req.pipeline_cap.has_value()) { + self->opts->pipeline_cap(req.pipeline_cap.value()); + } + if (req.client_context_id.has_value()) { + self->opts->client_context_id(req.client_context_id.value()); + } + if (req.scan_consistency.has_value()) { + self->opts->scan_consistency(req.scan_consistency.value()); + } + if (req.profile.has_value()) { + self->opts->profile(req.profile.value()); + } + if (req.raw.size() > 0) { + std::map<std::string, std::vector<std::byte>, std::less<>> raw_options{}; + for (auto& [name, option] : req.raw) { + raw_options[name] = std::move(option.bytes()); + } + self->opts->encoded_raw_options(raw_options); + } + if (req.positional_parameters.size() > 0) { + std::vector<std::vector<std::byte>> positional_params{}; + for (auto& param : req.positional_parameters) { + positional_params.emplace_back(std::move(param.bytes())); + } + self->opts->encoded_positional_parameters(positional_params); + } + if (req.named_parameters.size() > 0) { + std::map<std::string, std::vector<std::byte>, std::less<>> named_params{}; + for (auto& [name, param] : req.named_parameters) { + named_params[name] = std::move(param.bytes()); + } + self->opts->encoded_named_parameters(named_params); + } + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_transaction_query_options_type() +{ + PyTypeObject r = {}; + r.tp_name = "pycbc_core.transaction_query_options"; + r.tp_doc = "Transaction query options"; + r.tp_basicsize = sizeof(pycbc_txns::transaction_query_options); + r.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + r.tp_new = pycbc_txns::transaction_query_options__new__; + r.tp_dealloc = (destructor)pycbc_txns::transaction_query_options__dealloc__; + r.tp_methods = transaction_query_options_methods; + return r; +} + +static PyTypeObject transaction_query_options_type = init_transaction_query_options_type(); + +/* pycbc_txns::transaction_query_options type methods */ + +void +pycbc_txns::transaction_get_result__dealloc__(pycbc_txns::transaction_get_result* result) +{ + result->res.reset(); + Py_TYPE(result)->tp_free((PyObject*)result); + CB_LOG_DEBUG("dealloc transaction_get_result"); +} + +PyObject* +pycbc_txns::transaction_get_result__str__(pycbc_txns::transaction_get_result* result) +{ + if (result->res->content().data.size() != 0) { + auto value = reinterpret_cast<const char*>(result->res->content().data.data()); + auto flags = result->res->content().flags; + return PyUnicode_FromFormat("transaction_get_result:{key=%s, cas=%llu, value=%s, flags=%lu}", + result->res->id().key().c_str(), + result->res->cas(), + value, + flags); + } else { + return PyUnicode_FromFormat("transaction_get_result:{key=%s, cas=%llu}", + result->res->id().key().c_str(), + result->res->cas()); + } +} + +// TODO: a better way later, perhaps an exposed enum like operations +const std::string ID{ "id" }; +const std::string CAS{ "cas" }; +const std::string VALUE{ "value" }; + +PyObject* +pycbc_txns::transaction_get_result__get__(pycbc_txns::transaction_get_result* result, + PyObject* args) +{ + const char* field_name = nullptr; + PyObject* default_value = nullptr; + if (!PyArg_ParseTuple(args, "s|O", &field_name, &default_value)) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + Py_RETURN_NONE; + } + if (ID == field_name) { + return PyUnicode_FromString(result->res->id().key().c_str()); + } + if (CAS == field_name) { + return PyLong_FromUnsignedLongLong(result->res->cas().value()); + } + if (VALUE == field_name) { + PyObject* pyObj_value = nullptr; + PyObject* pyObj_flags = PyLong_FromUnsignedLong(result->res->content().flags); + try { + pyObj_value = binary_to_PyObject(result->res->content().data); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_TypeError, e.what()); + Py_RETURN_NONE; + } + PyObject* pyObj_result = PyTuple_Pack(2, pyObj_value, pyObj_flags); + Py_DECREF(pyObj_value); + Py_DECREF(pyObj_flags); + return pyObj_result; + } + PyErr_SetString(PyExc_ValueError, fmt::format("unknown field_name {}", field_name).c_str()); + Py_RETURN_NONE; +} + +static PyMethodDef transaction_get_result_methods[] = { + { "get", + (PyCFunction)pycbc_txns::transaction_get_result__get__, + METH_VARARGS, + PyDoc_STR("get field in result object") }, + { NULL, NULL, 0, NULL } +}; + +PyObject* +pycbc_txns::transaction_get_result__new__(PyTypeObject* type, PyObject*, PyObject*) +{ + auto self = reinterpret_cast<pycbc_txns::transaction_get_result*>(type->tp_alloc(type, 0)); + return reinterpret_cast<PyObject*>(self); +} + +static PyTypeObject +init_transaction_get_result_type() +{ + PyTypeObject r = {}; + r.tp_name = "pycbc_core.transaction_get_result"; + r.tp_doc = "Result of transaction operation on client"; + r.tp_basicsize = sizeof(pycbc_txns::transaction_get_result); + r.tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; + r.tp_new = pycbc_txns::transaction_get_result__new__; + r.tp_dealloc = (destructor)pycbc_txns::transaction_get_result__dealloc__; + r.tp_methods = transaction_get_result_methods; + r.tp_repr = (reprfunc)pycbc_txns::transaction_get_result__str__; + return r; +} + +static PyTypeObject transaction_get_result_type = init_transaction_get_result_type(); + +PyObject* +pycbc_txns::add_transaction_objects(PyObject* pyObj_module) +{ + PyObject* pyObj_enum_module = PyImport_ImportModule("enum"); + if (!pyObj_enum_module) { + return nullptr; + } + PyObject* pyObj_enum_class = PyObject_GetAttrString(pyObj_enum_module, "Enum"); + PyObject* pyObj_enum_values = PyUnicode_FromString(pycbc_txns::TxOperations::ALL_OPERATIONS()); + PyObject* pyObj_enum_name = PyUnicode_FromString("TransactionOperations"); + PyObject* pyObj_args = PyTuple_Pack(2, pyObj_enum_name, pyObj_enum_values); + Py_DECREF(pyObj_enum_name); + Py_DECREF(pyObj_enum_values); + + PyObject* pyObj_kwargs = PyDict_New(); + PyObject_SetItem( + pyObj_kwargs, PyUnicode_FromString("module"), PyModule_GetNameObject(pyObj_module)); + PyObject* transaction_operations = PyObject_Call(pyObj_enum_class, pyObj_args, pyObj_kwargs); + Py_DECREF(pyObj_args); + Py_DECREF(pyObj_kwargs); + + if (PyModule_AddObject(pyObj_module, "transaction_operations", transaction_operations)) { + Py_XDECREF(transaction_operations); + return nullptr; + } + Py_DECREF(pyObj_enum_class); + Py_DECREF(pyObj_enum_module); + if (PyType_Ready(&transaction_get_result_type) == 0) { + Py_INCREF(&transaction_get_result_type); + if (PyModule_AddObject(pyObj_module, + "transaction_get_result", + reinterpret_cast<PyObject*>(&transaction_get_result_type)) == 0) { + if (PyType_Ready(&transaction_config_type) == 0) { + Py_INCREF(&transaction_config_type); + if (PyModule_AddObject(pyObj_module, + "transaction_config", + reinterpret_cast<PyObject*>(&transaction_config_type)) == 0) { + if (PyType_Ready(&transaction_query_options_type) == 0) { + Py_INCREF(&transaction_query_options_type); + if (PyModule_AddObject(pyObj_module, + "transaction_query_options", + reinterpret_cast<PyObject*>(&transaction_query_options_type)) == + 0) { + if (PyType_Ready(&transaction_options_type) == 0) { + Py_INCREF(&transaction_options_type); + if (PyModule_AddObject(pyObj_module, + "transaction_options", + reinterpret_cast<PyObject*>(&transaction_options_type)) == + 0) { + return pyObj_module; + } + Py_DECREF(&transaction_options_type); + } + } + Py_DECREF(&transaction_query_options_type); + } + } + Py_DECREF(&transaction_config_type); + } + } + Py_DECREF(&transaction_get_result_type); + } + Py_DECREF(pyObj_module); + return nullptr; +} + +PyObject* +pycbc_txns::create_transactions([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // we expect it to be called like: + // create_transactions(conn, config) + PyObject* pyObj_conn = nullptr; + PyObject* pyObj_config = nullptr; + const char* kw_list[] = { "conn", "config", nullptr }; + const char* kw_format = "O!O"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &pyObj_config); + + if (!ret) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + Py_RETURN_NONE; + } + if (nullptr == pyObj_conn) { + PyErr_SetString(PyExc_ValueError, "expected a connection object"); + Py_RETURN_NONE; + } + if (nullptr == pyObj_config) { + PyErr_SetString(PyExc_ValueError, "expected a TransactionConfig object"); + Py_RETURN_NONE; + } + + std::pair<std::error_code, std::shared_ptr<tx_core::transactions>> res; + Py_BEGIN_ALLOW_THREADS auto conn = + reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + auto txn_config = reinterpret_cast<pycbc_txns::transaction_config*>(pyObj_config)->cfg; + std::future<std::pair<std::error_code, std::shared_ptr<tx_core::transactions>>> fut = + tx_core::transactions::create(conn->cluster_, *txn_config); + res = fut.get(); + Py_END_ALLOW_THREADS if (res.first.value()) + { + pycbc_set_python_exception(res.first, __FILE__, __LINE__, res.first.message().c_str()); + return nullptr; + } + pycbc_txns::transactions* txns = new pycbc_txns::transactions(res.second); + PyObject* pyObj_txns = PyCapsule_New(txns, "txns_", dealloc_transactions); + return pyObj_txns; +} + +PyObject* +pycbc_txns::destroy_transactions([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_txns = nullptr; + const char* kw_list[] = { "txns", nullptr }; + const char* kw_format = "O!"; + if (!PyArg_ParseTupleAndKeywords( + args, kwargs, kw_format, const_cast<char**>(kw_list), &PyCapsule_Type, &pyObj_txns)) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + Py_RETURN_NONE; + } + if (nullptr == pyObj_txns) { + PyErr_SetString(PyExc_ValueError, "expected a transactions object"); + Py_RETURN_NONE; + } + auto txns = + reinterpret_cast<pycbc_txns::transactions*>(PyCapsule_GetPointer(pyObj_txns, "txns_")); + if (nullptr == txns) { + PyErr_SetString(PyExc_ValueError, "passed null transactions"); + Py_RETURN_NONE; + } + Py_BEGIN_ALLOW_THREADS txns->txns->close(); + Py_END_ALLOW_THREADS Py_RETURN_NONE; +} + +PyObject* +init_transaction_exception_type(const char* klass) +{ + static PyObject* couchbase_exceptions = PyImport_ImportModule("couchbase.exceptions"); + assert(nullptr != couchbase_exceptions); + return PyObject_GetAttrString(couchbase_exceptions, klass); +} + +std::string +txn_external_exception_to_string(tx_core::external_exception ext_exception) +{ + switch (ext_exception) { + case tx_core::external_exception::UNKNOWN: + return "unknown"; + case tx_core::external_exception::COUCHBASE_EXCEPTION: + return "couchbase_exception"; + case tx_core::external_exception::NOT_SET: + return "not_set"; + case tx_core::external_exception::ACTIVE_TRANSACTION_RECORD_ENTRY_NOT_FOUND: + return "active_transaction_record_entry_not_found"; + case tx_core::external_exception::ACTIVE_TRANSACTION_RECORD_FULL: + return "active_transaction_record_full"; + case tx_core::external_exception::COMMIT_NOT_PERMITTED: + return "commit_not_permitted"; + case tx_core::external_exception::ACTIVE_TRANSACTION_RECORD_NOT_FOUND: + return "active_transaction_record_not_found"; + case tx_core::external_exception::CONCURRENT_OPERATIONS_DETECTED_ON_SAME_DOCUMENT: + return "concurrent_operations_detected_on_same_document"; + case tx_core::external_exception::DOCUMENT_ALREADY_IN_TRANSACTION: + return "document_already_in_transaction"; + case tx_core::external_exception::DOCUMENT_EXISTS_EXCEPTION: + return "document_exists_exception"; + case tx_core::external_exception::DOCUMENT_NOT_FOUND_EXCEPTION: + return "document_not_found_exception"; + case tx_core::external_exception::FEATURE_NOT_AVAILABLE_EXCEPTION: + return "feature_not_available_exception"; + case tx_core::external_exception::FORWARD_COMPATIBILITY_FAILURE: + return "forward_compatibility_failure"; + case tx_core::external_exception::ILLEGAL_STATE_EXCEPTION: + return "illegal_state_exception"; + case tx_core::external_exception::PARSING_FAILURE: + return "parsing_failure"; + case tx_core::external_exception::PREVIOUS_OPERATION_FAILED: + return "previous_operation_failed"; + case tx_core::external_exception::REQUEST_CANCELED_EXCEPTION: + return "request_canceled_exception"; + case tx_core::external_exception::ROLLBACK_NOT_PERMITTED: + return "rollback_not_permitted"; + case tx_core::external_exception::SERVICE_NOT_AVAILABLE_EXCEPTION: + return "service_not_available_exception"; + case tx_core::external_exception::TRANSACTION_ABORTED_EXTERNALLY: + return "transaction_aborted_externally"; + case tx_core::external_exception::TRANSACTION_ALREADY_ABORTED: + return "transaction_already_aborted"; + case tx_core::external_exception::TRANSACTION_ALREADY_COMMITTED: + return "transaction_already_committed"; + } + return "unknown"; +} + +PyObject* +create_python_exception(pycbc_txns::TxnExceptionType exc_type, + const char* message, + bool set_exception = false, + PyObject* pyObj_inner_exc = nullptr) +{ + static PyObject* pyObj_txn_failed = init_transaction_exception_type("TransactionFailed"); + static PyObject* pyObj_txn_expired = init_transaction_exception_type("TransactionExpired"); + static PyObject* pyObj_txn_ambig = init_transaction_exception_type("TransactionCommitAmbiguous"); + static PyObject* pyObj_txn_op_failed = + init_transaction_exception_type("TransactionOperationFailed"); + static PyObject* pyObj_document_exists_ex = + init_transaction_exception_type("DocumentExistsException"); + static PyObject* pyObj_document_not_found_ex = + init_transaction_exception_type("DocumentNotFoundException"); + static PyObject* pyObj_query_parsing_failure = + init_transaction_exception_type("ParsingFailedException"); + static PyObject* pyObj_couchbase_error = init_transaction_exception_type("CouchbaseException"); + static PyObject* pyObj_feature_not_available_error = + init_transaction_exception_type("FeatureUnavailableException"); + + PyObject* pyObj_final_error = nullptr; + PyObject* pyObj_exc_type = nullptr; + PyObject* pyObj_error_ctx = PyDict_New(); + + switch (exc_type) { + case pycbc_txns::TxnExceptionType::TRANSACTION_FAILED: { + pyObj_exc_type = pyObj_txn_failed; + break; + } + case pycbc_txns::TxnExceptionType::TRANSACTION_COMMIT_AMBIGUOUS: { + pyObj_exc_type = pyObj_txn_ambig; + break; + } + case pycbc_txns::TxnExceptionType::TRANSACTION_EXPIRED: { + pyObj_exc_type = pyObj_txn_expired; + break; + } + case pycbc_txns::TxnExceptionType::TRANSACTION_OPERATION_FAILED: { + pyObj_exc_type = pyObj_txn_op_failed; + break; + } + case pycbc_txns::TxnExceptionType::FEATURE_NOT_AVAILABLE: { + pyObj_exc_type = pyObj_feature_not_available_error; + break; + } + case pycbc_txns::TxnExceptionType::QUERY_PARSING_FAILURE: { + pyObj_exc_type = pyObj_query_parsing_failure; + break; + } + case pycbc_txns::TxnExceptionType::DOCUMENT_EXISTS: { + pyObj_exc_type = pyObj_document_exists_ex; + break; + } + case pycbc_txns::TxnExceptionType::DOCUMENT_NOT_FOUND: { + pyObj_exc_type = pyObj_document_not_found_ex; + break; + } + case pycbc_txns::TxnExceptionType::COUCHBASE_ERROR: + default: + pyObj_exc_type = pyObj_couchbase_error; + } + PyObject* pyObj_tmp = PyUnicode_FromString(message); + PyDict_SetItemString(pyObj_error_ctx, "message", pyObj_tmp); + Py_DECREF(pyObj_tmp); + if (pyObj_inner_exc != nullptr) { + pyObj_tmp = PyDict_GetItemString(pyObj_inner_exc, "inner_cause"); + if (pyObj_tmp != nullptr) { + PyDict_SetItemString(pyObj_error_ctx, "exc_info", pyObj_inner_exc); + Py_DECREF(pyObj_inner_exc); + } + Py_DECREF(pyObj_tmp); + } + PyObject* pyObj_args = PyTuple_New(0); + pyObj_final_error = PyObject_Call(pyObj_exc_type, pyObj_args, pyObj_error_ctx); + Py_DECREF(pyObj_args); + if (set_exception) { + PyErr_SetObject(pyObj_exc_type, pyObj_final_error); + return nullptr; + } + return pyObj_final_error; +} + +PyObject* +convert_to_python_exc_type(std::exception_ptr err, + bool set_exception = false, + PyObject* pyObj_inner_exc = nullptr) +{ + auto exc_type = pycbc_txns::TxnExceptionType::COUCHBASE_ERROR; + const char* message = nullptr; + + // Must be an error + assert(!!err); + + try { + std::rethrow_exception(err); + } catch (const tx_core::transaction_exception& e) { + switch (e.type()) { + case tx_core::failure_type::FAIL: + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_FAILED; + break; + case tx_core::failure_type::COMMIT_AMBIGUOUS: + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_COMMIT_AMBIGUOUS; + break; + case tx_core::failure_type::EXPIRY: + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_EXPIRED; + break; + } + message = e.what(); + } catch (const tx_core::transaction_operation_failed& e) { + if (e.cause() == tx_core::external_exception::FEATURE_NOT_AVAILABLE_EXCEPTION) { + exc_type = pycbc_txns::TxnExceptionType::FEATURE_NOT_AVAILABLE; + message = "Possibly attempting a binary transaction operation with a server version < 7.6.2"; + } else { + // follow logic that was used in C++ core transactions::wrap_run() to call + // transaction_context::handle_error() which boils down to (not exactly, should suffice + // for our purposes) calling transaction_operation_failed::get_final_exception() + if (e.to_raise() == tx_core::final_error::EXPIRED) { + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_EXPIRED; + } else if (e.to_raise() == tx_core::final_error::AMBIGUOUS) { + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_COMMIT_AMBIGUOUS; + } else { + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_OPERATION_FAILED; + } + message = e.what(); + } + } catch (const tx_core::query_parsing_failure& e) { + exc_type = pycbc_txns::TxnExceptionType::QUERY_PARSING_FAILURE; + message = e.what(); + } catch (const tx_core::document_exists& e) { + exc_type = pycbc_txns::TxnExceptionType::DOCUMENT_EXISTS; + message = e.what(); + } catch (const tx_core::document_not_found& e) { + exc_type = pycbc_txns::TxnExceptionType::DOCUMENT_NOT_FOUND; + message = e.what(); + } catch (const tx_core::op_exception& e) { + message = e.what(); + } catch (const std::exception& e) { + message = e.what(); + } catch (...) { + message = "Unknown error"; + } + return create_python_exception(exc_type, message, set_exception, pyObj_inner_exc); +} + +void +handle_returning_void(PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + std::exception_ptr err) +{ + auto state = PyGILState_Ensure(); + PyObject* pyObj_args = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_err = nullptr; + if (err) { + pyObj_err = convert_to_python_exc_type(err); + if (nullptr == pyObj_errback) { + barrier->set_value(pyObj_err); + } else { + pyObj_args = PyTuple_New(1); + PyTuple_SetItem(pyObj_args, 0, pyObj_err); + pyObj_func = pyObj_errback; + } + } else { + Py_INCREF(Py_None); + if (nullptr == pyObj_callback) { + barrier->set_value(Py_None); + } else { + pyObj_args = PyTuple_New(1); + PyTuple_SetItem(pyObj_args, 0, Py_None); + pyObj_func = pyObj_callback; + } + } + if (nullptr != pyObj_func) { + PyObject_CallObject(pyObj_func, pyObj_args); + Py_DECREF(pyObj_errback); + Py_DECREF(pyObj_callback); + Py_DECREF(pyObj_args); + } + PyGILState_Release(state); +} + +void +handle_returning_transaction_get_result( + PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + std::exception_ptr err, + std::optional<couchbase::core::transactions::transaction_get_result> res, + bool is_replica_get = false) +{ + // TODO: flesh out transaction_get_result and exceptions... + auto state = PyGILState_Ensure(); + PyObject* pyObj_args = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_err = nullptr; + if (err) { + pyObj_err = convert_to_python_exc_type(err); + if (nullptr == pyObj_errback) { + barrier->set_value(pyObj_err); + } else { + pyObj_args = PyTuple_New(1); + PyTuple_SetItem(pyObj_args, 0, pyObj_err); + pyObj_func = pyObj_errback; + } + } else { + PyObject* pyObj_get_result = nullptr; + + // BUG(PYCBC-1476): We should revert to using direct get + // operations once the underlying issue has been resolved. + if (!res.has_value()) { + pyObj_get_result = pycbc_build_exception( + couchbase::errc::make_error_code((is_replica_get) + ? couchbase::errc::key_value::document_irretrievable + : couchbase::errc::key_value::document_not_found), + __FILE__, + __LINE__, + "Txn get op: document not found."); + } else { + pyObj_get_result = + PyObject_CallObject(reinterpret_cast<PyObject*>(&transaction_get_result_type), nullptr); + auto result = reinterpret_cast<pycbc_txns::transaction_get_result*>(pyObj_get_result); + result->res = std::make_unique<tx_core::transaction_get_result>(std::move(res.value())); + } + + if (nullptr == pyObj_callback) { + barrier->set_value(pyObj_get_result); + } else { + pyObj_args = PyTuple_New(1); + PyTuple_SetItem(pyObj_args, 0, pyObj_get_result); + pyObj_func = pyObj_callback; + } + } + if (nullptr != pyObj_func) { + PyObject_CallObject(pyObj_func, pyObj_args); + Py_DECREF(pyObj_errback); + Py_DECREF(pyObj_callback); + Py_DECREF(pyObj_args); + } + PyGILState_Release(state); +} + +void +handle_returning_query_result(PyObject* pyObj_callback, + PyObject* pyObj_errback, + std::shared_ptr<std::promise<PyObject*>> barrier, + std::exception_ptr err, + std::optional<couchbase::core::operations::query_response> res) +{ + auto state = PyGILState_Ensure(); + PyObject* pyObj_args = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_err = nullptr; + if (err) { + pyObj_err = convert_to_python_exc_type(err); + if (nullptr == pyObj_errback) { + barrier->set_value(pyObj_err); + } else { + pyObj_args = PyTuple_New(1); + PyTuple_SetItem(pyObj_args, 0, pyObj_err); + pyObj_func = pyObj_errback; + } + } else { + PyObject* pyObj_json = PyBytes_FromString(res->ctx.http_body.c_str()); + if (nullptr == pyObj_callback) { + barrier->set_value(pyObj_json); + } else { + pyObj_args = PyTuple_New(1); + PyTuple_SetItem(pyObj_args, 0, pyObj_json); + pyObj_func = pyObj_callback; + } + } + if (nullptr != pyObj_func) { + PyObject_CallObject(pyObj_func, pyObj_args); + Py_DECREF(pyObj_errback); + Py_DECREF(pyObj_callback); + Py_DECREF(pyObj_args); + } + PyGILState_Release(state); +} + +PyObject* +pycbc_txns::transaction_query_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_ctx = nullptr; + PyObject* pyObj_options = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + const char* statement = nullptr; + + const char* kw_list[] = { "ctx", "statement", "options", "callback", "errback", nullptr }; + const char* kw_format = "O!sO|OO"; + if (!PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_ctx, + &statement, + &pyObj_options, + &pyObj_callback, + &pyObj_errback)) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + Py_RETURN_NONE; + } + if (nullptr == pyObj_ctx) { + PyErr_SetString(PyExc_ValueError, "expected transaction_context"); + Py_RETURN_NONE; + } + auto ctx = + reinterpret_cast<pycbc_txns::transaction_context*>(PyCapsule_GetPointer(pyObj_ctx, "ctx_")); + if (nullptr == ctx) { + PyErr_SetString(PyExc_ValueError, "passed null transaction_context"); + Py_RETURN_NONE; + } + if (nullptr == statement) { + PyErr_SetString(PyExc_ValueError, "expected query statement"); + Py_RETURN_NONE; + } + if (nullptr == pyObj_options) { + PyErr_SetString(PyExc_ValueError, "expected options"); + Py_RETURN_NONE; + } + auto opt = reinterpret_cast<pycbc_txns::transaction_query_options*>(pyObj_options); + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto fut = barrier->get_future(); + Py_BEGIN_ALLOW_THREADS ctx->ctx->query( + statement, + *opt->opts, + [pyObj_callback, pyObj_errback, barrier]( + std::exception_ptr err, std::optional<couchbase::core::operations::query_response> resp) { + handle_returning_query_result(pyObj_callback, pyObj_errback, barrier, err, resp); + }); + Py_END_ALLOW_THREADS if (nullptr == pyObj_callback || nullptr == pyObj_errback) + { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = fut.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} + +PyObject* +pycbc_txns::transaction_op([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_ctx = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_txn_get_result = nullptr; + PyObject* pyObj_value = nullptr; + const char* bucket = nullptr; + const char* scope = nullptr; + const char* collection = nullptr; + const char* key = nullptr; + couchbase::codec::encoded_value value{}; + TxOperations::TxOperationType op_type = TxOperations::UNKNOWN; + const char* kw_list[] = { "ctx", "bucket", "scope", "collection_name", "key", "op", + "callback", "errback", "value", "txn_get_result", nullptr }; + const char* kw_format = "O!|ssssIOOOO"; + + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_ctx, + &bucket, + &scope, + &collection, + &key, + &op_type, + &pyObj_callback, + &pyObj_errback, + &pyObj_value, + &pyObj_txn_get_result); + if (!ret) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + Py_RETURN_NONE; + } + if (nullptr != pyObj_value) { + PyObject* pyObj_data = PyTuple_GET_ITEM(pyObj_value, 0); + PyObject* pyObj_flags = PyTuple_GET_ITEM(pyObj_value, 1); + value.flags = static_cast<uint32_t>(PyLong_AsLong(pyObj_flags)); + try { + value.data = PyObject_to_binary(pyObj_data); + } catch (const std::exception& e) { + pycbc_set_python_exception(PycbcError::InvalidArgument, __FILE__, __LINE__, e.what()); + Py_RETURN_NONE; + } + } + if (nullptr == pyObj_ctx) { + PyErr_SetString(PyExc_ValueError, "no transaction_context passed in"); + Py_RETURN_NONE; + } + auto ctx = + reinterpret_cast<pycbc_txns::transaction_context*>(PyCapsule_GetPointer(pyObj_ctx, "ctx_")); + if (nullptr == ctx) { + PyErr_SetString(PyExc_ValueError, "passed null transaction_context"); + Py_RETURN_NONE; + } + + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + + auto barrier = std::make_shared<std::promise<PyObject*>>(); + auto fut = barrier->get_future(); + switch (op_type) { + case TxOperations::GET: { + if (nullptr == bucket || nullptr == scope || nullptr == collection || nullptr == key) { + PyErr_SetString(PyExc_ValueError, "couldn't create document id for get"); + Py_RETURN_NONE; + } + couchbase::core::document_id id{ bucket, scope, collection, key }; + Py_BEGIN_ALLOW_THREADS ctx->ctx->get_optional( + id, + [barrier, pyObj_callback, pyObj_errback]( + std::exception_ptr err, std::optional<tx_core::transaction_get_result> res) { + handle_returning_transaction_get_result(pyObj_callback, pyObj_errback, barrier, err, res); + }); + Py_END_ALLOW_THREADS break; + } + case TxOperations::GET_REPLICA_FROM_PREFERRED_SERVER_GROUP: { + if (nullptr == bucket || nullptr == scope || nullptr == collection || nullptr == key) { + PyErr_SetString(PyExc_ValueError, + "couldn't create document id for get_replica_from_preferred_server_group"); + Py_RETURN_NONE; + } + couchbase::core::document_id id{ bucket, scope, collection, key }; + Py_BEGIN_ALLOW_THREADS ctx->ctx->get_replica_from_preferred_server_group( + id, + [barrier, pyObj_callback, pyObj_errback]( + std::exception_ptr err, std::optional<tx_core::transaction_get_result> res) { + handle_returning_transaction_get_result( + pyObj_callback, pyObj_errback, barrier, err, res, true); + }); + Py_END_ALLOW_THREADS break; + } + case TxOperations::INSERT: { + if (nullptr == bucket || nullptr == scope || nullptr == collection || nullptr == key) { + PyErr_SetString(PyExc_ValueError, "couldn't create document id for insert"); + Py_RETURN_NONE; + } + couchbase::core::document_id id{ bucket, scope, collection, key }; + if (nullptr == pyObj_value) { + PyErr_SetString(PyExc_ValueError, + fmt::format("no value given for an insert of key {}", id.key()).c_str()); + Py_RETURN_NONE; + } + Py_BEGIN_ALLOW_THREADS ctx->ctx->insert( + id, + value, + [barrier, pyObj_callback, pyObj_errback]( + std::exception_ptr err, std::optional<tx_core::transaction_get_result> res) { + handle_returning_transaction_get_result(pyObj_callback, pyObj_errback, barrier, err, res); + }); + Py_END_ALLOW_THREADS break; + } + case TxOperations::REPLACE: { + if (nullptr == pyObj_value) { + PyErr_SetString(PyExc_ValueError, "replace expects a value"); + Py_RETURN_NONE; + } + if (nullptr == pyObj_txn_get_result || + 0 == PyObject_TypeCheck(pyObj_txn_get_result, &transaction_get_result_type)) { + PyErr_SetString(PyExc_ValueError, "replace expects to be passed a transaction_get_result"); + Py_RETURN_NONE; + } + auto tx_get_result = + reinterpret_cast<pycbc_txns::transaction_get_result*>(pyObj_txn_get_result); + Py_BEGIN_ALLOW_THREADS ctx->ctx->replace( + *tx_get_result->res, + value, + [pyObj_callback, pyObj_errback, barrier]( + std::exception_ptr err, std::optional<tx_core::transaction_get_result> res) { + handle_returning_transaction_get_result(pyObj_callback, pyObj_errback, barrier, err, res); + }); + Py_END_ALLOW_THREADS break; + } + case TxOperations::REMOVE: { + if (nullptr == pyObj_txn_get_result || + 0 == PyObject_TypeCheck(pyObj_txn_get_result, &transaction_get_result_type)) { + PyErr_SetString(PyExc_ValueError, "remove expects to be passed a transaction_get_result"); + Py_RETURN_NONE; + } + auto tx_get_result = + reinterpret_cast<pycbc_txns::transaction_get_result*>(pyObj_txn_get_result); + Py_BEGIN_ALLOW_THREADS ctx->ctx->remove( + *tx_get_result->res, [pyObj_callback, pyObj_errback, barrier](std::exception_ptr err) { + handle_returning_void(pyObj_callback, pyObj_errback, barrier, err); + }); + Py_END_ALLOW_THREADS break; + } + default: + // return error! + PyErr_SetString(PyExc_ValueError, "unknown txn operation"); + } + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = fut.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} + +PyObject* +transaction_result_to_dict(std::optional<tx::transaction_result> res) +{ + PyObject* dict = PyDict_New(); + if (res) { + PyObject* tmp = PyUnicode_FromString(res->transaction_id.c_str()); + PyDict_SetItemString(dict, "transaction_id", tmp); + Py_DECREF(tmp); + PyDict_SetItemString(dict, "unstaging_complete", res->unstaging_complete ? Py_True : Py_False); + } + return dict; +} + +PyObject* +pycbc_txns::create_new_attempt_context([[maybe_unused]] PyObject* self, + PyObject* args, + PyObject* kwargs) +{ + PyObject* pyObj_ctx = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + const char* kw_list[] = { "ctx", "callback", "errback", nullptr }; + const char* kw_format = "O!|OO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_ctx, + &pyObj_callback, + &pyObj_errback); + if (!ret) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + return nullptr; + } + auto ctx = + reinterpret_cast<pycbc_txns::transaction_context*>(PyCapsule_GetPointer(pyObj_ctx, "ctx_")); + + if (nullptr == ctx) { + PyErr_SetString(PyExc_ValueError, "passed null transaction context"); + return nullptr; + } + + std::shared_ptr<std::promise<PyObject*>> barrier = nullptr; + std::future<PyObject*> fut; + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + barrier = std::make_shared<std::promise<PyObject*>>(); + fut = barrier->get_future(); + } + Py_BEGIN_ALLOW_THREADS ctx->ctx->new_attempt_context( + [barrier, pyObj_callback, pyObj_errback](std::exception_ptr err) { + handle_returning_void(pyObj_callback, pyObj_errback, barrier, err); + }); + Py_END_ALLOW_THREADS if (nullptr == pyObj_callback || nullptr == pyObj_errback) + { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = fut.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} + +PyObject* +pycbc_txns::create_transaction_context([[maybe_unused]] PyObject* self, + PyObject* args, + PyObject* kwargs) +{ + PyObject* pyObj_txns = nullptr; + PyObject* pyObj_transaction_options = nullptr; + const char* kw_list[] = { "txns", "transaction_options", nullptr }; + const char* kw_format = "O!|O"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_txns, + &pyObj_transaction_options); + if (!ret) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + return nullptr; + } + auto txns = + reinterpret_cast<pycbc_txns::transactions*>(PyCapsule_GetPointer(pyObj_txns, "txns_")); + + if (nullptr == txns) { + PyErr_SetString(PyExc_ValueError, "passed null transactions"); + return nullptr; + } + if (nullptr != pyObj_transaction_options) { + if (!PyObject_IsInstance(pyObj_transaction_options, + reinterpret_cast<PyObject*>(&transaction_options_type))) { + PyErr_SetString(PyExc_ValueError, "expected a valid transaction_options object"); + return nullptr; + } + } + + auto tx_options = + nullptr != pyObj_transaction_options && Py_None != pyObj_transaction_options + ? *(reinterpret_cast<pycbc_txns::transaction_options*>(pyObj_transaction_options)->opts) + : tx::transaction_options(); + auto py_ctx = new pycbc_txns::transaction_context( + tx_core::transaction_context::create(*txns->txns, tx_options)); + PyObject* pyObj_ctx = PyCapsule_New(py_ctx, "ctx_", dealloc_transaction_context); + return pyObj_ctx; +} + +PyObject* +pycbc_txns::transaction_commit([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_ctx = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + const char* kw_list[] = { "ctx", "callback", "errback", nullptr }; + const char* kw_format = "O!|OO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_ctx, + &pyObj_callback, + &pyObj_errback); + if (!ret) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + return nullptr; + } + auto ctx = + reinterpret_cast<pycbc_txns::transaction_context*>(PyCapsule_GetPointer(pyObj_ctx, "ctx_")); + + if (nullptr == ctx) { + PyErr_SetString(PyExc_ValueError, "passed null transaction context"); + return nullptr; + } + + std::shared_ptr<std::promise<PyObject*>> barrier = nullptr; + std::future<PyObject*> fut; + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + barrier = std::make_shared<std::promise<PyObject*>>(); + fut = barrier->get_future(); + } + Py_BEGIN_ALLOW_THREADS ctx->ctx->finalize( + [pyObj_callback, pyObj_errback, barrier](std::optional<tx_core::transaction_exception> err, + std::optional<tx::transaction_result> res) { + auto state = PyGILState_Ensure(); + PyObject* pyObj_args = nullptr; + PyObject* pyObj_func = nullptr; + PyObject* pyObj_err = nullptr; + auto exc_type = pycbc_txns::TxnExceptionType::COUCHBASE_ERROR; + if (err) { + switch (err->type()) { + case tx_core::failure_type::FAIL: + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_FAILED; + break; + case tx_core::failure_type::COMMIT_AMBIGUOUS: + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_COMMIT_AMBIGUOUS; + break; + case tx_core::failure_type::EXPIRY: + exc_type = pycbc_txns::TxnExceptionType::TRANSACTION_EXPIRED; + break; + } + auto message = txn_external_exception_to_string(err->cause()); + pyObj_err = create_python_exception(exc_type, message.c_str()); + if (nullptr == pyObj_errback) { + barrier->set_value(pyObj_err); + } else { + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, pyObj_err); + pyObj_func = pyObj_errback; + } + } else { + PyObject* ret = transaction_result_to_dict(res); + if (nullptr == pyObj_callback) { + barrier->set_value(ret); + } else { + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, ret); + pyObj_func = pyObj_callback; + } + } + if (nullptr != pyObj_func) { + PyObject_CallObject(pyObj_func, pyObj_args); + Py_DECREF(pyObj_errback); + Py_DECREF(pyObj_callback); + Py_DECREF(pyObj_args); + } + PyGILState_Release(state); + }); + Py_END_ALLOW_THREADS if (nullptr == pyObj_callback || nullptr == pyObj_errback) + { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = fut.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} + +PyObject* +pycbc_txns::transaction_rollback([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + PyObject* pyObj_ctx = nullptr; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + const char* kw_list[] = { "ctx", "callback", "errback", nullptr }; + const char* kw_format = "O!|OO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_ctx, + &pyObj_callback, + &pyObj_errback); + if (!ret) { + PyErr_SetString(PyExc_ValueError, "couldn't parse args"); + return nullptr; + } + auto ctx = + reinterpret_cast<pycbc_txns::transaction_context*>(PyCapsule_GetPointer(pyObj_ctx, "ctx_")); + + if (nullptr == ctx) { + PyErr_SetString(PyExc_ValueError, "passed null transaction context"); + return nullptr; + } + std::shared_ptr<std::promise<PyObject*>> barrier = nullptr; + std::future<PyObject*> fut; + Py_XINCREF(pyObj_callback); + Py_XINCREF(pyObj_errback); + if (nullptr == pyObj_callback || nullptr == pyObj_errback) { + barrier = std::make_shared<std::promise<PyObject*>>(); + fut = barrier->get_future(); + } + Py_BEGIN_ALLOW_THREADS + { + ctx->ctx->rollback([pyObj_callback, pyObj_errback, barrier](std::exception_ptr err) { + handle_returning_void(pyObj_callback, pyObj_errback, barrier, err); + }); + } + Py_END_ALLOW_THREADS if (nullptr == pyObj_callback || nullptr == pyObj_errback) + { + PyObject* ret = nullptr; + Py_BEGIN_ALLOW_THREADS ret = fut.get(); + Py_END_ALLOW_THREADS return ret; + } + Py_RETURN_NONE; +} diff --git a/src/transactions/transactions.hxx b/src/transactions/transactions.hxx new file mode 100644 index 000000000..f40998763 --- /dev/null +++ b/src/transactions/transactions.hxx @@ -0,0 +1,194 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "../client.hxx" +#include "../exceptions.hxx" +#include <core/operations/document_query.hxx> +#include <core/transactions.hxx> +#include <core/transactions/internal/transaction_context.hxx> + +namespace tx = couchbase::transactions; +namespace tx_core = couchbase::core::transactions; + +namespace pycbc_txns +{ + +// @TODO: PYCBC-1425, is this the right approach? +using pycbc_txn_complete_callback = + std::function<void(std::optional<tx_core::transaction_exception>, + std::optional<tx::transaction_result>)>; + +class TxOperations +{ +public: + enum TxOperationType { + UNKNOWN, + GET, + GET_REPLICA_FROM_PREFERRED_SERVER_GROUP, + REPLACE, + INSERT, + REMOVE, + QUERY + }; + + TxOperations() + : TxOperations{ UNKNOWN } + { + } + constexpr TxOperations(TxOperationType op) + : operation_{ op } + { + } + + operator TxOperationType() const + { + return operation_; + } + // lets prevent the implicit promotion of bool to int + explicit operator bool() = delete; + constexpr bool operator==(TxOperations op) const + { + return operation_ == op.operation_; + } + constexpr bool operator!=(TxOperations op) const + { + return operation_ != op.operation_; + } + + static const char* ALL_OPERATIONS(void) + { + const char* ops = "GET " + "GET_REPLICA_FROM_PREFERRED_SERVER_GROUP " + "REPLACE " + "INSERT " + "REMOVE " + "QUERY"; + + return ops; + } + +private: + TxOperationType operation_; +}; + +enum TxnExceptionType { + TRANSACTION_FAILED, + TRANSACTION_COMMIT_AMBIGUOUS, + TRANSACTION_EXPIRED, + TRANSACTION_OPERATION_FAILED, + FEATURE_NOT_AVAILABLE, + QUERY_PARSING_FAILURE, + DOCUMENT_EXISTS, + DOCUMENT_NOT_FOUND, + COUCHBASE_ERROR +}; + +struct transaction_config { + PyObject_HEAD tx::transactions_config* cfg; +}; + +struct transaction_options { + PyObject_HEAD tx::transaction_options* opts; +}; + +static PyObject* +transaction_config__new__(PyTypeObject*, PyObject*, PyObject*); +static void +transaction_config__dealloc__(pycbc_txns::transaction_config*); +static PyObject* +transaction_config__to_dict__(PyObject*); + +static PyObject* +transaction_options__new__(PyTypeObject*, PyObject*, PyObject*); +static void +transaction_options__dealloc__(pycbc_txns::transaction_options*); +static PyObject* +transaction_options__to_dict__(PyObject*); +static PyObject* +transaction_options__str__(PyObject*); + +struct transactions { + std::shared_ptr<tx_core::transactions> txns; + + explicit transactions(std::shared_ptr<tx_core::transactions> transactions) + : txns(std::move(transactions)) + { + } +}; + +struct transaction_context { + std::shared_ptr<tx_core::transaction_context> ctx; + + explicit transaction_context(std::shared_ptr<tx_core::transaction_context> ctx) + : ctx(std::move(ctx)) + { + } +}; + +struct transaction_get_result { + PyObject_HEAD std::unique_ptr<tx_core::transaction_get_result> res; +}; + +struct transaction_query_options { + PyObject_HEAD tx::transaction_query_options* opts; +}; + +static PyObject* +transaction_query_options__new__(PyTypeObject*, PyObject*, PyObject*); +static void +transaction_query_options__dealloc__(pycbc_txns::transaction_query_options*); +static PyObject* +transaction_query_options__to_dict__(PyObject*); + +static PyObject* +transaction_get_result__new__(PyTypeObject*, PyObject*, PyObject*); +static PyObject* +transaction_get_result__str__(pycbc_txns::transaction_get_result* result); +static void +transaction_get_result__dealloc__(pycbc_txns::transaction_get_result* result); +static PyObject* +transaction_get_result__get__(pycbc_txns::transaction_get_result* result, PyObject* args); + +PyObject* +add_transaction_objects(PyObject* module); + +static void +dealloc_transactions(PyObject* txns); +static void +dealloc_transaction_context(PyObject* ctx); + +PyObject* +create_transactions(PyObject*, PyObject*, PyObject*); +PyObject* +create_transaction_context(PyObject*, PyObject*, PyObject*); +PyObject* +create_new_attempt_context(PyObject*, PyObject*, PyObject*); +PyObject* +transaction_op(PyObject*, PyObject*, PyObject*); +PyObject* +transaction_query_op(PyObject*, PyObject*, PyObject*); +PyObject* +transaction_commit(PyObject*, PyObject*, PyObject*); +PyObject* +transaction_rollback(PyObject*, PyObject*, PyObject*); +PyObject* +destroy_transactions(PyObject*, PyObject*, PyObject*); +void +add_transactions(PyObject* mod); +} // namespace pycbc_txns diff --git a/src/typeutil.c b/src/typeutil.c deleted file mode 100644 index 9b561f25e..000000000 --- a/src/typeutil.c +++ /dev/null @@ -1,204 +0,0 @@ -/** - * Copyright 2013 Couchbase, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - **/ - -/** - * This file contains various conversion utilities between C and Python - * types. This also does a lot of the porting and handling between Python - * major versions as well. - */ -#include "pycbc.h" -#if PY_MAJOR_VERSION == 2 - -unsigned PY_LONG_LONG -pycbc_IntAsULL(PyObject *o) -{ - if (PyLong_Check(o)) { - return PyLong_AsUnsignedLongLong(o); - } else { - long tmp = PyInt_AsLong(o); - if (tmp < 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, "value must be unsigned"); - } - return -1; - } - return tmp; - } -} - -PY_LONG_LONG -pycbc_IntAsLL(PyObject *o) -{ - if (PyLong_Check(o)) { - return PyLong_AsLongLong(o); - } else { - return PyInt_AsLong(o); - } -} - -long -pycbc_IntAsL(PyObject *o) -{ - if (PyInt_Check(o)) { - return PyInt_AsLong(o); - } - return PyLong_AsLong(o); -} - -unsigned long -pycbc_IntAsUL(PyObject *o) -{ - if (PyInt_Check(o)) { - long l = PyInt_AsLong(o); - if (l < 0) { - if (!PyErr_Occurred()) { - PyErr_SetString(PyExc_TypeError, "value must be unsigned"); - } - return -1; - } - return l; - } - return PyLong_AsUnsignedLong(o); -} - -#endif /* PY_MAJOR_VERSION == 2 */ - -PyObject * -pycbc_maybe_convert_to_int(PyObject *o) -{ - PyObject *args, *result; - args = Py_BuildValue("(O)", o); - - if (!args) { - return NULL; - } - -#if PY_MAJOR_VERSION == 2 - result = PyObject_CallObject((PyObject*)&PyInt_Type, args); - if (result) { - Py_DECREF(args); - return result; - } - - PyErr_Clear(); -#endif - - result = PyObject_CallObject((PyObject*)&PyLong_Type, args); - Py_DECREF(args); - - if (result) { - return result; - } - - return NULL; -} - -/** - * Py3-specific stuff - */ - -#if PY_MAJOR_VERSION == 3 -int -pycbc_BufFromString(PyObject *obj, char **key, Py_ssize_t *nkey, PyObject **newkey) -{ - int rv; - if (PyBytes_Check(obj)) { - *newkey = NULL; - return PyBytes_AsStringAndSize(obj, key, nkey); - } - - *newkey = PyUnicode_AsUTF8String(obj); - if (!*newkey) { - return -1; - } - rv = PyBytes_AsStringAndSize(*newkey, key, nkey); - if (rv < 0) { - Py_DECREF(*newkey); - *newkey = NULL; - } - return rv; -} - -#else -int -pycbc_BufFromString(PyObject *obj, char **key, Py_ssize_t *nkey, PyObject **newkey) -{ - int rv; - rv = PyBytes_AsStringAndSize(obj, key, nkey); - if (rv < 0) { - *newkey = NULL; - return -1; - } - *newkey = obj; - Py_INCREF(obj); - return 0; -} - -#endif /* PY_MAJOR_VERSION == 3*/ - - -int -pycbc_get_ttl(PyObject *obj, unsigned long *ttl, int nonzero) -{ - if (obj == NULL || PyObject_IsTrue(obj) == 0) { - if (!nonzero) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "TTL must be specified " - "and must not be 0 or False or None", - obj); - return -1; - } - *ttl = 0; - return 0; - } - - if (!PyNumber_Check(obj)) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, "TTL must be numeric", obj); - return -1; - } - *ttl = pycbc_IntAsUL(obj); - if (*ttl == (unsigned long)-1) { - PYCBC_EXC_WRAP_OBJ(PYCBC_EXC_ARGUMENTS, 0, - "TTL must be a valid Unix timestamp ", obj); - return -1; - } - return 0; -} - -int -pycbc_get_u32(PyObject *obj, lcb_uint32_t *out) -{ - - unsigned long val = pycbc_IntAsUL(obj); - if (PyErr_Occurred()) { - return -1; - } - - /** - * Python won't check for an overflow if our long type is not 32 bits, - * so we need to do it ourselves - */ -#if LONG_MAX > 0xFFFFFFFFUL - if ( (val & 0xFFFFFFFFUL) != val) { - PyErr_SetString(PyExc_OverflowError, "Value must be smaller " \ - "than 32 bits"); - return -1; - } -#endif - - *out = (lcb_uint32_t)val; - return 0; - -} diff --git a/src/utils.cxx b/src/utils.cxx new file mode 100644 index 000000000..543b48f61 --- /dev/null +++ b/src/utils.cxx @@ -0,0 +1,555 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "utils.hxx" + +couchbase::core::utils::binary +PyObject_to_binary(PyObject* pyObj_value) +{ + char* buf; + Py_ssize_t nbuf; + if (PyBytes_AsStringAndSize(pyObj_value, &buf, &nbuf) == -1) { + throw std::invalid_argument("Unable to determine bytes object from provided value."); + } + auto size = py_ssize_t_to_size_t(nbuf); + return couchbase::core::utils::to_binary(reinterpret_cast<const char*>(buf), size); +} + +PyObject* +binary_to_PyObject(couchbase::core::utils::binary value) +{ + auto buf = reinterpret_cast<const char*>(value.data()); + auto nbuf = size_t_to_py_ssize_t(value.size()); + return PyBytes_FromStringAndSize(buf, nbuf); +} + +PyObject* +binary_to_PyObject_unicode(couchbase::core::utils::binary value) +{ + auto buf = reinterpret_cast<const char*>(value.data()); + auto nbuf = size_t_to_py_ssize_t(value.size()); + return PyUnicode_FromStringAndSize(buf, nbuf); +} + +std::string +binary_to_string(couchbase::core::utils::binary value) +{ + auto json = couchbase::core::utils::json::parse_binary(value); + return couchbase::core::utils::json::generate(json); +} + +std::size_t +py_ssize_t_to_size_t(Py_ssize_t value) +{ + if (value < 0) { + throw std::invalid_argument("Cannot convert provided Py_ssize_t value to size_t."); + } + + return static_cast<std::size_t>(value); +} + +Py_ssize_t +size_t_to_py_ssize_t(std::size_t value) +{ + if (value > INT_MAX) { + throw std::invalid_argument("Cannot convert provided size_t value to Py_ssize_t."); + } + return static_cast<Py_ssize_t>(value); +} + +couchbase::replicate_to +PyObject_to_replicate_to(PyObject* pyObj_replicate_to) +{ + auto replicate_to = static_cast<uint8_t>(PyLong_AsLong(pyObj_replicate_to)); + if (replicate_to == 0) { + return couchbase::replicate_to::none; + } else if (replicate_to == 1) { + return couchbase::replicate_to::one; + } else if (replicate_to == 2) { + return couchbase::replicate_to::two; + } else if (replicate_to == 3) { + return couchbase::replicate_to::three; + } else { + return couchbase::replicate_to::none; + } +} + +couchbase::persist_to +PyObject_to_persist_to(PyObject* pyObj_persist_to) +{ + auto persist_to = static_cast<uint8_t>(PyLong_AsLong(pyObj_persist_to)); + if (persist_to == 0) { + return couchbase::persist_to::none; + } else if (persist_to == 1) { + return couchbase::persist_to::active; + } else if (persist_to == 2) { + return couchbase::persist_to::one; + } else if (persist_to == 3) { + return couchbase::persist_to::two; + } else if (persist_to == 4) { + return couchbase::persist_to::three; + } else if (persist_to == 5) { + return couchbase::persist_to::four; + } else { + return couchbase::persist_to::none; + } +} + +std::pair<couchbase::persist_to, couchbase::replicate_to> +PyObject_to_durability(PyObject* pyObj_durability) +{ + auto durability = std::make_pair(couchbase::persist_to::none, couchbase::replicate_to::none); + PyObject* pyObj_persist_to = PyDict_GetItemString(pyObj_durability, "persist_to"); + if (pyObj_persist_to) { + durability.first = PyObject_to_persist_to(pyObj_persist_to); + } + + PyObject* pyObj_replicate_to = PyDict_GetItemString(pyObj_durability, "replicate_to"); + if (pyObj_replicate_to) { + durability.second = PyObject_to_replicate_to(pyObj_replicate_to); + } + + return durability; +} + +couchbase::durability_level +PyObject_to_durability_level(PyObject* pyObj_durability_level) +{ + if (PyUnicode_Check(pyObj_durability_level)) { + auto durability = std::string(PyUnicode_AsUTF8(pyObj_durability_level)); + if (durability.compare("majorityAndPersistActive") == 0) { + return couchbase::durability_level::majority_and_persist_to_active; + } else if (durability.compare("majority") == 0) { + return couchbase::durability_level::majority; + } else if (durability.compare("persistToMajority") == 0) { + return couchbase::durability_level::persist_to_majority; + } else if (durability.compare("none") == 0) { + return couchbase::durability_level::none; + } else { + return couchbase::durability_level::none; + } + } else { + auto durability = static_cast<uint8_t>(PyLong_AsLong(pyObj_durability_level)); + if (durability == 0) { + return couchbase::durability_level::none; + } else if (durability == 1) { + return couchbase::durability_level::majority; + } else if (durability == 2) { + return couchbase::durability_level::majority_and_persist_to_active; + } else if (durability == 3) { + return couchbase::durability_level::persist_to_majority; + } else { + return couchbase::durability_level::none; + } + } +} + +couchbase::read_preference +PyObject_to_read_preference(PyObject* pyObj_read_preference) +{ + if (pyObj_read_preference == nullptr || pyObj_read_preference == Py_None) { + return couchbase::read_preference::no_preference; + } + + auto preference = static_cast<std::uint8_t>(PyLong_AsLong(pyObj_read_preference)); + if (preference == 1) { + return couchbase::read_preference::selected_server_group; + } else { + return couchbase::read_preference::no_preference; + } +} + +std::vector<couchbase::mutation_token> +get_mutation_state(PyObject* pyObj_mutation_state) +{ + std::vector<couchbase::mutation_token> mut_state{}; + size_t ntokens = static_cast<size_t>(PyList_Size(pyObj_mutation_state)); + for (size_t ii = 0; ii < ntokens; ++ii) { + + PyObject* pyObj_mut_token = PyList_GetItem(pyObj_mutation_state, ii); + PyObject* pyObj_bucket_name = PyDict_GetItemString(pyObj_mut_token, "bucket_name"); + auto bucket_name = std::string{ PyUnicode_AsUTF8(pyObj_bucket_name) }; + + PyObject* pyObj_partition_uuid = PyDict_GetItemString(pyObj_mut_token, "partition_uuid"); + auto partition_uuid = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_partition_uuid)); + + PyObject* pyObj_sequence_number = PyDict_GetItemString(pyObj_mut_token, "sequence_number"); + auto sequence_number = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_sequence_number)); + + PyObject* pyObj_partition_id = PyDict_GetItemString(pyObj_mut_token, "partition_id"); + auto partition_id = static_cast<uint16_t>(PyLong_AsUnsignedLong(pyObj_partition_id)); + + auto token = + couchbase::mutation_token{ partition_uuid, sequence_number, partition_id, bucket_name }; + mut_state.emplace_back(token); + } + return mut_state; +} + +couchbase::query_profile +str_to_profile_mode(std::string profile_mode) +{ + if (profile_mode.compare("off") == 0) { + return couchbase::query_profile::off; + } + if (profile_mode.compare("phases") == 0) { + return couchbase::query_profile::phases; + } + if (profile_mode.compare("timings") == 0) { + return couchbase::query_profile::timings; + } + // TODO: better exception + PyErr_SetString(PyExc_ValueError, "Invalid Profile Mode."); + return {}; +} + +couchbase::core::operations::query_request +build_query_request(PyObject* pyObj_query_args) +{ + couchbase::core::operations::query_request req; + PyObject* pyObj_statement = PyDict_GetItemString(pyObj_query_args, "statement"); + if (pyObj_statement != nullptr) { + if (PyUnicode_Check(pyObj_statement)) { + req.statement = std::string(PyUnicode_AsUTF8(pyObj_statement)); + } else { + PyErr_SetString(PyExc_ValueError, "Query statement is not a string."); + return {}; + } + } + + PyObject* pyObj_adhoc = PyDict_GetItemString(pyObj_query_args, "adhoc"); + if (pyObj_adhoc != nullptr) { + req.adhoc = pyObj_adhoc == Py_True ? true : false; + } + + PyObject* pyObj_metrics = PyDict_GetItemString(pyObj_query_args, "metrics"); + if (pyObj_metrics != nullptr) { + req.metrics = pyObj_metrics == Py_True ? true : false; + } + + PyObject* pyObj_readonly = PyDict_GetItemString(pyObj_query_args, "readonly"); + if (pyObj_readonly != nullptr) { + req.readonly = pyObj_readonly == Py_True ? true : false; + } + + PyObject* pyObj_flex_index = PyDict_GetItemString(pyObj_query_args, "flex_index"); + if (pyObj_flex_index != nullptr) { + req.flex_index = pyObj_flex_index == Py_True ? true : false; + } + + PyObject* pyObj_preserve_expiry = PyDict_GetItemString(pyObj_query_args, "preserve_expiry"); + if (pyObj_preserve_expiry != nullptr) { + req.preserve_expiry = pyObj_preserve_expiry == Py_True ? true : false; + } + + PyObject* pyObj_use_replica = PyDict_GetItemString(pyObj_query_args, "use_replica"); + if (pyObj_use_replica != nullptr) { + // If use_replica is None don't set the option (In this case None and False are not equivalent) + if (pyObj_use_replica == Py_True) { + req.use_replica = true; + } else if (pyObj_use_replica == Py_False) { + req.use_replica = false; + } + } + + PyObject* pyObj_max_parallelism = PyDict_GetItemString(pyObj_query_args, "max_parallelism"); + if (nullptr != pyObj_max_parallelism) { + req.max_parallelism = PyLong_AsUnsignedLongLong(pyObj_max_parallelism); + } + + PyObject* pyObj_scan_cap = PyDict_GetItemString(pyObj_query_args, "scan_cap"); + if (nullptr != pyObj_scan_cap) { + req.scan_cap = PyLong_AsUnsignedLongLong(pyObj_scan_cap); + } + + PyObject* pyObj_scan_wait = PyDict_GetItemString(pyObj_query_args, "scan_wait"); + if (nullptr != pyObj_scan_wait) { + // comes in as microseconds + req.scan_wait = std::chrono::milliseconds(PyLong_AsUnsignedLongLong(pyObj_scan_wait) / 1000ULL); + } + + PyObject* pyObj_pipeline_batch = PyDict_GetItemString(pyObj_query_args, "pipeline_batch"); + if (nullptr != pyObj_pipeline_batch) { + req.pipeline_batch = PyLong_AsUnsignedLongLong(pyObj_pipeline_batch); + } + + PyObject* pyObj_pipeline_cap = PyDict_GetItemString(pyObj_query_args, "pipeline_cap"); + if (nullptr != pyObj_pipeline_cap) { + req.pipeline_cap = PyLong_AsUnsignedLongLong(pyObj_pipeline_cap); + } + + PyObject* pyObj_scan_consistency = PyDict_GetItemString(pyObj_query_args, "scan_consistency"); + if (pyObj_scan_consistency != nullptr) { + if (PyUnicode_Check(pyObj_scan_consistency)) { + req.scan_consistency = str_to_scan_consistency_type<couchbase::query_scan_consistency>( + std::string(PyUnicode_AsUTF8(pyObj_scan_consistency))); + } else { + PyErr_SetString(PyExc_ValueError, "scan_consistency is not a string."); + } + if (PyErr_Occurred()) { + return {}; + } + } + + PyObject* pyObj_mutation_state = PyDict_GetItemString(pyObj_query_args, "mutation_state"); + if (pyObj_mutation_state != nullptr && PyList_Check(pyObj_mutation_state)) { + req.mutation_state = get_mutation_state(pyObj_mutation_state); + } + + PyObject* pyObj_query_context = PyDict_GetItemString(pyObj_query_args, "query_context"); + if (pyObj_query_context != nullptr) { + if (PyUnicode_Check(pyObj_query_context)) { + req.query_context = std::string(PyUnicode_AsUTF8(pyObj_query_context)); + } else { + PyErr_SetString(PyExc_ValueError, "query_context is not a string."); + return {}; + } + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(pyObj_query_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + if (PyUnicode_Check(pyObj_client_context_id)) { + req.client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + } else { + PyErr_SetString(PyExc_ValueError, "client_context_id is not a string."); + return {}; + } + } + + PyObject* pyObj_timeout = PyDict_GetItemString(pyObj_query_args, "timeout"); + if (nullptr != pyObj_timeout) { + // comes in as microseconds + req.timeout = std::chrono::milliseconds(PyLong_AsUnsignedLongLong(pyObj_timeout) / 1000ULL); + } + + PyObject* pyObj_profile_mode = PyDict_GetItemString(pyObj_query_args, "profile_mode"); + if (pyObj_profile_mode != nullptr) { + if (PyUnicode_Check(pyObj_profile_mode)) { + req.profile = str_to_profile_mode(std::string(PyUnicode_AsUTF8(pyObj_profile_mode))); + } else { + PyErr_SetString(PyExc_ValueError, "profile_mode is not a string."); + } + if (PyErr_Occurred()) { + return {}; + } + } + + PyObject* pyObj_send_to_node = PyDict_GetItemString(pyObj_query_args, "send_to_node"); + if (pyObj_send_to_node != nullptr) { + if (PyUnicode_Check(pyObj_send_to_node)) { + req.send_to_node = std::string(PyUnicode_AsUTF8(pyObj_send_to_node)); + } else { + PyErr_SetString(PyExc_ValueError, "send_to_node is not a string."); + return {}; + } + } + + PyObject* pyObj_span = PyDict_GetItemString(pyObj_query_args, "span"); + if (pyObj_span != nullptr) { + req.parent_span = std::make_shared<pycbc::request_span>(pyObj_span); + } + + PyObject* pyObj_raw = PyDict_GetItemString(pyObj_query_args, "raw"); + std::map<std::string, couchbase::core::json_string, std::less<>> raw_options{}; + if (pyObj_raw && PyDict_Check(pyObj_raw)) { + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_raw, &pos, &pyObj_key, &pyObj_value)) { + std::string k; + if (PyUnicode_Check(pyObj_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_key)); + } else { + PyErr_SetString( + PyExc_ValueError, + "Raw option key is not a string. The raw option should be a dict[str, JSONString]."); + return {}; + } + if (k.empty()) { + PyErr_SetString( + PyExc_ValueError, + "Raw option key is empty! The raw option should be a dict[str, JSONString]."); + return {}; + } + + if (PyBytes_Check(pyObj_value)) { + try { + auto res = PyObject_to_binary(pyObj_value); + // this will crash b/c txns query_options expects a std::vector<std::byte> + // auto res = std::string(PyBytes_AsString(pyObj_value)); + raw_options.emplace(k, couchbase::core::json_string{ std::move(res) }); + } catch (const std::exception& e) { + PyErr_SetString( + PyExc_ValueError, + "Unable to parse raw option value. The raw option should be a dict[str, JSONString]."); + } + } else { + PyErr_SetString( + PyExc_ValueError, + "Raw option value not a string. The raw option should be a dict[str, JSONString]."); + return {}; + } + } + } + if (raw_options.size() > 0) { + req.raw = raw_options; + } + + PyObject* pyObj_positional_parameters = + PyDict_GetItemString(pyObj_query_args, "positional_parameters"); + std::vector<couchbase::core::json_string> positional_parameters{}; + if (pyObj_positional_parameters && PyList_Check(pyObj_positional_parameters)) { + size_t nargs = static_cast<size_t>(PyList_Size(pyObj_positional_parameters)); + size_t ii; + for (ii = 0; ii < nargs; ++ii) { + PyObject* pyOb_param = PyList_GetItem(pyObj_positional_parameters, ii); + if (!pyOb_param) { + PyErr_SetString(PyExc_ValueError, "Unable to parse positional parameter."); + return {}; + } + // PyList_GetItem returns borrowed ref, inc while using, decr after done + Py_INCREF(pyOb_param); + if (PyBytes_Check(pyOb_param)) { + try { + auto res = PyObject_to_binary(pyOb_param); + positional_parameters.push_back(couchbase::core::json_string{ std::move(res) }); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_ValueError, + "Unable to parse positional paramter option value. Positional parameter " + "options must all be json strings."); + } + } else { + PyErr_SetString(PyExc_ValueError, + "Unable to parse positional parameter. Positional parameter options must " + "all be json strings."); + return {}; + } + Py_DECREF(pyOb_param); + pyOb_param = nullptr; + } + } + if (positional_parameters.size() > 0) { + req.positional_parameters = positional_parameters; + } + + PyObject* pyObj_named_parameters = PyDict_GetItemString(pyObj_query_args, "named_parameters"); + std::map<std::string, couchbase::core::json_string, std::less<>> named_parameters{}; + if (pyObj_named_parameters && PyDict_Check(pyObj_named_parameters)) { + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_named_parameters, &pos, &pyObj_key, &pyObj_value)) { + std::string k; + if (PyUnicode_Check(pyObj_key)) { + k = std::string(PyUnicode_AsUTF8(pyObj_key)); + } else { + PyErr_SetString(PyExc_ValueError, + "Named parameter key is not a string. Named parameters should be a " + "dict[str, JSONString]."); + return {}; + } + if (k.empty()) { + PyErr_SetString( + PyExc_ValueError, + "Named parameter key is empty. Named parameters should be a dict[str, JSONString]."); + return {}; + } + if (PyBytes_Check(pyObj_value)) { + try { + auto res = PyObject_to_binary(pyObj_value); + named_parameters.emplace(k, couchbase::core::json_string{ std::move(res) }); + } catch (const std::exception& e) { + PyErr_SetString(PyExc_ValueError, + "Unable to parse named parameter option. Named parameters should be a " + "dict[str, JSONString]."); + } + } else { + PyErr_SetString(PyExc_ValueError, + "Named parameter value not a string. Named parameters should be a " + "dict[str, JSONString]."); + return {}; + } + } + } + if (named_parameters.size() > 0) { + req.named_parameters = named_parameters; + } + + return req; +} + +std::string +service_type_to_str(couchbase::core::service_type t) +{ + switch (t) { + case couchbase::core::service_type::key_value: { + return "kv"; + } + case couchbase::core::service_type::query: { + return "query"; + } + case couchbase::core::service_type::analytics: { + return "analytics"; + } + case couchbase::core::service_type::search: { + return "search"; + } + case couchbase::core::service_type::management: { + return "mgmt"; + } + case couchbase::core::service_type::view: { + return "views"; + } + case couchbase::core::service_type::eventing: { + return "eventing"; + } + default: { + // TODO: better exception + PyErr_SetString(PyExc_ValueError, "Invalid service type."); + return {}; + } + } +} + +couchbase::core::service_type +str_to_service_type(std::string svc) +{ + if (svc.compare("kv") == 0) { + return couchbase::core::service_type::key_value; + } + if (svc.compare("query") == 0) { + return couchbase::core::service_type::query; + } + if (svc.compare("analytics") == 0) { + return couchbase::core::service_type::analytics; + } + if (svc.compare("search") == 0) { + return couchbase::core::service_type::search; + } + if (svc.compare("mgmt") == 0) { + return couchbase::core::service_type::management; + } + if (svc.compare("views") == 0) { + return couchbase::core::service_type::view; + } + + // TODO: better exception + PyErr_SetString(PyExc_ValueError, "Invalid service type."); + return {}; +} diff --git a/src/utils.hxx b/src/utils.hxx new file mode 100644 index 000000000..54662db04 --- /dev/null +++ b/src/utils.hxx @@ -0,0 +1,101 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "Python.h" // NOLINT + +#include "tracing.hxx" +#include <core/operations/document_query.hxx> +#include <core/utils/binary.hxx> +#include <core/utils/join_strings.hxx> +#include <core/utils/json.hxx> +#include <couchbase/durability_level.hxx> +#include <couchbase/mutation_token.hxx> +#include <couchbase/persist_to.hxx> +#include <couchbase/read_preference.hxx> +#include <couchbase/replicate_to.hxx> + +#include <tao/json/value.hpp> + +#include <chrono> +#include <stdexcept> +#include <string> + +constexpr std::chrono::seconds FIFTY_YEARS{ 50 * 365 * 24 * 60 * 60 }; + +couchbase::core::utils::binary +PyObject_to_binary(PyObject*); +PyObject* +binary_to_PyObject(couchbase::core::utils::binary value); + +PyObject* +binary_to_PyObject_unicode(couchbase::core::utils::binary value); + +std::string +binary_to_string(couchbase::core::utils::binary value); + +std::size_t py_ssize_t_to_size_t(Py_ssize_t); +Py_ssize_t size_t_to_py_ssize_t(std::size_t); + +couchbase::persist_to +PyObject_to_persist_to(PyObject* pyObj_persist_to); +couchbase::replicate_to +PyObject_to_replicate_to(PyObject* pyObj_replicate_to); +std::pair<couchbase::persist_to, couchbase::replicate_to> +PyObject_to_durability(PyObject*); +couchbase::durability_level +PyObject_to_durability_level(PyObject*); + +couchbase::read_preference +PyObject_to_read_preference(PyObject*); + +std::vector<couchbase::mutation_token> +get_mutation_state(PyObject* pyObj_mutation_state); + +std::string +profile_mode_to_str(couchbase::query_profile profile_mode); + +template<typename scan_consistency_type> +scan_consistency_type +str_to_scan_consistency_type(std::string consistency) +{ + if (consistency.compare("not_bounded") == 0) { + return scan_consistency_type::not_bounded; + } + if (consistency.compare("request_plus") == 0) { + return scan_consistency_type::request_plus; + } + + // TODO: better exception + PyErr_SetString(PyExc_ValueError, + fmt::format("Invalid Scan Consistency type {}", consistency).c_str()); + return {}; +} + +// TODO: consolidate these types of methods to another file that handles other requests as well +couchbase::core::operations::query_request +build_query_request(PyObject* pyObj_query_args); + +std::vector<couchbase::mutation_token> +get_mutation_state(PyObject* pyObj_mutation_state); + +std::string +service_type_to_str(couchbase::core::service_type t); + +couchbase::core::service_type +str_to_service_type(std::string svc); diff --git a/src/views.c b/src/views.c deleted file mode 100644 index 85eff9bc2..000000000 --- a/src/views.c +++ /dev/null @@ -1,387 +0,0 @@ -#include "pycbc.h" -#include "oputil.h" -#include "structmember.h" - -static int -should_call_async(const pycbc_ViewResult *vres, int flush_always) -{ - if (!flush_always) { - return vres->rows_per_call > -1 && - PyList_GET_SIZE(vres->rows) > vres->rows_per_call; - } else { - return PyList_GET_SIZE(vres->rows); - } -} - -void -pycbc_viewresult_addrow(pycbc_ViewResult *vres, pycbc_MultiResult *mres, - const void *data, size_t n) -{ - PyObject *j; - int rv; - - rv = pycbc_tc_simple_decode(&j, data, n, PYCBC_FMT_JSON); - if (rv != 0) { - pycbc_multiresult_adderr(mres); - pycbc_tc_simple_decode(&j, data, n, PYCBC_FMT_BYTES); - } - - PyList_Append(vres->rows, j); - Py_DECREF(j); -} - -void -pycbc_viewresult_step(pycbc_ViewResult *vres, pycbc_MultiResult *mres, - pycbc_Bucket *bucket, int force_callback) -{ - if ((bucket->flags & PYCBC_CONN_F_ASYNC) && - should_call_async(vres, force_callback)) { - pycbc_AsyncResult *ares = (pycbc_AsyncResult*)mres; - PyObject *args = PyTuple_Pack(1, mres); - PyObject *result; - - pycbc_assert(ares->callback); - - result = PyObject_CallObject(ares->callback, args); - Py_XDECREF(result); - if (!result) { - PyErr_Print(); - } - - Py_DECREF(args); - - Py_DECREF(vres->rows); - vres->rows = PyList_New(0); - } - - if (!bucket->nremaining) { - lcb_breakout(bucket->instance); - } -} - -static int -add_view_field(PyObject *dd, PyObject *k, const void *v, size_t n) -{ - PyObject *tmp; - int rv; - - if (!n) { - return 0; - } - - rv = pycbc_tc_simple_decode(&tmp, v, n, PYCBC_FMT_JSON); - if (rv != 0) { - return rv; - } - - PyDict_SetItem(dd, k, tmp); - Py_XDECREF(tmp); - return 0; -} - -static int -parse_row_json(pycbc_Bucket *bucket, pycbc_ViewResult *vres, - pycbc_MultiResult *mres, const lcb_RESPVIEWQUERY *resp) -{ - PyObject *dd = PyDict_New(); - PyObject *docid; - int is_ok, rv = 0; - - if (resp->ndocid) { - rv = pycbc_tc_decode_key(bucket, resp->docid, resp->ndocid, &docid); - if (rv == -1) { - goto GT_DONE; - } else { - PyDict_SetItem(dd, pycbc_helpers.vkey_id, docid); - Py_XDECREF(docid); - } - } - - #define ADD_FIELD(helpname, fbase) \ - add_view_field(dd, pycbc_helpers.helpname, resp->fbase, resp->n##fbase) - - is_ok = ADD_FIELD(vkey_key, key) == 0 && - ADD_FIELD(vkey_value, value) == 0 && - ADD_FIELD(vkey_geo, geometry) == 0; - - #undef ADD_FIELD - - if (!is_ok) { - rv = -1; - goto GT_DONE; - } else { - PyList_Append(vres->rows, dd); - } - - if (resp->docresp) { - /* include_docs */ - const lcb_RESPGET *rg = resp->docresp; - pycbc_ValueResult *docres = pycbc_valresult_new(bucket); - - docres->key = docid; - Py_INCREF(docid); - docres->rc = rg->rc; - - if (rg->rc == LCB_SUCCESS) { - docres->cas = rg->cas; - docres->flags = rg->itmflags; - rv = pycbc_tc_decode_value( - bucket, rg->value, rg->nvalue, rg->itmflags, &docres->value); - if (rv != 0) { - pycbc_multiresult_adderr(mres); - } - } - - PyDict_SetItem(dd, pycbc_helpers.vkey_docresp, (PyObject*)docres); - Py_DECREF(docres); - } - - GT_DONE: - Py_DECREF(dd); - return rv; -} - -static void -row_callback(lcb_t instance, int cbtype, const lcb_RESPVIEWQUERY *resp) -{ - pycbc_MultiResult *mres = (pycbc_MultiResult*)resp->cookie; - pycbc_Bucket *bucket = mres->parent; - const char * const * hdrs = NULL; - short htcode = 0; - pycbc_ViewResult *vres; - - if (resp->htresp != NULL) { - hdrs = resp->htresp->headers; - htcode = resp->htresp->htstatus; - } - - PYCBC_CONN_THR_END(bucket); - - vres = (pycbc_ViewResult*)PyDict_GetItem((PyObject*)mres, Py_None); - - if (resp->rflags & LCB_RESP_F_FINAL) { - pycbc_httpresult_add_data(mres, &vres->base, resp->value, resp->nvalue); - } else if (resp->rc == LCB_SUCCESS) { - if (parse_row_json(bucket, vres, mres, resp) != 0) { - pycbc_multiresult_adderr(mres); - } - } - - pycbc_viewresult_step(vres, mres, bucket, resp->rflags & LCB_RESP_F_FINAL); - - if (resp->rflags & LCB_RESP_F_FINAL) { - pycbc_httpresult_complete(&vres->base, mres, resp->rc, htcode, hdrs); - } else { - PYCBC_CONN_THR_BEGIN(bucket); - } - (void)instance; (void)cbtype; -} - -void -pycbc_views_callbacks_init(lcb_t instance) -{ - (void)instance; -} - -typedef struct { - const char *optstr; - pycbc_strlen_t noptstr; - const void *body; - pycbc_strlen_t nbody; - PyObject *bk; -} viewpath_st; - -static int -get_viewpath_str(pycbc_Bucket *self, viewpath_st *vp, PyObject *options) -{ - PyObject *args; - - if (!options) { - options = Py_None; - } - - args = PyTuple_Pack(1, options); - vp->bk = PyObject_CallObject(pycbc_helpers.view_path_helper, args); - Py_DECREF(args); - - if (!vp->bk) { - return -1; - } else { - int rv = PyArg_ParseTuple( - vp->bk, "s#s#", &vp->optstr, &vp->noptstr, - &vp->body, &vp->nbody); - - if (!rv) { - return -1; - } - } - return 0; -} - -PyObject * -pycbc_Bucket__view_request(pycbc_Bucket *self, PyObject *args, PyObject *kwargs) -{ - int rv; - PyObject *ret = NULL; - pycbc_MultiResult *mres = NULL; - pycbc_ViewResult *vres = NULL; - lcb_CMDVIEWQUERY vcmd = { 0 }; - viewpath_st vp = { NULL }; - lcb_error_t rc; - const char *view = NULL, *design = NULL; - PyObject *options = NULL; - int flags; - - static char *kwlist[] = { "design", "view", "options", "_flags", NULL }; - - rv = PyArg_ParseTupleAndKeywords(args, kwargs, "ss|Oi", kwlist, - &design, &view, &options, &flags); - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - if (-1 == pycbc_oputil_conn_lock(self)) { - return NULL; - } - - if (self->pipeline_queue) { - PYCBC_EXC_WRAP(PYCBC_EXC_PIPELINE, 0, - "HTTP/View Requests cannot be executed in " - "pipeline context"); - } - - mres = (pycbc_MultiResult *)pycbc_multiresult_new(self); - vres = (pycbc_ViewResult *)PYCBC_TYPE_CTOR(&pycbc_ViewResultType); - vres->base.htype = PYCBC_HTTP_HVIEW; - - pycbc_httpresult_init(&vres->base, mres); - - rv = get_viewpath_str(self, &vp, options); - if (rv != 0) { - goto GT_DONE; - } - - vcmd.ddoc = design; - vcmd.nddoc = strlen(design); - vcmd.view = view; - vcmd.nview = strlen(view); - vcmd.optstr = vp.optstr; - vcmd.noptstr = vp.noptstr; - vcmd.postdata = vp.body; - vcmd.npostdata = vp.nbody; - vcmd.handle = &vres->base.u.vh; - vcmd.callback = row_callback; - vcmd.cmdflags = flags; - - vres->rows = PyList_New(0); - vres->base.format = PYCBC_FMT_JSON; - - rc = lcb_view_query(self->instance, mres, &vcmd); - - if (rc != LCB_SUCCESS) { - PYCBC_EXC_WRAP(PYCBC_EXC_LCBERR, rc, "Couldn't schedule view"); - goto GT_DONE; - } - - ret = (PyObject*)mres; - mres = NULL; /* Avoid GT_DONE decref */ - - GT_DONE: - Py_XDECREF(mres); - Py_XDECREF(vp.bk); - pycbc_oputil_conn_unlock(self); - return ret; -} - -static PyObject * -ViewResult_fetch(pycbc_ViewResult *self, PyObject *args) -{ - PyObject *ret = NULL; - pycbc_MultiResult *mres = NULL; - pycbc_Bucket *bucket; - int rv; - - rv = PyArg_ParseTuple(args, "O", &mres); - if (!rv) { - PYCBC_EXCTHROW_ARGS(); - return NULL; - } - - bucket = mres->parent; - - if (bucket->flags & PYCBC_CONN_F_ASYNC) { - PYCBC_EXC_WRAP(PYCBC_EXC_INTERNAL, 0, "Cannot use fetch with async"); - return NULL; - } - - if (-1 == pycbc_oputil_conn_lock(bucket)) { - return NULL; - } - - if (!self->base.done) { - pycbc_oputil_wait_common(bucket); - } - - if (pycbc_multiresult_maybe_raise(mres)) { - goto GT_DONE; - } - - ret = self->rows ? self->rows : PyList_New(0); - self->rows = PyList_New(0); - - GT_DONE: - pycbc_oputil_conn_unlock(bucket); - return ret; -} - -static void -ViewResult_dealloc(pycbc_ViewResult *vres) -{ - Py_CLEAR(vres->rows); - Py_TYPE(vres)->tp_base->tp_dealloc((PyObject*)vres); -} - -PyTypeObject pycbc_ViewResultType = { - PYCBC_POBJ_HEAD_INIT(NULL) - 0 -}; - -static struct PyMemberDef ViewResult_TABLE_members[] = { - { "rows", - T_OBJECT_EX, offsetof(pycbc_ViewResult, rows), READONLY, - PyDoc_STR("Most recently fetched rows") - }, - { "rows_per_call", - T_LONG, offsetof(pycbc_ViewResult, rows_per_call), 0, - PyDoc_STR("Rate limit callbacks to this many rows at a time") - }, - { NULL } -}; - -static struct PyMethodDef ViewResult_TABLE_methods[] = { - { "fetch", (PyCFunction)ViewResult_fetch, METH_VARARGS, - PyDoc_STR("Call this to fetch items from the view") - }, - { NULL } -}; - -int -pycbc_ViewResultType_init(PyObject **ptr) -{ - PyTypeObject *p = &pycbc_ViewResultType; - *ptr = (PyObject*)p; - if (p->tp_name) { - return 0; - } - - p->tp_name = "ViewResult"; - p->tp_doc = PyDoc_STR("Low level view result object"); - p->tp_new = PyType_GenericNew; - p->tp_dealloc = (destructor)ViewResult_dealloc; - p->tp_basicsize = sizeof(pycbc_ViewResult); - p->tp_members = ViewResult_TABLE_members; - p->tp_methods = ViewResult_TABLE_methods; - p->tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE; - p->tp_base = &pycbc_HttpResultType; - return pycbc_ResultType_ready(p, PYCBC_HTRESULT_BASEFLDS); -} diff --git a/src/views.cxx b/src/views.cxx new file mode 100644 index 000000000..22e40a393 --- /dev/null +++ b/src/views.cxx @@ -0,0 +1,527 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "views.hxx" + +#include <core/design_document_namespace.hxx> +#include <core/management/design_document.hxx> +#include <core/operations/document_view.hxx> +#include <core/view_scan_consistency.hxx> +#include <core/view_sort_order.hxx> + +#include "exceptions.hxx" +#include "tracing.hxx" + +result* +create_result_from_view_response(couchbase::core::operations::document_view_response resp) +{ + PyObject* pyObj_result = create_result_obj(); + result* res = reinterpret_cast<result*>(pyObj_result); + + PyObject* pyObj_tmp = nullptr; + PyObject* pyObj_payload = PyDict_New(); + + if (resp.error.has_value()) { + PyObject* pyObj_error = PyDict_New(); + pyObj_tmp = PyUnicode_FromString(resp.error.value().code.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error, "code", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(resp.error.value().message.c_str()); + if (-1 == PyDict_SetItemString(pyObj_error, "message", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + if (-1 == PyDict_SetItemString(pyObj_payload, "error", pyObj_error)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_error); + } + + PyObject* pyObj_meta = PyDict_New(); + if (resp.meta.total_rows.has_value()) { + pyObj_tmp = PyLong_FromUnsignedLongLong(resp.meta.total_rows.value()); + if (-1 == PyDict_SetItemString(pyObj_meta, "total_rows", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + if (resp.meta.debug_info.has_value()) { + pyObj_tmp = PyUnicode_FromString(resp.meta.debug_info.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_meta, "debug_info", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + if (-1 == PyDict_SetItemString(pyObj_payload, "metadata", pyObj_meta)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_meta); + + // if (resp.rows.size() > 0) { + // PyObject* pyObj_rows = PyList_New(static_cast<Py_ssize_t>(0)); + // for (auto const& row : resp.rows) { + // PyObject* pyObj_row = PyDict_New(); + + // if (row.id.has_value()) { + // pyObj_tmp = PyUnicode_FromString(row.id.value().c_str()); + // if (-1 == PyDict_SetItemString(pyObj_row, "id", pyObj_tmp)) { + // PyErr_Print(); + // PyErr_Clear(); + // } + // Py_DECREF(pyObj_tmp); + // } + + // pyObj_tmp = PyUnicode_FromString(row.key.c_str()); + // if (-1 == PyDict_SetItemString(pyObj_row, "key", pyObj_tmp)) { + // PyErr_Print(); + // PyErr_Clear(); + // } + // Py_DECREF(pyObj_tmp); + + // pyObj_tmp = PyUnicode_FromString(row.value.c_str()); + // if (-1 == PyDict_SetItemString(pyObj_row, "value", pyObj_tmp)) { + // PyErr_Print(); + // PyErr_Clear(); + // } + // Py_DECREF(pyObj_tmp); + + // if (-1 == PyList_Append(pyObj_rows, pyObj_row)) { + // PyErr_Print(); + // PyErr_Clear(); + // } + // } + + // if (-1 == PyDict_SetItemString(pyObj_payload, "rows", pyObj_rows)) { + // PyErr_Print(); + // PyErr_Clear(); + // } + // Py_DECREF(pyObj_rows); + // } + + if (-1 == PyDict_SetItemString(res->dict, RESULT_VALUE, pyObj_payload)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_XDECREF(pyObj_payload); + + return res; +} + +void +create_view_result(couchbase::core::operations::document_view_response resp, + std::shared_ptr<rows_queue<PyObject*>> rows, + PyObject* pyObj_callback, + PyObject* pyObj_errback) +{ + + auto set_exception = false; + PyObject* pyObj_exc = nullptr; + PyObject* pyObj_args = NULL; + PyObject* pyObj_func = NULL; + PyObject* pyObj_callback_res = nullptr; + + PyGILState_STATE state = PyGILState_Ensure(); + + if (resp.ctx.ec.value()) { + pyObj_exc = + build_exception_from_context(resp.ctx, __FILE__, __LINE__, "Error doing views operation."); + // lets clear any errors + PyErr_Clear(); + rows->put(pyObj_exc); + } else { + for (auto const& row : resp.rows) { + PyObject* pyObj_row = PyDict_New(); + PyObject* pyObj_tmp = nullptr; + + if (row.id.has_value()) { + pyObj_tmp = PyUnicode_FromString(row.id.value().c_str()); + if (-1 == PyDict_SetItemString(pyObj_row, "id", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + } + + pyObj_tmp = PyUnicode_FromString(row.key.c_str()); + if (-1 == PyDict_SetItemString(pyObj_row, "key", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + pyObj_tmp = PyUnicode_FromString(row.value.c_str()); + if (-1 == PyDict_SetItemString(pyObj_row, "value", pyObj_tmp)) { + PyErr_Print(); + PyErr_Clear(); + } + Py_DECREF(pyObj_tmp); + + rows->put(pyObj_row); + } + + auto res = create_result_from_view_response(resp); + if (res == nullptr || PyErr_Occurred() != nullptr) { + set_exception = true; + } else { + // None indicates done (i.e. raise StopIteration) + Py_INCREF(Py_None); + rows->put(Py_None); + rows->put(reinterpret_cast<PyObject*>(res)); + } + } + + if (set_exception) { + pyObj_exc = pycbc_build_exception( + PycbcError::UnableToBuildResult, __FILE__, __LINE__, "Views operation error."); + rows->put(pyObj_exc); + } + + // This is for txcouchbase -- let it knows we're done w/ the query request + if (pyObj_callback != nullptr) { + pyObj_func = pyObj_callback; + pyObj_args = PyTuple_New(1); + PyTuple_SET_ITEM(pyObj_args, 0, PyBool_FromLong(static_cast<long>(1))); + } + + if (pyObj_func != nullptr) { + pyObj_callback_res = PyObject_CallObject(pyObj_func, pyObj_args); + if (pyObj_callback_res) { + Py_DECREF(pyObj_callback_res); + } else { + pycbc_set_python_exception( + PycbcError::InternalSDKError, __FILE__, __LINE__, "Views complete callback failed."); + } + Py_DECREF(pyObj_args); + Py_XDECREF(pyObj_callback); + Py_XDECREF(pyObj_errback); + } + + PyGILState_Release(state); +} + +couchbase::core::operations::document_view_request +get_view_request(PyObject* op_args) +{ + PyObject* pyObj_bucket_name = PyDict_GetItemString(op_args, "bucket_name"); + auto bucket_name = std::string(PyUnicode_AsUTF8(pyObj_bucket_name)); + + PyObject* pyObj_document_name = PyDict_GetItemString(op_args, "document_name"); + auto document_name = std::string(PyUnicode_AsUTF8(pyObj_document_name)); + + PyObject* pyObj_view_name = PyDict_GetItemString(op_args, "view_name"); + auto view_name = std::string(PyUnicode_AsUTF8(pyObj_view_name)); + + couchbase::core::operations::document_view_request req{ bucket_name, document_name, view_name }; + + PyObject* pyObj_namespace = PyDict_GetItemString(op_args, "namespace"); + if (pyObj_namespace != nullptr) { + auto ns = couchbase::core::design_document_namespace::development; + if (pyObj_namespace == Py_False) { + ns = couchbase::core::design_document_namespace::production; + } + req.ns = ns; + } + + PyObject* pyObj_limit = PyDict_GetItemString(op_args, "limit"); + if (pyObj_limit != nullptr) { + auto limit = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_limit)); + req.limit = limit; + } + + PyObject* pyObj_skip = PyDict_GetItemString(op_args, "skip"); + if (pyObj_skip != nullptr) { + auto skip = static_cast<uint64_t>(PyLong_AsUnsignedLongLong(pyObj_skip)); + req.skip = skip; + } + + PyObject* pyObj_scan_consistency = PyDict_GetItemString(op_args, "scan_consistency"); + if (pyObj_scan_consistency != nullptr) { + auto scan_consistency = std::string(PyUnicode_AsUTF8(pyObj_scan_consistency)); + if (scan_consistency.compare("ok") == 0) { + req.consistency = couchbase::core::view_scan_consistency::not_bounded; + } else if (scan_consistency.compare("update_after") == 0) { + req.consistency = couchbase::core::view_scan_consistency::update_after; + } else if (scan_consistency.compare("false") == 0) { + req.consistency = couchbase::core::view_scan_consistency::request_plus; + } + } + + PyObject* pyObj_keys = PyDict_GetItemString(op_args, "keys"); + if (pyObj_keys != nullptr && PyList_Check(pyObj_keys)) { + size_t nkeys = static_cast<size_t>(PyList_GET_SIZE(pyObj_keys)); + std::vector<std::string> keys{}; + size_t ii; + for (ii = 0; ii < nkeys; ++ii) { + PyObject* pyObj_key = PyList_GetItem(pyObj_keys, ii); + auto key = std::string(PyUnicode_AsUTF8(pyObj_key)); + keys.push_back(key); + } + + if (keys.size() > 0) { + req.keys = keys; + } + } + + PyObject* pyObj_key = PyDict_GetItemString(op_args, "key"); + if (pyObj_key != nullptr) { + auto key = std::string(PyUnicode_AsUTF8(pyObj_key)); + req.key = key; + } + + PyObject* pyObj_start_key = PyDict_GetItemString(op_args, "start_key"); + if (pyObj_start_key != nullptr) { + auto start_key = std::string(PyUnicode_AsUTF8(pyObj_start_key)); + req.start_key = start_key; + } + + PyObject* pyObj_end_key = PyDict_GetItemString(op_args, "end_key"); + if (pyObj_end_key != nullptr) { + auto end_key = std::string(PyUnicode_AsUTF8(pyObj_end_key)); + req.end_key = end_key; + } + + PyObject* pyObj_start_key_doc_id = PyDict_GetItemString(op_args, "start_key_doc_id"); + if (pyObj_start_key_doc_id != nullptr) { + auto start_key_doc_id = std::string(PyUnicode_AsUTF8(pyObj_start_key_doc_id)); + req.start_key_doc_id = start_key_doc_id; + } + + PyObject* pyObj_end_key_doc_id = PyDict_GetItemString(op_args, "end_key_doc_id"); + if (pyObj_end_key_doc_id != nullptr) { + auto end_key_doc_id = std::string(PyUnicode_AsUTF8(pyObj_end_key_doc_id)); + req.end_key_doc_id = end_key_doc_id; + } + + PyObject* pyObj_inclusive_end = PyDict_GetItemString(op_args, "inclusive_end"); + if (pyObj_inclusive_end != nullptr) { + if (pyObj_inclusive_end == Py_True) { + req.inclusive_end = true; + } else { + req.inclusive_end = false; + } + } + + PyObject* pyObj_reduce = PyDict_GetItemString(op_args, "reduce"); + if (pyObj_reduce != nullptr) { + if (pyObj_reduce == Py_True) { + req.reduce = true; + } else { + req.reduce = false; + } + } + + PyObject* pyObj_group = PyDict_GetItemString(op_args, "group"); + if (pyObj_group != nullptr) { + if (pyObj_group == Py_True) { + req.group = true; + } else { + req.group = false; + } + } + + PyObject* pyObj_group_level = PyDict_GetItemString(op_args, "group_level"); + if (pyObj_group_level != nullptr) { + auto group_level = static_cast<uint32_t>(PyLong_AsUnsignedLong(pyObj_group_level)); + req.group_level = group_level; + } + + PyObject* pyObj_debug = PyDict_GetItemString(op_args, "debug"); + if (pyObj_debug != nullptr && pyObj_debug == Py_True) { + req.debug = true; + } + + PyObject* pyObj_order = PyDict_GetItemString(op_args, "order"); + if (pyObj_order != nullptr) { + auto order = std::string(PyUnicode_AsUTF8(pyObj_order)); + if (order.compare("false") == 0) { + req.order = couchbase::core::view_sort_order::ascending; + } else if (order.compare("true") == 0) { + req.order = couchbase::core::view_sort_order::descending; + } + } + + PyObject* pyObj_query_string = PyDict_GetItemString(op_args, "query_string"); + if (pyObj_query_string != nullptr && PyList_Check(pyObj_query_string)) { + size_t nqstrings = static_cast<size_t>(PyList_GET_SIZE(pyObj_query_string)); + std::vector<std::string> query_string{}; + size_t ii; + for (ii = 0; ii < nqstrings; ++ii) { + PyObject* pyObj_q_string = PyList_GetItem(pyObj_query_string, ii); + auto q_string = std::string(PyUnicode_AsUTF8(pyObj_q_string)); + query_string.push_back(q_string); + } + + if (query_string.size() > 0) { + req.query_string = query_string; + } + } + + PyObject* pyObj_client_context_id = PyDict_GetItemString(op_args, "client_context_id"); + if (pyObj_client_context_id != nullptr) { + auto client_context_id = std::string(PyUnicode_AsUTF8(pyObj_client_context_id)); + req.client_context_id = client_context_id; + } + + PyObject* pyObj_timeout = PyDict_GetItemString(op_args, "timeout"); + if (nullptr != pyObj_timeout) { + // comes in as microseconds + req.timeout = std::chrono::milliseconds(PyLong_AsUnsignedLongLong(pyObj_timeout) / 1000ULL); + } + + PyObject* pyObj_full_set = PyDict_GetItemString(op_args, "full_set"); + if (pyObj_full_set != nullptr) { + req.full_set = pyObj_debug == Py_True ? true : false; + } + + PyObject* pyObj_raw = PyDict_GetItemString(op_args, "raw"); + if (pyObj_raw != nullptr && PyDict_Check(pyObj_raw)) { + std::map<std::string, std::string> raw_options{}; + PyObject *pyObj_key, *pyObj_value; + Py_ssize_t pos = 0; + + // PyObj_key and pyObj_value are borrowed references + while (PyDict_Next(pyObj_raw, &pos, &pyObj_key, &pyObj_value)) { + std::string key; + std::string val; + if (PyUnicode_Check(pyObj_key)) { + key = std::string(PyUnicode_AsUTF8(pyObj_key)); + } else { + PyErr_SetString( + PyExc_ValueError, + "Raw parameter key is not a string. Raw option should be a dict[str, str]."); + return {}; + } + if (key.empty()) { + PyErr_SetString(PyExc_ValueError, + "Raw parameter key is empty. Raw option should be a dict[str, str]."); + return {}; + } + if (PyUnicode_Check(pyObj_value)) { + val = std::string(PyUnicode_AsUTF8(pyObj_value)); + } else { + PyErr_SetString( + PyExc_ValueError, + "Raw parameter value is not a string. Raw option should be a dict[str, str]."); + return {}; + } + if (val.empty()) { + PyErr_SetString(PyExc_ValueError, + "Raw parameter value is empty. Raw option should be a dict[str, str]."); + return {}; + } + raw_options.emplace(key, val); + } + if (raw_options.size() > 0) { + req.raw = raw_options; + } + } + + return req; +} + +streamed_result* +handle_view_query([[maybe_unused]] PyObject* self, PyObject* args, PyObject* kwargs) +{ + // need these for all operations + PyObject* pyObj_conn = nullptr; + // optional + PyObject* pyObj_op_args = nullptr; + std::uint64_t streaming_timeout_us = 0; + PyObject* pyObj_callback = nullptr; + PyObject* pyObj_errback = nullptr; + PyObject* pyObj_row_callback = nullptr; + PyObject* pyObj_span = nullptr; + + static const char* kw_list[] = { "conn", "op_args", "streaming_timeout", + "callback", "errback", "row_callback", + "span", nullptr }; + + const char* kw_format = "O!|OKOOOO"; + int ret = PyArg_ParseTupleAndKeywords(args, + kwargs, + kw_format, + const_cast<char**>(kw_list), + &PyCapsule_Type, + &pyObj_conn, + &pyObj_op_args, + &streaming_timeout_us, + &pyObj_callback, + &pyObj_errback, + &pyObj_row_callback, + &pyObj_span); + if (!ret) { + PyErr_Print(); + PyErr_SetString(PyExc_ValueError, "Unable to parse arguments"); + return nullptr; + } + + connection* conn = nullptr; + conn = reinterpret_cast<connection*>(PyCapsule_GetPointer(pyObj_conn, "conn_")); + if (nullptr == conn) { + PyErr_SetString(PyExc_ValueError, "passed null connection"); + return nullptr; + } + PyErr_Clear(); + + auto req = get_view_request(pyObj_op_args); + + // timeout is always set either to default, or timeout provided in options + auto streaming_timeout = couchbase::core::timeout_defaults::view_timeout; + if (streaming_timeout_us > 0) { + streaming_timeout = std::chrono::milliseconds(streaming_timeout_us / 1000ULL); + } + streamed_result* streamed_res = create_streamed_result_obj(streaming_timeout); + + if (nullptr != pyObj_span) { + req.parent_span = std::make_shared<pycbc::request_span>(pyObj_span); + } + // TODO: let the couchbase++ streaming stabilize a bit more... + // req.row_callback = [rows = streamed_res->rows](std::string&& row) { + // PyGILState_STATE state = PyGILState_Ensure(); + // PyObject* pyObj_row = PyBytes_FromStringAndSize(row.c_str(), row.length()); + // rows->put(pyObj_row); + // PyGILState_Release(state); + // return couchbase::core::utils::json::stream_control::next_row; + // }; + + // we need the callback, errback, and logic to all stick around, so... + // use XINCREF b/c they _could_ be NULL + Py_XINCREF(pyObj_errback); + Py_XINCREF(pyObj_callback); + + { + Py_BEGIN_ALLOW_THREADS conn->cluster_.execute( + req, + [rows = streamed_res->rows, pyObj_callback, pyObj_errback]( + couchbase::core::operations::document_view_response resp) { + create_view_result(resp, rows, pyObj_callback, pyObj_errback); + }); + Py_END_ALLOW_THREADS + } + return streamed_res; +} diff --git a/src/views.hxx b/src/views.hxx new file mode 100644 index 000000000..2d78ec030 --- /dev/null +++ b/src/views.hxx @@ -0,0 +1,24 @@ +/* + * Copyright 2016-2022. Couchbase, Inc. + * All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include "client.hxx" +#include "result.hxx" + +streamed_result* +handle_view_query(PyObject* self, PyObject* args, PyObject* kwargs); diff --git a/tests.ini.sample b/tests.ini.sample deleted file mode 100644 index c168508d7..000000000 --- a/tests.ini.sample +++ /dev/null @@ -1,25 +0,0 @@ -[realserver] -host = localhost -port = 8091 -admin_username = Administrator - -; The administrative password. This is the password used to -; log into the admin console -admin_password = 123456 - -bucket_name = default -; If a SASL bucket is being used (i.e. buckets are set up -; per the script, then this is the *bucket* password -; bucket_password sasl_password -bucket_password = - -; Set this to true if there is a real cluster available -enabled = True - -[mock] -; Set this to enabled to use the mock -enabled = True -; Local path for the mock -path = /tmp/CouchbaseMock-1.0.0-g50cf222.jar -; Where to download it, if not available -url = http://packages.coucbase.com/clients/c/mock/CouchbaseMock-LATEST.jar diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..30affc24d --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1,17 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .test_features import EnvironmentFeatures # noqa: F401 +from .test_features import ServerFeatures # noqa: F401 diff --git a/tests/consistency.py b/tests/consistency.py new file mode 100644 index 000000000..dfe02e2eb --- /dev/null +++ b/tests/consistency.py @@ -0,0 +1,166 @@ +from datetime import datetime, timedelta +from time import sleep +from typing import Iterable + +import requests + +from tests.couchbase_config import CouchbaseConfig + + +class ConsistencyChecker: + RETRY_DELAY_SECS = 1 + DEFAULT_TIMEOUT = timedelta(seconds=10) + + def __init__(self, **kwargs): + self._hostnames: Iterable[str] = kwargs.get('hostnames', []) + self._config: CouchbaseConfig = kwargs.get('config') + + def fetch_hostnames(self): + path = f'http://{self._config.host}:{self._config.port}/pools/nodes' + resp = requests.get(path, auth=self._config.get_username_and_pw()) + self._hostnames = [node['configuredHostname'] for node in resp.json()['nodes']] + + def wait_until_user_present(self, name, domain='local', timeout=DEFAULT_TIMEOUT): + path = f'settings/rbac/users/{domain}/{name}' + error_msg = f'User {name} in the {domain} domain is not present in all nodes' + self._wait_until_resource_present(path, error_msg, timeout) + + def wait_until_user_dropped(self, name, domain='local', timeout=DEFAULT_TIMEOUT): + path = f'settings/rbac/users/{domain}/{name}' + error_msg = f'User {name} in the {domain} domain has not been dropped from all nodes' + self._wait_until_resource_dropped(path, error_msg, timeout) + + def wait_until_group_present(self, name, timeout=DEFAULT_TIMEOUT): + path = f'settings/rbac/groups/{name}' + error_msg = f'Group {name} is not present in all nodes' + self._wait_until_resource_present(path, error_msg, timeout) + + def wait_until_group_dropped(self, name, timeout=DEFAULT_TIMEOUT): + path = f'settings/rbac/groups/{name}' + error_msg = f'Group {name} has not been dropped from all nodes' + self._wait_until_resource_dropped(path, error_msg, timeout) + + def wait_until_bucket_present(self, name, timeout=DEFAULT_TIMEOUT): + path = f'pools/default/buckets/{name}' + error_msg = f'Bucket {name} is not present in all nodes' + self._wait_until_resource_present(path, error_msg, timeout) + + def wait_until_bucket_dropped(self, name, timeout=DEFAULT_TIMEOUT): + path = f'pools/default/buckets/{name}' + error_msg = f'Bucket {name} has not been dropped from all nodes' + self._wait_until_resource_dropped(path, error_msg, timeout) + + def wait_until_scope_present(self, bucket_name, scope_name, timeout=DEFAULT_TIMEOUT): + def predicate(resp): + return any(scope['name'] == scope_name for scope in resp['scopes']) + + path = f'pools/default/buckets/{bucket_name}/scopes' + error_msg = f'Scope {scope_name} in bucket {bucket_name} is not present in all nodes' + self._wait_until_resource_satisfies_predicate(path, predicate, error_msg, timeout) + + def wait_until_scope_dropped(self, bucket_name, scope_name, timeout=DEFAULT_TIMEOUT): + def predicate(resp): + return all(scope['name'] != scope_name for scope in resp['scopes']) + + path = f'pools/default/buckets/{bucket_name}/scopes' + error_msg = f'Scope {scope_name} in bucket {bucket_name} has not been dropped in all nodes' + self._wait_until_resource_satisfies_predicate(path, predicate, error_msg, timeout) + + def wait_until_collection_present(self, bucket_name, scope_name, collection_name, timeout=DEFAULT_TIMEOUT): + def predicate(resp): + return any( + scope['name'] == scope_name and any(coll['name'] == collection_name for coll in scope['collections']) + for scope in resp['scopes'] + ) + + path = f'pools/default/buckets/{bucket_name}/scopes' + error_msg = f'Collection {collection_name} in scope {scope_name}, bucket {bucket_name} is not present in all ' \ + f'nodes' + self._wait_until_resource_satisfies_predicate(path, predicate, error_msg, timeout) + + def wait_until_collection_dropped(self, bucket_name, scope_name, collection_name, timeout=DEFAULT_TIMEOUT): + def predicate(resp): + return all( + scope['name'] != scope_name or all(coll['name'] != collection_name for coll in scope['collections']) + for scope in resp['scopes'] + ) + + path = f'pools/default/buckets/{bucket_name}/scopes' + error_msg = f'Collection {collection_name} in scope {scope_name}, bucket {bucket_name} has not been dropped ' \ + f'in all nodes' + self._wait_until_resource_satisfies_predicate(path, predicate, error_msg, timeout) + + def wait_until_collection_has_settings(self, bucket_name, scope_name, collection_name, settings, + timeout=DEFAULT_TIMEOUT): + def satisfies_settings(coll): + return all(coll[k] == v for k, v in settings.items()) + + def predicate(resp): + return any( + scope['name'] == scope_name + and any(coll['name'] == collection_name and satisfies_settings(coll) for coll in scope['collections']) + for scope in resp['scopes'] + ) + + path = f'pools/default/buckets/{bucket_name}/scopes' + error_msg = f'Collection {collection_name} in scope {scope_name}, bucket {bucket_name} does not have the' \ + f'expected settings in all nodes ({settings})' + self._wait_until_resource_satisfies_predicate(path, predicate, error_msg, timeout) + + def _wait_until_resource_present(self, path, error_msg, timeout=DEFAULT_TIMEOUT): + deadline = datetime.now() + timeout + while datetime.now() < deadline: + if self._resource_is_present(path): + return + else: + sleep(self.RETRY_DELAY_SECS) + raise RuntimeError(error_msg) + + def _wait_until_resource_dropped(self, path, error_msg, timeout=DEFAULT_TIMEOUT): + deadline = datetime.now() + timeout + while datetime.now() < deadline: + if self._resource_is_dropped(path): + return + else: + sleep(self.RETRY_DELAY_SECS) + raise RuntimeError(error_msg) + + def _wait_until_resource_satisfies_predicate(self, path, predicate, error_msg, timeout=DEFAULT_TIMEOUT): + deadline = datetime.now() + timeout + while datetime.now() < deadline: + if self._resource_satisfies_predicate(path, predicate): + return + else: + sleep(self.RETRY_DELAY_SECS) + raise RuntimeError(error_msg) + + def _resource_is_present(self, path): + for hostname in self._hostnames: + url = f'http://{hostname}/{path}' + resp = requests.get(url, auth=self._config.get_username_and_pw()) + if resp.status_code != 200: + return False + return True + + def _resource_is_dropped(self, path): + for hostname in self._hostnames: + url = f'http://{hostname}/{path}' + resp = requests.get(url, auth=self._config.get_username_and_pw()) + if resp.status_code != 404: + return False + return True + + def _resource_satisfies_predicate(self, path, predicate): + for hostname in self._hostnames: + url = f'http://{hostname}/{path}' + resp = requests.get(url, auth=self._config.get_username_and_pw()) + if resp.status_code != 200 or not predicate(resp.json()): + return False + return True + + @classmethod + def from_test_environment(cls, env): + checker = cls(config=env.config) + if not (env.config.is_protostellar or env.config.is_mock_server): + checker.fetch_hostnames() + return checker diff --git a/tests/couchbase_config.py b/tests/couchbase_config.py new file mode 100644 index 000000000..8f692ca46 --- /dev/null +++ b/tests/couchbase_config.py @@ -0,0 +1,214 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pathlib +import time +from configparser import ConfigParser +from typing import Optional + +import pytest + +from tests.environments import CouchbaseTestEnvironmentException + +from .mock_server import (LegacyMockBucketSpec, + MockServer, + MockServerType) + +BASEDIR = pathlib.Path(__file__).parent.parent +CONFIG_FILE = os.path.join(pathlib.Path(__file__).parent, "test_config.ini") + + +class CouchbaseConfig: + def __init__(self): + self.host = "localhost" + self.port = 8091 + self.admin_username = "Administrator" + self.admin_password = "password" + self.bucket_name = "default" + self.bucket_password = "" + self.real_server_enabled = False + self.mock_server_enabled = False + self.mock_server = None + self.protostellar_enabled = False + + @property + def is_mock_server(self): + return self.mock_server_enabled + + @property + def is_real_server(self): + return self.real_server_enabled + + @property + def is_protostellar(self): + return self.protostellar_enabled + + def get_connection_string(self) -> str: + if self.mock_server_enabled: + if self.mock_server.mock_type == MockServerType.Legacy: + # What current client uses for mock: + # http://127.0.0.1:49696?ipv6=disabled + return f"http://{self.host}:{self.port}" + + return self.mock_server.connstr + elif self.is_protostellar: + return f"protostellar://{self.host}:{self.port}" + else: + return f"couchbase://{self.host}" + + def get_username_and_pw(self): + return self.admin_username, self.admin_password + + def shutdown(self): + if self.mock_server_enabled and self.mock_server.mock_type == MockServerType.GoCAVES: + self.mock_server.shutdown() + + @classmethod + def load_config(cls): # noqa: C901 + couchbase_config = cls() + try: + test_config = ConfigParser() + test_config.read(CONFIG_FILE) + + if test_config.getboolean('realserver', 'enabled', fallback=False): + couchbase_config.real_server_enabled = True + couchbase_config.protostellar_enabled = test_config.getboolean( + 'realserver', 'is_protostellar', fallback=False) + couchbase_config.host = test_config.get('realserver', 'host') + couchbase_config.port = test_config.getint('realserver', 'port') + couchbase_config.admin_username = test_config.get( + 'realserver', 'admin_username') + couchbase_config.admin_password = test_config.get( + 'realserver', 'admin_password') + couchbase_config.bucket_name = test_config.get('realserver', 'bucket_name') + couchbase_config.bucket_password = test_config.get( + 'realserver', 'bucket_password') + + mock_path = '' + mock_url = '' + mock_version = '' + # @TODO(jc): allow override of log dir and filename + # log_dir = '' + # log_filename = '' + if test_config.getboolean('gocaves', 'enabled'): + if couchbase_config.real_server_enabled: + raise CouchbaseTestEnvironmentException( + "Both real and mock servers cannot be enabled at the same time.") + + couchbase_config.mock_server_enabled = True + + if test_config.has_option('gocaves', 'path'): + mock_path = str( + BASEDIR.joinpath(test_config.get('gocaves', 'path'))) + if test_config.has_option('gocaves', 'url'): + mock_url = test_config.get('gocaves', 'url') + if test_config.has_option('gocaves', 'version'): + mock_version = test_config.get('gocaves', 'version') + + couchbase_config.mock_server = CouchbaseConfig.create_mock_server(MockServerType.GoCAVES, + mock_path, + mock_url, + mock_version) + couchbase_config.bucket_name = "default" + # cluster_info.port = cluster_info.mock_server.rest_port + # cluster_info.host = "127.0.0.1" + couchbase_config.admin_username = "Administrator" + couchbase_config.admin_password = "password" + + if test_config.has_section('legacymockserver') and test_config.getboolean('legacymockserver', 'enabled'): + if couchbase_config.real_server_enabled: + raise CouchbaseTestEnvironmentException( + "Both real and mock servers cannot be enabled at the same time.") + + if couchbase_config.mock_server_enabled: + raise CouchbaseTestEnvironmentException( + "Both java mock and gocaves mock cannot be enabled at the same time.") + + couchbase_config.mock_server_enabled = True + mock_path = str( + BASEDIR.joinpath(test_config.get("mockserver", "path"))) + if test_config.has_option("mockserver", "url"): + mock_url = test_config.get("mockserver", "url") + + couchbase_config.mock_server = CouchbaseConfig.create_mock_server(MockServerType.Legacy, + mock_path, + mock_url) + couchbase_config.bucket_name = "default" + couchbase_config.port = couchbase_config.mock_server.rest_port + couchbase_config.host = "127.0.0.1" + couchbase_config.admin_username = "Administrator" + couchbase_config.admin_password = "password" + + except CouchbaseTestEnvironmentException: + raise + except Exception as ex: + raise CouchbaseTestEnvironmentException( + f"Problem trying read/load test configuration:\n{ex}") + + return couchbase_config + + @staticmethod + def create_mock_server(mock_type, # type: MockServerType + mock_path, # type: str + mock_download_url, # type: Optional[str] + mock_version, # type: Optional[str] + log_dir=None, # type: Optional[str] + log_filename=None, # type: Optional[str] + ) -> MockServer: + + if mock_type == MockServerType.Legacy: + bspec_dfl = LegacyMockBucketSpec('default', 'couchbase') + mock = MockServer.create_legacy_mock_server([bspec_dfl], + mock_path, + mock_download_url, + replicas=2, + nodes=4) + else: + mock = MockServer.create_caves_mock_server(mock_path, + mock_download_url, + mock_version, + log_dir, + log_filename) + + try: + mock.start() + if mock_type == MockServerType.GoCAVES: + mock.create_cluster() + except Exception as ex: + raise CouchbaseTestEnvironmentException( + f"Problem trying to start mock server:\n{ex}") + + return mock + + @staticmethod + def restart_mock(mock) -> None: + try: + print("\nR.I.P. mock...") + mock.stop() + time.sleep(3) + mock.start() + return mock + except Exception: + import traceback + traceback.print_exc() + raise CouchbaseTestEnvironmentException('Error trying to restart mock') + + +@pytest.fixture(scope="session") +def couchbase_test_config(): + config = CouchbaseConfig.load_config() + yield config + config.shutdown() diff --git a/tests/data_provider.py b/tests/data_provider.py new file mode 100644 index 000000000..79a52bcd9 --- /dev/null +++ b/tests/data_provider.py @@ -0,0 +1,268 @@ +import datetime +import random +from typing import Any, Dict + +from faker import Faker +from faker.providers import BaseProvider +from faker_vehicle import VehicleProvider + +fake = Faker('en_US') + +# first, import a similar Provider or use the default one + +fake.add_provider(VehicleProvider) + + +class DealershipProvider(BaseProvider): + def dealership(self, dealer_uuid, vehicle_ids, idx) -> Dict[str, Any]: + dealer = { + 'batch': dealer_uuid, + 'id': f'{dealer_uuid}::{idx}', + 'name': fake.company(), + 'address': fake.street_address(), + 'city': fake.city(), + 'country': fake.current_country(), + 'country_code': fake.current_country_code(), + 'geo': self._get_geo(), + 'motto': fake.catch_phrase(), + 'general_manager': { + 'first_name': fake.first_name(), + 'last_name': fake.last_name(), + 'phone': fake.phone_number(), + 'email': fake.company_email(), + } + } + + today = datetime.datetime.now() + prev_year = (today - datetime.timedelta(days=365)) + num_reps = fake.random_int(min=2, max=5) + reps = [] + for _ in range(num_reps): + rep = { + 'first_name': fake.first_name(), + 'last_name': fake.last_name(), + 'phone': fake.phone_number(), + 'email': fake.company_email(), + 'rating': fake.random_int(min=1, max=5), + 'reviews': [], + } + num_reviews = fake.random_int(min=2, max=5) + for _ in range(num_reviews): + rep['reviews'].append({ + 'date': fake.date_time_between(prev_year).isoformat(), + 'comment': fake.sentence(nb_words=10), + 'categories': { + 'service': fake.random_int(min=1, max=5), + 'friendliness': fake.random_int(min=1, max=5), + 'honesty': fake.random_int(min=1, max=5), + 'knowledgeable': fake.random_int(min=1, max=5), + 'responsive': fake.random_int(min=1, max=5), + } + }) + reps.append(rep) + + dealer['sales_reps'] = reps + + num_vehicles = fake.random_int(min=5, max=10) + vehicles = random.choices(vehicle_ids, k=num_vehicles) # nosec + + prev_30_days = (today - datetime.timedelta(days=30)) + + dealer['inventory'] = { + 'num_vehicles': num_vehicles, + 'last_updated': fake.date_time_between(prev_30_days).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'vehicles': vehicles, + } + dealer['type'] = 'dealership' + return dealer + + def vehicle(self, vehicle_uuid, idx) -> Dict[str, Any]: + vehicle = {'batch': vehicle_uuid, 'id': f'{vehicle_uuid}::{idx}'} + base = fake.vehicle_object() + vehicle.update(**{k.lower(): v for k, v in base.items()}) + manufacturer = { + 'address': fake.street_address(), + 'city': fake.city(), + 'country': fake.current_country(), + 'country_code': fake.current_country_code(), + 'geo': self._get_geo(), + } + vehicle['manufacturer'] = manufacturer + + desc = fake.sentence(nb_words=10) + # add a key word for FTS operations to some of the docs + if fake.random_int(min=0, max=1) == 1: + tokens = desc.split(' ') + pos = fake.random_int(min=0, max=7) + tokens.insert(pos, 'great auto deal') + desc = ' '.join(tokens) + vehicle['description'] = desc + vehicle['rating'] = fake.random_int(min=1, max=10) + now = datetime.datetime.now() + today = datetime.datetime(year=now.year, month=now.month, day=now.day) + prev_year = (today - datetime.timedelta(days=365)) + vehicle['last_updated'] = fake.date_time_between(prev_year).strftime('%Y-%m-%dT%H:%M:%SZ') + vehicle['type'] = 'vehicle' + + return vehicle + + def _get_geo(self): + lat, ln, loc, country_code, tz = fake.local_latlng() + return { + 'lat': lat, + 'ln': ln, + 'accuracy': random.choice(['ROOFTOP', 'APPROXIMATE', 'RANGE_INTERPOLATED']), # nosec + 'location': { + 'loc': loc, + 'country_code': country_code, + 'tz': tz, + } + } + + +fake.add_provider(DealershipProvider) + + +class DataProvider: + def __init__(self, num_docs=100): + dealer_uuid = fake.uuid4() + vehicle_uuid = fake.uuid4() + + self._dealer_list = [] + self._dealer_uuid = dealer_uuid[:8] + self._new_dealer_list = [] + self._new_vehicle_list = [] + self._num_docs = num_docs + self._vehicle_list = [] + self._vehicle_uuid = vehicle_uuid[:8] + + def build_docs(self): + self._build_vehicles() + # future? + # self._build_dealerships() + + def generate_keys(self, num_keys): + uuid = fake.uuid4() + batch_uuid = uuid[:8] + return [f'{batch_uuid}::{i}' for i in range(num_keys)] + + def get_array_docs(self, num_docs): + uuid = fake.uuid4() + batch_uuid = uuid[:8] + docs = [] + for i in range(num_docs): + docs.append({ + 'batch': batch_uuid, + 'id': f'{batch_uuid}::{i}', + 'array': [1, 2, 3, 4, 5], + 'type': 'array' + }) + + return docs + + def get_array_only_docs(self, num_docs): + uuid = fake.uuid4() + batch_uuid = uuid[:8] + docs = {} + for i in range(num_docs): + key = f'{batch_uuid}::{i}' + docs[key] = [] + + return docs + + def get_bytes_docs(self, num_docs, start_value=None): + uuid = fake.uuid4() + batch_uuid = uuid[:8] + docs = {} + for i in range(num_docs): + if start_value: + key = f'{batch_uuid}::bytes::{i}' + docs[key] = start_value + else: + key = f'{batch_uuid}::bytes_empty::{i}' + docs[key] = b'' + + return docs + + def get_count_docs(self, num_docs): + uuid = fake.uuid4() + batch_uuid = uuid[:8] + docs = [] + for i in range(num_docs): + docs.append({ + 'batch': batch_uuid, + 'id': f'{batch_uuid}::{i}', + 'count': 100, + 'type': 'count' + }) + + return docs + + def get_counter_docs(self, num_docs, start_value=None): + uuid = fake.uuid4() + batch_uuid = uuid[:8] + if start_value is None: + return f'{batch_uuid}::counter_empty' + docs = {} + for i in range(num_docs): + key = f'{batch_uuid}::counter::{i}' + docs[key] = start_value + + return docs + + def get_dealerships(self, full_doc=False): + if full_doc is True: + return self._dealer_list + + return [{k: v for k, v in d.items() if k != 'sales_reps'} for d in self._dealer_list] + + def get_new_dealership(self, full_doc=False): + idx = len(self._dealer_list) + len(self._new_dealer_list) + vehicle_ids = list(map(lambda v: v['id'], self._vehicle_list + self._new_vehicle_list)) + dealer = fake.dealership(self._dealer_uuid, vehicle_ids, idx) + self._new_dealer_list.append(dealer) + if full_doc is True: + return dealer + + return {k: v for k, v in dealer.items() if k != 'sales_reps'} + + def get_new_vehicle(self): + idx = len(self._vehicle_list) + len(self._new_vehicle_list) + vehicle = fake.vehicle(self._vehicle_uuid, idx) + self._new_vehicle_list.append(vehicle) + return vehicle + + def get_simple_docs(self, num_docs): + uuid = fake.uuid4() + batch_uuid = uuid[:8] + docs = [] + for i in range(num_docs): + docs.append({'batch': batch_uuid, 'id': f'{batch_uuid}::{i}', 'type': 'simple'}) + + return docs + + def get_utf8_docs(self, num_docs, start_value=None): + uuid = fake.uuid4() + batch_uuid = uuid[:8] + docs = {} + for i in range(num_docs): + if start_value: + key = f'{batch_uuid}::utf8::{i}' + docs[key] = start_value + else: + key = f'{batch_uuid}::utf8_empty::{i}' + docs[key] = '' + + return docs + + def get_vehicles(self): + return self._vehicle_list + + def _build_vehicles(self): + for i in range(self._num_docs): + self._vehicle_list.append(fake.vehicle(self._vehicle_uuid, i)) + + def _build_dealerships(self): + vehicle_ids = list(map(lambda v: v['id'], self._vehicle_list)) + for i in range(self._num_docs): + self._dealer_list.append(fake.dealership(self._dealer_uuid, vehicle_ids, i)) diff --git a/tests/environments/__init__.py b/tests/environments/__init__.py new file mode 100644 index 000000000..0aa940925 --- /dev/null +++ b/tests/environments/__init__.py @@ -0,0 +1,38 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import namedtuple +from enum import IntEnum + + +class CollectionType(IntEnum): + DEFAULT = 1 + NAMED = 2 + + +class CouchbaseTestEnvironmentException(Exception): + """Raised when something with the test environment is incorrect.""" + + def __init__(self, message=None): + super().__init__(message) + + def __repr__(self): + return f"{type(self).__name__}({super().__str__()})" + + def __str__(self): + return self.__repr__() + + +KVPair = namedtuple("KVPair", "key value") diff --git a/tests/environments/analytics_environment.py b/tests/environments/analytics_environment.py new file mode 100644 index 000000000..490072ac3 --- /dev/null +++ b/tests/environments/analytics_environment.py @@ -0,0 +1,346 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import time +from typing import Optional + +from couchbase.result import AnalyticsResult +from tests.environments import CollectionType +from tests.environments.test_environment import AsyncTestEnvironment, TestEnvironment + + +class AnalyticsTestEnvironment(TestEnvironment): + DATASET_NAME = 'test-dataset' + + @property + def fqdn(self) -> Optional[str]: + return f'`{self.bucket.name}`.`{self.scope.name}`.`{self.collection.name}`' + + def assert_rows(self, + result, # type: AnalyticsResult + expected_count): + count = 0 + assert isinstance(result, (AnalyticsResult,)) + for row in result.rows(): + assert row is not None + count += 1 + assert count >= expected_count + + def create_analytics_collections(self): + """ + Setup queries: + Create dataverse: + CREATE DATAVERSE `default`.`test-scope` IF NOT EXISTS; + + Create dataset: + USE `default`.`test-scope`; + CREATE DATASET IF NOT EXISTS `test-collection` ON `default`.`test-scope`.`test-collection`; + + Connect Link: + USE `default`.`test-scope`; CONNECT LINK Local; + """ + + dv_fqdn = f'`{self.bucket.name}`.`{self.scope.name}`' + q_str = f'CREATE DATAVERSE {dv_fqdn} IF NOT EXISTS;' + res = self.cluster.analytics_query(q_str) + [_ for _ in res.rows()] + + q_str = f'USE {dv_fqdn}; CREATE DATASET IF NOT EXISTS `{self.collection.name}` ON {self.fqdn}' + res = self.cluster.analytics_query(q_str) + [_ for _ in res.rows()] + + q_str = f'USE {dv_fqdn}; CONNECT LINK Local;' + res = self.cluster.analytics_query(q_str) + [_ for _ in res.rows()] + + def get_batch_id(self): + if hasattr(self, '_batch_id'): + return self._batch_id + + doc = list(self._loaded_docs.values())[0] + self._batch_id = doc['batch'] + return self._batch_id + + def setup(self, + collection_type, # type: CollectionType + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + self.create_analytics_collections() + query_namespace = f'`{self.collection.name}`' + query_context = f'default:`{self.bucket.name}`.`{self.scope.name}`' + else: + TestEnvironment.try_n_times(10, + 3, + self.aixm.create_dataset, + self.DATASET_NAME, + self.bucket.name, + ignore_if_exists=True) + self.aixm.connect_link() + query_namespace = f'`{self.DATASET_NAME}`' + query_context = None + + TestEnvironment.try_n_times(5, 3, self.load_data) + + for _ in range(5): + row_count_good = self._check_row_count(self.cluster, + query_namespace, + 10, + query_context=query_context) + + if row_count_good: + break + print('Waiting for index to load, sleeping a bit...') + time.sleep(5) + + def teardown(self, + collection_type, # type: CollectionType + ): + + TestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + self.teardown_analytics_collections() + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + else: + self.aixm.disconnect_link() + TestEnvironment.try_n_times(10, + 3, + self.aixm.drop_dataset, + self.DATASET_NAME, + ignore_if_not_exists=True) + + def teardown_analytics_collections(self): + """ + Tear-down queries: + Disconnect Link: + USE `default`.`test-scope`; DISCONNECT LINK Local; + + Droo dataset: + USE `default`.`test-scope`; DROP DATASET `test-collection` IF EXISTS; + + Drop dataverse: + DROP DATAVERSE `default`.`test-scope` IF EXISTS; + """ + dv_fqdn = f'`{self.bucket.name}`.`{self.scope.name}`' + q_str = f'USE {dv_fqdn}; DISCONNECT LINK Local;' + res = self.cluster.analytics_query(q_str) + [_ for _ in res.rows()] + + q_str = f'USE {dv_fqdn}; DROP DATASET `{self.collection.name}` IF EXISTS;' + res = self.cluster.analytics_query(q_str) + [_ for _ in res.rows()] + + q_str = f'DROP DATAVERSE {dv_fqdn} IF EXISTS;' + res = self.cluster.analytics_query(q_str) + [_ for _ in res.rows()] + + def _check_row_count(self, + cb, + query_namespace, # type: str + min_count, # type: int + query_context=None, # type: Optional[str] + ) -> bool: + + q_str = f'SELECT COUNT(1) AS doc_count FROM {query_namespace}' + if query_context is not None: + result = cb.analytics_query(q_str, query_context=query_context) + else: + result = cb.analytics_query(q_str) + + rows = [r for r in result.rows()] + return len(rows) > 0 and rows[0].get('doc_count', 0) > min_count + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> AnalyticsTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env + + +class AsyncAnalyticsTestEnvironment(AsyncTestEnvironment): + DATASET_NAME = 'test-dataset' + + @property + def fqdn(self) -> Optional[str]: + return f'`{self.bucket.name}`.`{self.scope.name}`.`{self.collection.name}`' + + async def assert_rows(self, + result, # type: AnalyticsResult + expected_count): + count = 0 + assert isinstance(result, (AnalyticsResult,)) + async for row in result.rows(): + assert row is not None + count += 1 + assert count >= expected_count + + async def create_analytics_collections(self): + """ + Setup queries: + Create dataverse: + CREATE DATAVERSE `default`.`test-scope` IF NOT EXISTS; + + Create dataset: + USE `default`.`test-scope`; + CREATE DATASET IF NOT EXISTS `test-collection` ON `default`.`test-scope`.`test-collection`; + + Connect Link: + USE `default`.`test-scope`; CONNECT LINK Local; + """ + + dv_fqdn = f'`{self.bucket.name}`.`{self.scope.name}`' + q_str = f'CREATE DATAVERSE {dv_fqdn} IF NOT EXISTS;' + res = self.cluster.analytics_query(q_str) + [_ async for _ in res.rows()] + + q_str = f'USE {dv_fqdn}; CREATE DATASET IF NOT EXISTS `{self.collection.name}` ON {self.fqdn}' + res = self.cluster.analytics_query(q_str) + [_ async for _ in res.rows()] + + q_str = f'USE {dv_fqdn}; CONNECT LINK Local;' + res = self.cluster.analytics_query(q_str) + [_ async for _ in res.rows()] + + def get_batch_id(self): + if hasattr(self, '_batch_id'): + return self._batch_id + + doc = list(self._loaded_docs.values())[0] + self._batch_id = doc['batch'] + return self._batch_id + + async def setup(self, + collection_type, # type: CollectionType + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + await AsyncTestEnvironment.try_n_times(5, 3, self.setup_named_collections) + await self.create_analytics_collections() + query_namespace = f'`{self.collection.name}`' + query_context = f'default:`{self.bucket.name}`.`{self.scope.name}`' + else: + await AsyncTestEnvironment.try_n_times(10, + 3, + self.aixm.create_dataset, + self.DATASET_NAME, + self.bucket.name, + ignore_if_exists=True) + await self.aixm.connect_link() + query_namespace = f'`{self.DATASET_NAME}`' + query_context = None + + await AsyncTestEnvironment.try_n_times(5, 3, self.load_data) + + for _ in range(5): + row_count_good = await self._check_row_count(self.cluster, + query_namespace, + 10, + query_context=query_context) + + if row_count_good: + break + print('Waiting for index to load, sleeping a bit...') + await AsyncTestEnvironment.sleep(5) + + async def teardown(self, + collection_type, # type: CollectionType + ): + + await AsyncTestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + await self.teardown_analytics_collections() + await AsyncTestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + else: + await self.aixm.disconnect_link() + await AsyncTestEnvironment.try_n_times(10, + 3, + self.aixm.drop_dataset, + self.DATASET_NAME, + ignore_if_not_exists=True) + + async def teardown_analytics_collections(self): + """ + Tear-down queries: + Disconnect Link: + USE `default`.`test-scope`; DISCONNECT LINK Local; + + Droo dataset: + USE `default`.`test-scope`; DROP DATASET `test-collection` IF EXISTS; + + Drop dataverse: + DROP DATAVERSE `default`.`test-scope` IF EXISTS; + """ + dv_fqdn = f'`{self.bucket.name}`.`{self.scope.name}`' + q_str = f'USE {dv_fqdn}; DISCONNECT LINK Local;' + res = self.cluster.analytics_query(q_str) + [_ async for _ in res.rows()] + + q_str = f'USE {dv_fqdn}; DROP DATASET `{self.collection.name}` IF EXISTS;' + res = self.cluster.analytics_query(q_str) + [_ async for _ in res.rows()] + + q_str = f'DROP DATAVERSE {dv_fqdn} IF EXISTS;' + res = self.cluster.analytics_query(q_str) + [_ async for _ in res.rows()] + + async def _check_row_count(self, + cb, + query_namespace, # type: str + min_count, # type: int + query_context=None, # type: Optional[str] + ) -> bool: + + q_str = f'SELECT COUNT(1) AS doc_count FROM {query_namespace}' + if query_context is not None: + result = cb.analytics_query(q_str, query_context=query_context) + else: + result = cb.analytics_query(q_str) + + rows = [r async for r in result.rows()] + return len(rows) > 0 and rows[0].get('doc_count', 0) > min_count + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> AnalyticsTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/binary_environment.py b/tests/environments/binary_environment.py new file mode 100644 index 000000000..c53fe90da --- /dev/null +++ b/tests/environments/binary_environment.py @@ -0,0 +1,184 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import random +import time +from typing import Optional + +from couchbase.exceptions import AmbiguousTimeoutException, UnAmbiguousTimeoutException +from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment + + +class BinaryTestEnvironment(TestEnvironment): + + def get_existing_doc_by_type(self, doc_type, key_only=False): + if doc_type == 'bytes': + filtered_keys = set([k for k in self._loaded_docs.keys() if '::bytes::' in k]) + elif doc_type == 'bytes_empty': + filtered_keys = set([k for k in self._loaded_docs.keys() if 'bytes_empty' in k]) + elif doc_type == 'counter': + filtered_keys = set([k for k in self._loaded_docs.keys() if '::counter::' in k]) + elif doc_type == 'counter_empty': + # returns a key + key = self.data_provider.get_counter_docs(1) + self._used_extras.add(key) + return key + elif doc_type == 'utf8': + filtered_keys = set([k for k in self._loaded_docs.keys() if '::utf8::' in k]) + elif doc_type == 'utf8_empty': + filtered_keys = set([k for k in self._loaded_docs.keys() if 'utf8_empty' in k]) + + available_keys = filtered_keys.difference(self._used_docs) + key = random.choice(list(available_keys)) + self._used_docs.add(key) + if key_only is True: + return key + return key, self._loaded_docs[key] + + def get_multiple_existing_docs_by_type(self, doc_type, num_docs): + if doc_type == 'bytes': + filtered_keys = set([k for k in self._loaded_docs.keys() if '::bytes::' in k]) + elif doc_type == 'bytes_empty': + filtered_keys = set([k for k in self._loaded_docs.keys() if 'bytes_empty' in k]) + elif doc_type == 'counter': + filtered_keys = set([k for k in self._loaded_docs.keys() if '::counter::' in k]) + elif doc_type == 'counter_empty': + keys = [self.data_provider.get_counter_docs(1) for _ in range(num_docs)] + self._used_extras.update(keys) + return keys + elif doc_type == 'utf8': + filtered_keys = set([k for k in self._loaded_docs.keys() if '::utf8::' in k]) + elif doc_type == 'utf8_empty': + filtered_keys = set([k for k in self._loaded_docs.keys() if 'utf8_empty' in k]) + + available_keys = filtered_keys.difference(self._used_docs) + keys = random.choices(list(available_keys), k=num_docs) + self._used_docs.update(keys) + return keys + + def load_data(self, multi_tests_suite=False): # noqa: C901 + tc = RawBinaryTranscoder() + num_docs = 10 if multi_tests_suite is False else 20 + for k, v in self.data_provider.get_bytes_docs(num_docs).items(): + for _ in range(3): + try: + _ = self.collection.upsert(k, v, transcoder=tc) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + if multi_tests_suite is False: + for k, v in self.data_provider.get_bytes_docs(num_docs, b'XXXX').items(): + for _ in range(3): + try: + _ = self.collection.upsert(k, v, transcoder=tc) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for k, v in self.data_provider.get_counter_docs(num_docs, 100).items(): + for _ in range(3): + try: + _ = self.collection.upsert(k, v) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + tc = RawStringTranscoder() + for k, v in self.data_provider.get_utf8_docs(num_docs).items(): + for _ in range(3): + try: + _ = self.collection.upsert(k, v, transcoder=tc) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + if multi_tests_suite is False: + for k, v in self.data_provider.get_utf8_docs(num_docs, 'XXXX').items(): + for _ in range(3): + try: + _ = self.collection.upsert(k, v, transcoder=tc) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + self._doc_types = ['bytes', 'bytes_empty', 'counter', 'counter_empty', 'utf8', 'utf8_empty'] + + def setup(self, + collection_type, # type: CollectionType + test_suite=None, # type: Optional[str] + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + multi = test_suite.split('.')[-1] == 'binary_collection_multi_t' + TestEnvironment.try_n_times(5, 3, self.load_data, multi_tests_suite=multi) + + def teardown(self, + collection_type, # type: CollectionType + ): + + TestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> BinaryTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/bucket_mgmt_environment.py b/tests/environments/bucket_mgmt_environment.py new file mode 100644 index 000000000..0d1c967b9 --- /dev/null +++ b/tests/environments/bucket_mgmt_environment.py @@ -0,0 +1,83 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import random +import uuid + +from couchbase.exceptions import BucketDoesNotExistException +from tests.environments.test_environment import TestEnvironment + + +class BucketManagementTestEnvironment(TestEnvironment): + + def drop_bucket(self): + available_ids = self._used_ids.difference(self._dropped_ids) + bucket_name = available_ids.pop() + self._dropped_ids.add(bucket_name) + try: + self.bm.drop_bucket(bucket_name) + except BucketDoesNotExistException: + pass + self.consistency.wait_until_bucket_dropped(bucket_name) + + def get_bucket_name(self): + all_ids = set(self._bucket_ids) + available_ids = all_ids.difference(self._used_ids) + id = random.choice(list(available_ids)) + self._used_ids.add(id) + return id + + def get_bucket_names(self, num=3): + all_ids = set(self._bucket_ids) + available_ids = all_ids.difference(self._used_ids) + names = list(available_ids)[:num] + self._used_ids.update(names) + return names + + def setup(self): + self._batch_id = str(uuid.uuid4())[:8] + self._used_ids = set() + self._dropped_ids = set() + # 3 retries for 15 tests + self._bucket_ids = [f'{self._batch_id}_bucket_{i}' for i in range(50)] + + def teardown(self): + + for b in self._used_ids: + TestEnvironment.try_n_times_till_exception(10, + 3, + self.bm.drop_bucket, + b, + expected_exceptions=(BucketDoesNotExistException, )) + self._used_ids.clear() + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> BucketManagementTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/collection_mgmt_environment.py b/tests/environments/collection_mgmt_environment.py new file mode 100644 index 000000000..173e575ad --- /dev/null +++ b/tests/environments/collection_mgmt_environment.py @@ -0,0 +1,101 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import random +import uuid + +from couchbase.exceptions import (BucketDoesNotExistException, + CollectionNotFoundException, + ScopeNotFoundException) +from couchbase.management.collections import CollectionSpec +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class CollectionManagementTestEnvironment(TestEnvironment): + TEST_BUCKET = 'test-bucket' + + def add_dropped_scope(self, scope_name # type: str + ): + self._dropped_scope_ids.add(scope_name) + + def get_collection_name(self): + all_ids = set(self._collection_ids) + available_ids = all_ids.difference(self._used_collection_ids) + id = random.choice(list(available_ids)) + self._used_collection_ids.add(id) + return id + + def get_scope_name(self): + all_ids = set(self._scope_ids) + available_ids = all_ids.difference(self._used_scope_ids) + id = random.choice(list(available_ids)) + self._used_scope_ids.add(id) + return id + + def get_scope_names(self): + available_ids = self._used_scope_ids.difference(self._dropped_scope_ids) + return list(available_ids) + + def setup(self): + TestEnvironment.try_n_times(3, 5, self.setup_collection_mgmt, self.TEST_BUCKET) + self._batch_id = str(uuid.uuid4())[:8] + self._used_scope_ids = set() + self._dropped_scope_ids = set() + self._scope_ids = [f'{self._batch_id}_scope_{i}' for i in range(50)] + self._used_collection_ids = set() + self._dropped_collection_ids = set() + self._collection_ids = [f'{self._batch_id}_collection_{i}' for i in range(50)] + + def teardown(self): + for c in self._used_collection_ids: + TestEnvironment.try_n_times_till_exception(10, + 3, + self.cm.drop_collection, + CollectionSpec(c), + expected_exceptions=(CollectionNotFoundException, )) + self._collection_ids.clear() + for s in self._used_scope_ids: + TestEnvironment.try_n_times_till_exception(10, + 3, + self.cm.drop_scope, + s, + expected_exceptions=(ScopeNotFoundException, )) + self._collection_ids.clear() + if EnvironmentFeatures.is_feature_supported('bucket_mgmt', self.server_version_short, self.mock_server_type): + TestEnvironment.try_n_times_till_exception(10, + 3, + self.bm.drop_bucket, + self.TEST_BUCKET, + expected_exceptions=(BucketDoesNotExistException, )) + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> CollectionManagementTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/collection_multi_environment.py b/tests/environments/collection_multi_environment.py new file mode 100644 index 000000000..80f4373b8 --- /dev/null +++ b/tests/environments/collection_multi_environment.py @@ -0,0 +1,129 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import random +import time + +from couchbase.exceptions import (AmbiguousTimeoutException, + DocumentNotFoundException, + UnAmbiguousTimeoutException) +from couchbase.result import GetResult +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment + + +class CollectionMultiTestEnvironment(TestEnvironment): + + FAKE_DOCS = { + 'not-a-key1': {'what': 'a fake test doc!', 'id': 'not-a-key1'}, + 'not-a-key2': {'what': 'a fake test doc!', 'id': 'not-a-key2'}, + 'not-a-key3': {'what': 'a fake test doc!', 'id': 'not-a-key3'}, + 'not-a-key4': {'what': 'a fake test doc!', 'id': 'not-a-key4'} + } + + def check_all_not_found(self, cb_env, keys, okay_key=None): + not_found = 0 + for k in keys: + try: + cb_env.collection.get(k) + if okay_key and k == okay_key: + not_found += 1 # this is okay, it shouldn't have expired + except DocumentNotFoundException: + not_found += 1 + + if not_found != len(keys): + raise Exception('Not all docs were expired') + + def get_docs(self, num_docs): + filtered_keys = set(self._loaded_docs.keys()) + available_keys = filtered_keys.difference(self._used_docs) + keys = random.choices(list(available_keys), k=num_docs) + self._used_docs.update(keys) + return {k: self._loaded_docs[k] for k in keys} + + def get_new_docs(self, num_docs): + docs = {} + for v in self.data_provider.get_simple_docs(num_docs): + key = f'{v["id"]}' + self._used_extras.add(key) + docs[key] = v + + return docs + + def load_data(self): + for v in self.data_provider.get_simple_docs(100): + for _ in range(3): + try: + key = f'{v["id"]}' + _ = self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for k in self._loaded_docs.keys(): + TestEnvironment.try_n_times(5, 1, self.collection.get, k) + + def make_sure_docs_exists(self, cb_env, keys): + found = 0 + for k in keys: + doc = TestEnvironment.try_n_times(10, 3, cb_env.collection.get, k) + if isinstance(doc, GetResult): + found += 1 + + if len(keys) != found: + raise Exception('Unable to find all docs.') + + def setup(self, + collection_type, # type: CollectionType + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + TestEnvironment.try_n_times(5, 3, self.load_data) + + def teardown(self, + collection_type, # type: CollectionType + ): + + TestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> CollectionMultiTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/eventing_mgmt_environment.py b/tests/environments/eventing_mgmt_environment.py new file mode 100644 index 000000000..d283bc23e --- /dev/null +++ b/tests/environments/eventing_mgmt_environment.py @@ -0,0 +1,248 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +from dataclasses import fields +from typing import (List, + Optional, + get_type_hints) + +from couchbase.management.eventing import (EventingFunction, + EventingFunctionBucketAccess, + EventingFunctionBucketBinding, + EventingFunctionConstantBinding, + EventingFunctionDcpBoundary, + EventingFunctionDeploymentStatus, + EventingFunctionKeyspace, + EventingFunctionLanguageCompatibility, + EventingFunctionLogLevel, + EventingFunctionProcessingStatus, + EventingFunctionSettings, + EventingFunctionState, + EventingFunctionUrlAuthBasic, + EventingFunctionUrlAuthBearer, + EventingFunctionUrlAuthDigest, + EventingFunctionUrlBinding, + EventingFunctionUrlNoAuth) +from couchbase.management.logic.eventing_logic import QueryScanConsistency +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +class EventingFunctionManagementTestStatusException(Exception): + """Raised when waiting for a certained status does not happen within a specified timeframe""" + + +class EventingManagementTestEnvironment(TestEnvironment): + EVT_SRC_BUCKET_NAME = 'beer-sample' + EVT_META_BUCKET_NAME = 'default' + TEST_EVT_NAME = 'test-evt-func' + SIMPLE_EVT_CODE = ('function OnUpdate(doc, meta) {\n log("Doc created/updated", meta.id);\n}' + '\n\nfunction OnDelete(meta, options) {\n log("Doc deleted/expired", meta.id);\n}') + + BASIC_FUNC = EventingFunction( + TEST_EVT_NAME, + SIMPLE_EVT_CODE, + 'evt-7.0.0-5302-ee', + metadata_keyspace=EventingFunctionKeyspace(EVT_META_BUCKET_NAME), + source_keyspace=EventingFunctionKeyspace(EVT_SRC_BUCKET_NAME), + settings=EventingFunctionSettings.new_settings( + dcp_stream_boundary=EventingFunctionDcpBoundary.FromNow, + language_compatibility=EventingFunctionLanguageCompatibility.Version_6_6_2 + ), + bucket_bindings=[ + EventingFunctionBucketBinding( + alias='evtFunc', + name=EventingFunctionKeyspace(EVT_SRC_BUCKET_NAME), + access=EventingFunctionBucketAccess.ReadWrite + ) + ] + ) + + @property + def evt_version(self): + return self._version + + def setup(self, + collection_type=None, # type: Optional[CollectionType] + test_suite=None, # type: Optional[str] + num_docs=50, # type: Optional[int] + ): + super().setup(collection_type=collection_type, test_suite=test_suite, num_docs=num_docs) + + self._version = 'evt-{}'.format( + self.server_version_full.replace('enterprise', 'ee').replace('community', 'ce') + ) + self.enable_eventing_mgmt() + if EnvironmentFeatures.is_feature_supported('collections', self.server_version_short): + self.enable_collection_mgmt() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + def teardown(self, + collection_type=None, # type: Optional[CollectionType] + test_suite=None, # type: Optional[str] + ): + super().teardown(collection_type=collection_type, test_suite=test_suite) + + self.disable_eventing_mgmt() + if EnvironmentFeatures.is_feature_supported('collections', self.server_version_short): + TestEnvironment.try_n_times_till_exception(5, + 3, + self.teardown_named_collections, + raise_if_no_exception=False) + self.disable_collection_mgmt() + + def validate_bucket_bindings(self, bindings # type: List[EventingFunctionBucketBinding] + ) -> None: + binding_fields = fields(EventingFunctionBucketBinding) + type_hints = get_type_hints(EventingFunctionBucketBinding) + for binding in bindings: + assert isinstance(binding, EventingFunctionBucketBinding) # nosec + for field in binding_fields: + value = getattr(binding, field.name) + if value is not None: + if field.name == 'name': + assert isinstance(value, EventingFunctionKeyspace) # nosec + elif field.name == 'access': + assert isinstance(value, EventingFunctionBucketAccess) # nosec + else: + assert isinstance(value, type_hints[field.name]) # nosec + + def validate_constant_bindings(self, bindings # type: List[EventingFunctionConstantBinding] + ) -> None: + binding_fields = fields(EventingFunctionConstantBinding) + type_hints = get_type_hints(EventingFunctionConstantBinding) + for binding in bindings: + assert isinstance(binding, EventingFunctionConstantBinding) # nosec + for field in binding_fields: + value = getattr(binding, field.name) + if value is not None: + assert isinstance(value, type_hints[field.name]) # nosec + + def validate_eventing_function(self, func, # type: EventingFunction + shallow=False # type: Optional[bool] + ) -> None: + assert isinstance(func, EventingFunction) # nosec + if shallow is False: + func_fields = fields(EventingFunction) + type_hints = get_type_hints(EventingFunction) + for field in func_fields: + value = getattr(func, field.name) + if value is not None: + if field.name == 'bucket_bindings': + self.validate_bucket_bindings(value) + elif field.name == 'url_bindings': + self.validate_url_bindings(value) + elif field.name == 'constant_bindings': + self.validate_constant_bindings(value) + elif field.name == 'settings': + self.validate_settings(value) + else: + assert isinstance(value, type_hints[field.name]) # nosec + + def validate_settings(self, settings # type: EventingFunctionSettings # noqa: C901 + ) -> None: # noqa: C901 + assert isinstance(settings, EventingFunctionSettings) # nosec + settings_fields = fields(EventingFunctionSettings) + type_hints = get_type_hints(EventingFunctionSettings) + for field in settings_fields: + value = getattr(settings, field.name) + if value is not None: + if field.name == 'dcp_stream_boundary': + assert isinstance(value, EventingFunctionDcpBoundary) # nosec + elif field.name == 'deployment_status': + assert isinstance(value, EventingFunctionDeploymentStatus) # nosec + elif field.name == 'processing_status': + assert isinstance(value, EventingFunctionProcessingStatus) # nosec + elif field.name == 'language_compatibility': + assert isinstance(value, EventingFunctionLanguageCompatibility) # nosec + elif field.name == 'log_level': + assert isinstance(value, EventingFunctionLogLevel) # nosec + elif field.name == 'query_consistency': + assert isinstance(value, QueryScanConsistency) # nosec + elif field.name == 'handler_headers': + assert isinstance(value, list) # nosec + elif field.name == 'handler_footers': + assert isinstance(value, list) # nosec + else: + assert isinstance(value, type_hints[field.name]) # nosec + + def validate_url_bindings(self, bindings # type: List[EventingFunctionUrlBinding] + ) -> None: + binding_fields = fields(EventingFunctionUrlBinding) + type_hints = get_type_hints(EventingFunctionUrlBinding) + for binding in bindings: + assert isinstance(binding, EventingFunctionUrlBinding) # nosec + for field in binding_fields: + value = getattr(binding, field.name) + if value is not None: + if field.name == 'auth': + if isinstance(value, EventingFunctionUrlAuthBasic): + assert isinstance(value.username, str) # nosec + assert value.password is None # nosec + elif isinstance(value, EventingFunctionUrlAuthDigest): + assert isinstance(value.username, str) # nosec + assert value.password is None # nosec + elif isinstance(value, EventingFunctionUrlAuthBearer): + assert value.key is None # nosec + else: + assert isinstance(value, EventingFunctionUrlNoAuth) # nosec + else: + assert isinstance(value, type_hints[field.name]) # nosec + + def wait_until_status(self, + num_times, # type: int + seconds_between, # type: int + state, # type: EventingFunctionState + name # type: str + ) -> None: + + func_status = None + for _ in range(num_times): + func_status = self.efm._get_status(name) + if func_status is None or func_status.state != state: + TestEnvironment.sleep(seconds_between) + else: + break + + if func_status is None: + raise EventingFunctionManagementTestStatusException( + "Unable to obtain function status for {}".format(name) + ) + if func_status.state != state: + raise EventingFunctionManagementTestStatusException( + "Function {} status is {} which does not match desired status of {}.".format( + name, func_status.state.value, state.value + ) + ) + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> EventingManagementTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/query_environment.py b/tests/environments/query_environment.py new file mode 100644 index 000000000..2694738b6 --- /dev/null +++ b/tests/environments/query_environment.py @@ -0,0 +1,371 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import time +from datetime import timedelta +from typing import Optional + +import requests + +from couchbase.exceptions import QueryIndexAlreadyExistsException, QueryIndexNotFoundException +from couchbase.result import QueryResult +from tests.environments import CollectionType, CouchbaseTestEnvironmentException +from tests.environments.test_environment import AsyncTestEnvironment, TestEnvironment + + +class QueryTestEnvironment(TestEnvironment): + + @property + def fqdn(self) -> Optional[str]: + return f'`{self.bucket.name}`.`{self.scope.name}`.`{self.collection.name}`' + + def assert_rows(self, + result, # type: QueryResult + expected_count): + count = 0 + assert isinstance(result, (QueryResult,)) + for row in result.rows(): + assert row is not None + count += 1 + assert count >= expected_count + + def get_batch_id(self): + if hasattr(self, '_batch_id'): + return self._batch_id + + doc = list(self._loaded_docs.values())[0] + self._batch_id = doc['batch'] + return self._batch_id + + def setup(self, + collection_type, # type: CollectionType + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + TestEnvironment.try_n_times(10, + 3, + self.qixm.create_primary_index, + self.bucket.name, + scope_name=self.scope.name, + collection_name=self.collection.name, + timeout=timedelta(seconds=60), + ignore_if_exists=True) + query_namespace = self.fqdn + else: + if self.server_version_short > 6.6: + TestEnvironment.try_n_times(10, + 3, + self.qixm.create_primary_index, + self.bucket.name, + timeout=timedelta(seconds=60), + ignore_if_exists=True) + else: + self._create_primary_index() + query_namespace = f'`{self.bucket.name}`' + + TestEnvironment.try_n_times(5, 3, self.load_data) + + for _ in range(5): + row_count_good = self._check_row_count(self.cluster, query_namespace, 5) + + if row_count_good: + break + print('Waiting for index to load, sleeping a bit...') + time.sleep(5) + + def teardown(self, + collection_type, # type: CollectionType + ): + + TestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + TestEnvironment.try_n_times_till_exception(10, + 3, + self.qixm.drop_primary_index, + self.bucket.name, + scope_name=self.scope.name, + collection_name=self.collection.name, + expected_exceptions=(QueryIndexNotFoundException,)) + + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + else: + if self.server_version_short > 6.6: + TestEnvironment.try_n_times_till_exception(10, + 3, + self.qixm.drop_primary_index, + self.bucket.name, + expected_exceptions=(QueryIndexNotFoundException)) + else: + self._drop_primary_index() + + def _check_row_count(self, + cb, + query_namespace, # type: str + min_count # type: int + ) -> bool: + + batch = self.get_batch_id() + result = cb.query(f"SELECT * FROM {query_namespace} WHERE batch LIKE '{batch}%' LIMIT 5") + count = 0 + for _ in result.rows(): + count += 1 + return count >= min_count + + def _create_primary_index(self): + q_str = f'CREATE PRIMARY INDEX `#primary` on `{self.bucket.name}`' + for _ in range(10): + try: + self.cluster.query(q_str).execute() + except QueryIndexAlreadyExistsException: + break + + TestEnvironment.sleep(3) + + def _drop_primary_index(self): + q_str = f'DROP PRIMARY INDEX on `{self.bucket.name}`' + for _ in range(10): + try: + self.cluster.query(q_str).execute() + except QueryIndexNotFoundException: + break + + TestEnvironment.sleep(3) + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> QueryTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env + + +class AsyncQueryTestEnvironment(AsyncTestEnvironment): + UDF = """ + function runLoop(num){ + let count = 0 + for(let i = 0; i < num; i++){ + count++ + } + return count + } + """ + + UDF_URI_PATH = 'evaluator/v1/libraries/simple' + + @property + def fqdn(self) -> Optional[str]: + return f'`{self.bucket.name}`.`{self.scope.name}`.`{self.collection.name}`' + + async def assert_rows(self, + result, # type: QueryResult + expected_count) -> bool: + count = 0 + assert isinstance(result, (QueryResult,)) + async for row in result.rows(): + assert row is not None + count += 1 + assert count >= expected_count + + async def drop_udf(self): + try: + q_str = 'DROP FUNCTION loop IF EXISTS;' + await self.cluster.query(q_str).execute() + except Exception as ex: + raise CouchbaseTestEnvironmentException(f'Failed to drop UDF: {ex}') + + username, pw = self.config.get_username_and_pw() + url = f'http://{self.config.host}:8093/{self.UDF_URI_PATH}' + r = requests.delete(url, auth=(username, pw)) + if r.status_code != 200: + msg = f'Unable to delete UDF via REST. Status: {r.status_code}. Content: {r.content}' + raise CouchbaseTestEnvironmentException(msg) + + def get_batch_id(self): + if hasattr(self, '_batch_id'): + return self._batch_id + + doc = list(self._loaded_docs.values())[0] + self._batch_id = doc['batch'] + return self._batch_id + + def get_udf(self): + username, pw = self.config.get_username_and_pw() + url = f'http://{self.config.host}:8093/{self.UDF_URI_PATH}' + r = requests.get(url, data=self.UDF.encode('-utf-8'), auth=(username, pw)) + if r.status_code != 200: + msg = f'Unable to get UDF. Status: {r.status_code}. Content: {r.content}' + raise CouchbaseTestEnvironmentException(msg) + + udf = None + try: + udf = r.json() + except Exception as ex: + raise CouchbaseTestEnvironmentException(f'Failed to serialize UDF: {ex}') + + if 'simple' not in udf: + raise CouchbaseTestEnvironmentException(f'unexpected UDF content: {udf}') + + async def load_udf(self, get_retries=3): + username, pw = self.config.get_username_and_pw() + url = f'http://{self.config.host}:8093/{self.UDF_URI_PATH}' + r = requests.post(url, data=self.UDF.encode('-utf-8'), auth=(username, pw)) + if r.status_code != 200: + msg = f'Unable to load UDF via REST. Status: {r.status_code}. Content: {r.content}' + raise CouchbaseTestEnvironmentException(msg) + + try: + q_str = 'CREATE FUNCTION loop(num) IF NOT EXISTS LANGUAGE JAVASCRIPT AS "runLoop" AT "simple";' + await self.cluster.query(q_str).execute() + except Exception as ex: + raise CouchbaseTestEnvironmentException(f'Failed to create UDF: {ex}') + + retry_count = 1 + while True: + try: + self.get_udf() + except Exception as ex: + if retry_count == get_retries: + raise ex + retry_count += 1 + else: + break + + async def setup(self, + collection_type, # type: CollectionType + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + await AsyncTestEnvironment.try_n_times(5, 3, self.setup_named_collections) + await AsyncTestEnvironment.try_n_times(10, + 3, + self.qixm.create_primary_index, + self.bucket.name, + scope_name=self.scope.name, + collection_name=self.collection.name, + timeout=timedelta(seconds=60), + ignore_if_exists=True) + query_namespace = self.fqdn + else: + if self.server_version_short > 6.6: + await AsyncTestEnvironment.try_n_times(10, + 3, + self.qixm.create_primary_index, + self.bucket.name, + timeout=timedelta(seconds=60), + ignore_if_exists=True) + else: + await self._create_primary_index() + query_namespace = f'`{self.bucket.name}`' + + await AsyncTestEnvironment.try_n_times(5, 3, self.load_data) + + for _ in range(5): + row_count_good = await self._check_row_count(self.cluster, query_namespace, 5) + + if row_count_good: + break + print('Waiting for index to load, sleeping a bit...') + await AsyncTestEnvironment.sleep(5) + + async def teardown(self, + collection_type, # type: CollectionType + ): + + await AsyncTestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + await AsyncTestEnvironment.try_n_times_till_exception(10, + 3, + self.qixm.drop_primary_index, + self.bucket.name, + scope_name=self.scope.name, + collection_name=self.collection.name, + expected_exceptions=(QueryIndexNotFoundException,)) + + await AsyncTestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + else: + if self.server_version_short > 6.6: + await AsyncTestEnvironment.try_n_times_till_exception(10, + 3, + self.qixm.drop_primary_index, + self.bucket.name, + expected_exceptions=(QueryIndexNotFoundException)) + else: + await self._drop_primary_index() + + async def _check_row_count(self, + cb, + query_namespace, # type: str + min_count # type: int + ) -> bool: + + batch = self.get_batch_id() + result = cb.query(f"SELECT * FROM {query_namespace} WHERE batch LIKE '{batch}%' LIMIT 5") + count = 0 + async for _ in result.rows(): + count += 1 + return count >= min_count + + async def _create_primary_index(self): + q_str = f'CREATE PRIMARY INDEX `#primary` on `{self.bucket.name}`' + for _ in range(10): + try: + await self.cluster.query(q_str).execute() + except QueryIndexAlreadyExistsException: + break + + await AsyncTestEnvironment.sleep(3) + + async def _drop_primary_index(self): + q_str = f'DROP PRIMARY INDEX on `{self.bucket.name}`' + for _ in range(10): + try: + await self.cluster.query(q_str).execute() + except QueryIndexNotFoundException: + break + + await AsyncTestEnvironment.sleep(3) + + @classmethod + def from_environment(cls, + env # type: AsyncTestEnvironment + ) -> AsyncQueryTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/query_index_mgmt_environment.py b/tests/environments/query_index_mgmt_environment.py new file mode 100644 index 000000000..dadddeba0 --- /dev/null +++ b/tests/environments/query_index_mgmt_environment.py @@ -0,0 +1,162 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +from typing import List, Optional + +import pytest + +from couchbase.management.options import (DropPrimaryQueryIndexOptions, + DropQueryIndexOptions, + GetAllQueryIndexOptions) +from couchbase.management.queries import QueryIndex +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment + + +class QueryIndexManagementTestEnvironment(TestEnvironment): + + def clear_all_indexes(self, + collection_type=None, # type: Optional[CollectionType] + ignore_fail=False # type: Optional[bool] + ): + pairs = [('_default', '_default')] + if not collection_type and self._collection_type: + collection_type = self._collection_type + + if collection_type and collection_type == CollectionType.NAMED: + pairs.append((self.TEST_SCOPE, self.TEST_COLLECTION)) + + for scope_col in pairs: + + indexes = self._get_all_indexes(scope_col[0], scope_col[1]) + success = self.drop_all_indexes(indexes, + scope_name=scope_col[0], + collection_name=scope_col[1]) + if success: + continue + elif ignore_fail is True: + continue + else: + pytest.xfail( + "Indexes were not dropped after {} waits of {} seconds each".format(10, 2)) + + def drop_all_indexes(self, + indexes, # type: List[QueryIndex] + scope_name='_default', # type: Optional[str] + collection_name='_default', # type: Optional[str] + ): + for index in indexes: + self._drop_index(index, scope_name, collection_name) + for _ in range(10): + indexes = self._get_all_indexes(scope_name, collection_name) + if 0 == len(indexes): + return True + TestEnvironment.sleep(2) + + return False + + def get_fqdn(self): + return f'`{self.bucket.name}`.`{self.scope.name}`.`{self.collection.name}`' + + def get_batch_id(self): + if hasattr(self, '_batch_id'): + return self._batch_id + + doc = list(self._loaded_docs.values())[0] + self._batch_id = doc['batch'] + return self._batch_id + + def setup(self, + collection_type, # type: CollectionType + test_suite=None, # type: Optional[str] + ): + + # TODO: will change once updated query_context + if self.server_version_short <= 6.6: + pytest.skip((f'Query Index Management only supported on server versions > 6.6. ' + f'Using server version: {self.server_version}.')) + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + self._collection_type = collection_type + self._from_collection = test_suite == 'ClassicCollectionQueryIndexManagementTests' + self.enable_query_mgmt(from_collection=self._from_collection) + + def teardown(self, + collection_type, # type: CollectionType + ): + + TestEnvironment.try_n_times(5, + 3, + self.clear_all_indexes, + collection_type, + ignore_fail=True) + self.disable_query_mgmt() + TestEnvironment.try_n_times(5, 3, self.purge_data) + if collection_type == CollectionType.NAMED: + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + self.disable_collection_mgmt() + + def _drop_index(self, index, scope_name, collection_name): + if self._from_collection: + if index.is_primary: + opts = DropPrimaryQueryIndexOptions() + if index.name != '#primary': + opts['index_name'] = index.name + self.qixm.drop_primary_index(opts) + else: + self.qixm.drop_index(index.name) + else: + if index.is_primary: + opts = DropPrimaryQueryIndexOptions(scope_name=scope_name, collection_name=collection_name) + if index.name != '#primary': + opts['index_name'] = index.name + self.qixm.drop_primary_index(self.bucket.name, opts) + else: + self.qixm.drop_index(self.bucket.name, + index.name, + DropQueryIndexOptions(scope_name=scope_name, collection_name=collection_name)) + + def _get_all_indexes(self, + scope_name=None, # type: Optional[str] + collection_name=None, # type: Optional[str] + ) -> List[QueryIndex]: + if self._from_collection: + return self.qixm.get_all_indexes() + else: + return self.qixm.get_all_indexes(self.bucket.name, + GetAllQueryIndexOptions(scope_name=scope_name, + collection_name=collection_name)) + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> QueryIndexManagementTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/rate_limit_environment.py b/tests/environments/rate_limit_environment.py new file mode 100644 index 000000000..46a63ec59 --- /dev/null +++ b/tests/environments/rate_limit_environment.py @@ -0,0 +1,300 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import json +import random +import time +from dataclasses import dataclass +from typing import List, Optional +from urllib.parse import urlparse + +import requests + +from couchbase.exceptions import (CouchbaseException, + DocumentNotFoundException, + ScopeNotFoundException) +from couchbase.management.options import GetUserOptions +from tests.environments import CollectionType, CouchbaseTestEnvironmentException +from tests.environments.test_environment import TestEnvironment +from tests.test_features import EnvironmentFeatures + + +@dataclass +class RateLimitData: + url: str = None + username: str = None + pw: str = None + fts_indexes: List[str] = None + + +class RateLimitTestEnvironment(TestEnvironment): + CONTENT = {'some': 'content'} + KEY = 'imakey' + NOKEY = 'somerandomkey' + USERNAME = 'rate-limit-user' + RATE_LIMIT_SCOPE_NAME = 'rate-limit-scope' + + # def __init__(self, + # **kwargs # type: Dict[str, Any] + # ): + # self._tracer = kwargs.pop('tracer', None) + # self._meter = kwargs.pop('meter', None) + # super().__init__(**kwargs) + + @property + def rate_limit_params(self) -> Optional[RateLimitData]: + """Returns the rate limit testing data""" + return self._rate_limit_params if hasattr(self, '_rate_limit_params') else None + + def create_rate_limit_scope(self, scope_name, limits): + params = { + 'name': scope_name + } + + scope_limits = {} + kv_limits = limits.get('kv_limits', None) + if kv_limits: + scope_limits['kv'] = { + 'data_size': kv_limits['data_size'] + } + + index_limits = limits.get('index_limits', None) + if index_limits: + scope_limits['index'] = { + 'num_indexes': index_limits['num_indexes'] + } + + fts_limits = limits.get('fts_limits', None) + if fts_limits: + scope_limits['fts'] = { + 'num_fts_indexes': fts_limits['num_fts_indexes'] + } + + cluster_mgr_limits = limits.get('cluster_mgr_limits', None) + if cluster_mgr_limits: + scope_limits['clusterManager'] = { + 'num_collections': cluster_mgr_limits['num_collections'] + } + + if scope_limits: + params['limits'] = json.dumps(scope_limits) + + path = f'/pools/default/buckets/{self.bucket.name}/scopes' + url = f'{self.rate_limit_params.url}/{path}' + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + requests.post(url, + headers=headers, + data=params, + auth=(self.rate_limit_params.username, + self.rate_limit_params.pw)) + # verify the scope exists + all_scopes = self.cm.get_all_scopes() + assert scope_name in map(lambda s: s.name, all_scopes) + + def create_rate_limit_user(self, username, limits): + params = { + 'password': 'password', + 'roles': 'admin' + } + + user_limits = {} + kv_limits = limits.get('kv_limits', None) + if kv_limits: + user_limits['kv'] = { + 'num_connections': kv_limits['num_connections'], + 'num_ops_per_min': kv_limits['num_ops_per_min'], + 'ingress_mib_per_min': kv_limits['ingress_mib_per_min'], + 'egress_mib_per_min': kv_limits['egress_mib_per_min'] + } + + query_limits = limits.get('query_limits', None) + if query_limits: + user_limits['query'] = { + 'num_queries_per_min': query_limits['num_queries_per_min'], + 'num_concurrent_requests': query_limits['num_concurrent_requests'], + 'ingress_mib_per_min': query_limits['ingress_mib_per_min'], + 'egress_mib_per_min': query_limits['egress_mib_per_min'] + } + + fts_limits = limits.get('fts_limits', None) + if fts_limits: + user_limits['fts'] = { + 'num_queries_per_min': fts_limits['num_queries_per_min'], + 'num_concurrent_requests': fts_limits['num_concurrent_requests'], + 'ingress_mib_per_min': fts_limits['ingress_mib_per_min'], + 'egress_mib_per_min': fts_limits['egress_mib_per_min'] + } + + if user_limits: + params['limits'] = json.dumps(user_limits) + + path = f'/settings/rbac/users/local/{username}' + + url = f'{self.rate_limit_params.url}/{path}' + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + r = requests.put(url, + headers=headers, + data=params, + auth=(self.rate_limit_params.username, + self.rate_limit_params.pw)) + + if r.status_code != 200: + raise CouchbaseTestEnvironmentException('Unable to create rate-limit-user.') + + # lets verify user exists + user_metadata = TestEnvironment.try_n_times(10, + 1, + self.um.get_user, + username, + GetUserOptions(domain_name='local')) + + assert user_metadata is not None + assert username == user_metadata.user.username + + def disable_rate_limiting(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('rate_limiting', + self.server_version_short, + self.mock_server_type) + if hasattr(self, '_rate_limit_params'): + del self._rate_limit_params + return self + + def drop_rate_limit_user(self): + TestEnvironment.try_n_times_till_exception(10, 3, self.um.drop_user, self.USERNAME) + + def drop_scope(self): + TestEnvironment.try_n_times_till_exception(5, + 1, + self.cm.drop_scope, + self.RATE_LIMIT_SCOPE_NAME, + expected_exceptions=(ScopeNotFoundException,)) + + def enable_rate_limiting(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('rate_limiting', + self.server_version_short, + self.mock_server_type) + parsed_conn = urlparse(self._config.get_connection_string()) + url = f'http://{parsed_conn.netloc}:8091' + u, p = self.config.get_username_and_pw() + self._rate_limit_params = RateLimitData(url, u, p) + return self + + def enforce_rate_limits(self, enforce=True): + url = f'{self.rate_limit_params.url}/internalSettings' + payload = {'enforceLimits': f'{"true" if enforce is True else "false"}'} + headers = {'Content-Type': 'application/x-www-form-urlencoded'} + r = requests.post(url, + headers=headers, + data=payload, + auth=(self.rate_limit_params.username, + self.rate_limit_params.pw)) + + if r.status_code != 200: + raise CouchbaseTestEnvironmentException('Unable to enforce rate limits.') + + def random_doc_by_size(self, size): + doc = bytearray((random.getrandbits(8) for i in range(size))) + return doc.hex() + + def remove_docs(self): + try: + self.collection.remove('ratelimit-ingress') + TestEnvironment.try_n_times_till_exception(10, + 3, + self.collection.get, + 'ratelimit-ingress', + (DocumentNotFoundException,)) + except CouchbaseException: + pass + + try: + self.collection.remove('ratelimit-egress') + TestEnvironment.try_n_times_till_exception(10, + 3, + self.collection.get, + 'ratelimit-egress', + (DocumentNotFoundException,)) + except CouchbaseException: + pass + + def setup(self, + collection_type=None, # type: Optional[CollectionType] + num_docs=None, # type: Optional[int] + ): + self.enable_rate_limiting().enable_bucket_mgmt().enable_collection_mgmt().enable_user_mgmt() + self._fts_indexes = [] + self.enforce_rate_limits(True) + TestEnvironment.try_n_times(5, 3, self.load_data) + + def teardown(self): + self.enforce_rate_limits(False) + self.disable_rate_limiting().disable_bucket_mgmt().disable_collection_mgmt().disable_user_mgmt() + TestEnvironment.try_n_times(5, 3, self.purge_data) + + def try_until_timeout(self, timeout, interval, func, *args, **kwargs): + """Execute provided func until specified timeout has been reached. + + :param timeout: timeout in seconds + :type timeout: int + :param interval: sleep interval in milliseconds + :type interval: int + :param func: function to execute periodically, sleeping interval milliseconds between each execution + :type func: function + """ + timeout_ms = timeout * 1000 + time_left = timeout_ms + interval_ms = float(interval / 1000) + start = time.perf_counter() + is_query = kwargs.pop('query', False) + is_fts = kwargs.pop('fts', False) + + while True: + if is_query is True: + func(*args, **kwargs).execute() + elif is_fts is True: + res = func(*args, **kwargs) + res.rows() + else: + func(*args, **kwargs) + time_left = timeout_ms - ((time.perf_counter() - start) * 1000) + if time_left <= 0: + break + + time.sleep(interval_ms) + + @classmethod + def from_environment(cls, + env, # type: TestEnvironment + ) -> RateLimitTestEnvironment: + + base_env_args = { + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + # we have to create a new environment b/c we need a new cluster in order to set the tracer + cb_env = TestEnvironment.get_environment(**base_env_args) + env_args = { + 'bucket': cb_env.bucket, + 'cluster': cb_env.cluster, + 'default_collection': cb_env.default_collection, + 'couchbase_config': cb_env.config, + 'data_provider': cb_env.data_provider, + } + + return cls(**env_args) diff --git a/tests/environments/search_environment.py b/tests/environments/search_environment.py new file mode 100644 index 000000000..316033dd5 --- /dev/null +++ b/tests/environments/search_environment.py @@ -0,0 +1,455 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import json +import pathlib +import time +from datetime import timedelta +from os import path +from typing import (List, + Optional, + Union) + +import couchbase.search as search +from couchbase.exceptions import (AmbiguousTimeoutException, + SearchIndexNotFoundException, + UnAmbiguousTimeoutException) +from couchbase.management.collections import CollectionSpec +from couchbase.management.search import SearchIndex +from couchbase.result import SearchResult +from couchbase.search import SearchRow +from tests.environments import CollectionType, CouchbaseTestEnvironmentException +from tests.environments.test_environment import AsyncTestEnvironment, TestEnvironment + + +class SearchTestEnvironment(TestEnvironment): + OTHER_COLLECTION = 'other-collection' + TEST_COLLECTION_INDEX_NAME = 'test-search-coll-index' + TEST_COLLECTION_INDEX_PATH = path.join(pathlib.Path(__file__).parent.parent, + 'test_cases', + f'{TEST_COLLECTION_INDEX_NAME}-params-new.json') + TEST_INDEX_NAME = 'test-search-index' + TEST_INDEX_PATH = path.join(pathlib.Path(__file__).parent.parent, + 'test_cases', + f'{TEST_INDEX_NAME}-params-new.json') + + def assert_rows(self, + result, # type: SearchResult + expected_count, # type: int + return_rows=False # type: bool + ) -> Optional[List[Union[SearchRow, dict]]]: + rows = [] + assert isinstance(result, SearchResult) # nosec + for row in result.rows(): + assert row is not None # nosec + self.validate_search_row(row) + rows.append(row) + assert len(rows) >= expected_count # nosec + + self.validate_metadata(result, expected_count) + + if return_rows is True: + return rows + + def create_and_load_other_collection(self): + collection_spec = CollectionSpec(self.OTHER_COLLECTION, self.scope.name) + self.cm.create_collection(collection_spec) + collection = None + for _ in range(5): + collection = self.get_collection(self.scope.name, self.OTHER_COLLECTION, bucket_name=self.bucket.name) + if collection: + break + TestEnvironment.sleep(5) + + if not collection: + raise CouchbaseTestEnvironmentException("Unabled to create other-collection for FTS collection testing") + + coll = self.scope.collection(self.OTHER_COLLECTION) + for _ in range(25): + new_doc = self.data_provider.get_new_vehicle() + for _ in range(3): + try: + key = f'{new_doc["id"]}' + _ = coll.upsert(key, new_doc) + self._loaded_docs[key] = new_doc + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + raise ex + + def get_encoded_query(self, search_query): + encoded_q = search_query.as_encodable() + encoded_q['query'] = json.loads(encoded_q['query']) + if 'facets' in encoded_q: + encoded_q['facets'] = json.loads(encoded_q['facets']) + if 'sort_specs' in encoded_q: + encoded_q['sort'] = json.loads(encoded_q['sort_specs']) + if 'vector_search' in encoded_q: + encoded_q['vector_search'] = json.loads(encoded_q['vector_search']) + + return encoded_q + + def load_search_index(self, + sixm, + collection_type, # type: CollectionType + ): + if collection_type == CollectionType.NAMED: + idx_name = self.TEST_COLLECTION_INDEX_NAME + idx_path = self.TEST_COLLECTION_INDEX_PATH + else: + idx_name = self.TEST_INDEX_NAME + idx_path = self.TEST_INDEX_PATH + + with open(idx_path) as params_file: + input = params_file.read() + params_json = json.loads(input) + TestEnvironment.try_n_times(10, + 3, + sixm.upsert_index, + SearchIndex(name=idx_name, + idx_type='fulltext-index', + source_name='default', + source_type='couchbase', + params=params_json)) + + def setup(self, + collection_type, # type: CollectionType + num_docs=None, # type: Optional[int] + test_suite=None, # type: Optional[str] + ): + + if test_suite in ['ClassicSearchParamTests', 'ClassicVectorSearchParamTests']: + return + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + TestEnvironment.try_n_times(5, 3, self.load_data) + self.create_and_load_other_collection() + self.load_search_index(self.sixm, collection_type) + # make sure the index loads... + num_docs = self._check_doc_count(self.sixm, self.TEST_COLLECTION_INDEX_NAME, 20, retries=10, delay=3) + else: + TestEnvironment.try_n_times(5, 3, self.load_data) + self.load_search_index(self.sixm, collection_type) + # make sure the index loads... + num_docs = self._check_doc_count(self.sixm, self.TEST_INDEX_NAME, 20, retries=10, delay=3) + + if num_docs == 0: + raise CouchbaseTestEnvironmentException('No docs loaded into the index') + + def teardown(self, + collection_type, # type: CollectionType + test_suite=None, # type: Optional[str] + ): + if test_suite in ['ClassicSearchParamTests', 'ClassicVectorSearchParamTests']: + return + + TestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + TestEnvironment.try_n_times_till_exception(10, + 3, + self.sixm.drop_index, + self.TEST_COLLECTION_INDEX_NAME, + expected_exceptions=(SearchIndexNotFoundException, )) + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + else: + TestEnvironment.try_n_times_till_exception(10, + 3, + self.sixm.drop_index, + self.TEST_INDEX_NAME, + expected_exceptions=(SearchIndexNotFoundException, )) + + def validate_metadata(self, + result, # type: SearchResult + expected_count # type: int + ) -> None: + meta = result.metadata() + assert isinstance(meta, search.SearchMetaData) # nosec + metrics = meta.metrics() + assert isinstance(metrics, search.SearchMetrics) # nosec + assert isinstance(metrics.error_partition_count(), int) # nosec + assert isinstance(metrics.max_score(), float) # nosec + assert isinstance(metrics.success_partition_count(), int) # nosec + assert isinstance(metrics.total_partition_count(), int) # nosec + total_count = metrics.error_partition_count() + metrics.success_partition_count() + assert total_count == metrics.total_partition_count() # nosec + assert isinstance(metrics.took(), timedelta) # nosec + assert metrics.took().total_seconds() >= 0 # nosec + assert metrics.total_rows() >= expected_count # nosec + + def validate_search_row(self, row): + assert isinstance(row, SearchRow) # nosec + assert isinstance(row.index, str) # nosec + assert isinstance(row.id, str) # nosec + assert isinstance(row.score, float) # nosec + assert isinstance(row.explanation, dict) # nosec + assert isinstance(row.fragments, dict) # nosec + + if row.locations: + assert isinstance(row.locations, search.SearchRowLocations) # nosec + + if row.fields: + assert isinstance(row.fields, search.SearchRowFields) # nosec + + def _check_doc_count(self, + sixm, + idx_name, # type: str + min_count, # type: int + retries=20, # type: int + delay=30 # type: int + ) -> bool: + + indexed_docs = 0 + no_docs_cutoff = 300 + for i in range(retries): + # if no docs after waiting for a period of time, exit + if indexed_docs == 0 and i * delay >= no_docs_cutoff: + return 0 + indexed_docs = TestEnvironment.try_n_times(10, + 10, + sixm.get_indexed_documents_count, + idx_name) + if indexed_docs >= min_count: + break + print(f'Found {indexed_docs} indexed docs, waiting a bit...') + TestEnvironment.sleep(delay) + + return indexed_docs + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> SearchTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env + + +class AsyncSearchTestEnvironment(AsyncTestEnvironment): + OTHER_COLLECTION = 'other-collection' + TEST_COLLECTION_INDEX_NAME = 'test-search-coll-index' + TEST_COLLECTION_INDEX_PATH = path.join(pathlib.Path(__file__).parent.parent, + 'test_cases', + f'{TEST_COLLECTION_INDEX_NAME}-params-new.json') + TEST_INDEX_NAME = 'test-search-index' + TEST_INDEX_PATH = path.join(pathlib.Path(__file__).parent.parent, + 'test_cases', + f'{TEST_INDEX_NAME}-params-new.json') + + async def assert_rows(self, + result, # type: SearchResult + expected_count, # type: int + return_rows=False # type: bool + ) -> Optional[List[Union[SearchRow, dict]]]: + rows = [] + assert isinstance(result, SearchResult) # nosec + async for row in result.rows(): + assert row is not None # nosec + self.validate_search_row(row) + rows.append(row) + assert len(rows) >= expected_count # nosec + + self.validate_metadata(result, expected_count) + + if return_rows is True: + return rows + + async def create_and_load_other_collection(self): + collection_spec = CollectionSpec(self.OTHER_COLLECTION, self.scope.name) + await self.cm.create_collection(collection_spec) + collection = None + for _ in range(5): + collection = await self.get_collection(self.scope.name, self.OTHER_COLLECTION, bucket_name=self.bucket.name) + if collection: + break + await AsyncTestEnvironment.sleep(5) + + if not collection: + raise CouchbaseTestEnvironmentException("Unabled to create other-collection for FTS collection testing") + + coll = self.scope.collection(self.OTHER_COLLECTION) + for _ in range(25): + new_doc = self.data_provider.get_new_vehicle() + for _ in range(3): + try: + key = f'{new_doc["id"]}' + _ = await coll.upsert(key, new_doc) + self._loaded_docs[key] = new_doc + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + await AsyncTestEnvironment.sleep(3) + continue + except Exception as ex: + raise ex + + async def load_search_index(self, + sixm, + collection_type, # type: CollectionType + ): + if collection_type == CollectionType.NAMED: + idx_name = self.TEST_COLLECTION_INDEX_NAME + idx_path = self.TEST_COLLECTION_INDEX_PATH + else: + idx_name = self.TEST_INDEX_NAME + idx_path = self.TEST_INDEX_PATH + + with open(idx_path) as params_file: + input = params_file.read() + params_json = json.loads(input) + await AsyncTestEnvironment.try_n_times(10, + 3, + sixm.upsert_index, + SearchIndex(name=idx_name, + idx_type='fulltext-index', + source_name='default', + source_type='couchbase', + params=params_json)) + + async def setup(self, + collection_type, # type: CollectionType + num_docs=None, # type: Optional[int] + test_suite=None, # type: Optional[str] + ): + + if test_suite == 'ClassicSearchParamTests': + return + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + await AsyncTestEnvironment.try_n_times(5, 3, self.setup_named_collections) + await AsyncTestEnvironment.try_n_times(5, 3, self.load_data) + await self.create_and_load_other_collection() + await self.load_search_index(self.sixm, collection_type) + # make sure the index loads... + num_docs = await self._check_doc_count(self.sixm, self.TEST_COLLECTION_INDEX_NAME, 20, retries=10, delay=3) + else: + await AsyncTestEnvironment.try_n_times(5, 3, self.load_data) + await self.load_search_index(self.sixm, collection_type) + # make sure the index loads... + num_docs = await self._check_doc_count(self.sixm, self.TEST_INDEX_NAME, 20, retries=10, delay=3) + + if num_docs == 0: + raise CouchbaseTestEnvironmentException('No docs loaded into the index') + + async def teardown(self, + collection_type, # type: CollectionType + test_suite=None, # type: Optional[str] + ): + if test_suite == 'ClassicSearchParamTests': + return + + await AsyncTestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + await AsyncTestEnvironment.try_n_times_till_exception(10, + 3, + self.sixm.drop_index, + self.TEST_COLLECTION_INDEX_NAME, + expected_exceptions=(SearchIndexNotFoundException, )) + await AsyncTestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + else: + await AsyncTestEnvironment.try_n_times_till_exception(10, + 3, + self.sixm.drop_index, + self.TEST_INDEX_NAME, + expected_exceptions=(SearchIndexNotFoundException, )) + + def validate_metadata(self, + result, # type: SearchResult + expected_count # type: int + ) -> None: + meta = result.metadata() + assert isinstance(meta, search.SearchMetaData) # nosec + metrics = meta.metrics() + assert isinstance(metrics, search.SearchMetrics) # nosec + assert isinstance(metrics.error_partition_count(), int) # nosec + assert isinstance(metrics.max_score(), float) # nosec + assert isinstance(metrics.success_partition_count(), int) # nosec + assert isinstance(metrics.total_partition_count(), int) # nosec + total_count = metrics.error_partition_count() + metrics.success_partition_count() + assert total_count == metrics.total_partition_count() # nosec + assert isinstance(metrics.took(), timedelta) # nosec + assert metrics.took().total_seconds() >= 0 # nosec + assert metrics.total_rows() >= expected_count # nosec + + def validate_search_row(self, row): + assert isinstance(row, SearchRow) # nosec + assert isinstance(row.index, str) # nosec + assert isinstance(row.id, str) # nosec + assert isinstance(row.score, float) # nosec + assert isinstance(row.explanation, dict) # nosec + assert isinstance(row.fragments, dict) # nosec + + if row.locations: + assert isinstance(row.locations, search.SearchRowLocations) # nosec + + if row.fields: + assert isinstance(row.fields, search.SearchRowFields) # nosec + + async def _check_doc_count(self, + sixm, + idx_name, # type: str + min_count, # type: int + retries=20, # type: int + delay=30 # type: int + ) -> bool: + + indexed_docs = 0 + no_docs_cutoff = 300 + for i in range(retries): + # if no docs after waiting for a period of time, exit + if indexed_docs == 0 and i * delay >= no_docs_cutoff: + return 0 + indexed_docs = await AsyncTestEnvironment.try_n_times(10, + 10, + sixm.get_indexed_documents_count, + idx_name) + if indexed_docs >= min_count: + break + print(f'Found {indexed_docs} indexed docs, waiting a bit...') + await AsyncTestEnvironment.sleep(delay) + + return indexed_docs + + @classmethod + def from_environment(cls, + env # type: AsyncTestEnvironment + ) -> AsyncSearchTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/subdoc_environment.py b/tests/environments/subdoc_environment.py new file mode 100644 index 000000000..23f1724da --- /dev/null +++ b/tests/environments/subdoc_environment.py @@ -0,0 +1,299 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import asyncio +import random +import time + +from couchbase.exceptions import AmbiguousTimeoutException, UnAmbiguousTimeoutException +from tests.environments import CollectionType +from tests.environments.test_environment import AsyncTestEnvironment, TestEnvironment + + +class SubdocTestEnvironment(TestEnvironment): + + def get_existing_doc_by_type(self, doc_type, key_only=False): + if not self._loaded_docs: + self.load_data() + + if doc_type == 'array_only': + filtered_keys = set([k for k, v in self._loaded_docs.items() if isinstance(v, list)]) + else: + filtered_keys = set([k for k, v in self._loaded_docs.items() + if not isinstance(v, list) and v['type'] in doc_type]) + available_keys = filtered_keys.difference(self._used_docs) + key = random.choice(list(available_keys)) + self._used_docs.add(key) + if key_only is True: + return key + return key, self._loaded_docs[key] + + def get_new_doc(self, key_only=False): + return self.get_new_doc_by_type('vehicle', key_only=key_only) + + def get_new_doc_by_type(self, doc_type, key_only=False): + if doc_type == 'array': + doc = self.data_provider.get_array_docs(1)[0] + elif doc_type == 'array_only': + docs = self.data_provider.get_array_only_docs(1) + key = docs.keys()[0] + self._used_extras.add(key) + if key_only is True: + return key + return key, docs[key] + elif doc_type == 'count': + doc = self.data_provider.get_count_docs(1)[0] + elif doc_type == 'vehicle': + doc = self.data_provider.get_new_vehicle() + self._used_extras.add(doc['id']) + if key_only is True: + return doc['id'] + return doc['id'], doc + + def load_data(self): # noqa: C901 + for v in self.data_provider.get_array_docs(25): + for _ in range(3): + try: + key = f'{v["id"]}' + _ = self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for k, v in self.data_provider.get_array_only_docs(15).items(): + for _ in range(3): + try: + _ = self.collection.upsert(k, v) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for v in self.data_provider.get_count_docs(15): + for _ in range(3): + try: + key = f'{v["id"]}' + _ = self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for v in self.data_provider.get_vehicles()[:100]: + for _ in range(3): + try: + key = f'{v["id"]}' + _ = self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for k in self._loaded_docs.keys(): + TestEnvironment.try_n_times(5, 1, self.collection.get, k) + + self._doc_types = ['array', 'array_only', 'count', 'vehicle'] + + def setup(self, + collection_type, # type: CollectionType + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + TestEnvironment.try_n_times(5, 3, self.load_data) + + def teardown(self, + collection_type, # type: CollectionType + ): + + TestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> SubdocTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env + + +class AsyncSubdocTestEnvironment(AsyncTestEnvironment): + + def get_existing_doc_by_type(self, doc_type, key_only=False): + if not self._loaded_docs: + self.load_data() + + if doc_type == 'array_only': + filtered_keys = set([k for k, v in self._loaded_docs.items() if isinstance(v, list)]) + else: + filtered_keys = set([k for k, v in self._loaded_docs.items() + if not isinstance(v, list) and v['type'] in doc_type]) + available_keys = filtered_keys.difference(self._used_docs) + key = random.choice(list(available_keys)) + self._used_docs.add(key) + if key_only is True: + return key + return key, self._loaded_docs[key] + + def get_new_doc(self, key_only=False): + return self.get_new_doc_by_type('vehicle', key_only=key_only) + + def get_new_doc_by_type(self, doc_type, key_only=False): + if doc_type == 'array': + doc = self.data_provider.get_array_docs(1)[0] + elif doc_type == 'array_only': + docs = self.data_provider.get_array_only_docs(1) + key = docs.keys()[0] + self._used_extras.add(key) + if key_only is True: + return key + return key, docs[key] + elif doc_type == 'count': + doc = self.data_provider.get_count_docs(1)[0] + elif doc_type == 'vehicle': + doc = self.data_provider.get_new_vehicle() + self._used_extras.add(doc['id']) + if key_only is True: + return doc['id'] + return doc['id'], doc + + async def load_data(self): # noqa: C901 + for v in self.data_provider.get_array_docs(15): + for _ in range(3): + try: + key = f'{v["id"]}' + _ = await self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + await asyncio.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for k, v in self.data_provider.get_array_only_docs(5).items(): + for _ in range(3): + try: + _ = await self.collection.upsert(k, v) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + await asyncio.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for v in self.data_provider.get_count_docs(15): + for _ in range(3): + try: + key = f'{v["id"]}' + _ = await self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + await asyncio.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for v in self.data_provider.get_vehicles()[:50]: + for _ in range(3): + try: + key = f'{v["id"]}' + _ = await self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + await asyncio.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for k in self._loaded_docs.keys(): + await AsyncTestEnvironment.try_n_times(5, 1, self.collection.get, k) + + self._doc_types = ['array', 'array_only', 'count', 'vehicle'] + + async def setup(self, + collection_type, # type: CollectionType + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + await AsyncTestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + await AsyncTestEnvironment.try_n_times(5, 3, self.load_data) + + async def teardown(self, + collection_type, # type: CollectionType + ): + + await AsyncTestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + await AsyncTestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + + @classmethod + def from_environment(cls, + env # type: AsyncTestEnvironment + ) -> AsyncSubdocTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/test_environment.py b/tests/environments/test_environment.py new file mode 100644 index 000000000..4967374ca --- /dev/null +++ b/tests/environments/test_environment.py @@ -0,0 +1,1127 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import asyncio +import random +import time +from typing import (TYPE_CHECKING, + Any, + Callable, + Dict, + Optional, + Tuple, + Type, + Union) + +import pytest +import pytest_asyncio + +from acouchbase.cluster import Cluster as AsyncCluster +from acouchbase.cluster import get_event_loop +from couchbase.auth import PasswordAuthenticator +from couchbase.cluster import Cluster +from couchbase.durability import DurabilityLevel, ServerDurability +from couchbase.exceptions import (AmbiguousTimeoutException, + BucketAlreadyExistsException, + CollectionAlreadyExistsException, + CouchbaseException, + ScopeAlreadyExistsException, + ScopeNotFoundException, + UnAmbiguousTimeoutException) +from couchbase.management.buckets import (BucketType, + CreateBucketSettings, + StorageBackend) +from couchbase.management.collections import CollectionSpec +from couchbase.options import ClusterOptions, TransactionConfig +from tests.consistency import ConsistencyChecker +from tests.data_provider import DataProvider +from tests.environments import CollectionType, CouchbaseTestEnvironmentException +from tests.test_features import EnvironmentFeatures + +if TYPE_CHECKING: + from tests.mock_server import MockServerType + + +class TestEnvironment: + NOT_A_KEY = 'not-a-key' + TEST_SCOPE = "test-scope" + TEST_COLLECTION = "test-collection" + + def __init__(self, + **kwargs # type: Dict[str, Any] + ): + + self._bucket = kwargs.pop('bucket', None) + self._cluster = kwargs.pop('cluster', None) + self._config = kwargs.pop('couchbase_config', None) + self._data_provider = kwargs.pop('data_provider', None) + self._default_collection = kwargs.pop('default_collection', None) + self._default_scope = self._default_collection._scope if self._default_collection else None + self._extra_docs = {} + self._loaded_docs = {} + self._named_collection = None + self._named_scope = None + self._test_bucket = None + self._test_bucket_cm = None + self._use_named_collections = False + self._used_docs = set() + self._used_extras = set() + self._doc_types = ['dealership', 'vehicle'] + self._consistency = ConsistencyChecker.from_test_environment(self) + self._use_scope_search_mgmt = False + self._use_scope_eventing_mgmt = False + + @property + def aixm(self) -> Optional[Any]: + """Returns the default AnalyticsIndexManager""" + return self._aixm if hasattr(self, '_aixm') else None + + @property + def bm(self) -> Optional[Any]: + """Returns the cluster's BucketManager""" + return self._bm if hasattr(self, '_bm') else None + + @property + def bucket(self): + return self._bucket + + @property + def config(self): + return self._config + + @property + def consistency(self): + return self._consistency + + @property + def cluster(self): + return self._cluster + + @property + def cm(self) -> Optional[Any]: + """Returns the default CollectionManager""" + return self._cm if hasattr(self, '_cm') else None + + @property + def collection(self): + if self._use_named_collections: + return self._named_collection + return self._default_collection + + @property + def data_provider(self): + return self._data_provider + + @property + def default_collection(self): + return self._default_collection + + @property + def default_scope(self): + return self._default_scope + + @property + def efm(self) -> Optional[Any]: + """Returns the default EventingFunctionManager""" + return self._efm if hasattr(self, '_efm') else None + + @property + def is_developer_preview(self) -> Optional[bool]: + return self._cluster.is_developer_preview + + @property + def is_mock_server(self) -> bool: + return self._config.is_mock_server + + @property + def is_real_server(self): + return not self._config.is_mock_server + + @property + def mock_server_type(self) -> MockServerType: + if self.is_mock_server: + return self._config.mock_server.mock_type + return None + + @property + def named_collection(self): + return self._named_collection + + @property + def named_scope(self): + return self._named_scope + + @property + def qixm(self) -> Optional[Any]: + """Returns the default QueryIndexManager""" + return self._qixm if hasattr(self, '_qixm') else None + + @property + def scope(self): + if self._use_named_collections: + return self._named_scope + return self._default_scope + + @property + def server_version(self) -> Optional[str]: + return self._cluster.server_version + + @property + def server_version_full(self) -> Optional[str]: + return self._cluster.server_version_full + + @property + def server_version_short(self) -> Optional[float]: + return self._cluster.server_version_short + + @property + def server_version_patch(self) -> Optional[int]: + if self.server_version: + try: + return int(self.server_version.split('-')[0].split('.')[2]) + except Exception: + return None + return None + + @property + def sixm(self) -> Optional[Any]: + """Returns the default SearchIndexManager""" + return self._sixm if hasattr(self, '_sixm') else None + + @property + def test_bucket(self) -> Optional[Any]: + """Returns the test bucket object""" + return self._test_bucket if hasattr(self, '_test_bucket') else None + + @property + def test_bucket_cm(self) -> Optional[Any]: + """Returns the test bucket's CollectionManager""" + return self._test_bucket_cm if hasattr(self, '_test_bucket_cm') else None + + @property + def um(self) -> Optional[Any]: + """Returns the default UserManager""" + return self._um if hasattr(self, '_um') else None + + @property + def use_scope_search_mgmt(self) -> bool: + return self._use_scope_search_mgmt + + @property + def use_scope_eventing_mgmt(self) -> bool: + return self._use_scope_eventing_mgmt + + @property + def use_named_collections(self) -> bool: + return self._use_named_collections + + @property + def vixm(self) -> Optional[Any]: + """Returns the default ViewIndexManager""" + return self._vixm if hasattr(self, '_vixm') else None + + def create_bucket(self, bucket_name, storage_backend=None): + try: + settings_kwargs = { + 'name': bucket_name, + 'bucket_type': BucketType.COUCHBASE, + 'ram_quota_mb': 100, + } + if storage_backend is not None: + settings_kwargs['storage_backend'] = storage_backend + if storage_backend is StorageBackend.MAGMA: + settings_kwargs['ram_quota_mb'] = 1024 + + self.bm.create_bucket(CreateBucketSettings(**settings_kwargs)) + except BucketAlreadyExistsException: + pass + TestEnvironment.try_n_times(10, 1, self.bm.get_bucket, bucket_name) + + def disable_analytics_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('analytics', + self.server_version_short, + self.mock_server_type) + if hasattr(self, '_aixm'): + del self._aixm + + return self + + def disable_bucket_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('basic_bucket_mgmt', + self.server_version_short, + self.mock_server_type) + if not hasattr(self, '_bm'): + del self._bm + + return self + + def disable_collection_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('collections', + self.server_version_short, + self.mock_server_type) + if not hasattr(self, '_cm'): + del self._cm + + return self + + def disable_scope_eventing_mgmt(self) -> TestEnvironment: + self._use_scope_eventing_mgmt = False + return self + + def disable_eventing_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('eventing_function_mgmt', + self.server_version_short, + self.mock_server_type) + if hasattr(self, '_efm'): + del self._efm + + return self + + def disable_named_collections(self) -> TestEnvironment: + self._use_named_collections = False + return self + + def disable_query_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('query_index_mgmt', + self.server_version_short, + self.mock_server_type) + if not hasattr(self, '_qixm'): + del self._qixm + + return self + + def disable_scope_search_mgmt(self) -> TestEnvironment: + self._use_scope_search_mgmt = False + return self + + def disable_search_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('search_index_mgmt', + self.server_version_short, + self.mock_server_type) + if not hasattr(self, '_sixm'): + del self._sixm + + return self + + def disable_views_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('view_index_mgmt', + self.server_version_short, + self.mock_server_type) + if not hasattr(self, '_vixm'): + del self._vixm + + return self + + def disable_user_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('user_mgmt', + self.server_version_short, + self.mock_server_type) + if not hasattr(self, '_um'): + del self._um + + return self + + def enable_analytics_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('analytics', + self.server_version_short, + self.mock_server_type) + if not hasattr(self.cluster, 'analytics_indexes'): + pytest.skip('Analytics index management not available on cluster.') + + self._aixm = self.cluster.analytics_indexes() + return self + + def enable_bucket_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('basic_bucket_mgmt', + self.server_version_short, + self.mock_server_type) + if not hasattr(self.cluster, 'buckets'): + pytest.skip('Bucket management not available on cluster.') + + self._bm = self.cluster.buckets() + return self + + def enable_named_collections(self) -> None: + self._use_named_collections = True + + def enable_collection_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('collections', + self.server_version_short, + self.mock_server_type) + if not hasattr(self.bucket, 'collections'): + pytest.skip('Collection management not available on bucket.') + + self._cm = self.bucket.collections() + return self + + def enable_scope_eventing_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('scope_eventing_function_mgmt', + self.server_version_short, + self.mock_server_type) + + self._use_scope_eventing_mgmt = True + return self + + def enable_eventing_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('eventing_function_mgmt', + self.server_version_short, + self.mock_server_type) + + if self._use_scope_eventing_mgmt: + if not hasattr(self.scope, 'eventing_functions'): + pytest.skip('Eventing functions management not available on scope.') + self._efm = self.scope.eventing_functions() + else: + if not hasattr(self.cluster, 'eventing_functions'): + pytest.skip('Eventing functions management not available on cluster.') + self._efm = self.cluster.eventing_functions() + + return self + + def enable_query_mgmt(self, from_collection=False) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('query_index_mgmt', + self.server_version_short, + self.mock_server_type) + if not from_collection and not hasattr(self.cluster, 'query_indexes'): + pytest.skip('Query index management not available on cluster.') + + if not from_collection and not hasattr(self.collection, 'query_indexes'): + pytest.skip('Query index management not available on collection.') + + if from_collection: + self._qixm = self.collection.query_indexes() + else: + self._qixm = self.cluster.query_indexes() + return self + + def enable_scope_search_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('scope_search_index_mgmt', + self.server_version_short, + self.mock_server_type) + + self._use_scope_search_mgmt = True + return self + + def enable_search_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('search_index_mgmt', + self.server_version_short, + self.mock_server_type) + if self.use_scope_search_mgmt: + if not hasattr(self.scope, 'search_indexes'): + pytest.skip('Search index management not available on scope.') + self._sixm = self.scope.search_indexes() + else: + if not hasattr(self.cluster, 'search_indexes'): + pytest.skip('Search index management not available on cluster.') + self._sixm = self.cluster.search_indexes() + return self + + def enable_views_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('view_index_mgmt', + self.server_version_short, + self.mock_server_type) + if not hasattr(self.bucket, 'view_indexes'): + pytest.skip('View index not available on bucket.') + + self._vixm = self.bucket.view_indexes() + return self + + def enable_user_mgmt(self) -> TestEnvironment: + EnvironmentFeatures.check_if_feature_supported('user_mgmt', + self.server_version_short, + self.mock_server_type) + if not hasattr(self.cluster, 'users'): + pytest.skip('User management not available on cluster.') + + self._um = self.cluster.users() + return self + + def get_collection(self, scope_name, coll_name, bucket_name=None): + scope = self.get_scope(scope_name, bucket_name=bucket_name) + if scope: + return next( + (c for c in scope.collections if c.name == coll_name), None) + + return None + + def get_existing_doc(self, key_only=False): + if not self._loaded_docs: + self.load_data() + + all_keys = set(self._loaded_docs.keys()) + available_keys = all_keys.difference(self._used_docs) + key = random.choice(list(available_keys)) + self._used_docs.add(key) + if key_only is True: + return key + return key, self._loaded_docs[key] + + def get_new_doc(self, key_only=False): + doc = self.data_provider.get_new_vehicle() + self._used_extras.add(doc['id']) + if key_only is True: + return doc['id'] + return doc['id'], doc + + def get_scope(self, scope_name, bucket_name=None): + if bucket_name is None and self._test_bucket is None: + raise CouchbaseTestEnvironmentException("Must provide a bucket name or have a valid test_bucket available") + + bucket_names = ['default'] + if self._test_bucket is not None: + bucket_names.append(self._test_bucket.name) + + bucket = bucket_name or self._test_bucket.name + if bucket not in bucket_names + [bucket_name]: + raise CouchbaseTestEnvironmentException( + f"{bucket} is an invalid bucket name.") + + scopes = [] + if bucket_name is not None: + scopes = self.cluster.bucket(bucket_name).collections().get_all_scopes() + elif bucket == self.bucket.name: + scopes = self.cm.get_all_scopes() + else: + scopes = self.test_bucket_cm.get_all_scopes() + return next((s for s in scopes if s.name == scope_name), None) + + def load_data(self, num_docs=50, test_suite=None): + if test_suite and test_suite == 'datastructures_t': + for k in self.data_provider.generate_keys(num_docs): + self._loaded_docs[k] = {} + return + + # for d in self.data_provider.get_dealerships(): + # for _ in range(3): + # try: + # key = f'{d["id"]}' + # _ = self.collection.upsert(key, d) + # self._loaded_docs[key] = d + # break + # except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + # time.sleep(3) + # continue + # except Exception as ex: + # print(ex) + # raise + + for v in self.data_provider.get_vehicles()[:num_docs]: + for _ in range(3): + try: + key = f'{v["id"]}' + _ = self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + self._doc_types = ['vehicle'] + + def purge_data(self): + for k in self._loaded_docs.keys(): + try: + self.collection.remove(k) + except CouchbaseException: + pass + except Exception: + raise + + for k in self._used_extras: + try: + self.collection.remove(k) + except CouchbaseException: + pass + except Exception: + raise + + self._loaded_docs.clear() + self._used_docs.clear() + self._used_extras.clear() + + def setup(self, + collection_type=None, # type: Optional[CollectionType] + test_suite=None, # type: Optional[str] + num_docs=50, # type: Optional[int] + ): + if collection_type is None: + collection_type = CollectionType.DEFAULT + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + if test_suite: + suite_name = test_suite.split('.')[-1] + TestEnvironment.try_n_times(5, + 3, + self.load_data, + num_docs=num_docs, + test_suite=suite_name) + if suite_name == 'transactions_t': + self.setup_transactions_query() + + else: + TestEnvironment.try_n_times(5, 3, self.load_data, num_docs=num_docs) + + def setup_collection_mgmt(self, bucket_name): + self.create_bucket(bucket_name) + self._test_bucket = TestEnvironment.try_n_times(3, 5, self.cluster.bucket, bucket_name) + self._test_bucket_cm = self._test_bucket.collections() + + def setup_named_collections(self): + try: + self.cm.create_scope(self.TEST_SCOPE) + except ScopeAlreadyExistsException: + self.cm.drop_scope(self.TEST_SCOPE) + self.cm.create_scope(self.TEST_SCOPE) + + self.consistency.wait_until_scope_present(self.bucket.name, self.TEST_SCOPE) + + self._collection_spec = CollectionSpec(self.TEST_COLLECTION, self.TEST_SCOPE) + try: + self.cm.create_collection(self._collection_spec) + except CollectionAlreadyExistsException: + self.cm.drop_collection(self._collection_spec) + self.cm.create_collection(self._collection_spec) + + self.consistency.wait_until_collection_present(self.bucket.name, self.TEST_SCOPE, self.TEST_COLLECTION) + + c = self.get_collection(self.TEST_SCOPE, self.TEST_COLLECTION, bucket_name=self.bucket.name) + if c is None: + raise CouchbaseTestEnvironmentException("Unable to create collection for named collection testing") + + self._named_scope = self.bucket.scope(self.TEST_SCOPE) + self._named_collection = self._named_scope.collection(self.TEST_COLLECTION) + + # TODO: move to a new evironment?? + def setup_transactions_query(self): + def _check_row_count(query_namespace, # type: str + min_count # type: int + ) -> bool: + result = self.cluster.query(f"SELECT id FROM {query_namespace};") + count = 0 + for _ in result.rows(): + count += 1 + return count >= min_count + + self.enable_query_mgmt() + fqdn = self.bucket.name + if self._use_named_collections: + self.qixm.create_primary_index(self.bucket.name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION, + ignore_if_exists=True) + fqdn = f'`{fqdn}`.`{self.TEST_SCOPE}`.`{self.TEST_COLLECTION}`' + else: + self.qixm.create_primary_index(self.bucket.name, ignore_if_exists=True) + + for _ in range(5): + row_count_good = _check_row_count(fqdn, 20) + + if row_count_good: + break + print('Waiting for index to load, sleeping a bit...') + TestEnvironment.sleep(5) + + def teardown(self, + collection_type=None, # type: Optional[CollectionType] + test_suite=None, # type: Optional[str] + ): + + if collection_type is None: + collection_type = CollectionType.DEFAULT + + TestEnvironment.try_n_times(5, 3, self.purge_data) + + if test_suite and test_suite.split('.')[-1] == 'transactions_t': + if self._use_named_collections: + self.qixm.drop_primary_index(self.bucket.name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + else: + self.qixm.drop_primary_index(self.bucket.name) + self.disable_query_mgmt() + + if collection_type == CollectionType.NAMED: + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + self.disable_named_collections() + + def teardown_named_collections(self): + self.cm.drop_scope(self.TEST_SCOPE) + TestEnvironment.try_n_times_till_exception(10, + 1, + self.cm.drop_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeNotFoundException,) + ) + self._collection_spec = None + self._scope = None + self._named_collection = None + + def verify_mutation_tokens(self, bucket_name, result): + mutation_token = result.mutation_token() + assert mutation_token is not None + partition_id, partition_uuid, sequence_number, mt_bucket_name = mutation_token.as_tuple() + assert isinstance(partition_id, int) + assert partition_id != 0 + assert isinstance(partition_uuid, int) + assert partition_uuid != 0 + assert isinstance(sequence_number, int) + assert sequence_number != 0 + assert bucket_name == mt_bucket_name + + def verify_mutation_tokens_disabled(self, bucket_name, result): + mutation_token = result.mutation_token() + assert mutation_token is not None + partition_id, partition_uuid, sequence_number, mt_bucket_name = mutation_token.as_tuple() + assert partition_id != 0 + assert partition_uuid == 0 + assert sequence_number == 0 + assert bucket_name == mt_bucket_name + + @classmethod # noqa: C901 + def get_environment(cls, **kwargs # type: Dict[str, Any] # noqa: C901 + ) -> TestEnvironment: # noqa: C901 + + config = kwargs.get('couchbase_config', None) + if config is None: + raise CouchbaseTestEnvironmentException('No test config provided.') + + conn_string = config.get_connection_string() + username, pw = config.get_username_and_pw() + opts = ClusterOptions(PasswordAuthenticator(username, pw)) + + enable_mutation_tokens = kwargs.pop('enable_mutation_tokens', None) + if enable_mutation_tokens is not None: + opts['enable_mutation_tokens'] = enable_mutation_tokens + + meter = kwargs.pop('meter', None) + if meter: + opts['meter'] = meter + + tracer = kwargs.pop('tracer', None) + if tracer: + opts['tracer'] = tracer + + transaction_config = kwargs.pop('transaction_config', None) + if transaction_config: + opts['transaction_config'] = transaction_config + + env_args = {} + for _ in range(3): + try: + cluster = Cluster.connect(conn_string, opts) + env_args['cluster'] = cluster + print(f'Cluster: {id(cluster)}') + bucket = cluster.bucket(f'{config.bucket_name}') + env_args['bucket'] = bucket + cluster.cluster_info() + env_args['default_collection'] = bucket.default_collection() + break + except (UnAmbiguousTimeoutException, AmbiguousTimeoutException): + continue + env_args.update(**kwargs) + cb_env = cls(**env_args) + return cb_env + + @staticmethod + def sleep(num_seconds # type: float + ) -> None: + time.sleep(num_seconds) + + @staticmethod + def try_n_times(num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + **kwargs # type: Dict[str, Any] + ) -> Any: + for _ in range(num_times): + try: + return func(*args, **kwargs) + except Exception as e: + print(f'trying {func} failed with {type(e).__name__}, sleeping for {seconds_between} seconds...') + time.sleep(seconds_between) + + @staticmethod + def try_n_times_till_exception(num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + expected_exceptions=(Exception,), # type: Tuple[Type[Exception],...] + raise_exception=False, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ) -> None: + for _ in range(num_times): + try: + func(*args, **kwargs) + time.sleep(seconds_between) + except expected_exceptions: + if raise_exception: + raise + # helpful to have this print statement when tests fail + return + except Exception: + raise + + +class AsyncTestEnvironment(TestEnvironment): + + async def create_bucket(self, bucket_name): + try: + await self.bm.create_bucket( + CreateBucketSettings( + name=bucket_name, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100)) + except BucketAlreadyExistsException: + pass + await AsyncTestEnvironment.try_n_times(10, 1, self.bm.get_bucket, bucket_name) + + async def get_collection(self, scope_name, coll_name, bucket_name=None): + scope = await self.get_scope(scope_name, bucket_name=bucket_name) + if scope: + return next( + (c for c in scope.collections if c.name == coll_name), None) + + return None + + async def get_scope(self, scope_name, bucket_name=None): + if bucket_name is None and self._test_bucket is None: + raise CouchbaseTestEnvironmentException("Must provide a bucket name or have a valid test_bucket available") + + bucket_names = ['default'] + if self._test_bucket is not None: + bucket_names.append(self._test_bucket.name) + + bucket = bucket_name or self._test_bucket.name + if bucket not in bucket_names: + raise CouchbaseTestEnvironmentException( + f"{bucket} is an invalid bucket name.") + + scopes = [] + if bucket == self.bucket.name: + scopes = await self.cm.get_all_scopes() + else: + scopes = await self.test_bucket_cm.get_all_scopes() + return next((s for s in scopes if s.name == scope_name), None) + + async def load_data(self, num_docs=50, test_suite=None): + if test_suite and test_suite == 'datastructures_t': + for k in self.data_provider.generate_keys(num_docs): + self._loaded_docs[k] = {} + return + + for v in self.data_provider.get_vehicles()[:num_docs]: + for _ in range(3): + try: + key = f'{v["id"]}' + _ = await self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + await asyncio.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + self._doc_types = ['vehicle'] + + async def purge_data(self): + for k in self._loaded_docs.keys(): + try: + await self.collection.remove(k) + except CouchbaseException: + pass + except Exception: + raise + + for k in self._used_extras: + try: + await self.collection.remove(k) + except CouchbaseException: + pass + except Exception: + raise + + self._loaded_docs.clear() + self._used_docs.clear() + self._used_extras.clear() + + async def setup(self, + collection_type=None, # type: Optional[CollectionType] + test_suite=None, # type: Optional[str] + num_docs=50, # type: Optional[int] + ): + if collection_type is None: + collection_type = CollectionType.DEFAULT + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + await AsyncTestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + if test_suite: + suite_name = test_suite.split('.')[-1] + await AsyncTestEnvironment.try_n_times(5, + 3, + self.load_data, + num_docs=num_docs, + test_suite=suite_name) + if suite_name == 'transactions_t': + await self.setup_transactions_query() + else: + await AsyncTestEnvironment.try_n_times(5, 3, self.load_data, num_docs=num_docs) + + async def setup_collection_mgmt(self, bucket_name): + await self.create_bucket(bucket_name) + self._test_bucket = await AsyncTestEnvironment.try_n_times(3, 5, self.cluster.bucket, bucket_name) + self._test_bucket_cm = self._test_bucket.collections() + + async def setup_named_collections(self): + try: + await self.cm.create_scope(self.TEST_SCOPE) + except ScopeAlreadyExistsException: + await self.cm.drop_scope(self.TEST_SCOPE) + await self.cm.create_scope(self.TEST_SCOPE) + + await AsyncTestEnvironment.try_n_times_till_exception(5, + 1, + self.cm.create_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeAlreadyExistsException,)) + + self._collection_spec = CollectionSpec(self.TEST_COLLECTION, self.TEST_SCOPE) + try: + await self.cm.create_collection(self._collection_spec) + except CollectionAlreadyExistsException: + await self.cm.drop_collection(self._collection_spec) + await self.cm.create_collection(self._collection_spec) + + await AsyncTestEnvironment.try_n_times_till_exception(5, + 1, + self.cm.create_collection, + self._collection_spec, + expected_exceptions=(CollectionAlreadyExistsException,)) + + c = await self.get_collection(self.TEST_SCOPE, self.TEST_COLLECTION, bucket_name=self.bucket.name) + if c is None: + raise CouchbaseTestEnvironmentException("Unabled to create collection for name collection testing") + + self._named_scope = self.bucket.scope(self.TEST_SCOPE) + self._named_collection = self._named_scope.collection(self.TEST_COLLECTION) + + # TODO: move to a new evironment?? + async def setup_transactions_query(self): + async def _check_row_count(query_namespace, # type: str + min_count # type: int + ) -> bool: + result = self.cluster.query(f"SELECT id FROM {query_namespace};") + count = 0 + async for _ in result.rows(): + count += 1 + return count >= min_count + + self.enable_query_mgmt() + fqdn = self.bucket.name + if self._use_named_collections: + await self.qixm.create_primary_index(self.bucket.name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION, + ignore_if_exists=True) + fqdn = f'`{fqdn}`.`{self.TEST_SCOPE}`.`{self.TEST_COLLECTION}`' + else: + await self.qixm.create_primary_index(self.bucket.name, ignore_if_exists=True) + + for _ in range(5): + row_count_good = await _check_row_count(fqdn, 20) + + if row_count_good: + break + print('Waiting for index to load, sleeping a bit...') + await TestEnvironment.sleep(5) + + async def teardown(self, + collection_type=None, # type: Optional[CollectionType] + test_suite=None, # type: Optional[str] + ): + + if collection_type is None: + collection_type = CollectionType.DEFAULT + + await AsyncTestEnvironment.try_n_times(5, 3, self.purge_data) + + if test_suite and test_suite.split('.')[-1] == 'transactions_t': + if self._use_named_collections: + await self.qixm.drop_primary_index(self.bucket.name, + scope_name=self.TEST_SCOPE, + collection_name=self.TEST_COLLECTION) + else: + await self.qixm.drop_primary_index(self.bucket.name) + self.disable_query_mgmt() + + if collection_type == CollectionType.NAMED: + await AsyncTestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + self.disable_named_collections() + + async def teardown_named_collections(self): + await self.cm.drop_scope(self.TEST_SCOPE) + await AsyncTestEnvironment.try_n_times_till_exception(10, + 1, + self.cm.drop_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeNotFoundException,) + ) + self._collection_spec = None + self._scope = None + self._named_collection = None + + @classmethod # noqa: C901 + async def get_environment(cls, **kwargs # type: Dict[str, Any] # noqa: C901 + ) -> TestEnvironment: # noqa: C901 + + config = kwargs.get('couchbase_config', None) + if config is None: + raise CouchbaseTestEnvironmentException('No test config provided.') + + conn_string = config.get_connection_string() + username, pw = config.get_username_and_pw() + opts = ClusterOptions(PasswordAuthenticator(username, pw)) + + enable_mutation_tokens = kwargs.pop('enable_mutation_tokens', None) + if enable_mutation_tokens is not None: + opts['enable_mutation_tokens'] = enable_mutation_tokens + + meter = kwargs.pop('meter', None) + if meter: + opts['meter'] = meter + + tracer = kwargs.pop('tracer', None) + if tracer: + opts['tracer'] = tracer + + transaction_config = kwargs.pop('transaction_config', None) + if transaction_config: + opts['transaction_config'] = transaction_config + + env_args = {} + for _ in range(3): + try: + cluster = await AsyncCluster.connect(conn_string, opts) + env_args['cluster'] = cluster + print(f'Cluster: {id(cluster)}') + bucket = cluster.bucket(f'{config.bucket_name}') + await bucket.on_connect() + env_args['bucket'] = bucket + await cluster.cluster_info() + env_args['default_collection'] = bucket.default_collection() + break + except (UnAmbiguousTimeoutException, AmbiguousTimeoutException): + continue + env_args.update(**kwargs) + cb_env = cls(**env_args) + return cb_env + + @staticmethod + async def sleep(num_seconds # type: float + ) -> None: + await asyncio.sleep(num_seconds) + + @staticmethod + async def try_n_times(num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + **kwargs # type: Dict[str, Any] + ) -> Any: + for _ in range(num_times): + try: + return await func(*args, **kwargs) + except Exception: + print(f'trying {func} failed, sleeping for {seconds_between} seconds...') + await asyncio.sleep(seconds_between) + + @staticmethod + async def try_n_times_till_exception(num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + expected_exceptions=(Exception,), # type: Tuple[Type[Exception],...] + raise_exception=False, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ) -> None: + for _ in range(num_times): + try: + await func(*args, **kwargs) + await asyncio.sleep(seconds_between) + except expected_exceptions: + if raise_exception: + raise + # helpful to have this print statement when tests fail + return + except Exception: + raise + + +@pytest.fixture(scope='session', name='test_env') +def base_test_environment(couchbase_config): + data_provider = DataProvider(100) + data_provider.build_docs() + return couchbase_config, data_provider + + +@pytest.fixture(scope='session', name='cb_base_env') +def couchbase_base_environment(test_env): + couchbase_config, data_provider = test_env + return TestEnvironment.get_environment(couchbase_config=couchbase_config, data_provider=data_provider) + + +@pytest.fixture(scope='session', name='cb_base_txn_env') +def couchbase_base_txn_environment(test_env): + couchbase_config, data_provider = test_env + transaction_config = TransactionConfig(durability=ServerDurability(DurabilityLevel.NONE)) + return TestEnvironment.get_environment(couchbase_config=couchbase_config, + data_provider=data_provider, + transaction_config=transaction_config) + + +@pytest_asyncio.fixture(scope='session') +def event_loop(): + loop = get_event_loop() + yield loop + loop.close() + + +@pytest_asyncio.fixture(scope='session', name='acb_base_env') +async def acouchbase_base_environment(test_env): + couchbase_config, data_provider = test_env + return await AsyncTestEnvironment.get_environment(couchbase_config=couchbase_config, + data_provider=data_provider) + + +@pytest_asyncio.fixture(scope='session', name='acb_base_txn_env') +async def acouchbase_base_txn_environment(test_env): + couchbase_config, data_provider = test_env + transaction_config = TransactionConfig(durability=ServerDurability(DurabilityLevel.NONE)) + return await AsyncTestEnvironment.get_environment(couchbase_config=couchbase_config, + data_provider=data_provider, + transaction_config=transaction_config) diff --git a/tests/environments/tracing_and_metrics_environment.py b/tests/environments/tracing_and_metrics_environment.py new file mode 100644 index 000000000..8bca45bf6 --- /dev/null +++ b/tests/environments/tracing_and_metrics_environment.py @@ -0,0 +1,202 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +from typing import (Any, + Dict, + List) + +from couchbase.metrics import CouchbaseMeter, CouchbaseValueRecorder +from couchbase.tracing import CouchbaseSpan, CouchbaseTracer +from tests.environments.test_environment import TestEnvironment + + +class BasicMeter(CouchbaseMeter): + _CB_OPERATION = 'db.couchbase.operations' + _CB_SERVICE = 'db.couchbase.service' + _CB_OP = 'db.operation' + + def __init__(self): + self._recorders = {'NOOP': NoOpRecorder()} + super().__init__() + + def value_recorder(self, + name, # type: str + tags # type: Dict[str, str] + ) -> CouchbaseValueRecorder: + + if name != self._CB_OPERATION: + return self._recorders['NOOP'] + + svc = tags.get(self._CB_SERVICE, None) + if not svc: + return self._recorders['NOOP'] + + op_type = tags.get(self._CB_OP, None) + if not op_type: + return self._recorders['NOOP'] + + key = f'{svc}::{op_type}' + recorder = self._recorders.get(key, None) + if recorder: + return recorder + + recorder = BasicValueRecorder() + self._recorders[key] = recorder + return recorder + + def recorders(self) -> Dict[str, CouchbaseValueRecorder]: + return self._recorders + + def reset(self) -> None: + self._recorders = {'NOOP': NoOpRecorder()} + + +class BasicValueRecorder(CouchbaseValueRecorder): + def __init__(self): + self._values = [] + super().__init__() + + @property + def values(self) -> List[int]: + return self._values + + def record_value(self, value: int) -> None: + self._values.append(value) + + +class NoOpRecorder(CouchbaseValueRecorder): + def __init__(self): + super().__init__() + + def record_value(self, value: int) -> None: + pass + + +class TestSpan(CouchbaseSpan): + def __init__(self, name): + super().__init__(None) + self.finished_ = False + self.name_ = name + self.attributes_ = dict() + self.parent_ = None + self._span = None + + def set_attribute(self, key, value): + self.attributes_[key] = value + + def set_parent(self, parent): + self.parent_ = parent + + def get_parent(self): + return self.parent_ + + def finish(self): + self.finished_ = True + + def is_finished(self): + return self.finished_ + + def get_attributes(self): + return self.attributes_ + + def get_name(self): + return self.name_ + + +class TestTracer(CouchbaseTracer): + def __init__(self): + self.spans_ = list() + + def start_span(self, name, parent=None, **kwargs): + span = TestSpan(name) + span.set_parent(parent) + self.spans_.append(span) + return span + + def reset(self): + self.spans_ = list() + + def spans(self): + return self.spans_ + + +class TracingAndMetricsTestEnvironment(TestEnvironment): + + def __init__(self, + **kwargs # type: Dict[str, Any] + ): + self._tracer = kwargs.pop('tracer', None) + self._meter = kwargs.pop('meter', None) + super().__init__(**kwargs) + + @property + def meter(self): + return self._meter + + @property + def tracer(self): + return self._tracer + + def validate_metrics(self, op): + # default recorder is NOOP + keys = list(self.meter.recorders().keys()) + values = list(self.meter.recorders().values()) + assert len(self.meter.recorders()) == 2 + assert op in keys[1] + assert isinstance(values[1], BasicValueRecorder) + assert len(values[1].values) == 1 + assert isinstance(values[1].values[0], int) + + @classmethod + def from_environment(cls, + env, # type: TestEnvironment + **kwargs, # type: Dict[str, Any] + ) -> TracingAndMetricsTestEnvironment: + + base_env_args = { + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + meter = None + if 'create_meter' in kwargs: + meter = BasicMeter() + base_env_args['meter'] = meter + + tracer = None + if 'create_tracer' in kwargs: + tracer = TestTracer() + base_env_args['tracer'] = tracer + + # we have to create a new environment b/c we need a new cluster in order to set the tracer + cb_env = TestEnvironment.get_environment(**base_env_args) + env_args = { + 'bucket': cb_env.bucket, + 'cluster': cb_env.cluster, + 'default_collection': cb_env.default_collection, + 'couchbase_config': cb_env.config, + 'data_provider': cb_env.data_provider, + } + + if meter: + env_args['meter'] = meter + + if tracer: + env_args['tracer'] = tracer + + return cls(**env_args) diff --git a/tests/environments/transcoder_environment.py b/tests/environments/transcoder_environment.py new file mode 100644 index 000000000..6b5071540 --- /dev/null +++ b/tests/environments/transcoder_environment.py @@ -0,0 +1,161 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import random +import time + +from couchbase.exceptions import AmbiguousTimeoutException, UnAmbiguousTimeoutException +from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment + + +class FakeTestObj: + PROP = "fake prop" + PROP1 = 12345 + + +class TranscoderTestEnvironment(TestEnvironment): + + def get_existing_doc_by_type(self, doc_type, key_only=False): + if doc_type == 'bytes': + filtered_keys = set([k for k in self._loaded_docs.keys() if '::bytes::' in k]) + elif doc_type == 'json': + filtered_keys = set([k for k, v in self._loaded_docs.items() + if isinstance(v, dict) and v['type'] == 'simple']) + elif doc_type == 'utf8': + filtered_keys = set([k for k in self._loaded_docs.keys() if '::utf8::' in k]) + + available_keys = filtered_keys.difference(self._used_docs) + key = random.choice(list(available_keys)) + self._used_docs.add(key) + if key_only is True: + return key + return key, self._loaded_docs[key] + + def get_new_doc_by_type(self, doc_type, key_only=False): + key = None + doc = None + if doc_type == 'bytes': + for k, v in self.data_provider.get_bytes_docs(1, 'bytes content'.encode('utf-8')).items(): + key = k + doc = v + break + if doc_type == 'hex': + key = self.data_provider.generate_keys(1)[0] + hex_arr = ['ff0102030405060708090a0b0c0d0e0f', + '101112131415161718191a1b1c1d1e1f', + '202122232425262728292a2b2c2d2e2f', + '303132333435363738393a3b3c3d3e3f'] + doc = bytes.fromhex(''.join(hex_arr)) + elif doc_type == 'json': + doc = self.data_provider.get_simple_docs(1)[0] + key = f'{doc["id"]}' + elif doc_type == 'obj': + key = self.data_provider.generate_keys(1)[0] + doc = FakeTestObj() + elif doc_type == 'utf8': + for k, v in self.data_provider.get_utf8_docs(1, 'utf8 content').items(): + key = k + doc = v + break + + self._used_extras.add(key) + if key_only is True: + return key + return key, doc + + def load_data(self): # noqa: C901 + tc = RawBinaryTranscoder() + for k, v in self.data_provider.get_bytes_docs(10, 'bytes content'.encode('utf-8')).items(): + for _ in range(3): + try: + _ = self.collection.upsert(k, v, transcoder=tc) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + tc = RawStringTranscoder() + for k, v in self.data_provider.get_utf8_docs(10, 'utf8 content').items(): + for _ in range(3): + try: + _ = self.collection.upsert(k, v, transcoder=tc) + self._loaded_docs[k] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + for v in self.data_provider.get_simple_docs(10): + for _ in range(3): + try: + key = f'{v["id"]}' + _ = self.collection.upsert(key, v) + self._loaded_docs[key] = v + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + time.sleep(3) + continue + except Exception as ex: + print(ex) + raise + + self._doc_types = ['bytes', 'json', 'utf8'] + + def setup(self, + collection_type, # type: CollectionType + ): + + if collection_type == CollectionType.NAMED: + self.enable_collection_mgmt().enable_named_collections() + TestEnvironment.try_n_times(5, 3, self.setup_named_collections) + + TestEnvironment.try_n_times(5, 3, self.load_data) + + def teardown(self, + collection_type, # type: CollectionType + ): + + TestEnvironment.try_n_times(5, 3, self.purge_data) + + if collection_type == CollectionType.NAMED: + TestEnvironment.try_n_times(5, 3, self.teardown_named_collections) + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> TranscoderTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/user_mgmt_environment.py b/tests/environments/user_mgmt_environment.py new file mode 100644 index 000000000..e3856c4a1 --- /dev/null +++ b/tests/environments/user_mgmt_environment.py @@ -0,0 +1,156 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +from datetime import datetime +from typing import Optional + +from couchbase.auth import AuthDomain +from couchbase.management.users import (Role, + RoleAndOrigins, + User) +from tests.environments import CollectionType +from tests.environments.test_environment import TestEnvironment + + +class UserManagementTestEnvironment(TestEnvironment): + + def setup(self, + collection_type=None, # type: Optional[CollectionType] + ): + self.enable_user_mgmt() + + def teardown(self, + collection_type=None # type: Optional[CollectionType] + ): + + self.disable_user_mgmt() + + def validate_group(self, group, roles=None): + property_list = [ + 'name', 'description', 'roles', 'ldap_group_reference' + ] + properties = list(n for n in dir(group) if n in property_list) + for p in properties: + value = getattr(group, p) + if p == 'name': + assert value is not None # nosec + elif p == 'description' and value: + assert isinstance(value, str) # nosec + elif p == 'roles': + assert isinstance(value, set) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, Role), value)) is True # nosec + elif p == 'ldap_group_reference' and value: + assert isinstance(value, str) # nosec + + if roles: + assert len(roles) == len(group.roles) # nosec + assert set(roles) == group.roles # nosec + + return True + + def validate_user(self, user, user_roles=None): + # password is write-only + property_list = ['username', 'groups', 'roles'] + properties = list(n for n in dir(user) if n in property_list) + for p in properties: + value = getattr(user, p) + if p == 'username': + assert value is not None # nosec + elif p == 'groups' and value: + assert isinstance(value, set) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, str), value)) is True # nosec + + elif p == 'roles': + assert isinstance(value, set) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, Role), value)) is True # nosec + + if user_roles: + assert len(user_roles) == len(user.roles) # nosec + diff = set(user_roles).difference(user.roles) + assert diff == set() # nosec + + return True + + def validate_user_and_metadata(self, # noqa: C901 + user_metadata, + user_roles=None, + groups=None): + property_list = [ + 'domain', 'user', 'effective_roles', 'password_changed', + 'external_groups' + ] + properties = list(n for n in dir(user_metadata) if n in property_list) + for p in properties: + value = getattr(user_metadata, p) + if p == 'domain': + assert isinstance(value, AuthDomain) # nosec + elif p == 'user': + assert isinstance(value, User) # nosec + # per RFC, user property should return a mutable User object + # that will not have an effect on the UserAndMetadata object + assert id(value) != id(user_metadata._user) # nosec + self.validate_user(value, user_roles) + elif p == 'effective_roles': + assert isinstance(value, list) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, RoleAndOrigins), value)) is True # nosec + elif p == 'password_changed' and value: + assert isinstance(value, datetime) # nosec + elif p == 'external_groups' and value: + assert isinstance(value, set) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, str), value)) is True # nosec + + if user_roles: + actual_roles = set( + r.role for r in user_metadata.effective_roles + if any(map(lambda o: o.type == 'user', r.origins)) + or len(r.origins) == 0) + assert len(user_roles) == len(actual_roles) # nosec + assert set(user_roles) == actual_roles # nosec + + if groups: + actual_roles = set( + r.role for r in user_metadata.effective_roles + if any(map(lambda o: o.type != 'user', r.origins))) + group_roles = set() + for g in groups: + group_roles.update(g.roles) + + assert len(group_roles) == len(actual_roles) # nosec + + return True + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> UserManagementTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/environments/views_environment.py b/tests/environments/views_environment.py new file mode 100644 index 000000000..1d034ab99 --- /dev/null +++ b/tests/environments/views_environment.py @@ -0,0 +1,336 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from __future__ import annotations + +import pathlib +import time +from os import path +from textwrap import wrap +from typing import List, Optional + +from couchbase.exceptions import DesignDocumentNotFoundException +from couchbase.management.views import (DesignDocument, + DesignDocumentNamespace, + View) +from couchbase.result import ViewResult +from tests.environments import CollectionType +from tests.environments.test_environment import AsyncTestEnvironment, TestEnvironment + + +class ViewsTestEnvironment(TestEnvironment): + + TEST_VIEW_NAME = 'test-view' + TEST_VIEW_PATH = path.join(pathlib.Path(__file__).parent.parent, + 'test_cases', + f'{TEST_VIEW_NAME}-new.txt') + + DOCNAME = 'test-ddoc' + + @property + def test_ddoc(self): + return self._test_ddoc + + @property + def num_docs(self) -> int: + return self._num_docs + + def add_test_ddoc(self): + TestEnvironment.try_n_times(3, + 5, + self.vixm.upsert_design_document, + self.test_ddoc, + DesignDocumentNamespace.DEVELOPMENT) + + def assert_rows(self, + result, # type: ViewResult + expected_count, + return_rows=False): + assert isinstance(result, ViewResult) + rows = [] + for row in result.rows(): + assert row is not None + rows.append(row) + assert len(rows) >= expected_count + + if return_rows is True: + return rows + + def create_test_ddoc(self): + view_data = None + with open(self.TEST_VIEW_PATH) as view_file: + view_data = view_file.read() + + view = View(map=view_data) + self._test_ddoc = DesignDocument(name=self.DOCNAME, views={self.TEST_VIEW_NAME: view}) + + def drop_ddoc(self, from_prod=False): + if from_prod is True: + TestEnvironment.try_n_times_till_exception(10, + 3, + self.vixm.drop_design_document, + self.test_ddoc.name, + DesignDocumentNamespace.PRODUCTION, + expected_exceptions=(DesignDocumentNotFoundException, )) + TestEnvironment.try_n_times_till_exception(10, + 3, + self.vixm.drop_design_document, + self.test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + expected_exceptions=(DesignDocumentNotFoundException, )) + + def get_batch_id(self): + if hasattr(self, '_batch_id'): + return self._batch_id + + doc = list(self._loaded_docs.values())[0] + self._batch_id = doc['batch'] + return self._batch_id + + def get_keys(self) -> List[str]: + return wrap(self._batch_id, 2) + + def get_docids_by_key(self, key: str) -> List[str]: + keys = self.get_keys() + key_idx = keys.index(key) + if key_idx == 0: + return [f'{self._batch_id}::{i}' for i in range(10)] + elif key_idx == 1: + return [f'{self._batch_id}::{i}' for i in range(10, 20)] + elif key_idx == 2: + return [f'{self._batch_id}::{i}' for i in range(20, 30)] + else: + return [f'{self._batch_id}::{i}' for i in range(30, self.num_docs)] + + def setup(self, + collection_type, # type: CollectionType + test_suite=None, # type: Optional[str] + num_docs=50, # type: Optional[int] + ): + + self._num_docs = num_docs + if test_suite == 'ClassicViewsParamTests': + return + elif test_suite == 'ClassicViewIndexManagementTests': + self.enable_views_mgmt() + self.create_test_ddoc() + TestEnvironment.try_n_times(5, 3, self.load_data, num_docs=num_docs) + else: + self.create_test_ddoc() + self.add_test_ddoc() + TestEnvironment.try_n_times(5, 3, self.load_data, num_docs=num_docs) + + for _ in range(5): + try: + row_count_good = self._check_row_count(num_docs) + except DesignDocumentNotFoundException: + row_count_good = False + if row_count_good: + break + print('Waiting for view to load, sleeping a bit...') + time.sleep(5) + + def teardown(self, + collection_type, # type: CollectionType + test_suite=None, # type: Optional[str] + ): + if test_suite == 'ClassicViewsParamTests': + return + elif test_suite == 'ClassicViewIndexManagementTests': + self.disable_views_mgmt() + TestEnvironment.try_n_times(5, 3, self.purge_data) + else: + TestEnvironment.try_n_times(5, 3, self.purge_data) + self.drop_ddoc() + + def _check_row_count(self, + min_count # type: int + ) -> bool: + view_result = self.bucket.view_query(self.DOCNAME, + self.TEST_VIEW_NAME, + limit=min_count, + namespace=DesignDocumentNamespace.DEVELOPMENT) + count = 0 + for _ in view_result: + count += 1 + return count >= min_count + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> ViewsTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env + + +class AsyncViewsTestEnvironment(AsyncTestEnvironment): + + TEST_VIEW_NAME = 'test-view' + TEST_VIEW_PATH = path.join(pathlib.Path(__file__).parent.parent, + 'test_cases', + f'{TEST_VIEW_NAME}-new.txt') + + DOCNAME = 'test-ddoc' + + @property + def test_ddoc(self): + return self._test_ddoc + + @property + def num_docs(self) -> int: + return self._num_docs + + async def add_test_ddoc(self): + await AsyncTestEnvironment.try_n_times(3, + 5, + self.vixm.upsert_design_document, + self.test_ddoc, + DesignDocumentNamespace.DEVELOPMENT) + + async def assert_rows(self, + result, # type: ViewResult + expected_count, + return_rows=False): + assert isinstance(result, ViewResult) + rows = [] + async for row in result.rows(): + assert row is not None + rows.append(row) + assert len(rows) >= expected_count + + if return_rows is True: + return rows + + def create_test_ddoc(self): + view_data = None + with open(self.TEST_VIEW_PATH) as view_file: + view_data = view_file.read() + + view = View(map=view_data) + self._test_ddoc = DesignDocument(name=self.DOCNAME, views={self.TEST_VIEW_NAME: view}) + + async def drop_ddoc(self, from_prod=False): + if from_prod is True: + await AsyncTestEnvironment.try_n_times_till_exception(10, + 3, + self.vixm.drop_design_document, + self.test_ddoc.name, + DesignDocumentNamespace.PRODUCTION, + expected_exceptions=(DesignDocumentNotFoundException, )) # noqa: E501 + await AsyncTestEnvironment.try_n_times_till_exception(10, + 3, + self.vixm.drop_design_document, + self.test_ddoc.name, + DesignDocumentNamespace.DEVELOPMENT, + expected_exceptions=(DesignDocumentNotFoundException, )) + + def get_batch_id(self): + if hasattr(self, '_batch_id'): + return self._batch_id + + doc = list(self._loaded_docs.values())[0] + self._batch_id = doc['batch'] + return self._batch_id + + def get_keys(self) -> List[str]: + return wrap(self._batch_id, 2) + + def get_docids_by_key(self, key: str) -> List[str]: + keys = self.get_keys() + key_idx = keys.index(key) + if key_idx == 0: + return [f'{self._batch_id}::{i}' for i in range(10)] + elif key_idx == 1: + return [f'{self._batch_id}::{i}' for i in range(10, 20)] + elif key_idx == 2: + return [f'{self._batch_id}::{i}' for i in range(20, 30)] + else: + return [f'{self._batch_id}::{i}' for i in range(30, self.num_docs)] + + async def setup(self, + collection_type, # type: CollectionType + test_suite=None, # type: Optional[str] + num_docs=50, # type: Optional[int] + ): + self._num_docs = num_docs + if test_suite == 'ClassicViewsParamTests': + return + elif test_suite == 'ClassicViewIndexManagementTests': + self.enable_views_mgmt() + self.create_test_ddoc() + await AsyncTestEnvironment.try_n_times(5, 3, self.load_data, num_docs=num_docs) + else: + self.create_test_ddoc() + await self.add_test_ddoc() + await AsyncTestEnvironment.try_n_times(5, 3, self.load_data, num_docs=num_docs) + + for _ in range(5): + row_count_good = await self._check_row_count(5) + if row_count_good: + break + print('Waiting for view to load, sleeping a bit...') + await AsyncTestEnvironment.sleep(5) + + async def teardown(self, + collection_type, # type: CollectionType + test_suite=None, # type: Optional[str] + ): + if test_suite == 'ClassicViewsParamTests': + return + elif test_suite == 'ClassicViewIndexManagementTests': + self.disable_views_mgmt() + await AsyncTestEnvironment.try_n_times(5, 3, self.purge_data) + else: + await AsyncTestEnvironment.try_n_times(5, 3, self.purge_data) + await self.drop_ddoc() + + async def _check_row_count(self, + min_count # type: int + ) -> bool: + + view_result = self.bucket.view_query(self.DOCNAME, + self.TEST_VIEW_NAME, + limit=min_count, + namespace=DesignDocumentNamespace.DEVELOPMENT) + count = 0 + async for _ in view_result: + count += 1 + return count >= min_count + + @classmethod + def from_environment(cls, + env # type: TestEnvironment + ) -> ViewsTestEnvironment: + + env_args = { + 'bucket': env.bucket, + 'cluster': env.cluster, + 'default_collection': env.default_collection, + 'couchbase_config': env.config, + 'data_provider': env.data_provider, + } + + cb_env = cls(**env_args) + return cb_env diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 000000000..e68b75eb9 --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,1082 @@ + +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import pathlib +import time +from collections import namedtuple +from configparser import ConfigParser +from dataclasses import dataclass, fields +from datetime import datetime, timedelta +from enum import Enum, IntEnum +from typing import (List, + Optional, + Union, + get_type_hints) + +import pytest + +import couchbase.search as search +from couchbase.auth import AuthDomain +from couchbase.management.eventing import (EventingFunction, + EventingFunctionBucketAccess, + EventingFunctionBucketBinding, + EventingFunctionConstantBinding, + EventingFunctionDcpBoundary, + EventingFunctionDeploymentStatus, + EventingFunctionKeyspace, + EventingFunctionLanguageCompatibility, + EventingFunctionLogLevel, + EventingFunctionProcessingStatus, + EventingFunctionSettings, + EventingFunctionUrlAuthBasic, + EventingFunctionUrlAuthBearer, + EventingFunctionUrlAuthDigest, + EventingFunctionUrlBinding, + EventingFunctionUrlNoAuth) +from couchbase.management.logic.eventing_logic import QueryScanConsistency +from couchbase.management.users import (Role, + RoleAndOrigins, + User) +from couchbase.result import SearchResult +from couchbase.search import SearchRow + +from .mock_server import (LegacyMockBucketSpec, + MockServer, + MockServerType) + +BASEDIR = pathlib.Path(__file__).parent.parent + +CONFIG_FILE = os.path.join(pathlib.Path(__file__).parent, "test_config.ini") + +KVPair = namedtuple("KVPair", "key value") + + +class CouchbaseTestEnvironmentException(Exception): + """Raised when something with the test environment is incorrect.""" + + +class EventingFunctionManagementTestStatusException(Exception): + """Raised when waiting for a certained status does not happen within a specified timeframe""" + + +class DataSize(IntEnum): + """Determine amount of documents to load + for testing. + + Increases in 20% increments + """ + EXTRA_SMALL = 1 + SMALL = 2 + MEDIUM = 3 + LARGE = 4 + EXTRA_LARGE = 5 + + +class CollectionType(IntEnum): + DEFAULT = 1 + NAMED = 2 + + +class ServerFeatures(Enum): + KeyValue = 'kv' + SSL = 'ssl' + Views = 'views' + SpatialViews = 'spatial_views' + Diagnostics = 'diagnostics' + SynchronousDurability = 'sync_durability' + Query = 'query' + Subdoc = 'subdoc' + Xattr = 'xattr' + Search = 'search' + Analytics = 'analytics' + Collections = 'collections' + Replicas = 'replicas' + UserManagement = 'user_mgmt' + BasicBucketManagement = 'basic_bucket_mgmt' + BucketManagement = 'bucket_mgmt' + BucketMinDurability = 'bucket_min_durability' + BucketStorageBackend = 'bucket_storage_backend' + CustomConflictResolution = 'custom_conflict_resolution' + QueryIndexManagement = 'query_index_mgmt' + SearchIndexManagement = 'search_index_mgmt' + ViewIndexManagement = 'view_index_mgmt' + GetMeta = 'get_meta' + AnalyticsPendingMutations = 'analytics_pending_mutations' + AnalyticsLinkManagement = 'analytics_link_mgmt' + UserGroupManagement = 'user_group_mgmt' + PreserveExpiry = 'preserve_expiry' + SearchDisableScoring = 'search_disable_scoring' + Eventing = 'eventing' + EventingFunctionManagement = 'eventing_function_mgmt' + RateLimiting = 'rate_limiting' + Txns = 'txns' + TxnQueries = 'txn_queries' + KvRangeScan = 'kv_range_scan' + SubdocReplicaRead = 'subdoc_replica_read' + UpdateCollection = 'update_collection' + UpdateCollectionMaxExpiry = 'update_collection_max_expiry' + NegativeCollectionMaxExpiry = 'negative_collection_max_expiry' + NonDedupedHistory = 'non_deduped_history' + QueryWithoutIndex = 'query_without_index' + NotLockedKVStatus = 'kv_not_locked' + ScopeSearch = 'scope_search' + ScopeSearchIndexManagement = 'scope_search_index_mgmt' + ScopeEventingFunctionManagement = 'scope_eventing_function_mgmt' + ServerGroups = 'server_groups' + Magma128Buckets = 'magma_128_buckets' + + +# mock and real server (all versions) should have these features +BASIC_FEATURES = [ServerFeatures.KeyValue, + ServerFeatures.Diagnostics, + ServerFeatures.SSL, + ServerFeatures.SpatialViews, + ServerFeatures.Subdoc, + ServerFeatures.Views, + ServerFeatures.Replicas] + +# mock related feature lists +FEATURES_NOT_IN_MOCK = [ServerFeatures.Analytics, + ServerFeatures.BucketManagement, + ServerFeatures.EventingFunctionManagement, + ServerFeatures.GetMeta, + ServerFeatures.Query, + ServerFeatures.QueryIndexManagement, + ServerFeatures.RateLimiting, + ServerFeatures.Search, + ServerFeatures.SearchIndexManagement, + ServerFeatures.TxnQueries, + ServerFeatures.UserGroupManagement, + ServerFeatures.UserManagement, + ServerFeatures.ViewIndexManagement, + ServerFeatures.KvRangeScan, + ServerFeatures.ScopeSearch, + ServerFeatures.ScopeSearchIndexManagement, + ServerFeatures.ScopeEventingFunctionManagement, + ServerFeatures.ServerGroups] + +FEATURES_IN_MOCK = [ServerFeatures.Txns] + +# separate features into CBS versions, lets make 5.5 the earliest +AT_LEAST_V5_5_0_FEATURES = [ServerFeatures.BucketManagement, + ServerFeatures.GetMeta, + ServerFeatures.Query, + ServerFeatures.QueryIndexManagement, + ServerFeatures.Search, + ServerFeatures.SearchIndexManagement, + ServerFeatures.ViewIndexManagement] + +AT_LEAST_V6_0_0_FEATURES = [ServerFeatures.Analytics, + ServerFeatures.UserManagement] + +AT_LEAST_V6_5_0_FEATURES = [ServerFeatures.AnalyticsPendingMutations, + ServerFeatures.UserGroupManagement, + ServerFeatures.SynchronousDurability, + ServerFeatures.SearchDisableScoring] + +AT_LEAST_V6_6_0_FEATURES = [ServerFeatures.BucketMinDurability, + ServerFeatures.Txns] + +AT_LEAST_V7_0_0_FEATURES = [ServerFeatures.Collections, + ServerFeatures.AnalyticsLinkManagement, + ServerFeatures.TxnQueries] + +AT_LEAST_V7_1_0_FEATURES = [ServerFeatures.RateLimiting, + ServerFeatures.BucketStorageBackend, + ServerFeatures.CustomConflictResolution, + ServerFeatures.EventingFunctionManagement, + ServerFeatures.PreserveExpiry, + ServerFeatures.ScopeEventingFunctionManagement] + +AT_LEAST_V7_2_0_FEATURES = [ServerFeatures.NonDedupedHistory, + ServerFeatures.UpdateCollection] + +AT_LEAST_V7_5_0_FEATURES = [ServerFeatures.KvRangeScan, + ServerFeatures.SubdocReplicaRead, + ServerFeatures.UpdateCollectionMaxExpiry, + ServerFeatures.QueryWithoutIndex] + +AT_LEAST_V7_6_0_FEATURES = [ServerFeatures.NotLockedKVStatus, + ServerFeatures.NegativeCollectionMaxExpiry, + ServerFeatures.ScopeSearch, + ServerFeatures.ScopeSearchIndexManagement] + +AT_LEAST_V7_6_2_FEATURES = [ServerFeatures.ServerGroups] + +AT_LEAST_V8_0_0_FEATURES = [ServerFeatures.Magma128Buckets] + +# Only set the baseline needed +TEST_SUITE_MAP = { + 'analytics_t': [ServerFeatures.Analytics], + 'analyticsmgmt_t': [ServerFeatures.Analytics], + 'binary_collection_multi_t': [ServerFeatures.KeyValue], + 'binary_collection_t': [ServerFeatures.KeyValue], + 'binary_durability_t': [ServerFeatures.KeyValue], + 'bucket_t': [ServerFeatures.Diagnostics], + 'bucketmgmt_t': [ServerFeatures.BucketManagement], + 'cluster_t': [ServerFeatures.Diagnostics], + 'collection_multi_t': [ServerFeatures.KeyValue], + 'collection_t': [ServerFeatures.KeyValue], + 'collectionmgmt_t': [ServerFeatures.Collections], + 'connection_t': [ServerFeatures.Diagnostics], + 'datastructures_t': [ServerFeatures.Subdoc], + 'durability_t': [ServerFeatures.KeyValue], + 'eventingmgmt_t': [ServerFeatures.EventingFunctionManagement], + 'kv_range_scan_t': [ServerFeatures.KvRangeScan], + 'metrics_t': [ServerFeatures.Collections], + 'mutation_tokens_t': [ServerFeatures.KeyValue], + 'query_t': [ServerFeatures.Query, ServerFeatures.QueryIndexManagement], + 'querymgmt_t': [ServerFeatures.QueryIndexManagement], + 'rate_limit_t': [ServerFeatures.RateLimiting, + ServerFeatures.BucketManagement, + ServerFeatures.UserManagement, + ServerFeatures.Collections], + 'search_t': [ServerFeatures.Search, ServerFeatures.SearchIndexManagement], + 'searchmgmt_t': [ServerFeatures.SearchIndexManagement], + 'subdoc_t': [ServerFeatures.Subdoc], + 'tracing_t': [ServerFeatures.Collections], + 'transactions_t': [ServerFeatures.Txns], + 'transcoder_t': [ServerFeatures.KeyValue], + 'usermgmt_t': [ServerFeatures.UserManagement], + 'viewmgmt_t': [ServerFeatures.ViewIndexManagement], + 'views_t': [ServerFeatures.Views, ServerFeatures.ViewIndexManagement] +} + + +class FakeTestObj: + PROP = "fake prop" + PROP1 = 12345 + + +class CouchbaseTestEnvironment: + KEY = "airport_3830" + CONTENT = { + "airportname": "Chicago Ohare Intl", + "city": "Chicago", + "country": "United States", + "faa": "ORD", + "geo": + { + "alt": 668, + "lat": 41.978603, + "lon": -87.904842 + }, + "icao": "KORD", + "id": 3830, + "type": "airport", + "tz": "America/Chicago" + } + DURABLE_KEY = "airport_3876" + DURABLE_CONTENT = { + "airportname": "Charlotte Douglas Intl", + "city": "Charlotte", + "country": "United States", + "faa": "CLT", + "geo": + { + "alt": 748, + "lat": 35.214, + "lon": -80.943139 + }, + "icao": "KCLT", + "id": 3876, + "type": "airport", + "tz": "America/New_York" + } + NEW_KEY = "airport_3469" + NEW_CONTENT = { + "airportname": "San Francisco Intl", + "city": "San Francisco", + "country": "United States", + "faa": "SFO", + "geo": { + "alt": 13, + "lat": 37.618972, + "lon": -122.374889 + }, + "icao": "KSFO", + "id": 3469, + "type": "airport", + "tz": "America/Los_Angeles" + } + UTF8_KEY = "bc_tests_utf8" + BYTES_KEY = "bc_tests_bytes" + COUNTER_KEY = "bc_tests_counter" + + def __init__(self, cluster, bucket, collection, cluster_config): + self._cluster = cluster + self._bucket = bucket + self._collection = collection + self._loaded_keys = None + self._cluster_config = cluster_config + + @property + def cluster(self): + return self._cluster + + @property + def bucket(self): + return self._bucket + + @property + def collection(self): + return self._collection + + @property + def loaded_keys(self): + return self._loaded_keys + + @property + def is_mock_server(self) -> MockServer: + return self._cluster_config.is_mock_server + + @property + def mock_server_type(self) -> MockServerType: + if self.is_mock_server: + return self._cluster_config.mock_server.mock_type + return None + + @property + def is_real_server(self): + return not self._cluster_config.is_mock_server + + @property + def server_version(self) -> Optional[str]: + return self._cluster.server_version + + @property + def server_version_short(self) -> Optional[float]: + return self._cluster.server_version_short + + @property + def server_version_full(self) -> Optional[str]: + return self._cluster.server_version_full + + @property + def server_version_patch(self) -> Optional[int]: + if self.server_version: + try: + return int(self.server_version.split('-')[0].split('.')[2]) + except Exception: + return None + return None + + @property + def is_developer_preview(self) -> Optional[bool]: + return self._cluster.is_developer_preview + + def get_default_key_value(self): + return self.KEY, self.CONTENT + + def default_durable_key_value(self): + return self.DURABLE_KEY, self.DURABLE_CONTENT + + def get_binary_keys(self): + return self.UTF8_KEY, self.BYTES_KEY, self.COUNTER_KEY + + def get_binary_key(self, type_): + if type_ == "UTF8": + return self.UTF8_KEY + if type_ == "BYTES": + return self.BYTES_KEY + if type_ == "COUNTER": + return self.COUNTER_KEY + + def load_data_from_file(self): + # TODO: config # of documents loaded and set default key/doc + data_types = ["airports", "airlines", "routes", "hotels", "landmarks"] + if not self._loaded_keys: + self._loaded_keys = [] + sample_json = [] + with open(os.path.join(pathlib.Path(__file__).parent, "travel_sample_data.json")) as data_file: + sample_data = data_file.read() + sample_json = json.loads(sample_data) + + return data_types, sample_json + + def is_feature_supported(self, feature # type: str + ) -> bool: + try: + supported = self.supports_feature(feature) + return supported + except Exception: + return False + + def check_if_feature_supported(self, features # type: Union[str,List[str]] + ) -> None: + + features_list = [] + if isinstance(features, str): + features_list.append(features) + else: + features_list.extend(features) + + for feature in features_list: + try: + supported = self.supports_feature(feature) + if not supported: + pytest.skip(self.feature_not_supported_text(feature)) + except TypeError: + pytest.skip("Unable to determine server version") + except Exception: + raise + + def supports_feature(self, feature # type: str # noqa: C901 + ) -> bool: + + if feature in map(lambda f: f.value, BASIC_FEATURES): + return True + + if self.is_mock_server and feature in map(lambda f: f.value, FEATURES_NOT_IN_MOCK): + return False + + if self.is_mock_server and feature in map(lambda f: f.value, FEATURES_IN_MOCK): + return True + + if feature == ServerFeatures.Diagnostics.value: + if self.is_real_server: + return True + + return self.mock_server_type == MockServerType.GoCAVES + + if feature == ServerFeatures.Xattr.value: + if self.is_real_server: + return True + + return self.mock_server_type == MockServerType.GoCAVES + + if feature == ServerFeatures.BasicBucketManagement.value: + if self.is_real_server: + return True + + return self.mock_server_type == MockServerType.GoCAVES + + if feature in map(lambda f: f.value, AT_LEAST_V5_5_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 5.5 + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V6_0_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 6.0 + # @TODO: couchbase++ looks to choke w/ CAVES + # if feature == ServerFeatures.UserManagement.value: + # return self.mock_server_type == MockServerType.GoCAVES + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V6_5_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 6.5 + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V6_6_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 6.6 + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V7_0_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 7.0 + if feature == ServerFeatures.Collections.value: + return self.mock_server_type == MockServerType.GoCAVES + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V7_1_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 7.1 + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V7_2_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 7.2 + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V7_5_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 7.5 + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V7_6_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 7.6 + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V7_6_2_FEATURES): + if self.is_real_server: + return ((self.server_version_short >= 7.6 and self.server_version_patch >= 2) + or self.server_version_short > 7.6) + return not self.is_mock_server + + if feature in map(lambda f: f.value, AT_LEAST_V8_0_0_FEATURES): + if self.is_real_server: + return self.server_version_short >= 8.0 + return not self.is_mock_server + + raise CouchbaseTestEnvironmentException(f"Unable to determine if server has provided feature: {feature}") + + def feature_not_supported_text(self, feature # type: str # noqa: C901 + ) -> str: + + if self.is_mock_server and feature in map(lambda f: f.value, FEATURES_NOT_IN_MOCK): + return f'Mock server does not support feature: {feature}' + + if feature == ServerFeatures.Diagnostics.value: + if self.mock_server_type == MockServerType.Legacy: + return f'LegacyMockServer does not support feature: {feature}' + + if feature == ServerFeatures.Xattr.value: + if self.mock_server_type == MockServerType.Legacy: + return f'LegacyMockServer does not support feature: {feature}' + + if feature == ServerFeatures.BucketManagement.value: + if self.mock_server_type == MockServerType.Legacy: + return f'LegacyMockServer does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V5_5_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 5.5. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V6_0_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 6.0. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V6_5_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 6.5. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V6_6_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 6.6. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V7_0_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 7.0. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V7_1_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 7.1. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V7_2_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 7.2. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V7_5_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 7.5. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + if feature in map(lambda f: f.value, AT_LEAST_V7_6_0_FEATURES): + if self.is_real_server: + return (f'Feature: {feature} only supported on server versions >= 7.6. ' + f'Using server version: {self.server_version}.') + return f'Mock server does not support feature: {feature}' + + @staticmethod + def mock_supports_feature(test_suite, # type: str + is_mock # type: bool + ) -> bool: + if not is_mock: + return True + + test_suite_features = TEST_SUITE_MAP.get(test_suite, None) + if not test_suite_features: + raise CouchbaseTestEnvironmentException(f"Unable to determine features for test suite: {test_suite}") + + for feature in test_suite_features: + if feature.value in map(lambda f: f.value, FEATURES_NOT_IN_MOCK): + return False + + return True + + def skip_if_mock_unstable(self, stable): + if not stable and self.is_mock_server: + pytest.skip(('CAVES does not seem to be happy. Skipping tests as failure is not' + ' an accurate representation of the state of the test, but rather' + ' there is an environment issue.')) + + # common Search validation + + def validate_search_row(self, row): + assert isinstance(row, SearchRow) # nosec + assert isinstance(row.index, str) # nosec + assert isinstance(row.id, str) # nosec + assert isinstance(row.score, float) # nosec + assert isinstance(row.explanation, dict) # nosec + assert isinstance(row.fragments, dict) # nosec + + if row.locations: + assert isinstance(row.locations, search.SearchRowLocations) # nosec + + if row.fields: + assert isinstance(row.fields, search.SearchRowFields) # nosec + + def validate_search_metadata(self, + result, # type: SearchResult + expected_count # type: int + ) -> None: + meta = result.metadata() + assert isinstance(meta, search.SearchMetaData) # nosec + metrics = meta.metrics() + assert isinstance(metrics, search.SearchMetrics) # nosec + assert isinstance(metrics.error_partition_count(), int) # nosec + assert isinstance(metrics.max_score(), float) # nosec + assert isinstance(metrics.success_partition_count(), int) # nosec + assert isinstance(metrics.total_partition_count(), int) # nosec + total_count = metrics.error_partition_count() + metrics.success_partition_count() + assert total_count == metrics.total_partition_count() # nosec + assert isinstance(metrics.took(), timedelta) # nosec + assert metrics.took().total_seconds() >= 0 # nosec + assert metrics.total_rows() >= expected_count # nosec + + def assert_search_rows(self, + result, # type: SearchResult + expected_count, # type: int + return_rows=False # type: bool + ) -> Optional[List[Union[SearchRow, dict]]]: + rows = [] + assert isinstance(result, SearchResult) # nosec + for row in result.rows(): + assert row is not None # nosec + self.validate_search_row(row) + rows.append(row) + assert len(rows) >= expected_count # nosec + + self.validate_search_metadata(result, expected_count) + + if return_rows is True: + return rows + + async def assert_search_rows_async(self, + result, # type: SearchResult + expected_count, # type: int + return_rows=False # type: bool + ) -> Optional[List[Union[SearchRow, dict]]]: + rows = [] + assert isinstance(result, SearchResult) # nosec + async for row in result.rows(): + assert row is not None # nosec + self.validate_search_row(row) + rows.append(row) + assert len(rows) >= expected_count # nosec + + self.validate_search_metadata(result, expected_count) + + if return_rows is True: + return rows + + # common User mgmt validation + + def validate_user(self, user, user_roles=None): + # password is write-only + property_list = ['username', 'groups', 'roles'] + properties = list(n for n in dir(user) if n in property_list) + for p in properties: + value = getattr(user, p) + if p == 'username': + assert value is not None # nosec + elif p == 'groups' and value: + assert isinstance(value, set) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, str), value)) is True # nosec + + elif p == 'roles': + assert isinstance(value, set) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, Role), value)) is True # nosec + + if user_roles: + assert len(user_roles) == len(user.roles) # nosec + diff = set(user_roles).difference(user.roles) + assert diff == set() # nosec + + return True + + def validate_user_and_metadata(self, # noqa: C901 + user_metadata, + user_roles=None, + groups=None): + property_list = [ + 'domain', 'user', 'effective_roles', 'password_changed', + 'external_groups' + ] + properties = list(n for n in dir(user_metadata) if n in property_list) + for p in properties: + value = getattr(user_metadata, p) + if p == 'domain': + assert isinstance(value, AuthDomain) # nosec + elif p == 'user': + assert isinstance(value, User) # nosec + # per RFC, user property should return a mutable User object + # that will not have an effect on the UserAndMetadata object + assert id(value) != id(user_metadata._user) # nosec + self.validate_user(value, user_roles) + elif p == 'effective_roles': + assert isinstance(value, list) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, RoleAndOrigins), value)) is True # nosec + elif p == 'password_changed' and value: + assert isinstance(value, datetime) # nosec + elif p == 'external_groups' and value: + assert isinstance(value, set) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, str), value)) is True # nosec + + if user_roles: + actual_roles = set( + r.role for r in user_metadata.effective_roles + if any(map(lambda o: o.type == 'user', r.origins)) + or len(r.origins) == 0) + assert len(user_roles) == len(actual_roles) # nosec + assert set(user_roles) == actual_roles # nosec + + if groups: + actual_roles = set( + r.role for r in user_metadata.effective_roles + if any(map(lambda o: o.type != 'user', r.origins))) + group_roles = set() + for g in groups: + group_roles.update(g.roles) + + assert len(group_roles) == len(actual_roles) # nosec + + return True + + def validate_group(self, group, roles=None): + property_list = [ + 'name', 'description', 'roles', 'ldap_group_reference' + ] + properties = list(n for n in dir(group) if n in property_list) + for p in properties: + value = getattr(group, p) + if p == 'name': + assert value is not None # nosec + elif p == 'description' and value: + assert isinstance(value, str) # nosec + elif p == 'roles': + assert isinstance(value, set) # nosec + if len(value) > 0: + assert all(map(lambda r: isinstance(r, Role), value)) is True # nosec + elif p == 'ldap_group_reference' and value: + assert isinstance(value, str) # nosec + + if roles: + assert len(roles) == len(group.roles) # nosec + assert set(roles) == group.roles # nosec + + return True + + # commong eventing function validation + def validate_bucket_bindings(self, bindings # type: List[EventingFunctionBucketBinding] + ) -> None: + binding_fields = fields(EventingFunctionBucketBinding) + type_hints = get_type_hints(EventingFunctionBucketBinding) + for binding in bindings: + assert isinstance(binding, EventingFunctionBucketBinding) # nosec + for field in binding_fields: + value = getattr(binding, field.name) + if value is not None: + if field.name == "name": + assert isinstance(value, EventingFunctionKeyspace) # nosec + elif field.name == "access": + assert isinstance(value, EventingFunctionBucketAccess) # nosec + else: + assert isinstance(value, type_hints[field.name]) # nosec + + def validate_url_bindings(self, bindings # type: List[EventingFunctionUrlBinding] + ) -> None: + binding_fields = fields(EventingFunctionUrlBinding) + type_hints = get_type_hints(EventingFunctionUrlBinding) + for binding in bindings: + assert isinstance(binding, EventingFunctionUrlBinding) # nosec + for field in binding_fields: + value = getattr(binding, field.name) + if value is not None: + if field.name == "auth": + if isinstance(value, EventingFunctionUrlAuthBasic): + assert isinstance(value.username, str) # nosec + assert value.password is None # nosec + elif isinstance(value, EventingFunctionUrlAuthDigest): + assert isinstance(value.username, str) # nosec + assert value.password is None # nosec + elif isinstance(value, EventingFunctionUrlAuthBearer): + assert value.key is None # nosec + else: + assert isinstance(value, EventingFunctionUrlNoAuth) # nosec + else: + assert isinstance(value, type_hints[field.name]) # nosec + + def validate_constant_bindings(self, bindings # type: List[EventingFunctionConstantBinding] + ) -> None: + binding_fields = fields(EventingFunctionConstantBinding) + type_hints = get_type_hints(EventingFunctionConstantBinding) + for binding in bindings: + assert isinstance(binding, EventingFunctionConstantBinding) # nosec + for field in binding_fields: + value = getattr(binding, field.name) + if value is not None: + assert isinstance(value, type_hints[field.name]) # nosec + + def validate_settings(self, settings # type: EventingFunctionSettings # noqa: C901 + ) -> None: # noqa: C901 + assert isinstance(settings, EventingFunctionSettings) # nosec + settings_fields = fields(EventingFunctionSettings) + type_hints = get_type_hints(EventingFunctionSettings) + for field in settings_fields: + value = getattr(settings, field.name) + if value is not None: + if field.name == "dcp_stream_boundary": + assert isinstance(value, EventingFunctionDcpBoundary) # nosec + elif field.name == "deployment_status": + assert isinstance(value, EventingFunctionDeploymentStatus) # nosec + elif field.name == "processing_status": + assert isinstance(value, EventingFunctionProcessingStatus) # nosec + elif field.name == "language_compatibility": + assert isinstance(value, EventingFunctionLanguageCompatibility) # nosec + elif field.name == "log_level": + assert isinstance(value, EventingFunctionLogLevel) # nosec + elif field.name == "query_consistency": + assert isinstance(value, QueryScanConsistency) # nosec + elif field.name == "handler_headers": + assert isinstance(value, list) # nosec + elif field.name == "handler_footers": + assert isinstance(value, list) # nosec + else: + assert isinstance(value, type_hints[field.name]) # nosec + + def validate_eventing_function(self, func, # type: EventingFunction + shallow=False # type: Optional[bool] + ) -> None: + assert isinstance(func, EventingFunction) # nosec + if shallow is False: + func_fields = fields(EventingFunction) + type_hints = get_type_hints(EventingFunction) + for field in func_fields: + value = getattr(func, field.name) + if value is not None: + if field.name == "bucket_bindings": + self.validate_bucket_bindings(value) + elif field.name == "url_bindings": + self.validate_url_bindings(value) + elif field.name == "constant_bindings": + self.validate_constant_bindings(value) + elif field.name == "settings": + self.validate_settings(value) + else: + assert isinstance(value, type_hints[field.name]) # nosec + + +class ClusterInformation(): + def __init__(self): + self.host = "localhost" + self.port = 8091 + self.admin_username = "Administrator" + self.admin_password = "password" + self.bucket_name = "default" + self.bucket_password = "" + self.real_server_enabled = False + self.mock_server_enabled = False + self.mock_server = None + # self.mock_path = "" + # self.mock_url = None + # self.mock_server = None + # self.mock_version = None + + @property + def is_mock_server(self): + return self.mock_server_enabled + + @property + def is_real_server(self): + return self.real_server_enabled + + def get_connection_string(self): + if self.mock_server_enabled: + if self.mock_server.mock_type == MockServerType.Legacy: + # What current client uses for mock: + # http://127.0.0.1:49696?ipv6=disabled + return f"http://{self.host}:{self.port}" + + return self.mock_server.connstr + else: + return f"couchbase://{self.host}" + + def get_username_and_pw(self): + return self.admin_username, self.admin_password + + def shutdown(self): + if self.mock_server_enabled and self.mock_server.mock_type == MockServerType.GoCAVES: + self.mock_server.shutdown() + + +def create_mock_server(mock_type, # type: MockServerType + mock_path, # type: str + mock_download_url, # type: Optional[str] + mock_version, # type: Optional[str] + log_dir=None, # type: Optional[str] + log_filename=None, # type: Optional[str] + ) -> MockServer: + + if mock_type == MockServerType.Legacy: + bspec_dfl = LegacyMockBucketSpec('default', 'couchbase') + mock = MockServer.create_legacy_mock_server([bspec_dfl], + mock_path, + mock_download_url, + replicas=2, + nodes=4) + else: + mock = MockServer.create_caves_mock_server(mock_path, + mock_download_url, + mock_version, + log_dir, + log_filename) + + try: + mock.start() + if mock_type == MockServerType.GoCAVES: + mock.create_cluster() + except Exception as ex: + raise CouchbaseTestEnvironmentException( + f"Problem trying to start mock server:\n{ex}") + + return mock + + +def restart_mock(mock) -> None: + try: + print("\nR.I.P. mock...") + mock.stop() + time.sleep(3) + mock.start() + return mock + except Exception: + import traceback + traceback.print_exc() + raise CouchbaseTestEnvironmentException('Error trying to restart mock') + + +@dataclass +class RateLimitData: + url: str = None + username: str = None + pw: str = None + fts_indexes: List[str] = None + + +def load_config(): # noqa: C901 + + cluster_info = ClusterInformation() + try: + config = ConfigParser() + config.read(CONFIG_FILE) + + if config.getboolean('realserver', 'enabled'): + cluster_info.real_server_enabled = True + cluster_info.host = config.get('realserver', 'host') + cluster_info.port = config.getint('realserver', 'port') + cluster_info.admin_username = config.get( + 'realserver', 'admin_username') + cluster_info.admin_password = config.get( + 'realserver', 'admin_password') + cluster_info.bucket_name = config.get('realserver', 'bucket_name') + cluster_info.bucket_password = config.get( + 'realserver', 'bucket_password') + + mock_path = '' + mock_url = '' + mock_version = '' + # @TODO(jc): allow override of log dir and filename + # log_dir = '' + # log_filename = '' + if config.getboolean('gocaves', 'enabled'): + if cluster_info.real_server_enabled: + raise CouchbaseTestEnvironmentException( + "Both real and mock servers cannot be enabled at the same time.") + + cluster_info.mock_server_enabled = True + + if config.has_option('gocaves', 'path'): + mock_path = str( + BASEDIR.joinpath(config.get('gocaves', 'path'))) + if config.has_option('gocaves', 'url'): + mock_url = config.get('gocaves', 'url') + if config.has_option('gocaves', 'version'): + mock_version = config.get('gocaves', 'version') + + cluster_info.mock_server = create_mock_server(MockServerType.GoCAVES, + mock_path, + mock_url, + mock_version) + cluster_info.bucket_name = "default" + # cluster_info.port = cluster_info.mock_server.rest_port + # cluster_info.host = "127.0.0.1" + cluster_info.admin_username = "Administrator" + cluster_info.admin_password = "password" + + if config.has_section('legacymockserver') and config.getboolean('legacymockserver', 'enabled'): + if cluster_info.real_server_enabled: + raise CouchbaseTestEnvironmentException( + "Both real and mock servers cannot be enabled at the same time.") + + if cluster_info.mock_server_enabled: + raise CouchbaseTestEnvironmentException( + "Both java mock and gocaves mock cannot be enabled at the same time.") + + cluster_info.mock_server_enabled = True + mock_path = str( + BASEDIR.joinpath(config.get("mockserver", "path"))) + if config.has_option("mockserver", "url"): + mock_url = config.get("mockserver", "url") + + cluster_info.mock_server = create_mock_server(MockServerType.Legacy, + mock_path, + mock_url) + cluster_info.bucket_name = "default" + cluster_info.port = cluster_info.mock_server.rest_port + cluster_info.host = "127.0.0.1" + cluster_info.admin_username = "Administrator" + cluster_info.admin_password = "password" + + except CouchbaseTestEnvironmentException: + raise + except Exception as ex: + raise CouchbaseTestEnvironmentException( + f"Problem trying read/load test configuration:\n{ex}") + + return cluster_info + + +@pytest.fixture(scope="session") +def couchbase_test_config(): + config = load_config() + yield config + config.shutdown() diff --git a/tests/mock_server.py b/tests/mock_server.py new file mode 100644 index 000000000..9478cadb2 --- /dev/null +++ b/tests/mock_server.py @@ -0,0 +1,426 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +import json +import os +import pathlib +import platform +import select +import socket +import sys +import time +from enum import IntEnum +from subprocess import (STDOUT, + Popen, + call) +from typing import List, Optional +from urllib.request import urlretrieve +from uuid import uuid4 + + +class MockServerException(Exception): + pass + + +class MockServerType(IntEnum): + GoCAVES = 1, + Legacy = 2 + + +class MockServer: + + TEST_DIR = pathlib.Path(__file__).parent + + def __init__(self, log_dir=None, log_filename=None): + self._rest_port = None + self._connstr = None + + if log_dir is None: + self._log_dir = os.path.join(self.TEST_DIR, "test_logs") + else: + self._log_dir = log_dir + + if not os.path.exists(self._log_dir): + os.mkdir(self._log_dir) + + if log_filename is None: + self._log_filename = os.path.join(self._log_dir, 'test_logs.txt') + else: + self._log_filename = log_filename + + @property + def rest_port(self): + return self._rest_port + + @property + def connstr(self): + return self._connstr + + def start(self): + raise NotImplementedError("Mock doesn't implement start().") + + def shutdown(self): + raise NotImplementedError("Mock doesn't implement shutdown().") + + @staticmethod + def create_legacy_mock_server(buckets, # type: List[LegacyMockBucketSpec] + jar_path, # type: str + url=None, # type: str + replicas=None, # type: int + vbuckets=None, # type: int + nodes=4 # type: int + ) -> LegacyMockServer: + return LegacyMockServer(buckets=buckets, + jar_path=jar_path, + url=url, + replicas=replicas, + vbuckets=vbuckets, + nodes=nodes + ) + + @staticmethod + def create_caves_mock_server(caves_path=None, # type: Optional[str] + caves_url=None, # type: Optional[str] + caves_version=None, # type: Optional[str] + log_dir=None, # type: Optional[str] + log_filename=None, # type: Optional[str] + ) -> CavesMockServer: + return CavesMockServer(caves_path=caves_path, + caves_url=caves_url, + caves_version=caves_version, + log_dir=log_dir, + log_filename=log_filename) + + +class LegacyMockBucketSpec(): + def __init__(self, + name='default', # type: str + bucket_type='couchbase', # type: str + password='', # type: str + ): + self._name = name + self._bucket_type = bucket_type + self._password = password + + def __str__(self): + return ':'.join([self._name, self._password, self._bucket_type]) + + +class LegacyMockServer(MockServer): + + def __init__(self, + buckets, # type: List[LegacyMockBucketSpec] + jar_path, # type: str + url=None, # type: str + replicas=None, # type: int + vbuckets=None, # type: int + nodes=4 # type: int + ): + """ + Creates a new instance of the mock server. You must actually call + 'start()' for it to be invoked. + :param list buckets: A list of BucketSpec items + :param string runpath: a string pointing to the location of the mock + :param string url: The URL to use to download the mock. This is only + used if runpath does not exist + :param int replicas: How many replicas should each bucket have + :param int vbuckets: How many vbuckets should each bucket have + :param int nodes: How many total nodes in the cluster + + Note that you must have ``java`` in your `PATH` + """ + + super().__init__() + self._validate_jar(jar_path, url) + self._jar_path = jar_path + self._url = url + self._buckets = buckets + self._nodes = nodes + self._vbuckets = vbuckets + self._replicas = replicas + + # @property + # def rest_port(self): + # return self._rest_port + @property + def mock_type(self) -> MockServerType: + return MockServerType.Legacy + + def _validate_jar(self, jar_path, url): + if not os.path.exists(jar_path): + if not url: + raise MockServerException( + f"{jar_path} does not exist and no URL specified.") + # fp = open(runpath, "wb") + # ulp = urlopen(url) + # jarblob = ulp.read() + # fp.write(jarblob) + # fp.close() + urlretrieve(url, jar_path) + + def _setup_listener(self): + sock = socket.socket() + sock.bind(('', 0)) + sock.listen(5) + + _, port = sock.getsockname() + self._listen = sock + self._port = port + + def _invoke(self): + self._setup_listener() + args = [ + "java", "-client", "-jar", self._runpath, + "--port", "0", "--with-beer-sample", + "--harakiri-monitor", "127.0.0.1:" + str(self._port), + "--nodes", str(self._nodes) + ] + + if self._vbuckets is not None: + args += ["--vbuckets", str(self._vbuckets)] + + if self._replicas is not None: + args += ["--replicas", str(self._replicas)] + + bspec = ",".join([str(x) for x in self._buckets]) + args += ["--buckets", bspec] + + self._po = Popen(args) + + # Sometimes we get an invalid JAR file. Unfortunately there is no + # way to determine or "wait for completion". The next best thing + # is to set a maximum of 15 seconds for the process to start (and + # connect to the listening socket); + + rlist, _, _ = select.select([self._listen], [], [], 15) + if not rlist: + raise MockServerException( + 'Mock server was not ready in time') + + self._harakiri_sock, _ = self._listen.accept() + self._ctlfp = self._harakiri_sock.makefile() + + sbuf = "" + while True: + c = self._ctlfp.read(1) + if c == '\0': + break + sbuf += c + self._rest_port = int(sbuf) + + def _attempt_shutdown(self): + try: + self._listen.shutdown(socket.SHUT_RDWR) + except OSError: + pass + finally: + self._listen.close() + + try: + self._harakiri_sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + finally: + self._harakiri_sock.close() + + try: + self._po.terminate() + self._po.kill() + self._po.communicate() + except OSError: + pass + + def start(self): + self._invoke() + + def shutdown(self): + self._attempt_shutdown() + + +class CavesMockServer(MockServer): + def __init__(self, + caves_path=None, # type: Optional[str] + caves_url=None, # type: Optional[str] + caves_version=None, # type: Optional[str] + log_dir=None, # type: Optional[str] + log_filename=None, # type: Optional[str] + ): + + super().__init__(log_dir=log_dir, log_filename=log_filename) + + if caves_url is None: + caves_url = 'https://github.com/couchbaselabs/gocaves/releases/download' + + self._caves_version = caves_version + if self._caves_version is None: + self._caves_version = 'v0.0.1-74' + + self._build_caves_url(caves_url) + self._validate_caves_path(caves_path) + + # @property + # def rest_port(self): + # return self._rest_port + + # @property + # def connstr(self): + # return self._connstr + + @property + def mock_type(self) -> MockServerType: + return MockServerType.GoCAVES + + def _build_caves_url(self, url): + + if sys.platform.startswith('linux'): + if platform.machine() == 'aarch64': + self._caves_url = f"{url}/{self._caves_version}/gocaves-linux-arm64" + else: + self._caves_url = f"{url}/{self._caves_version}/gocaves-linux-amd64" + elif sys.platform.startswith('darwin'): + if platform.machine() == 'arm64': + self._caves_url = f"{url}/{self._caves_version}/gocaves-macos-arm64" + else: + self._caves_url = f"{url}/{self._caves_version}/gocaves-macos" + elif sys.platform.startswith('win32'): + self._caves_url = f"{url}/{self._caves_version}/gocaves-windows.exe" + else: + raise MockServerException("Unrecognized platform for running GoCAVES mock server.") + + def _validate_caves_path(self, caves_path=None): + if not (caves_path and not caves_path.isspace()): + if sys.platform.startswith('linux'): + caves_path = 'gocaves-linux-arm64' if platform.machine() == 'aarch64' else 'gocaves-linux-amd64' + elif sys.platform.startswith('darwin'): + caves_path = 'gocaves-macos-arm64' if platform.machine() == 'arm64' else 'gocaves-macos' + elif sys.platform.startswith('win32'): + caves_path = 'gocaves-windows.exe' + + self._caves_path = str(self.TEST_DIR.parent.joinpath(caves_path)) + + if not os.path.exists(self._caves_path): + urlretrieve(self._caves_url, self._caves_path) + if not sys.platform.startswith('win32'): + # make executable + call(['chmod', 'a+x', self._caves_path]) + + def _setup_listener(self): + sock = socket.socket() + sock.bind(('', 0)) + sock.listen(10) + + _, port = sock.getsockname() + self._listen = sock + self._port = port + + def _invoke(self): + self._setup_listener() + args = [self._caves_path, f"--control-port={self._port}"] + self._output_log = open(self._log_filename, 'w') + self._po = Popen(args, stdout=self._output_log, stderr=STDOUT) + self._caves_sock, self._caves_addr = self._listen.accept() + self._rest_port = self._caves_addr[1] + + def _attempt_shutdown(self): + try: + self._listen.shutdown(socket.SHUT_RDWR) + except OSError: + pass + finally: + self._listen.close() + + try: + self._caves_sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + finally: + self._caves_sock.close() + + try: + self._output_log.close() + self._po.terminate() + self._po.kill() + self._po.communicate() + except OSError: + pass + + def start(self): + self._invoke() + hello = self._read_command() + if hello['type'] != 'hello': + raise MockServerException("There was a problem, CAVES didn't greet us.") + + def shutdown(self): + self._attempt_shutdown() + + def create_cluster(self): + self._cluster_id = str(uuid4()) + res = self._round_trip_command({ + 'type': 'createcluster', + 'id': self._cluster_id + }) + if res is not None: + self._connstr = res['connstr'] + self._mgmt_addrs = res['mgmt_addrs'] + + def _read_command(self): + result = self._recv() + return json.loads(result) + + def _write_command(self, cmd): + cmd_str = json.dumps(cmd) + cmd_array = bytearray(cmd_str, 'utf-8') + # append null byte + cmd_array += b'\x00' + self._caves_sock.send(bytes(cmd_array)) + + def _round_trip_command(self, cmd): + self._write_command(cmd) + resp = self._read_command() + return resp + + def _recv(self, timeout=2, end=b'\x00'): + self._caves_sock.setblocking(0) + buf = [] + data = '' + chunk = 4096 + start = time.time() + while True: + # if received data and passed timeout + # return the received data + if buf and time.time() - start > timeout: + break + # if no data received, allow a bit more time + elif time.time() - start > timeout * 1.1: + break + + try: + data = self._caves_sock.recv(chunk) + if data: + if end and end in data: + buf.append(str(data[:len(data)-1], encoding='utf-8')) + break + + buf.append(str(data, encoding='utf-8')) + start = time.time() + else: + time.sleep(0.1) + except Exception: + pass + + result = ''.join(buf) + return result diff --git a/tests/test_cases/test-scope-search-index-full.json b/tests/test_cases/test-scope-search-index-full.json new file mode 100644 index 000000000..e4116a61a --- /dev/null +++ b/tests/test_cases/test-scope-search-index-full.json @@ -0,0 +1,63 @@ +{ + "type": "fulltext-index", + "name": "default.test-scope.test-scope-search-index-from-ui", + "uuid": "1b7227c4f8dc2592", + "sourceType": "gocbcore", + "sourceName": "default", + "sourceUUID": "08624d642386673b5baaf4b5d4b2fbee", + "planParams": { + "maxPartitionsPerPIndex": 1024, + "indexPartitions": 1 + }, + "params": { + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "scope.collection.type_field", + "type_field": "type" + }, + "mapping": { + "analysis": {}, + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": true, + "index_dynamic": false, + "store_dynamic": false, + "type_field": "_type", + "types": { + "test-scope.test-collection": { + "dynamic": false, + "enabled": true, + "properties": { + "make": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "make", + "store": true, + "type": "text" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceParams": null +} diff --git a/tests/test_cases/test-search-coll-index-params-new.json b/tests/test_cases/test-search-coll-index-params-new.json new file mode 100644 index 000000000..733667e4e --- /dev/null +++ b/tests/test_cases/test-search-coll-index-params-new.json @@ -0,0 +1,219 @@ +{ + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "scope.collection.type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": true, + "index_dynamic": false, + "store_dynamic": false, + "type_field": "_type", + "types": { + "test-scope.test-collection": { + "dynamic": false, + "enabled": true, + "properties": { + "make": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "make", + "store": true, + "type": "text" + } + ] + }, + "model": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "model", + "store": true, + "type": "text" + } + ] + }, + "year": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "year", + "store": true, + "type": "number" + } + ] + }, + "description": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "description", + "store": true, + "type": "text" + } + ] + }, + "rating": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "rating", + "store": true, + "type": "number" + } + ] + }, + "last_updated": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "last_updated", + "store": true, + "type": "datetime" + } + ] + } + } + }, + "test-scope.other-collection": { + "dynamic": false, + "enabled": true, + "properties": { + "make": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "make", + "store": true, + "type": "text" + } + ] + }, + "model": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "model", + "store": true, + "type": "text" + } + ] + }, + "year": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "year", + "store": true, + "type": "number" + } + ] + }, + "description": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "description", + "store": true, + "type": "text" + } + ] + }, + "rating": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "rating", + "store": true, + "type": "number" + } + ] + }, + "last_updated": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "last_updated", + "store": true, + "type": "datetime" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch" + } +} diff --git a/tests/test_cases/test-search-coll-index-params.json b/tests/test_cases/test-search-coll-index-params.json new file mode 100644 index 000000000..9e8f165c1 --- /dev/null +++ b/tests/test_cases/test-search-coll-index-params.json @@ -0,0 +1,219 @@ +{ + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "scope.collection.type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": true, + "index_dynamic": false, + "store_dynamic": false, + "type_field": "_type", + "types": { + "test-scope.test-collection": { + "dynamic": false, + "enabled": true, + "properties": { + "activity": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "activity", + "store": true, + "type": "text" + } + ] + }, + "content": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "content", + "store": true, + "type": "text" + } + ] + }, + "name": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "name", + "store": true, + "type": "text" + } + ] + }, + "title": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "title", + "store": true, + "type": "text" + } + ] + }, + "rating": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "rating", + "store": true, + "type": "number" + } + ] + }, + "updated": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "updated", + "store": true, + "type": "datetime" + } + ] + } + } + }, + "test-scope.other-collection": { + "dynamic": false, + "enabled": true, + "properties": { + "activity": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "activity", + "store": true, + "type": "text" + } + ] + }, + "content": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "content", + "store": true, + "type": "text" + } + ] + }, + "name": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "name", + "store": true, + "type": "text" + } + ] + }, + "title": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "title", + "store": true, + "type": "text" + } + ] + }, + "rating": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "rating", + "store": true, + "type": "number" + } + ] + }, + "updated": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "updated", + "store": true, + "type": "datetime" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch" + } +} diff --git a/tests/test_cases/test-search-index-full.json b/tests/test_cases/test-search-index-full.json new file mode 100644 index 000000000..cab9355f2 --- /dev/null +++ b/tests/test_cases/test-search-index-full.json @@ -0,0 +1,138 @@ +{ + "type": "fulltext-index", + "name": "test-search-index-from-ui", + "uuid": "24b4a143417c7c22", + "sourceType": "gocbcore", + "sourceName": "default", + "sourceUUID": "367a3da917a4d29dfd28d75fe755c280", + "planParams": { + "maxPartitionsPerPIndex": 1024, + "indexPartitions": 1 + }, + "params": { + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "analysis": {}, + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": true, + "index_dynamic": false, + "store_dynamic": false, + "type_field": "_type", + "types": { + "vehicle": { + "dynamic": false, + "enabled": true, + "properties": { + "description": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "description", + "store": true, + "type": "text" + } + ] + }, + "last_updated": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "last_updated", + "store": true, + "type": "datetime" + } + ] + }, + "make": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "make", + "store": true, + "type": "text" + } + ] + }, + "model": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "model", + "store": true, + "type": "text" + } + ] + }, + "rating": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "rating", + "store": true, + "type": "number" + } + ] + }, + "year": { + "dynamic": false, + "enabled": true, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "year", + "store": true, + "type": "number" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 15 + } + }, + "sourceParams": null + } diff --git a/tests/test_cases/test-search-index-params-new.json b/tests/test_cases/test-search-index-params-new.json new file mode 100644 index 000000000..253c63ab7 --- /dev/null +++ b/tests/test_cases/test-search-index-params-new.json @@ -0,0 +1,123 @@ +{ + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": true, + "index_dynamic": false, + "store_dynamic": false, + "type_field": "_type", + "types": { + "vehicle": { + "dynamic": false, + "enabled": true, + "properties": { + "make": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "make", + "store": true, + "type": "text" + } + ] + }, + "model": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "model", + "store": true, + "type": "text" + } + ] + }, + "year": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "year", + "store": true, + "type": "number" + } + ] + }, + "description": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "description", + "store": true, + "type": "text" + } + ] + }, + "rating": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "rating", + "store": true, + "type": "number" + } + ] + }, + "last_updated": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "last_updated", + "store": true, + "type": "datetime" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch" + } +} diff --git a/tests/test_cases/test-search-index-params.json b/tests/test_cases/test-search-index-params.json new file mode 100644 index 000000000..a55a4d83e --- /dev/null +++ b/tests/test_cases/test-search-index-params.json @@ -0,0 +1,123 @@ +{ + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": true, + "index_dynamic": false, + "store_dynamic": false, + "type_field": "_type", + "types": { + "landmark": { + "dynamic": false, + "enabled": true, + "properties": { + "activity": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "activity", + "store": true, + "type": "text" + } + ] + }, + "content": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "content", + "store": true, + "type": "text" + } + ] + }, + "name": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "name", + "store": true, + "type": "text" + } + ] + }, + "title": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "title", + "store": true, + "type": "text" + } + ] + }, + "rating": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "rating", + "store": true, + "type": "number" + } + ] + }, + "updated": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "docvalues": true, + "include_in_all": true, + "include_term_vectors": true, + "index": true, + "name": "updated", + "store": true, + "type": "datetime" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch" + } +} diff --git a/tests/test_cases/test-search-vector-coll-index-params.json b/tests/test_cases/test-search-vector-coll-index-params.json new file mode 100644 index 000000000..fe51d4603 --- /dev/null +++ b/tests/test_cases/test-search-vector-coll-index-params.json @@ -0,0 +1,60 @@ +{ + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "scope.collection.type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": true, + "store_dynamic": false, + "type_field": "_type", + "types": { + "test-scope.test-collection": { + "dynamic": false, + "enabled": true, + "properties": { + "text": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "index": true, + "name": "text", + "store": true, + "type": "text" + } + ] + }, + "vector_field": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "dims": 1536, + "index": true, + "name": "vector_field", + "similarity": "l2_norm", + "store": true, + "type": "vector" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 16 + } +} diff --git a/tests/test_cases/test-search-vector-index-params.json b/tests/test_cases/test-search-vector-index-params.json new file mode 100644 index 000000000..2b42d2fc7 --- /dev/null +++ b/tests/test_cases/test-search-vector-index-params.json @@ -0,0 +1,60 @@ +{ + "doc_config": { + "docid_prefix_delim": "", + "docid_regexp": "", + "mode": "type_field", + "type_field": "type" + }, + "mapping": { + "default_analyzer": "standard", + "default_datetime_parser": "dateTimeOptional", + "default_field": "_all", + "default_mapping": { + "dynamic": true, + "enabled": false + }, + "default_type": "_default", + "docvalues_dynamic": false, + "index_dynamic": true, + "store_dynamic": false, + "type_field": "_type", + "types": { + "vector": { + "dynamic": false, + "enabled": true, + "properties": { + "text": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "index": true, + "name": "text", + "store": true, + "type": "text" + } + ] + }, + "vector_field": { + "enabled": true, + "dynamic": false, + "fields": [ + { + "dims": 1536, + "index": true, + "name": "vector_field", + "similarity": "l2_norm", + "store": true, + "type": "vector" + } + ] + } + } + } + } + }, + "store": { + "indexType": "scorch", + "segmentVersion": 16 + } +} diff --git a/tests/test_cases/test-search-vector.json b/tests/test_cases/test-search-vector.json new file mode 100644 index 000000000..f205fc64f --- /dev/null +++ b/tests/test_cases/test-search-vector.json @@ -0,0 +1 @@ +[-0.014653487130999565, -0.008658270351588726, 0.017129190266132355, -0.015563474968075752, -0.02059517428278923, 0.019551364704966545, -0.03565012663602829, -0.0021344576962292194, 0.01641993410885334, -0.014666869305074215, -0.009340761229395866, 0.010538466274738312, -0.014318931847810745, -0.0037169004790484905, -0.018132854253053665, -0.0053796363063156605, 0.03904920071363449, 0.0031983410008251667, 0.004399391356855631, -0.024275273084640503, -0.03072548471391201, 0.03439220041036606, -0.02064870297908783, -0.02404777705669403, -0.014305549673736095, 0.01071243453770876, 0.015924794599413872, -0.00922701321542263, 0.01243873592466116, 0.01149529218673706, 0.02513173408806324, 0.004513140302151442, 0.01349592860788107, -0.027125678956508636, -0.028289927169680595, -0.03350897878408432, -0.012933876365423203, 0.006851675920188427, -0.006209331564605236, 0.020019741728901863, -0.024556299671530724, -0.012766599655151367, -0.014519665390253067, -0.014305549673736095, -0.008397317491471767, 0.0032100502867251635, 0.004901223350316286, -0.004449574742466211, -7.919532072264701e-05, 0.018801962956786156, 0.02343219704926014, 0.012753217481076717, -0.013395561836659908, -0.01007678173482418, 0.024957764893770218, -0.01077265478670597, 0.015443035401403904, 0.0005081046256236732, 0.015202156268060207, -0.040735356509685516, 0.006075509358197451, -0.008832238614559174, -0.005647279787808657, 0.02387380786240101, 0.0012085781199857593, 0.0024890853092074394, -0.008102909661829472, -0.005694117397069931, 0.020581793040037155, 0.006751309614628553, 0.013355415314435959, 0.01397099532186985, -0.019805625081062317, -0.0070256441831588745, 0.0009208612609654665, -0.029654910787940025, -0.015242302790284157, 0.008076145313680172, 0.008457537740468979, 0.009789064526557922, 0.009454510174691677, 3.5285043850308284e-05, -0.018387114629149437, 0.015335978008806705, 0.010397953912615776, 0.02933373861014843, -0.014278785325586796, 0.011361470445990562, -0.008658270351588726, -0.004315753001719713, 0.007634533569216728, 0.029173152521252632, 0.01680801808834076, 0.006851675920188427, 0.0060922373086214066, 0.015416271053254604, -0.009969723410904408, 0.024583064019680023, 0.006580686662346125, -0.0066609797067940235, 0.004837657790631056, 0.026188924908638, -0.01158896740525961, -0.016874928027391434, -0.01166256982833147, -0.018052561208605766, 0.003810575697571039, -0.006313043180853128, 0.020220473408699036, -0.01010354608297348, 0.0006837457185611129, 0.028825214132666588, 0.015162009745836258, -0.017664477229118347, -0.006032017525285482, -0.017008749768137932, 0.016286112368106842, 0.009400981478393078, 0.006557268090546131, -0.03963801637291908, 0.003102992894127965, 0.02317793481051922, 0.003569696331396699, -0.02072899602353573, 0.021505162119865417, 0.01633964106440544, -0.008397317491471767, 0.009695389308035374, -0.0054532382637262344, -0.007032335270196199, 0.024596447125077248, 0.018601229414343834, 0.0018183037173002958, -0.0140111418440938, -0.024302037432789803, 0.016045233234763145, -0.0044261557050049305, 0.02167913131415844, -0.017463743686676025, -0.008598050102591515, 0.012692997232079506, 0.01504157017916441, -0.0060888915322721004, -0.01917666383087635, -0.002182967960834503, 0.027567289769649506, 0.012037270702421665, 0.027674347162246704, -0.005209013354033232, -0.015683915466070175, 0.004680417478084564, -0.013449090532958508, 0.015202156268060207, 0.017878592014312744, 0.01171609852463007, 0.013000787235796452, -0.0066710165701806545, -0.0009835902601480484, -0.0013900739140808582, -0.023726604878902435, 0.005704154260456562, 0.01078603696078062, 0.021893246099352837, 0.00836386252194643, 0.04378649219870567, 0.03227781876921654, 0.02070223167538643, 0.003563005244359374, 0.018681522458791733, 0.00674127321690321, 0.012211238965392113, 0.022642647847533226, -0.013589603826403618, 0.01884210854768753, -0.02070223167538643, 0.022722940891981125, -0.009387599304318428, 0.01605861634016037, -0.03136783093214035, -0.022870145738124847, -0.011635805480182171, -0.015202156268060207, -0.0005900705000385642, 0.010190529748797417, -0.014305549673736095, -0.0280758123844862, 0.019832389429211617, 0.009053044952452183, -0.006858367007225752, -0.027674347162246704, -0.007507402915507555, 0.019738715142011642, 0.020327530801296234, -0.009682007133960724, -0.6753183007240295, -0.023003967478871346, 0.01817299984395504, -0.04394707828760147, 0.009882739745080471, 0.010832874104380608, 0.018306821584701538, 0.014894366264343262, -0.017316540703177452, -0.034713372588157654, -0.005556950345635414, 0.019243573769927025, 0.017798298969864845, 0.008256805129349232, -0.009829211048781872, -0.021331194788217545, 0.003061173716560006, -0.007875412702560425, 0.01153543870896101, 0.031689003109931946, -0.01878857985138893, 0.02418159879744053, 0.00023376995522994548, 0.021665748208761215, 0.013743498362600803, 0.02343219704926014, 0.015335978008806705, 0.004024690482765436, -0.02193339169025421, 0.008598050102591515, -0.022508826106786728, 0.015309213660657406, -0.030216962099075317, -0.015603621490299702, 0.05020993947982788, 0.028477277606725693, -0.017517272382974625, 0.014198492281138897, -0.001395092229358852, 0.06503739207983017, -0.01502818800508976, -0.00837724469602108, 0.029494324699044228, -0.004389354959130287, -0.005456583574414253, 0.014546429738402367, 0.023284992203116417, -0.009775682352483273, -0.004258878529071808, -0.012291532009840012, 0.008270187303423882, -0.024502770975232124, 0.012793364003300667, 0.003890868742018938, -0.00656061340123415, 0.0009200248750858009, 0.024368949234485626, -0.003036082023754716, 0.008437464013695717, 0.00822334922850132, -0.010946623049676418, 0.004382663872092962, -0.02593466453254223, -0.0005081046256236732, -0.011281177401542664, 0.01667419634759426, -0.01170271635055542, -0.02646995149552822, -0.0016301167197525501, -0.022187653928995132, 0.017303157597780228, 0.004416119307279587, -0.025867752730846405, -0.0010530102299526334, 0.0038038846105337143, 0.031046656891703606, 0.03519513085484505, -0.013616368174552917, 0.0059584155678749084, -0.009842593222856522, 0.015228920616209507, 0.02874492108821869, -0.03361603617668152, 0.006550577003508806, 0.01848079077899456, 0.0018735051853582263, -0.01989930123090744, -0.013462472707033157, 0.005781101528555155, 0.004041418433189392, 0.0033890369813889265, 0.019350631162524223, -0.006119001656770706, -0.01014369260519743, -0.0055970968678593636, 0.009012898430228233, -0.00057752471184358, 0.023392049595713615, 0.01504157017916441, -0.02933373861014843, -0.03864773362874985, -0.02505144104361534, -0.004677071701735258, -0.0008798782946541905, 0.005888158921152353, 0.002686472609639168, -0.0032167413737624884, 0.026148779317736626, 0.009006206877529621, -0.006466938182711601, 0.010444791056215763, 0.0027182551566511393, -0.016460081562399864, 0.017410214990377426, 0.009668624959886074, -0.03281310573220253, 0.020635321736335754, 0.003111356869339943, -0.004710527136921883, -0.017490508034825325, 0.027540525421500206, 0.025091586634516716, 0.013890702277421951, -0.0037369737401604652, -0.016580520197749138, 0.0013524364912882447, -0.002032418502494693, -0.014559811912477016, -0.006062127184122801, -0.022160889580845833, 0.02404777705669403, -0.0007167829899117351, 0.011368161998689175, -0.006597414612770081, 0.01886887289583683, 0.01892240159213543, 0.014934512786567211, -0.004392700269818306, 0.02296382002532482, -0.029842261224985123, -0.027032002806663513, -0.01261939574033022, -0.024087922647595406, -0.002850404242053628, 0.0009827538160607219, -0.02407454140484333, -0.007875412702560425, -0.006771382875740528, -0.01714257150888443, 0.0014268748927861452, -0.026617154479026794, -0.010210603475570679, -0.02373998612165451, -0.007433800958096981, -0.00839062687009573, 0.02157207392156124, 0.0074003455229103565, 0.010304278694093227, -0.007159465923905373, 0.0006051254458725452, 0.021183989942073822, -0.00912664644420147, -0.03431190922856331, -0.025667021051049232, -0.007072481792420149, -0.02126428298652172, -0.02404777705669403, 0.01221792958676815, -0.008598050102591515, -0.040226832032203674, -0.015202156268060207, -0.020782524719834328, -0.009414363652467728, 0.016379788517951965, 0.005908232182264328, 0.026148779317736626, -0.014425989240407944, -0.02098325826227665, -0.013890702277421951, -0.006236095912754536, 0.03650658577680588, -0.012304914183914661, -0.02994931861758232, -0.0058011747896671295, 0.029494324699044228, -0.007875412702560425, 0.03337515518069267, 0.021612219512462616, -0.015911411494016647, 0.021478397771716118, 0.007855339907109737, 0.024061158299446106, -0.009554876014590263, -0.009079809300601482, -0.0025727241300046444, -0.025372613221406937, -0.021197373047471046, 0.00653384905308485, -1.048749891197076e-05, 0.029467560350894928, 0.02590790018439293, 0.002629598369821906, 0.015456417575478554, 0.0013616367941722274, 0.0008966060122475028, -0.029842261224985123, 0.010036635212600231, -0.01819976419210434, 0.029039330780506134, 0.031742531806230545, 0.01786521077156067, -0.042260922491550446, -0.007360199000686407, -0.019778860732913017, 0.012084107846021652, 0.012177783064544201, 0.008611432276666164, 0.012699688784778118, -0.003156521590426564, -0.02944079600274563, -0.028825214132666588, -0.009206939488649368, 0.03008314035832882, 0.0007640388212166727, -0.0028169488068670034, -0.0010203912388533354, -0.006266205571591854, 0.016433317214250565, -0.004824275616556406, -0.020166944712400436, -0.02855757065117359, 0.021879862993955612, 0.012806746177375317, 0.0033003799617290497, -0.007554240524768829, 0.007875412702560425, 0.01814623549580574, -0.023927336558699608, 0.04705174267292023, 0.022749705240130424, 0.02098325826227665, 0.03642629459500313, 0.004151821136474609, 0.007126010488718748, 0.00460681552067399, -0.012177783064544201, 0.020809289067983627, 0.007206303533166647, -0.005580368917435408, 0.010036635212600231, -0.0018835418159142137, -0.013743498362600803, -0.02925344556570053, 0.013703351840376854, 0.011281177401542664, 0.001262943260371685, 0.03072548471391201, -0.0023669730871915817, 0.03337515518069267, 0.02699185721576214, -0.007935632951557636, -0.005118683911859989, 0.01172948069870472, -0.013094462454319, 0.007239759434014559, 0.0063799540512263775, 0.004881150089204311, -0.00836386252194643, -0.013863937929272652, 0.002939061261713505, 0.005995216313749552, -0.01327512226998806, -0.004673726391047239, 0.018627993762493134, -0.004205349832773209, -0.00904635339975357, 0.0005035045323893428, -0.015857882797718048, 0.007574313785880804, 0.028236398473381996, -0.016794634982943535, -0.03482042998075485, -0.0009919541189447045, 0.021531926468014717, -0.005503421183675528, -0.019711950793862343, -0.027674347162246704, 0.004804202355444431, -0.01504157017916441, 0.016192438080906868, 0.033963970839977264, 0.009260468184947968, -0.0038273034151643515, -0.020421206951141357, -0.0010789382504299283, -0.008705107495188713, 0.01065890584141016, -0.020274002104997635, 0.006931968964636326, -0.000532359816133976, 0.008778709918260574, -0.013703351840376854, -0.03008314035832882, -0.023258227854967117, 0.033080749213695526, 0.01670096069574356, 0.0050149718299508095, 0.0009442800655961037, -0.013783644884824753, 0.013154682703316212, -0.006393336225301027, 0.0006385808810591698, 0.004225423093885183, -0.011950286105275154, 0.008899149484932423, -0.005945033393800259, 0.009414363652467728, -0.035275425761938095, 0.017637712880969048, 0.027058767154812813, -0.0026111977640539408, -0.017155954614281654, -0.01778491772711277, -0.00020104633586015552, 0.08468243479728699, -0.00815643835812807, -0.0140111418440938, 0.02716582454741001, -0.0031966681126505136, 0.014573194086551666, -0.028905509039759636, 0.014211874455213547, 0.00042509331251494586, 0.001477058045566082, 0.006466938182711601, 0.0010563557734712958, 0.010518393479287624, -0.017356686294078827, 0.00028855324489995837, -0.015991704538464546, -0.017718005925416946, -0.010438100434839725, 0.025038057938218117, -0.02232147566974163, -0.014359078370034695, -0.008076145313680172, 0.02855757065117359, 0.012405280955135822, -0.0011341397184878588, -0.009548185393214226, 0.033161040395498276, 0.04135093465447426, 0.01897593028843403, -0.02944079600274563, 0.006998879835009575, -0.006647597532719374, -0.002939061261713505, 0.005904886871576309, -0.043037090450525284, -0.014185110107064247, 0.014707015827298164, -0.009300614707171917, -0.0037001727614551783, -0.011314633302390575, 0.03283986821770668, 0.010217294096946716, 0.009320688433945179, -0.013208211399614811, 0.0034040920436382294, -0.00914002861827612, -0.005620515439659357, 0.022281330078840256, -0.0016961912624537945, -0.03565012663602829, 0.02479717880487442, -0.00574764609336853, 0.00992288626730442, -0.0026530171744525433, 0.021197373047471046, 0.002666399348527193, -0.012853583320975304, 0.007721517700701952, 0.006393336225301027, -0.03626570850610733, 0.007935632951557636, -0.010458173230290413, -0.004533213563263416, 0.009380907751619816, -0.021451633423566818, -0.015335978008806705, -0.030190197750926018, -0.02301734872162342, -0.01967180334031582, -0.009501347318291664, -0.010384571738541126, -0.014144963584840298, -0.0140111418440938, -0.023833662271499634, -0.0020826016552746296, 0.015228920616209507, -0.007708135526627302, -0.01172948069870472, 0.02482394315302372, 0.01572406105697155, 0.017008749768137932, -0.009421054273843765, -0.028851978480815887, -0.020662086084485054, -0.012278149835765362, 0.006396682001650333, 0.012378516606986523, 0.01079941913485527, -0.019029458984732628, 0.00010632559860823676, 0.017356686294078827, -0.016941839829087257, 0.02214750647544861, -0.01923019252717495, 0.011736171320080757, 0.008270187303423882, 0.02663053758442402, 0.0279152262955904, -0.021157225593924522, -0.007734899874776602, 0.0313945934176445, 0.008470919914543629, 0.023726604878902435, -0.005085228476673365, 0.0313410647213459, -0.010424718260765076, -0.01171609852463007, 0.0050250086933374405, -0.010899784974753857, 0.020274002104997635, 0.0027968755457550287, -0.011883375234901905, -0.01884210854768753, -0.01984577253460884, -0.005700808484107256, 0.001559023978188634, -0.03760392591357231, 0.016071997582912445, -0.005583714693784714, -0.027647582814097404, 0.00462019769474864, -0.010357807390391827, 0.0381927415728569, 0.013094462454319, -0.007012262009084225, 0.02242853306233883, -0.00010376416321378201, 0.0004750673833768815, -0.016232583671808243, 0.007942323572933674, -0.0013716734247282147, 0.009300614707171917, -0.008738563396036625, -0.0036633717827498913, -0.03348221257328987, -0.012692997232079506, 0.009300614707171917, 0.002622907282784581, -0.0035529686138033867, -0.019444307312369347, -0.011374852620065212, 0.01261939574033022, 0.016179054975509644, -0.02317793481051922, -0.027460232377052307, -0.022361623123288155, 0.00920024886727333, 0.016540374606847763, 0.00028458042652346194, 0.020635321736335754, -0.01583111844956875, 0.018266675993800163, -0.02034091390669346, 0.00974222645163536, -0.004851039964705706, -0.027098914608359337, -0.029628146439790726, -0.00421538669615984, 0.01747712679207325, 0.02299058437347412, 0.0261353962123394, 0.018574465066194534, 0.012311605736613274, 0.00011876684584422037, -0.01595155894756317, -0.010551848448812962, -0.008424081839621067, 0.0018099398585036397, -0.018306821584701538, 0.011361470445990562, 0.0005210686358623207, -0.009983105584979057, -0.0004909587441943586, 0.000363828003173694, -0.00252588652074337, -0.009193557314574718, 0.00023690640227869153, -0.022896910086274147, -0.01961827464401722, -0.003937706351280212, 0.0018835418159142137, 0.00014887674478814006, -0.025118350982666016, 0.00031176296761259437, 0.010980078019201756, 0.019270338118076324, 0.01925695687532425, 0.0045867422595620155, 0.004994898568838835, -0.0014896038919687271, 0.02546628750860691, -0.017972268164157867, 0.0279419906437397, 0.008912531659007072, 0.005118683911859989, 0.012110872194170952, -0.03626570850610733, -0.007627842482179403, -0.005630552303045988, 0.001751392730511725, 0.04132417216897011, 0.0031665582209825516, -0.0015339324017986655, -0.006032017525285482, 0.005657316651195288, 0.015335978008806705, 0.0018166309455409646, -0.004921296611428261, 0.005496730096638203, -0.02031414955854416, -0.022174270823597908, -0.03075224906206131, -0.02028738521039486, -0.04528529569506645, -0.0017631021328270435, -0.008745254017412663, -0.040735356509685516, 0.023418813943862915, -0.0021846408490091562, -0.01245211809873581, -0.007647915743291378, 0.01078603696078062, 0.026255836710333824, 0.01737006939947605, 0.022000303491950035, -0.013181447051465511, -0.027567289769649506, -0.020782524719834328, -0.003400746500119567, -0.011448455043137074, -0.01149529218673706, 0.02944079600274563, 0.017276393249630928, -0.03147488832473755, -0.007775046397000551, 0.005416437052190304, -0.014733780175447464, -0.01502818800508976, -0.022910291329026222, -0.007560931611806154, -0.009882739745080471, 0.034472495317459106, -0.02106355130672455, -0.0025208680890500546, 0.013797027058899403, 0.013783644884824753, -0.000516050320584327, 0.0005390509031713009, 0.0014636758714914322, -0.009528111666440964, -0.015523328445851803, 0.007922250777482986, -0.04274268075823784, 0.02201368473470211, 0.008343788795173168, -0.028396984562277794, -0.01678125374019146, 0.004168549086898565, -0.011515365913510323, -0.018735051155090332, 0.005456583574414253, 0.018213147297501564, 0.02992255426943302, -0.002457302762195468, 0.036078356206417084, 0.013516001403331757, -0.01714257150888443, -0.011428381316363811, -0.0005515966913662851, 0.03428514301776886, -0.012987405061721802, 0.012585939839482307, -0.009347452782094479, 0.014171727932989597, 0.00620598578825593, -0.006185912527143955, -0.01675448939204216, -0.00978237297385931, -0.012458809651434422, 0.013475854881107807, 0.022334858775138855, -0.0028403676114976406, -0.003847376676276326, 0.0036633717827498913, -0.014345696195960045, -0.006406718399375677, -0.011441763490438461, -0.02245529741048813, 0.005312725435942411, -0.011441763490438461, -0.025198644027113914, -0.026282601058483124, 0.005242468789219856, 0.0349007248878479, 0.011194193735718727, -0.012010506354272366, -0.000753583968617022, 0.027125678956508636, -0.003726937109604478, 0.014265403151512146, -0.012365134432911873, 0.003245178610086441, -0.023592783138155937, 0.0022749705240130424, 0.006436828523874283, -0.026014957576990128, 0.013823791407048702, -0.0015080043813213706, -0.024569682776927948, -0.01987253688275814, 0.01340225338935852, 0.010819491930305958, 0.018855491653084755, -0.018641376867890358, -0.008812164887785912, -0.003633261891081929, 0.01923019252717495, -0.009374217130243778, -0.023913955315947533, 0.02106355130672455, -0.02062193863093853, -0.0031213934998959303, 0.0060152895748615265, -0.0038339945022016764, -0.0019705260638147593, -0.0011232667602598667, -0.001321490271948278, 0.004299025051295757, -0.03300045430660248, -0.009815828874707222, -0.011341397650539875, 0.016874928027391434, -0.013308577239513397, -0.0031732493080198765, -0.011093826964497566, -0.007748282048851252, -0.011582276783883572, 0.01397099532186985, -0.00583797600120306, -0.004610160831362009, 0.030886070802807808, 0.015590239316225052, 0.0051554846577346325, 0.041591815650463104, -0.004767401609569788, -0.007119319401681423, -0.023833662271499634, 0.00822334922850132, -0.024757033213973045, -0.0006072163814678788, -0.003720246022567153, 0.025171879678964615, 0.011207575909793377, -0.012017196975648403, -0.02582760713994503, 0.01747712679207325, -0.029654910787940025, -0.007621151395142078, -0.00978237297385931, 0.00021934228425379843, 0.03704187273979187, -0.0028169488068670034, 0.005212359130382538, 0.003713554935529828, 0.010518393479287624, 0.015148627571761608, 0.0041551669128239155, -0.011983742006123066, 0.019042842090129852, 0.005319416522979736, -0.022950438782572746, -0.00289389630779624, -0.01817299984395504, -0.025667021051049232, 0.020461352542042732, 0.006878440268337727, 0.0009752263431437314, 0.023418813943862915, -0.008598050102591515, 0.0003496094432193786, -0.01324835792183876, 0.004713872913271189, 0.009380907751619816, 0.014947894960641861, 0.012311605736613274, -0.024636592715978622, -0.02944079600274563, -0.027152443304657936, 0.006259514484554529, -0.0030929562635719776, -0.004141784738749266, 0.00032974526402540505, -0.017436979338526726, -0.004486375954002142, -0.014546429738402367, -0.0003926833451259881, 0.00989612191915512, 0.009066427126526833, 0.0030762285459786654, -0.0065305037423968315, -0.0015799335669726133, 0.012545793317258358, 0.01257924921810627, -0.004539904650300741, -0.011247722432017326, -0.012920494191348553, -0.0034760211128741503, 0.03069872036576271, -0.014573194086551666, -0.022803233936429024, -0.007607769221067429, -0.010739198885858059, -0.0032050320878624916, -0.0029758622404187918, -0.0033003799617290497, -0.007039026357233524, 0.008062763139605522, -0.035328954458236694, 0.013255048543214798, 0.010123618878424168, -0.010324351489543915, -0.008959369733929634, -0.004258878529071808, -1.9994859030703083e-05, -0.0013716734247282147, 0.004171894397586584, -0.0015272411983460188, -0.025225408375263214, -0.011341397650539875, 0.027834933251142502, -0.012846892699599266, -0.011515365913510323, -0.0059416876174509525, 0.004335826262831688, -0.01241197157651186, -0.0004771583480760455, 0.18713639676570892, -0.01402452401816845, 0.004074873868376017, 0.025091586634516716, -0.013810409232974052, 0.02925344556570053, 0.010665597394108772, -0.011241030879318714, -0.0024890853092074394, 0.031715765595436096, 0.0013189810561016202, 0.009869357571005821, -0.008270187303423882, 0.001988926436752081, 0.014104817062616348, 0.0010362825123593211, -0.013562839478254318, -0.026336129754781723, -0.025011293590068817, 0.0036366074346005917, 0.007587695959955454, -0.04247503727674484, -0.010277514345943928, -0.012820128351449966, 0.02577407844364643, -0.009882739745080471, -0.003266924526542425, 0.005981834139674902, 0.011722789146006107, -0.0008050217293202877, -0.01260601356625557, -0.02015356346964836, -0.012686306610703468, -0.021103696897625923, 0.005108647048473358, -0.008945987559854984, -0.0010479919146746397, -0.0038875231985002756, 0.01324835792183876, 0.03573042154312134, -0.014747162349522114, 0.012485573999583721, -0.0037369737401604652, -0.009902812540531158, 0.00464696204289794, 0.024984529241919518, 0.0030176814179867506, 0.00408491026610136, 0.003176594851538539, -0.005834630224853754, -0.027259500697255135, -0.019939446821808815, 0.022027067840099335, 0.023927336558699608, -0.0015732424799352884, 0.011816464364528656, 0.006212676875293255, -0.01683478243649006, -0.014292167499661446, 0.02337866835296154, -0.004472993779927492, 0.0313410647213459, -0.020166944712400436, -0.013475854881107807, -0.012231312692165375, 0.013208211399614811, -0.003102992894127965, -0.010344425216317177, -0.011361470445990562, 0.003971161786466837, 0.008564595133066177, -0.01781168207526207, -0.001863468554802239, 0.0016092071309685707, 0.004737291485071182, -0.02193339169025421, 0.029467560350894928, 0.011127282865345478, 0.02209397777915001, 0.00990950409322977, -0.03337515518069267, 0.0044462294317781925, -0.021558690816164017, -0.013241666369140148, 0.0006745454738847911, -0.042876504361629486, 0.0262156892567873, -2.828031028911937e-05, 0.004339171573519707, 0.005275924224406481, 0.013863937929272652, -0.007413727696985006, 0.014854219742119312, -0.005710845347493887, 0.022548973560333252, 0.012907112017273903, -0.012933876365423203, 0.01848079077899456, -0.009635169059038162, -0.010893094353377819, -0.01253910269588232, -0.03722922503948212, 0.028236398473381996, 0.013582912273705006, -0.0131479911506176, 0.005975143052637577, -0.016580520197749138, 0.01956474594771862, 0.017999032512307167, -0.004392700269818306, -0.0024472661316394806, -0.017356686294078827, 0.029039330780506134, 0.005473311524838209, 0.004365935921669006, -0.0033271443098783493, 0.029842261224985123, -0.004804202355444431, -0.011843228712677956, 0.009916194714605808, -0.0014352387515828013, -0.017463743686676025, 0.0040079629980027676, 0.004677071701735258, -0.007373581174761057, -0.011120591312646866, -0.01611214503645897, 0.0131479911506176, -0.0052391234785318375, -0.02546628750860691, -0.002303407760336995, -0.017570801079273224, 0.020608557388186455, -0.03425838053226471, -0.017918739467859268, 0.00887907575815916, -0.0014486209256574512, -0.02278985269367695, 0.012144328095018864, -0.010404644533991814, -0.006985497660934925, 0.0026546898297965527, -0.01064552366733551, 0.0002245696960017085, 0.006697780918329954, -0.01308108028024435, 0.007627842482179403, 0.013904084451496601, -0.007567622698843479, -0.022361623123288155, 0.0021143844351172447, -0.008009234443306923, -0.0017095734365284443, -0.01917666383087635, 0.03982536494731903, -0.011435072869062424, -0.037363044917583466, -0.025225408375263214, -0.006878440268337727, 0.018346969038248062, -0.020193709060549736, 0.01828005723655224, 0.018306821584701538, 0.004583396483212709, -0.011147355660796165, -0.01964503899216652, -0.17204129695892334, 0.011515365913510323, -0.0031147024128586054, -0.014921130612492561, 0.008470919914543629, -0.03433867171406746, 0.010137001052498817, 0.008778709918260574, 0.006470283959060907, 0.007072481792420149, 0.016513610258698463, 0.0061758761294186115, -0.04528529569506645, -0.017544036731123924, 0.016888311132788658, 0.011956977657973766, 0.001239524339325726, -0.02237500436604023, 0.01569729670882225, 0.02376675046980381, 0.017303157597780228, -0.0023552635684609413, 0.00020031450549140573, 0.0009961359901353717, -0.0025409413501620293, 0.009996487759053707, 0.008189894258975983, 0.013529383577406406, 0.0055970968678593636, -0.0016593902837485075, 0.005543567705899477, -0.006985497660934925, 0.013797027058899403, 0.03192988038063049, 0.003576387418434024, 0.006557268090546131, -0.010906476527452469, -0.006058781873434782, 0.007788428571075201, 0.022816617041826248, 0.0022448606323450804, -0.0007640388212166727, 0.0029591345228254795, -0.011448455043137074, -0.00541978282853961, 0.032438404858112335, -0.010290896520018578, 0.01310784462839365, 0.010625450871884823, 0.003368963720276952, 0.01789197511970997, -0.027125678956508636, 0.0131479911506176, -0.004626888781785965, 0.019056223332881927, 0.015536710619926453, 0.024007629603147507, 0.018882256001234055, 0.011053680442273617, -0.011087135411798954, -0.003392382524907589, -0.04054800420999527, -0.003061173716560006, -0.020247237756848335, -0.005376290529966354, -0.02858433499932289, -0.014211874455213547, 0.005546913482248783, -0.0263227466493845, 0.0055201491340994835, -0.010183839127421379, 0.008296951651573181, 0.014211874455213547, -0.02209397777915001, 0.02348572574555874, 0.011140665039420128, -0.003400746500119567, 0.01308108028024435, 0.017209483310580254, -0.025145115330815315, -0.025399377569556236, 0.05529516562819481, -0.002159549156203866, -0.015523328445851803, 0.023057496175169945, 0.021853098645806313, -0.009648551233112812, -0.0018450680654495955, -0.009454510174691677, 0.011501983739435673, 0.03142135962843895, -0.025613492354750633, -0.003016008762642741, -0.03289339691400528, 0.010471555404365063, 0.014907748438417912, -0.006095582619309425, 0.019457688555121422, 0.008644888177514076, -0.006045399699360132, 0.0020541646517813206, -0.003258560784161091, -0.013449090532958508, 0.014064670540392399, 0.010638833045959473, 0.020528264343738556, 0.014680251479148865, 0.03974507376551628, 0.03864773362874985, 0.0006486175116151571, -0.0034559478517621756, 0.03206370398402214, 0.007079172879457474, 0.028959037736058235, 0.008477610535919666, 0.007808501832187176, 0.016406552866101265, -0.0033756548073142767, 0.01861461251974106, -0.004780783783644438, -0.006212676875293255, -0.007982470095157623, -0.027834933251142502, 0.0005591242224909365, 0.014760544523596764, -0.0019287066534161568, -0.11326676607131958, -0.008537830784916878, 0.03152841702103615, -0.0006653452292084694, -0.052083443850278854, 0.004493067041039467, 0.0006051254458725452, 0.0029424068052321672, -0.017155954614281654, 0.028102576732635498, 0.008584667928516865, -0.010464864782989025, -0.010029943659901619, 0.006794801913201809, 0.02487747184932232, 7.067463593557477e-05, -0.014399224892258644, -0.006058781873434782, -0.024810561910271645, 0.0015414598165079951, -0.016098761931061745, -0.010819491930305958, 0.008711799047887325, 0.0131479911506176, 0.006155802868306637, -0.011642496101558208, -0.007306670304387808, 0.04424148425459862, 0.002174604218453169, 0.0007644570432603359, -0.006865058094263077, 0.021638983860611916, -0.011207575909793377, -0.001953798346221447, -0.04044094681739807, -0.006356535479426384, -0.024810561910271645, 0.005048427265137434, 0.026362894102931023, -0.02026062086224556, 0.01498804148286581, -0.010384571738541126, 0.007279905956238508, -0.005188940092921257, -0.015496564097702503, -0.007955705747008324, -0.004506449215114117, 0.04480353742837906, 0.025118350982666016, -0.01892240159213543, -0.016098761931061745, -0.015550092794001102, 0.007239759434014559, -0.005724227521568537, 0.043679434806108475, -0.042876504361629486, 0.008042690344154835, 0.03755039721727371, -0.0040079629980027676, 0.009233703836798668, 0.013214902020990849, -0.015135245397686958, -0.008966060355305672, 0.025533199310302734, 0.009715462103486061, 0.017209483310580254, -0.019082987681031227, 0.003310416592285037, -0.0023803552612662315, -0.028236398473381996, -0.009494656696915627, 0.024636592715978622, -0.034713372588157654, 0.0010053362930193543, -0.01323497574776411, 0.004770746920257807, -0.028396984562277794, 0.010545157827436924, 0.01953798159956932, -0.040226832032203674, -0.015576857142150402, -0.012632777914404869, -0.0016702633583918214, -0.03278633952140808, 0.026148779317736626, 0.0012453790986910462, 0.009474582970142365, 0.0006435991963371634, -0.002303407760336995, 0.011441763490438461, 0.008102909661829472, 0.031180478632450104, 0.03725598752498627, -0.03142135962843895, -0.012639468535780907, 0.02410130575299263, 0.01631287671625614, 0.004375972785055637, 0.0021762768737971783, 0.01876181550323963, -0.022080596536397934, -0.03412455692887306, -0.03647982329130173, 0.008002543821930885, 0.02123751863837242, -0.010190529748797417, 0.011100517585873604, -0.014332314021885395, 0.024141451343894005, -0.008852311410009861, 0.010351115837693214, -0.01714257150888443, -0.01595155894756317, -0.007199612446129322, -0.0036265708040446043, -0.0024589754175394773, -0.007567622698843479, -0.005918269045650959, 0.030886070802807808, 0.013014169409871101, 0.006142420694231987, 0.010377880185842514, 0.0011742862407118082, 0.016874928027391434, 0.022548973560333252, 0.002397082978859544, -0.029146388173103333, -0.0010530102299526334, -0.022950438782572746, 0.024649975821375847, -0.020836053416132927, 0.00674127321690321, 0.02404777705669403, -0.008591359481215477, -0.0018902329029515386, 0.01737006939947605, -0.021277666091918945, -0.01786521077156067, -0.01152205653488636, 0.018935784697532654, -0.015536710619926453, -0.027286265045404434, -0.027433468028903008, -0.02454291842877865, -0.011093826964497566, 0.001997290411964059, -0.008805474266409874, -0.008136365562677383, 0.007701444439589977, -0.002738328417763114, 0.003415801329538226, 0.008751945570111275, 0.006115656346082687, 0.02260250225663185, 0.011916831135749817, -0.031876351684331894, -0.0033087439369410276, -0.02925344556570053, 0.029761968180537224, 0.007420418784022331, -0.010050017386674881, -0.03201017528772354, 0.019042842090129852, -0.0057275728322565556, 0.01683478243649006, -0.015536710619926453, 0.011783009395003319, 0.013877320103347301, -0.015737444162368774, -0.010792727582156658, 0.021465016528964043, 0.014292167499661446, 0.008256805129349232, 0.00457001430913806, 0.012130945920944214, 0.0035964606795459986, 0.006891822442412376, -0.010551848448812962, 0.002308425959199667, -0.007821884006261826, -0.03556983545422554, 0.01814623549580574, 0.012800054624676704, -0.006172530353069305, -0.04405413568019867, 0.004636925179511309, 0.015683915466070175, 0.014064670540392399, -0.017624329775571823, 0.00924039538949728, 0.01061875931918621, 0.03570365533232689, -0.034633081406354904, -0.006824911572039127, -0.01992606557905674, -0.01078603696078062, -0.006865058094263077, 0.015898030251264572, -0.015108481049537659, 0.00496144313365221, 0.010491629131138325, 0.03717569634318352, 0.009682007133960724, 0.0032317964360117912, 0.004667035304009914, 0.008450846187770367, 0.01081280130892992, 0.024302037432789803, -0.015416271053254604, -0.016219202429056168, 0.006490357220172882, 0.017289776355028152, -0.0050250086933374405, -0.004017999395728111, -0.008270187303423882, 0.0023803552612662315, 0.010163765400648117, -0.007072481792420149, 0.0005415600608102977, -0.026081867516040802, -0.000506431853864342, 0.013034243136644363, -0.002597815589979291, -0.017008749768137932, -0.012264767661690712, -0.027112295851111412, 0.013101154007017612, 0.006410064175724983, 0.006771382875740528, -0.040039483457803726, 0.005105301737785339, -0.005931651219725609, 0.010190529748797417, -0.02262926660478115, -0.019404159858822823, -0.019992975518107414, -0.011167429387569427, 0.01842726208269596, -0.0014109836192801595, 0.02858433499932289, -0.010518393479287624, 0.032491933554410934, 0.02020709216594696, -0.017704622820019722, 0.03415132313966751, -0.014118199236690998, 0.01964503899216652, -0.018681522458791733, -0.0008631505770608783, -0.023753369227051735, -0.019912682473659515, 0.02415483444929123, 0.00221976893953979, 0.0038440311327576637, 0.004476339090615511, -0.023927336558699608, -0.0008539503323845565, -0.01497465930879116, 0.013395561836659908, -0.009541493840515614, -0.004774092696607113, 0.0279419906437397, -0.002530904719606042, 0.018293440341949463, 0.01496127713471651, 0.0021628946997225285, -0.03773774579167366, 0.019216809421777725, 0.004921296611428261, -0.03787156939506531, -0.0140111418440938, 0.0061658392660319805, 0.01330188661813736, -0.04935348033905029, 0.006212676875293255, -0.003904250916093588, -0.011234340257942677, -0.004135093651711941, 0.0003182449727319181, -0.006430137436836958, 0.02858433499932289, 0.007360199000686407, 0.026536861434578896, -0.01984577253460884, -0.007447183132171631, 0.01639316976070404, -0.0033672910649329424, -0.013797027058899403, -0.0037537014577537775, -0.027781404554843903] diff --git a/tests/test_cases/test-vector-search-docs.json b/tests/test_cases/test-vector-search-docs.json new file mode 100644 index 000000000..bef40f482 --- /dev/null +++ b/tests/test_cases/test-vector-search-docs.json @@ -0,0 +1,100 @@ +{"id": "test:10000000", "text": "\"Beginners BBQ Class Taking Place in Missoula!\\nDo you want to get better at making delicious BBQ? You will have the opportunity, put this on your calendar now. Thursday, September 22nd join World Class BBQ Champion, Tony Balay from Lonestar Smoke Rangers. He will be teaching a beginner level class for everyone who wants to get better with their culinary skills.\\nHe will teach you everything you need to know to compete in a KCBS BBQ competition, including techniques, recipes, timelines, meat selection and trimming, plus smoker and fire information.\\nThe cost to be in the class is $35 per person, and for spectators it is free. Included in the cost will be either a t-shirt or apron and you will be tasting samples of each meat that is prepared.\"", "vector_field": [-0.0002675256982911378, 0.0025827737990766764, 0.00492906104773283, -0.0586487278342247, -0.008114876225590706, 0.002511827740818262, -0.02136489376425743, -0.0012212854344397783, -0.03913518786430359, -0.006195954512804747, -0.01574326492846012, 0.02181084081530571, -0.025486521422863007, -0.007851362228393555, 0.015202723443508148, 0.01620272547006607, 0.022189220413565636, 0.00892568752169609, -0.007432442624121904, -0.015391913242638111, -0.008729741908609867, 0.008756768889725208, 0.006726360414177179, -0.018135160207748413, -0.02044597454369068, 0.01279055792838335, 0.0173378624022007, -0.00780406454578042, -0.010702717117965221, 0.014621641486883163, 0.02322976104915142, 0.02864868752658367, -0.001638515735976398, -0.010243257507681847, -0.0194729994982481, -0.028973013162612915, 0.005665548145771027, 0.00240878714248538, 0.03800005093216896, 0.0025641927495598793, 0.02391895093023777, 0.018459483981132507, 0.012581097893416882, -0.011067582294344902, -0.015851372852921486, 0.0024223006330430508, -0.00797298364341259, -0.03686491400003433, -0.008378390222787857, 0.002555746817961335, 0.03918924182653427, -0.008486498147249222, -0.009087850339710712, 0.0001275339600397274, 0.009668932296335697, 0.004239870700985193, 0.011128393933176994, 0.035756804049015045, 0.012587855570018291, -0.007817578501999378, -0.016013534739613533, 0.007743253838270903, -0.023756789043545723, -0.0006837002583779395, -0.0009206093964166939, -0.008425687439739704, -0.022243274375796318, 0.013831100426614285, -0.021040569990873337, 0.01337839663028717, -0.003184125991538167, 0.002282097702845931, 0.023608140647411346, -0.002869936404749751, 0.03443247824907303, 0.007608118467032909, -0.028270309790968895, 0.02183786779642105, 0.029702743515372276, -0.0035000047646462917, 0.01560812909156084, -0.017932457849383354, -0.01644596830010414, 0.03724329546093941, 0.003625005017966032, 0.01739191636443138, -0.00047550740418955684, 0.0032195989042520523, -0.023567600175738335, -0.009040553122758865, 0.02393246442079544, 0.03448653221130371, 0.009783796966075897, 0.00655744131654501, -0.0479460135102272, -0.005253385752439499, -0.034810859709978104, 0.028189226984977722, -0.015675697475671768, -0.04310816898941994, 0.0031604773830622435, -0.004233113955706358, 0.009236498735845089, -0.020283810794353485, -0.015472994185984135, 0.0005147811025381088, 0.004554060287773609, -0.007527037523686886, 0.020986516028642654, 0.010513528250157833, -0.032297343015670776, 0.00819595716893673, 0.002135138027369976, -0.04645952209830284, -0.02018921636044979, -0.004358114209026098, -0.0017601375002413988, -0.0007981429807841778, 0.0075878482311964035, -0.03140544891357422, 0.031054096296429634, 0.012047314085066319, 0.026810847222805023, -0.005270277615636587, 0.018013538792729378, -0.015337859280407429, -0.04816222935914993, -0.004793925676494837, 0.010263527743518353, 0.014756777323782444, -0.005016898736357689, 0.028783822432160378, 0.018716242164373398, -0.0014104748843237758, 0.0010836164001375437, 0.03181085363030434, -0.007527037523686886, -0.0180946197360754, -0.046081144362688065, -0.01207434106618166, 0.002714530797675252, -0.007141901645809412, -0.006945955567061901, -0.01279055792838335, -0.023148680105805397, 0.019581107422709465, 0.04143248870968819, 0.033405452966690063, 0.0041891951113939285, 0.003543923841789365, -0.001195947639644146, -0.002000002656131983, -0.024027060717344284, -0.004185816738754511, 0.025445980951189995, 0.029648689553141594, 0.003976356703788042, 0.03729734942317009, 0.027378415688872337, -0.029837878420948982, 0.015202723443508148, -0.007290550507605076, 0.004820952657610178, 0.0005920616094954312, 0.008398660458624363, 0.03416220843791962, 0.015270290896296501, -0.00866893120110035, 0.010317581705749035, -0.024013547226786613, 0.01408110000193119, 0.026108143851161003, -0.018594620749354362, 0.033189233392477036, -0.0007280415156856179, 0.029459500685334206, -0.020581109449267387, -0.0023935844656080008, -0.011858124285936356, 0.0039527080953121185, -0.02227030135691166, -0.0008644437766633928, 0.01335136964917183, 0.021716246381402016, -0.023797329515218735, -0.010351365432143211, 0.0057466295547783375, 0.012378395535051823, 0.008702714927494526, 0.009574337862432003, -0.0030523689929395914, 0.03181085363030434, -0.004135140683501959, 0.017027050256729126, -0.630487322807312, 0.004540546797215939, -0.0009653729503042996, -0.01708110421895981, 0.012493260204792023, 0.001036318950355053, 0.023405438289046288, -0.004145275801420212, -0.0009831094648689032, 0.027162199839949608, 0.016364887356758118, -0.01594596728682518, 0.00743919936940074, -0.0025608143769204617, -0.001104731229133904, -0.016256779432296753, 0.03497302159667015, -0.02582436054944992, -0.026202738285064697, 0.004540546797215939, -0.009189201518893242, 0.032486531883478165, 0.015189209952950478, 0.009364877827465534, 0.01081082597374916, -0.004064194858074188, -0.01207434106618166, -0.02162165194749832, 0.00768919987604022, -0.013621640391647816, -0.01758110523223877, 0.028567606583237648, 0.02581084705889225, -0.005939197260886431, 0.04686493054032326, 0.010432446375489235, -0.048432499170303345, 0.012986504472792149, 0.014337857253849506, 0.025175711140036583, -0.019054079428315163, -0.04545952007174492, 0.04035140573978424, -0.005902035161852837, 0.0038716269191354513, 0.007662172894924879, -0.002555746817961335, 0.0068040634505450726, -0.006689198315143585, -0.019797325134277344, 0.008351363241672516, 0.017662186175584793, 0.0011097987880930305, 0.004489871207624674, 0.009432445280253887, -0.0037973024882376194, 0.021716246381402016, 0.004226357210427523, -0.007310820743441582, 0.006351360119879246, -0.008229740895330906, 0.0021638544276356697, -0.002890206640586257, 0.011304069310426712, -0.0004030833370052278, -0.021932462230324745, 0.005364872049540281, 0.00973649974912405, 0.01993246003985405, -0.011608123779296875, 0.020729757845401764, 0.038567621260881424, -0.024229763075709343, 0.005807440262287855, 0.006537171080708504, 0.027567604556679726, -0.008804066106677055, -0.005645277909934521, -0.018000025302171707, 0.007885145954787731, 0.0020912191830575466, -0.008628389798104763, 0.0031908827368170023, -0.021202731877565384, 0.013054071925580502, -0.01019596029073, 0.007574334740638733, -0.012513530440628529, 0.028945986181497574, 0.0125270439311862, 0.0029408824630081654, -0.0033260181080549955, -0.013763532042503357, -0.0012128395028412342, -0.01501353457570076, 0.018932458013296127, -0.011797313578426838, -0.0002907520974986255, 0.0019121647346764803, -0.04627033323049545, -0.023270301520824432, 0.0016021981136873364, 0.04270276054739952, -0.004793925676494837, 0.022648679092526436, 0.00031418961589224637, -0.020216243341565132, 0.012020287103950977, 0.04037843272089958, -0.010837852954864502, -0.005418926477432251, -0.0031891935504972935, -0.01383785717189312, 0.014743263833224773, -0.005246629007160664, -0.02301354520022869, 0.022635165601968765, 0.02159462496638298, 0.025716250762343407, -0.024027060717344284, 0.014621641486883163, -0.011763529852032661, 0.016297319903969765, -0.03886491805315018, -0.01831083558499813, 0.03791896998882294, 0.003645275253802538, -0.0016410495154559612, 0.008858120068907738, -0.009540553204715252, 0.006564198061823845, -0.0005033790948800743, 0.017972998321056366, -0.01827029511332512, 0.013222991488873959, 0.015297317877411842, 0.033864911645650864, -0.014445966109633446, 0.01591894030570984, -0.0007195955840870738, -0.01875678263604641, -0.00575676467269659, 0.00042652085539884865, 0.00031397846760228276, -0.03305409848690033, -0.006337846629321575, -0.02745949663221836, -0.0015717926435172558, -0.027256794273853302, 0.0008179909782484174, -0.004780412185937166, 0.009925689548254013, -0.0037973024882376194, -0.0016089548589661717, 0.014459479600191116, -0.007162171881645918, -0.0037973024882376194, -0.018918944522738457, -0.017743267118930817, -0.0276486873626709, -0.015405426733195782, 0.005888521671295166, -0.020945975556969643, 0.02183786779642105, 0.011635150760412216, -0.025270305573940277, 0.015108129009604454, 0.00399324856698513, -0.0007736497209407389, -0.0008800687501206994, 0.000653717084787786, -0.004520276561379433, 0.006800685077905655, -0.007067577447742224, -0.014445966109633446, -0.0018412187928333879, -0.0025793954264372587, 0.009554067626595497, -0.01524326391518116, -0.005023655481636524, 0.006229738239198923, 0.00844595767557621, 0.0077635240741074085, 0.0060304137878119946, 0.015635157003998756, -0.004722979385405779, -0.008844606578350067, -0.010236500762403011, -0.019175702705979347, 0.020810838788747787, -0.0023243275936692953, 0.03494599461555481, 0.002621625317260623, 0.015121642500162125, 0.0239730067551136, 0.02700003795325756, -0.008743255399167538, 0.003395274979993701, -0.0038885187823325396, 0.020486515015363693, 0.010189203545451164, 0.030783826485276222, 0.03100004233419895, -0.010283797979354858, -0.01489191222935915, -0.01454056054353714, 0.022148679941892624, 0.00013217923697084188, 0.028783822432160378, 0.02183786779642105, 0.0177973210811615, -0.03000004030764103, 0.011432448402047157, 0.008317578583955765, 0.03845951333642006, 0.03140544891357422, -0.021202731877565384, 0.0059932516887784, -0.0028209497686475515, -0.007756767328828573, 0.0076959566213190556, -0.0012567584635689855, -0.00973649974912405, -0.01502704806625843, -0.046108171343803406, 0.02624327875673771, 0.009040553122758865, 0.01690542884171009, 0.0013994951732456684, -0.0007500010542571545, 0.010635149665176868, 0.010405419394373894, 0.022445976734161377, 0.011594610288739204, 0.0375676192343235, -0.028243282809853554, -0.009425688534975052, -0.001885137753561139, 0.03702707961201668, 0.0019662189297378063, 0.006608117371797562, -0.0029695986304432154, 0.018635161221027374, -0.006817576941102743, 0.01570272445678711, 0.007797307800501585, 0.0021875030361115932, -0.020581109449267387, -0.007094604428857565, 0.02908112108707428, -0.00371959968470037, 0.008533795364201069, -0.020945975556969643, 0.019540566951036453, -0.006229738239198923, -0.011608123779296875, -0.008756768889725208, 0.0011233123950660229, 0.019108135253190994, 0.01057433895766735, 0.00546284532174468, 0.018621647730469704, -0.00012468344357330352, -0.02208111062645912, 0.016013534739613533, 0.0020777054596692324, 0.0002415543858660385, -0.025500034913420677, -0.011121637187898159, -0.024054087698459625, -0.014837858267128468, -0.0332973413169384, 0.0065675764344632626, -0.020108135417103767, -0.00351689662784338, 0.01825678162276745, -0.01708110421895981, -0.0032162205316126347, 0.00021399944671429694, 0.009547309949994087, -0.02929733693599701, -0.035270318388938904, 0.014608127996325493, 0.00434122234582901, -0.022837869822978973, -0.025662196800112724, -0.0033631802070885897, -0.02016218937933445, -0.01782434806227684, 0.029648689553141594, 0.014148668386042118, 0.010506771504878998, -0.0033479775302112103, 0.007155415136367083, 0.01825678162276745, 0.026175711303949356, 0.02794598415493965, -0.013662180863320827, -0.0016393603291362524, 0.008000010624527931, 0.0041283839382231236, -0.005729737691581249, 0.010472987778484821, -0.014810831286013126, 0.024675710126757622, 0.023851383477449417, 0.0036959510762244463, -0.003486491274088621, -0.013567586429417133, -0.00017557034152559936, 0.01895948499441147, 0.003591221058741212, -0.029675716534256935, 0.01827029511332512, 0.03970275819301605, -0.013621640391647816, 0.006898657884448767, 0.0119865033775568, -0.0024510168004781008, -0.009445958770811558, -0.029405444860458374, -0.0026858143974095583, -0.039405457675457, -0.007243253290653229, 0.12021638453006744, -0.026554090902209282, -0.003699329448863864, 0.016459481790661812, -0.02533787302672863, 0.007466226350516081, -0.010378392413258553, -0.02741895616054535, 0.01644596830010414, -0.001255913870409131, -0.011202718131244183, 0.01856759376823902, -0.0034932480193674564, 0.0006135987932793796, 0.02509462833404541, 0.005290547851473093, -0.00025147839915007353, -0.02883787825703621, 0.029837878420948982, -0.016729753464460373, -0.015567588619887829, -0.006621630862355232, -0.0003686660493258387, 0.019297324120998383, 0.013290558941662312, 0.0068040634505450726, 0.008304065093398094, 0.0004826864751521498, 0.03281085565686226, -0.04313519597053528, 0.0002609801013022661, 0.01997300051152706, 0.021729759871959686, -0.015162182971835136, -0.0030439230613410473, 0.015864886343479156, 0.00019404587510507554, 0.014702722430229187, 0.02066219039261341, -0.008452714420855045, 0.008270281367003918, 0.009452715516090393, 0.003942572977393866, 0.005033790599554777, 0.00914866104722023, -0.02089191973209381, -0.025716250762343407, 0.024945979937911034, 0.0022854760754853487, -0.005364872049540281, 0.0024780440144240856, 0.016972996294498444, -0.0016604752745479345, -0.03305409848690033, -0.0022246651351451874, -0.015202723443508148, 0.008743255399167538, -0.00018591663683764637, 0.002366557251662016, -0.023256788030266762, -0.03313517943024635, -0.04389195144176483, 0.053594667464494705, 0.009493255987763405, -0.04197303205728531, -0.009993257001042366, -0.03356761485338211, 0.015648670494556427, -0.01877029612660408, -0.0033378424122929573, -0.009993257001042366, -0.015621643513441086, 0.008202713914215565, 0.01113515067845583, -0.006787171587347984, -0.01898651197552681, 0.0017043942352756858, 0.01559461560100317, 0.01639191433787346, 0.006611495744436979, 0.021013543009757996, -0.0002373314055148512, 0.024229763075709343, -0.04151356965303421, -0.010304068215191364, 0.014567587524652481, 0.03305409848690033, -0.006777036469429731, -0.00397973507642746, 0.0029121660627424717, -0.0022500031627714634, 0.0024966250639408827, 0.039054106920957565, -0.01921624317765236, -0.004989871755242348, -0.03548653423786163, 0.0016393603291362524, 0.00951352622359991, 0.0024070979561656713, -0.008128389716148376, -0.00779055105522275, -0.009006769396364689, -0.019094619899988174, -0.007364875171333551, 0.0002850510645657778, -0.0033564234618097544, 0.0023716248106211424, -0.003091220511123538, -0.020000027492642403, 0.019851379096508026, 0.01350001897662878, -0.03267572075128555, 0.02837841771543026, 0.014986506663262844, 0.005834467243403196, 0.002680746838450432, 0.003625005017966032, 0.027635173872113228, -0.011729746125638485, 0.003182436805218458, -0.015000020153820515, -0.04532438516616821, 0.018121646717190742, 0.03781086206436157, -0.0005823487881571054, -0.011439205147325993, -0.026418955996632576, -0.0253513865172863, -0.015310831367969513, -0.024324357509613037, -0.00422973558306694, 0.019121648743748665, 0.010351365432143211, -0.029135175049304962, 0.013513532467186451, -0.028000038117170334, 0.004827709402889013, 0.024000033736228943, 0.0026064224075526, -0.027837876230478287, -0.02740544266998768, -0.00018697237828746438, 0.008918930776417255, 0.014797317795455456, 0.014702722430229187, -0.022864896804094315, -0.014702722430229187, 0.006489873863756657, -0.001245778752490878, 0.028000038117170334, -0.007722983602434397, -0.0029662202578037977, -0.017743267118930817, 0.006182441022247076, 0.012702720239758492, -0.016486508771777153, -0.011864881031215191, -0.0213784072548151, 0.018000025302171707, 0.03324328735470772, 0.009885149076581001, -0.01314190961420536, 0.014851371757686138, 0.011466232128441334, -0.0016706103924661875, 0.014337857253849506, -0.018472997471690178, 0.016270292922854424, 0.0009755080682225525, -0.010391905903816223, 0.012141908518970013, 0.013959478586912155, -0.034135181456804276, 0.009229741990566254, 0.017432456836104393, 0.014851371757686138, 0.0004641053674276918, -0.041837893426418304, -0.016094617545604706, -0.01829732209444046, 0.020810838788747787, -0.003614869900047779, -0.012358125299215317, -0.002716219983994961, -0.009668932296335697, -0.02536490000784397, 0.0281081460416317, -0.010236500762403011, -0.006256765220314264, 0.011229745112359524, 0.019824352115392685, -0.020013540983200073, 0.04508114233613014, -0.024121655151247978, 0.012804071418941021, 0.012054070830345154, -0.0035473022144287825, -0.026121657341718674, -0.017229752615094185, 0.005668926518410444, 0.01217569224536419, -0.006290548946708441, -0.0020388541743159294, -0.018229754641652107, -0.0281081460416317, 0.019081106409430504, -0.028324363753199577, -0.015445967204868793, -0.012695963494479656, -0.004067573230713606, -0.01406758651137352, -0.01739191636443138, -0.012297314591705799, -0.0036317617632448673, 0.012412179261446, -0.0033091262448579073, 0.00580068351700902, 0.013662180863320827, 0.004327708855271339, -0.013817586936056614, 0.023770302534103394, -0.01900002546608448, 0.03429734334349632, 0.012628396041691303, 0.007641902193427086, 0.02721625380218029, -0.015635157003998756, -0.006229738239198923, 0.0342162624001503, 0.03643248230218887, -0.0011444272240623832, 0.02489192597568035, 0.0027111524250358343, -0.037648700177669525, 0.0063919005915522575, 0.020000027492642403, 0.0052668992429971695, -0.025418953970074654, -0.02232435531914234, 0.019554080441594124, 0.005472980439662933, -0.019135162234306335, -0.023824356496334076, -0.016108131036162376, 0.002354732947424054, 0.011621637269854546, -0.003386829048395157, -0.0010194270871579647, -0.003543923841789365, -0.030324365943670273, -0.035324372351169586, 0.02485138550400734, -0.012047314085066319, -0.0019645297434180975, 0.012418936006724834, -0.02956760860979557, -0.008635146543383598, -0.004239870700985193, -0.012533800676465034, 0.015216236934065819, 0.0133648831397295, 0.031351394951343536, -0.0038614918012171984, 0.04291897639632225, -0.0012238192139193416, 0.03208112344145775, -0.010013527236878872, -0.009581094607710838, 0.018675701692700386, 0.02905409410595894, -0.030108150094747543, -0.0053952778689563274, -0.022229760885238647, -0.01151352934539318, 0.009074336849153042, 0.00546284532174468, -0.01574326492846012, -0.024513546377420425, 0.0017567591276019812, 0.006547306198626757, 0.007385145407170057, 0.019337864592671394, -0.012750017456710339, 0.008979742415249348, -0.004631763324141502, -0.011587853543460369, -0.03548653423786163, -0.004827709402889013, -0.011290555819869041, -0.014418939128518105, 0.0028091254644095898, -0.0197027288377285, -0.0194729994982481, 0.009594608098268509, 0.005766899790614843, -0.0009146971860900521, -0.01045271661132574, 0.01207434106618166, -0.03686491400003433, 0.018594620749354362, -0.013750018551945686, 0.020689217373728752, -0.004472979344427586, 0.002179057104513049, -0.020000027492642403, -0.012033800594508648, 0.015648670494556427, -0.004202708601951599, -0.005668926518410444, -0.03983789309859276, -0.01312839612364769, -0.012412179261446, -0.0009543931810185313, -0.009013526141643524, 0.02089191973209381, 0.0056891972199082375, 0.019135162234306335, -0.0007107272977009416, -0.03454058617353439, 0.02509462833404541, -0.004655411932617426, -0.002447638427838683, 0.017702726647257805, 0.0033158829901367426, 0.023256788030266762, -0.0080946059897542, 0.004527033306658268, 0.0033902074210345745, -0.0017077726079151034, 0.010858123190701008, 0.009560824371874332, 0.027040578424930573, -0.0023682464379817247, -0.010479744523763657, 0.0023935844656080008, 0.005202709697186947, -0.010567582212388515, 0.03275680169463158, -0.022445976734161377, -0.008101362735033035, 0.013527045957744122, 0.027864903211593628, 0.009898662567138672, 0.03100004233419895, -0.0033547342754900455, -0.023121653124690056, 0.006675684824585915, -0.011195961385965347, -0.05605413019657135, -0.023621654137969017, -0.019864892587065697, 0.04543249309062958, -0.02391895093023777, -0.03405410051345825, -0.00810811948031187, -0.0037973024882376194, -0.04454060271382332, -0.004587844014167786, -0.013331099413335323, 0.014986506663262844, 0.003489869646728039, 0.004263519309461117, -0.008297308348119259, 0.01195271871984005, 0.0032533828634768724, -0.002983112121000886, -0.014513533562421799, -0.0001629014004720375, -0.01968921534717083, 0.005702710710465908, -0.004429060034453869, 0.010750014334917068, -0.026378413662314415, -0.022216247394680977, 0.041378434747457504, 0.007290550507605076, 0.017959484830498695, 0.01183785405009985, -0.027540577575564384, -0.02413516864180565, 0.007250010035932064, -0.006668928079307079, 0.007918929681181908, 0.002763517200946808, 0.036945994943380356, -0.015283804386854172, -0.006287170574069023, 0.005222979933023453, -0.002532097976654768, -0.003268585540354252, 0.008527038618922234, 0.010918933898210526, -0.003608113154768944, 0.00475000636652112, 0.01574326492846012, -0.035540588200092316, 0.008236497640609741, 0.008804066106677055, 0.019310837611556053, 0.013662180863320827, -0.007013523019850254, 0.01901353895664215, 0.010952717624604702, -0.023581113666296005, -0.02135138027369976, -0.03027031198143959, -0.019337864592671394, -0.022635165601968765, -0.005314196459949017, -0.020094621926546097, -0.015500021167099476, -0.019310837611556053, -0.011074339970946312, 0.0036418968811631203, -0.007067577447742224, -0.023797329515218735, 0.000316512247081846, -0.003986491821706295, 0.02202705666422844, 0.005543926730751991, -0.027283821254968643, 0.00575676467269659, -0.020824352279305458, -0.025689223781228065, -0.005067574325948954, -0.019851379096508026, 0.005533791147172451, -0.02437841147184372, 0.0043141948990523815, 0.012317584827542305, -0.018162187188863754, -0.010351365432143211, 0.009364877827465534, 0.009506769478321075, 0.010668933391571045, 0.010743257589638233, 0.20670299232006073, -0.005250007379800081, -0.02160813845694065, 0.03654059022665024, -0.003976356703788042, 0.020689217373728752, -0.0007732274243608117, -0.004209465347230434, 0.0005249162786640227, 0.013344612903892994, -0.006179062649607658, -0.00668244156986475, -0.030081123113632202, 0.00243750330992043, -0.016837861388921738, -0.015554075129330158, -0.020243270322680473, -0.024675710126757622, -0.009506769478321075, -0.01782434806227684, 0.014810831286013126, 0.008317578583955765, -0.0161892119795084, -0.005293926224112511, 0.01358785666525364, 0.00972298625856638, -0.014810831286013126, 0.02435138449072838, 0.026918955147266388, 0.023513546213507652, -0.027756795287132263, -0.003371626138687134, -0.0017221306916326284, -0.008662174455821514, -0.012668936513364315, 0.009979743510484695, 0.006574333179742098, -0.048216283321380615, 0.03305409848690033, 0.03416220843791962, 0.0059797377325594425, -0.0064628468826413155, 0.013662180863320827, -0.006689198315143585, -0.02647300995886326, -0.015472994185984135, -0.013918938115239143, 0.014378397725522518, -0.0007048151455819607, -0.0030608149245381355, -0.032027069479227066, -0.017459483817219734, 0.02724328078329563, 0.0213784072548151, 0.02722976729273796, -0.010716230608522892, 0.023175707086920738, 0.005185817833989859, 0.009554067626595497, -0.0183243490755558, -0.011871637776494026, 0.026189224794507027, -0.016310833394527435, 0.02017570286989212, -0.010567582212388515, -0.0010625014547258615, 0.009601364843547344, 0.026364900171756744, -0.003302369499579072, 0.0010853055864572525, 0.015094615519046783, -0.0022331110667437315, -0.030675718560814857, -0.02208111062645912, -0.03589194267988205, -0.024945979937911034, 0.025486521422863007, 0.017000023275613785, 0.03564869612455368, 0.027729768306016922, -0.01408110000193119, 0.02039192058146, 0.002984801307320595, -0.009405418299138546, -0.011702719144523144, -0.036027077585458755, 0.00795947015285492, -0.020229756832122803, -0.0004229313344694674, -0.03535139933228493, 0.011432448402047157, 0.009378391318023205, -0.0007204401772469282, -0.009594608098268509, 0.01994597353041172, -0.024337871000170708, -0.010027040727436543, 0.01927029713988304, -0.005540548358112574, 0.011297312565147877, -0.00023902059183456004, 0.010493258014321327, -0.007567577995359898, -0.006854739040136337, -0.02718922682106495, -0.006280413828790188, 0.004320951644331217, -0.011297312565147877, 0.014283803291618824, 0.0021672328002750874, -0.002533787162974477, -0.0036216266453266144, -0.002935814904049039, -0.0007326868362724781, -0.0161892119795084, 0.00950001273304224, 0.009648662060499191, 0.0019543946254998446, 0.02225678786635399, -0.016986509785056114, 0.012054070830345154, 0.0027668955735862255, 0.004067573230713606, -0.01068920362740755, -0.0009856432443484664, 0.00902028288692236, -0.005837846081703901, -0.01536488626152277, -0.008385146968066692, -0.04151356965303421, -0.007310820743441582, -0.0005380074726417661, 0.006131764966994524, 0.006138521712273359, -0.0044831144623458385, -0.02722976729273796, 0.00098057568538934, -0.006074332632124424, -0.004239870700985193, -0.007358118426054716, -0.010641906410455704, 0.015108129009604454, -0.002472976455464959, -0.0008910485194064677, 0.012405422516167164, -0.02789193019270897, 0.029648689553141594, 0.003689194330945611, 0.0009400350973010063, 0.005540548358112574, -0.02272976003587246, -0.0030320987571030855, -0.021932462230324745, -0.01125001534819603, 0.00371959968470037, -0.0004721290315501392, -0.011033798567950726, -0.033189233392477036, -0.005831088870763779, 0.0017466240096837282, -0.021270299330353737, 0.009101363830268383, 0.04732438921928406, -0.010952717624604702, -0.021432461217045784, -0.03702707961201668, -0.17189212143421173, 0.018405430018901825, 0.03213517740368843, -0.023081112653017044, 0.015216236934065819, -0.0061081163585186005, 0.01874326914548874, 0.007709470111876726, -0.034378424286842346, 0.009297310374677181, -0.02387841045856476, -0.017270293086767197, -0.001706928014755249, -0.0075878482311964035, 0.016324346885085106, 0.014608127996325493, 0.01687840186059475, 0.021000029519200325, 0.007250010035932064, 0.001559968339279294, 0.026635171845555305, 0.009716229513287544, 0.0023243275936692953, 0.004087843466550112, 0.008168930187821388, 0.03024328500032425, 9.005502215586603e-05, -0.034324370324611664, -0.023297328501939774, 0.001932435086928308, -0.009297310374677181, 0.001477197976782918, -0.02302705869078636, -0.0153513727709651, 0.018878404051065445, 0.0060979812406003475, -0.0010827718069776893, -0.014391911216080189, -0.019351378083229065, 0.037432484328746796, -0.001298988237977028, 0.011445961892604828, 0.004283789545297623, -0.015891913324594498, -0.03332436829805374, 0.009479742497205734, 0.006486495491117239, 0.009729743003845215, 0.01920272968709469, -0.005594602320343256, 0.015554075129330158, 0.006033792160451412, 0.02959463559091091, 0.0046689254231750965, 0.015270290896296501, 0.0025608143769204617, -0.023054085671901703, -0.013182450085878372, 0.012905423529446125, -0.015081102028489113, 0.00016385158232878894, -0.00679392833262682, -0.010121635161340237, 0.0011140217538923025, -0.014513533562421799, -0.029675716534256935, -0.018810836598277092, 0.028432471677660942, -0.005128385499119759, -0.003211152972653508, 0.025189224630594254, -0.025445980951189995, 0.004891898483037949, -0.012716233730316162, -0.01524326391518116, -0.006138521712273359, -0.00552027765661478, 0.011533799581229687, 0.01501353457570076, -0.02272976003587246, 0.02162165194749832, 0.03543248027563095, -0.007371631916612387, 0.0053243315778672695, -0.00458108726888895, 0.006797306705266237, 0.00196115137077868, 0.005327709950506687, 0.0017052388284355402, -0.021189218387007713, 0.006844603922218084, -0.02017570286989212, -0.006229738239198923, -0.0038682485464960337, -0.0022973003797233105, 0.03662167116999626, -0.013256775215268135, -0.025162195786833763, 0.008648660965263844, -0.01831083558499813, 0.005527034401893616, 0.014716236852109432, -0.03094598837196827, 0.02060813643038273, 0.012986504472792149, 0.0155811021104455, -0.028243282809853554, 0.023837869986891747, 0.005425683222711086, 0.0008505079313181341, -0.019310837611556053, 0.02536490000784397, 0.020229756832122803, 0.0250540878623724, 0.012709476985037327, 0.006722982041537762, -0.0010895285522565246, -0.0035777075681835413, 0.008148659951984882, 0.004263519309461117, 0.030621662735939026, -0.008804066106677055, 0.015567588619887829, 0.010472987778484821, -0.008641903288662434, -0.021973002701997757, -0.09172985702753067, -0.02162165194749832, 0.014040559530258179, 0.03440545126795769, -0.024256790056824684, 0.015513534657657146, 0.005891900043934584, 0.014716236852109432, -0.0015244953101500869, 0.011304069310426712, 0.007932443171739578, -0.027783822268247604, 0.023378411307930946, 0.003331085667014122, 0.014216235838830471, 0.015040561556816101, 0.008824336342513561, -0.00810811948031187, -0.003543923841789365, 0.009797310456633568, 0.013074342161417007, -0.01831083558499813, -0.01758110523223877, -0.009297310374677181, 0.025959495455026627, -0.035513561218976974, -0.038081131875514984, 0.014648668467998505, -0.0013665559235960245, -0.002168921986594796, -0.01759461872279644, -0.0004915547324344516, 0.009067580103874207, -0.015189209952950478, -0.003895275527611375, -0.0023125032894313335, -0.029648689553141594, -0.009182444773614407, 0.009817580692470074, -0.021067596971988678, 0.005347980186343193, 0.019418945536017418, 0.01232434157282114, -0.057837918400764465, 0.006861495785415173, 0.003908789250999689, -0.03321626037359238, -0.007993253879249096, 0.035756804049015045, -0.025162195786833763, -0.015310831367969513, -0.020121648907661438, -0.03783788904547691, -0.029459500685334206, 0.0009391905041411519, -0.005841224454343319, -0.026162197813391685, 0.014621641486883163, -0.03254058584570885, -0.003604734782129526, -0.01537839975208044, -0.009790553711354733, 0.013182450085878372, 0.03286490961909294, 0.004462843760848045, 0.005945954006165266, 0.00015646136307623237, -0.020000027492642403, 0.016959482803940773, -0.021202731877565384, -0.0056858183816075325, 0.006540549453347921, -0.005246629007160664, 0.010581095702946186, -0.03540545329451561, -0.005966224242001772, -0.042567625641822815, -0.007466226350516081, 0.00225507072173059, -0.020337866619229317, 0.007391902152448893, -0.01879732310771942, 6.268483957683202e-06, 0.0043851411901414394, 0.02554057538509369, -0.009472985751926899, 0.00371959968470037, 0.017729753628373146, -0.0065101440995931625, -0.02813517302274704, -0.0035574373323470354, 0.011533799581229687, 0.010216230526566505, 0.025189224630594254, -0.01877029612660408, 0.017256779596209526, 0.009581094607710838, -0.0007516902405768633, -0.005182439461350441, 0.01150001585483551, -0.011317582800984383, -0.00798649713397026, -0.07654064893722534, 0.011817583814263344, -0.006861495785415173, -0.012493260204792023, 0.017405429854989052, 0.009952716529369354, 0.028027065098285675, -0.026405442506074905, 0.008114876225590706, 0.011270285584032536, -0.009574337862432003, 0.029918959364295006, -0.0002550679200794548, -0.017216239124536514, -0.008885147050023079, -0.023986520245671272, 0.016081104055047035, -0.016108131036162376, 0.01595948077738285, 0.0070608207024633884, -0.0031908827368170023, -0.00044087899732403457, -0.012250016443431377, -0.0016790563240647316, -0.004253384191542864, 0.026797333732247353, -0.0166486706584692, 0.0008471295586787164, 0.014405425637960434, 0.006537171080708504, 0.026459496468305588, -0.019378405064344406, 0.006006765179336071, 0.037162214517593384, -0.0035033831372857094, -0.021918948739767075, -0.018702728673815727, 0.022243274375796318, 0.004540546797215939, -0.00026414732565172017, -0.01737840287387371, 0.0035473022144287825, -0.003089531324803829, -0.002483111573383212, -0.03540545329451561, -0.007608118467032909, -0.006672306451946497, 0.027270307764410973, 0.03827032446861267, 0.007250010035932064, 0.0010312513913959265, 0.004837844520807266, 0.01266217976808548, -0.00995947327464819, -0.015540561638772488, 0.005597980692982674, -0.008540552109479904, -0.019175702705979347, -0.018108133226633072, -0.015391913242638111, 0.051729802042245865, 0.0375676192343235, 0.01185136754065752, 0.005415548104792833, -0.0022702733986079693, -0.016459481790661812, -0.0017652051756158471, 0.008060822263360023, -0.011094610206782818, -0.03348653391003609, -0.01785137504339218, 0.01950002647936344, -0.0142027223482728, -0.021067596971988678, 0.017000023275613785, 0.004435816779732704, 0.006979739293456078, -0.011979746632277966, -0.035729777067899704, 0.03300004452466965, 0.013945965096354485, 0.007945956662297249, -0.012506773695349693, 0.0014619952999055386, 0.01173650287091732, 0.00779055105522275, -0.00962163507938385, -0.00819595716893673, 0.0070000095292925835, 0.010277041234076023, -0.017000023275613785, 0.0119865033775568, 0.013054071925580502, 0.005496629048138857, 0.010966231115162373, 0.018675701692700386, 0.02416219562292099, -0.038810864090919495, 0.022135164588689804, 0.01067569013684988, 0.011283799074590206, -0.005766899790614843, -0.03445950523018837, 0.0022956111934036016, -0.024918952956795692, -0.021716246381402016, -0.01713515818119049, -0.018000025302171707, -0.008216227404773235, -0.017310835421085358, 0.019864892587065697, -0.0030878421384841204, -0.0024172330740839243, 0.01617569848895073, -0.017283806577324867, 0.024486519396305084, 0.02552706189453602, -0.019594620913267136, -0.016567589715123177, 0.026108143851161003, 0.010331095196306705, 0.010405419394373894, 0.020783811807632446, 0.005412169732153416, 0.004415546543896198, -0.006320954766124487, 0.011425691656768322, -0.012925693765282631, 0.011716232635080814, -0.005087845027446747, 0.00745271285995841, -0.018162187188863754, -0.042324382811784744, -0.010459473356604576, -0.006195954512804747, 0.010385149158537388, 0.02932436391711235, 0.03002706915140152, -0.030189231038093567, 0.0656757652759552, 0.012891910038888454, 0.01302704494446516, 0.006760144606232643, -0.00645609013736248, 0.025608142837882042, 0.02622976526618004, -0.0035135182552039623, 0.0007652037311345339, -0.010256770998239517, 0.01945948600769043, -0.012506773695349693, -0.0104256896302104, -0.01948651298880577, -0.00985136441886425, -0.022689219564199448, 0.025162195786833763, 0.0319189615547657, -0.008790552616119385, -0.025702737271785736, 0.0049223038367927074, 0.01020947378128767, 0.015824345871806145, 0.029189229011535645, -0.008750012144446373, -0.023851383477449417, 0.005750007927417755, 0.00345946429297328, 0.016324346885085106, -0.03935140371322632, 0.0017550699412822723, -0.0017804078524932265, -0.026418955996632576, -0.012128395028412342, 0.015121642500162125, -0.025891928002238274, 0.0028834498953074217, 0.014824344776570797, 0.008844606578350067, 0.01161488052457571, 0.005554061848670244, -0.006104737985879183, -0.01385137066245079, -0.030621662735939026, -0.010750014334917068, 0.030081123113632202, -0.008520281873643398, -0.0015591237461194396, -0.020824352279305458]} +{"id": "test:10000001", "text": "\"Discussion in 'Mac OS X Lion (10.7)' started by axboi87, Jan 20, 2012.\\nI've got a 500gb internal drive and a 240gb SSD.\\nWhen trying to restore using disk utility i'm given the error \\\"Not enough space on disk ____ to restore\\\"\\nBut I shouldn't have to do that!!!\\nAny ideas or workarounds before resorting to the above?\\nUse Carbon Copy Cloner to copy one drive to the other. I've done this several times going from larger HDD to smaller SSD and I wound up with a bootable SSD drive. One step you have to remember not to skip is to use Disk Utility to partition the SSD as GUID partition scheme HFS+ before doing the clone. If it came Apple Partition Scheme, even if you let CCC do the clone, the resulting drive won't be bootable. CCC usually works in \\\"file mode\\\" and it can easily copy a larger drive (that's mostly empty) onto a smaller drive. If you tell CCC to clone a drive you did NOT boot from, it can work in block copy mode where the destination drive must be the same size or larger than the drive you are cloning from (if I recall).\\nI've actually done this somehow on Disk Utility several times (booting from a different drive (or even the dvd) so not running disk utility from the drive your cloning) and had it work just fine from larger to smaller bootable clone. Definitely format the drive cloning to first, as bootable Apple etc..\\nThanks for pointing this out. My only experience using DU to go larger to smaller was when I was trying to make a Lion install stick and I was unable to restore InstallESD.dmg to a 4 GB USB stick but of course the reason that wouldn't fit is there was slightly more than 4 GB of data.\"", "vector_field": [0.007919649593532085, -0.0029073364567011595, 0.018931794911623, -0.006880154833197594, -0.01578732393682003, 0.0237654447555542, -0.00991417933255434, -0.022115247324109077, 0.0025240229442715645, -0.035082943737506866, 0.007256971672177315, 0.0297815203666687, -0.014345025643706322, -0.004538043402135372, -0.010362462140619755, 0.011239535175263882, 0.008134045638144016, 0.024012325331568718, 0.0200752392411232, 0.001243332982994616, -0.007412896025925875, 0.008822710253298283, -0.01860695332288742, -0.001369209261611104, 0.028118329122662544, -0.0038266396149992943, 0.004833649843931198, 0.0015706113772466779, 0.011674824170768261, 0.007172512821853161, 0.017814338207244873, 0.002894342876970768, -0.022725950926542282, -0.019165681675076485, -0.026442144066095352, -0.01203215029090643, 0.013695341534912586, -0.0071790097281336784, 0.02194632962346077, -0.036226388067007065, 0.005233205854892731, -0.010446920990943909, -0.016670893877744675, -0.0053598941303789616, -0.014604899100959301, 0.020387087017297745, -0.0016631913604214787, -0.03170458599925041, 0.007646782323718071, 0.0061265211552381516, 0.01583929918706417, 0.009693287312984467, -0.022751938551664352, 0.0015819808468222618, -0.023258691653609276, -0.029963431879878044, -0.01676185056567192, 0.009764752350747585, 0.004690719302743673, -0.002210550243034959, -0.017372554168105125, 0.01314311008900404, 0.013500436209142208, -0.005895883310586214, -0.018970776349306107, -0.015540444292128086, 0.0048856246285140514, -0.028222277760505676, 0.0027302976232022047, -0.0067437211982905865, 0.018074212595820427, 0.010258512571454048, 0.026104308664798737, 0.01077176257967949, 0.006487096194177866, -0.008387422189116478, -0.005921870935708284, -0.006084291730076075, 0.024700989946722984, -0.0017249113880097866, 0.020880848169326782, -0.0027091829106211662, 0.005301422439515591, 0.018593959510326385, 0.012870242819190025, 0.003456319449469447, 0.004037786740809679, 0.009647808969020844, -0.01760643906891346, -0.00025581312365829945, -0.01489076018333435, -0.013435468077659607, 0.0073999022133648396, 0.0038656205870211124, 0.006906142458319664, 0.013000179082155228, 0.02406430058181286, 0.0030129102524369955, -0.0006346601876430213, -0.023401621729135513, 0.0006967862136662006, 0.007848184555768967, -0.0130911348387599, -0.021361613646149635, -0.0044081066735088825, -0.006620281375944614, 0.0260913148522377, -0.0042846668511629105, -0.007952134124934673, 0.025181757286190987, 0.009193030185997486, 0.019295617938041687, -0.0012343998532742262, -0.008744748309254646, 0.03144471347332001, -0.02112772688269615, 0.0151766212657094, -0.007451876997947693, -0.021166708320379257, -0.031236812472343445, 0.04469826817512512, -0.011057623662054539, 0.03037923015654087, -0.013890246860682964, -0.007003594655543566, 0.0010654819197952747, -0.03710995614528656, -0.0033686121460050344, -0.01583929918706417, 0.008010605350136757, 0.033731598407030106, 0.04080016165971756, 0.0073739150539040565, 0.017671408131718636, -0.00994016695767641, -0.01800924353301525, 0.0012222182704135776, -0.006513083353638649, -0.011187560856342316, -0.013903240673244, 0.0151766212657094, 0.020244156941771507, 0.0038233911618590355, 0.0018239881610497832, 0.009245005436241627, 0.021036772057414055, 0.015748342499136925, -0.01760643906891346, 0.022427096962928772, -0.004658235237002373, 0.005571041256189346, -0.0006277572829276323, 0.0031672101467847824, -0.0016047197859734297, 0.018749883398413658, 0.027572594583034515, 0.0010671061463654041, 0.0031590890139341354, -0.005389129742980003, -0.009245005436241627, -0.00012445510947145522, 0.017645420506596565, 0.007828693836927414, 0.011233038268983364, 0.007698757108300924, 0.02004925161600113, 0.03266611695289612, -0.012564891017973423, -0.011609855107963085, -0.010882209055125713, 0.00949188508093357, 0.02251805178821087, -0.03656422346830368, 0.04394463449716568, 0.03885111212730408, 0.0017655165866017342, 0.028871962800621986, 0.017216630280017853, 0.0060030813328921795, -0.01295470166951418, -0.012746802531182766, 0.029989419505000114, 0.005843908526003361, 0.024623028934001923, -0.02479194663465023, -0.028716038912534714, 0.012090621516108513, 0.0025971122086048126, 0.019334599375724792, -0.02234913408756256, -0.015722356736660004, 0.006425376050174236, -0.02286888100206852, -0.0026523354463279247, -0.6303495168685913, -0.017827332019805908, 0.01020004041492939, -0.014371013268828392, -0.0037876584101468325, 0.032484207302331924, -0.006821683142334223, -0.0033848544117063284, -0.03581058979034424, 0.012512916699051857, 0.0029446932021528482, 0.025883415713906288, 0.012421960942447186, -0.00017632833623792976, -0.012350494973361492, -0.004934350959956646, 0.014371013268828392, -0.018438035622239113, -0.02292085625231266, 0.01110959891229868, -0.016852805390954018, 0.013208078220486641, -0.019334599375724792, 0.0053306580521166325, 0.02156951278448105, -0.008647295646369457, 0.0145399309694767, -0.0027254249434918165, -0.0004115342744626105, 0.031912483274936676, -0.022686969488859177, -0.005860150791704655, 0.011915206909179688, -0.017892301082611084, 0.04108602553606033, -0.020543012768030167, -0.028352215886116028, 0.02062097378075123, 0.022128241136670113, 0.021062759682536125, -0.0034108415711671114, -0.00838092528283596, 0.03464115783572197, -0.003781161503866315, -0.006691746413707733, 0.006064801476895809, 0.00991417933255434, -0.000969653541687876, 0.00013003834465052933, -0.012551897205412388, 0.011824251152575016, 0.0376037172973156, 0.010817240923643112, 0.012909223325550556, -0.005320913158357143, -0.021361613646149635, 0.031106876209378242, -0.005795182194560766, 0.04812860116362572, -0.014384006150066853, -0.0031525923404842615, 0.010245518758893013, -0.027806481346488, -0.01722962222993374, -0.013266549445688725, 0.008426402695477009, -0.01942555606365204, -0.02713080868124962, 0.031262800097465515, -0.011752786114811897, 0.03721390664577484, -0.00901761557906866, -0.0028001386672258377, -0.02042606845498085, -0.00821850448846817, 0.010381951928138733, 0.02663704939186573, 0.00961532536894083, -0.017515484243631363, 0.00790665578097105, 0.00714002875611186, -0.02531169354915619, -0.007101047318428755, -0.012701325118541718, 0.02514277584850788, 0.004921357147395611, 0.0002306378592038527, -0.010869215242564678, -0.016514969989657402, 0.004898618441075087, 0.027026860043406487, 0.02775450609624386, 0.012551897205412388, -0.03445924445986748, -0.005499576218426228, 0.025818446651101112, -0.025532586500048637, -0.01613815315067768, -0.005320913158357143, -0.013552410528063774, 0.014591905288398266, -0.019009757786989212, -0.007497354876250029, 0.015774330124258995, 0.05023357644677162, -0.015384520404040813, -0.02731272019445896, -0.0020416323095560074, 0.0468292310833931, -0.034433260560035706, 0.01865892857313156, -0.005840660072863102, 0.0041027553379535675, -0.013383492827415466, 0.01238297950476408, -0.02042606845498085, 0.010557367466390133, -0.01330553088337183, -0.003313389141112566, -0.028196291998028755, 0.03290000557899475, -0.004654986783862114, -0.010830234736204147, -0.011349981650710106, -0.02381742000579834, 0.03892907127737999, 0.03082101419568062, -0.014046170748770237, 0.0037226900458335876, 0.0045900181867182255, -0.029417697340250015, -0.007536335848271847, 0.015228595584630966, -0.00014891978935338557, -0.0030356491915881634, 0.029365722090005875, 0.04131991043686867, -0.00400205422192812, 0.007497354876250029, -0.030769040808081627, -0.026896921917796135, -0.0020513776689767838, 0.014812798239290714, -0.01676185056567192, 0.0008372804149985313, 0.008907169103622437, -0.009641312062740326, -0.03365363925695419, -0.007835190743207932, -0.003292274195700884, -0.016099173575639725, -0.026740998029708862, -0.021491551771759987, -0.006847670767456293, -0.010557367466390133, -0.027338707819581032, -0.007334933616220951, -0.04004652798175812, -0.010278003290295601, -0.04446438327431679, -0.015566431917250156, 0.0383053757250309, -0.02731272019445896, 0.01795727014541626, -0.0021277156192809343, -0.011194057762622833, 0.004684222396463156, 0.0010995903285220265, -0.012480432167649269, -0.029833495616912842, 0.0021861870773136616, -0.0021130975801497698, 0.012318011373281479, -0.01988033391535282, -0.0013984451070427895, -0.005535308737307787, -0.003972818609327078, 0.003950079437345266, 0.0010849725222215056, -0.00866678636521101, 0.030093368142843246, -0.00396632170304656, 0.00555479945614934, 0.008121051825582981, 0.0077377380803227425, 0.009225514717400074, 0.016073185950517654, -0.013981202617287636, -0.05753602460026741, 0.0029463174287229776, -0.003917595371603966, 0.017515484243631363, 0.011038132943212986, -0.0250388253480196, 0.003482306841760874, 0.01600821688771248, -0.003240299643948674, 0.011609855107963085, 0.003557020565494895, -0.006808689795434475, 0.014319038018584251, -0.00026271602837368846, 0.016540957614779472, -0.018165167421102524, 0.03700600937008858, -0.01635904610157013, -0.0050188098102808, 0.00990118645131588, 0.017099685966968536, 0.005090275313705206, 0.0018532240064814687, -0.025077806785702705, 0.00012130820687161759, -0.029287761077284813, 0.01101214624941349, 0.017177648842334747, -0.0040312898345291615, 0.02791042998433113, 0.004596515092998743, 0.0026198511477559805, -0.02506481297314167, 0.004391864873468876, 0.00861481111496687, 0.009439910762012005, -0.011629345826804638, 0.03414739668369293, 0.01190871000289917, 0.003394599538296461, 0.00835493765771389, -0.017047710716724396, 0.006139514967799187, 0.00394033407792449, -0.015046684071421623, -7.633788482053205e-05, 0.01793128252029419, 0.006071298383176327, 0.023531559854745865, -0.006470853928476572, 0.03393949940800667, -0.031860508024692535, 0.009504878893494606, -0.008062579669058323, 0.0290798619389534, -0.015280570834875107, 0.03474510833621025, 0.02232314646244049, 0.0029723048210144043, 0.024558059871196747, -0.01295470166951418, 0.020880848169326782, -0.012486929073929787, -0.004287915304303169, -0.030483178794384003, 0.0014544803416356444, 0.0056977299973368645, 0.011181063950061798, 0.0062824455089867115, 0.005389129742980003, 0.007666272576898336, 0.02449309080839157, 0.01298068929463625, 0.010278003290295601, 0.008114554919302464, -0.034849055111408234, 0.012603872455656528, -0.00023875891929492354, -0.012486929073929787, -0.01773637719452381, -0.011791766621172428, 0.013357505202293396, 0.013299033977091312, -0.0017395291943103075, 0.02471398375928402, -0.0213486198335886, 0.01852899044752121, 0.002277142833918333, 0.023284679278731346, -0.014474961906671524, 0.029859483242034912, 0.012649349868297577, -0.014409993775188923, -0.028222277760505676, 0.021114734932780266, -0.008335446938872337, -0.011629345826804638, 0.011752786114811897, -0.024830928072333336, 0.008842200972139835, 0.006074546370655298, 0.030639102682471275, -9.826472523855045e-05, 0.013981202617287636, 0.018048224970698357, 0.006659262347966433, -0.00021967444627080113, -0.03139273822307587, 0.023076780140399933, -1.988388794416096e-05, 0.005944609642028809, 0.017294591292738914, 0.01583929918706417, 0.016255097463726997, -0.029365722090005875, 0.004235940519720316, 0.037395820021629333, -0.0026328449603170156, -0.0020578745752573013, -0.0012530782260000706, 0.031132863834500313, -0.01394222117960453, 0.0051649888046085835, -0.02715679630637169, -0.035056956112384796, 0.020867854356765747, -0.01665790192782879, 0.011986671946942806, -0.019932309165596962, 0.009374941699206829, 0.056600481271743774, 0.019243644550442696, 0.0007373914704658091, -0.01855497807264328, -0.034173384308815, 0.01641102135181427, 0.046777255833148956, 0.012499922886490822, -0.025558574125170708, 0.011629345826804638, -0.013227568939328194, 0.011064120568335056, 0.016969749704003334, -0.04529597610235214, 0.03596651181578636, -0.007731241174042225, 0.010992655530571938, 0.015553438104689121, 0.0133315185084939, -0.006474102381616831, 0.006630026735365391, 0.01037545595318079, 0.03981264308094978, 0.006201235111802816, -0.03336777538061142, -0.020906835794448853, -0.006110279355198145, 0.010693800635635853, 0.013682347722351551, 0.022790919989347458, 0.011551383882761002, 0.010550870560109615, 0.03445924445986748, 0.01022602804005146, -0.00587314460426569, -0.023089773952960968, -0.03367962688207626, 0.03396548703312874, 0.022011298686265945, 0.026792973279953003, 0.006120024248957634, -0.0121231060475111, 0.0011824250686913729, -0.015137639828026295, -0.005782188847661018, 0.01548846997320652, 0.01643700897693634, 0.0007020648918114603, -0.004554285667836666, 0.013195084407925606, 0.0477907657623291, -0.02316773682832718, -0.0250388253480196, 0.021933335810899734, -1.031627289194148e-05, -0.015280570834875107, 0.011356478556990623, -0.0020692439284175634, -0.0048856246285140514, 0.01683981344103813, -0.003641479415819049, -0.0126948282122612, -0.010992655530571938, -0.02199830487370491, -0.012162086553871632, -0.015059677883982658, -0.026156282052397728, -0.011025140061974525, 0.00914105586707592, -0.002080613514408469, 0.0061005339957773685, -0.01219457108527422, -0.013890246860682964, 0.0025581312365829945, -0.04204755648970604, 0.008062579669058323, 0.004349635448306799, -0.02623424492776394, 0.004125494044274092, 0.02224518358707428, 0.015072671696543694, 0.008842200972139835, 0.02419423684477806, -0.016216116026043892, 0.014526937156915665, 0.015085665509104729, -0.004677725490182638, -0.019074726849794388, -0.011655333451926708, -0.026130296289920807, -0.016125159338116646, 0.023947356268763542, 0.009504878893494606, 0.03292599320411682, 0.019711416214704514, 0.001458540908060968, 0.04503610357642174, -0.006938626524060965, -0.014695854857563972, -0.0279364176094532, 0.01776236481964588, -0.00415797857567668, 0.0069841044023633, 0.023986337706446648, 0.029573621228337288, -0.0327700674533844, 0.0026133544743061066, -0.0027595332358032465, -0.001408190350048244, -0.0013976329937577248, -0.011863232590258121, -0.012421960942447186, -0.00861481111496687, -0.029209798201918602, -0.00048523282748647034, 0.0006155757000669837, 0.008725257590413094, -0.014565917663276196, -0.013903240673244, 0.0064481147564947605, -0.00011866886779898778, 0.010804247111082077, 0.023778438568115234, 0.0010914693120867014, 0.017541471868753433, -0.010044116526842117, -0.015774330124258995, -0.054157670587301254, 0.014267063699662685, 0.022622000426054, -0.019113706424832344, 0.01988033391535282, -0.01486477255821228, -0.0320684090256691, -0.019009757786989212, 0.013890246860682964, 0.019984284415841103, 0.00899162795394659, 0.0008015477797016501, 0.0018564723432064056, -0.02142658270895481, -0.025506598874926567, 0.00990118645131588, 0.04095608741044998, 0.0015949745429679751, -0.01216858346015215, -0.0016477613244205713, 0.005626264493912458, 0.01117456704378128, -0.006409133784472942, 0.003362115239724517, -0.019633455201983452, -0.02357053942978382, 0.02379143238067627, 0.016320064663887024, 0.01630707085132599, 0.017021724954247475, -0.015371526591479778, -0.04389265924692154, -0.0027903933078050613, -0.010167556814849377, -0.031912483274936676, -0.010713291354477406, 0.004745942540466785, 0.034355297684669495, 0.014721842482686043, 0.014215088449418545, 0.0100376196205616, 0.014994709752500057, 0.009011118672788143, -0.005895883310586214, -0.01885383389890194, -0.010882209055125713, -0.019893327727913857, -0.025454623624682426, -0.006360407453030348, 0.011505905538797379, 0.00394033407792449, 0.015696369111537933, -0.017450515180826187, 0.00975175853818655, 0.042489342391490936, -8.669222734170035e-05, -0.017996249720454216, -0.012675337493419647, -0.036226388067007065, 0.018996763974428177, -0.014176107943058014, -0.0017444018740206957, -0.00603556539863348, -0.02598736435174942, -0.02715679630637169, 0.017502490431070328, 0.006945123430341482, 0.02661106176674366, -0.016229109838604927, 0.04202156886458397, 0.015072671696543694, -0.0009233635501004755, 0.023089773952960968, 0.011278516612946987, -0.011388963088393211, -0.008445893414318562, -0.013461454771459103, 0.013474448584020138, 0.007438883185386658, 0.004791420418769121, 0.0024574301205575466, 0.005070784594863653, 0.01283126138150692, 0.006607287563383579, 0.012266036123037338, -0.016540957614779472, -0.008731754496693611, 0.00013947905972599983, -0.04100806266069412, 0.00271567958407104, -0.02658507414162159, -0.006220725364983082, 0.000818601984065026, -0.01273380871862173, 0.019386574625968933, -0.011375969275832176, 0.007101047318428755, -0.004619254264980555, -0.008731754496693611, 0.007789712864905596, -0.00014556985115632415, 0.03570663928985596, 0.04701114445924759, 0.020140208303928375, 0.007705253548920155, 0.022180216386914253, -0.032458219677209854, 0.011713804677128792, 0.03708396852016449, -0.016917774453759193, 0.007289455737918615, 0.011018643155694008, 0.008874685503542423, -0.0056555001065135, 0.0016323314048349857, -0.017541471868753433, -0.02409028820693493, -0.048648346215486526, 0.00775073142722249, -0.004203456453979015, 0.031288787722587585, -0.008933156728744507, -0.006113527808338404, -0.03357567638158798, 0.02137460745871067, 0.015553438104689121, 0.004083264619112015, -0.0031119869090616703, -0.0074258893728256226, -0.02219321019947529, 0.00564900366589427, -0.01330553088337183, 0.0034660648088902235, -0.016774844378232956, 0.00682818004861474, -0.010466411709785461, -0.04776477813720703, 0.007854681462049484, 0.017125673592090607, -0.02102377824485302, 0.013149606995284557, -0.018074212595820427, -0.014461969025433064, 0.0022040533367544413, 0.011824251152575016, 0.006012826692312956, -0.04308705031871796, -0.013838271610438824, 0.027572594583034515, -0.025974370539188385, -0.0007317067356780171, -0.01635904610157013, 0.006834676954895258, -0.008153535425662994, -0.003332879627123475, -0.0032874017488211393, -0.029573621228337288, -0.003917595371603966, 0.006279197055846453, 0.01586528681218624, 0.009953160770237446, 0.00239733443595469, -0.011739792302250862, -0.0034075933508574963, -0.018100200220942497, -0.0016437008744105697, -0.003123356495052576, 0.002358353463932872, -0.03666817396879196, -0.007380411494523287, -0.031132863834500313, -0.0058633992448449135, 0.010446920990943909, -0.016151146963238716, -0.009206023998558521, 0.011603358201682568, 0.020296132192015648, -0.03796754032373428, 0.018451029434800148, 0.03523886576294899, 0.017723383381962776, -0.010362462140619755, 0.019568486139178276, 0.0024736723862588406, -0.0309249646961689, 0.005769195035099983, -0.0018905807519331574, -0.0231937225908041, -0.02145257033407688, 0.0013326645130291581, -0.002761157462373376, -0.00039833757909946144, 0.0057432078756392, 0.029573621228337288, -0.0019051986746490002, -0.0007820572936907411, -0.022660981863737106, -0.03895505890250206, 0.001908447127789259, -0.04095608741044998, 0.0126948282122612, 0.006730727385729551, 0.010154563002288342, 0.010615838691592216, -0.020880848169326782, 0.027572594583034515, 0.011265522800385952, -0.020036257803440094, -0.030223306268453598, 0.0018142429180443287, 0.02189435437321663, -0.029521645978093147, -0.007010091561824083, -0.03100292570888996, -0.006808689795434475, -0.016540957614779472, -0.0047719296999275684, 0.005700977984815836, 0.0074843610636889935, -0.0006980043835937977, -0.011330490931868553, -0.001726535614579916, 0.031262800097465515, -0.012058136984705925, -0.009504878893494606, 0.010005135089159012, -0.005210466682910919, -0.023713471367955208, -0.021361613646149635, -0.002559755463153124, 0.034407272934913635, -0.011700810864567757, -0.011642339639365673, -0.015345538966357708, -0.010856221430003643, -0.009868701919913292, 0.0034335805103182793, 0.01833408698439598, 0.004499062430113554, 0.03341975063085556, 0.007360921241343021, -0.007893661968410015, 0.03037923015654087, -0.0020659954752773046, 0.024207230657339096, 0.0011840492952615023, -0.019841352477669716, -0.015605412423610687, 0.02127065882086754, 0.012272533029317856, -0.01760643906891346, -0.0013310404028743505, -0.010057110339403152, -0.007497354876250029, 0.02915782295167446, -0.004996071103960276, -0.003453071229159832, 0.005525563377887011, -0.008017102256417274, -0.001394384540617466, 0.02227117121219635, 0.04261928051710129, 0.011544886976480484, -0.0006602414650842547, -0.04126793518662453, -0.0006890712538734078, 0.011317497119307518, 0.008608315140008926, -0.01706070452928543, -0.023609520867466927, -0.0016477613244205713, 0.007633788511157036, 0.002165072364732623, -0.0013740819413214922, 0.007354424335062504, -0.00019155530026182532, -0.02566252276301384, 0.007809203118085861, -0.001419559819623828, -0.006630026735365391, 0.008816213347017765, 0.0071790097281336784, -0.030665090307593346, 0.010901699773967266, 0.007133531849831343, 0.0019799123983830214, 0.006246712990105152, -0.0009071214590221643, -0.04033239185810089, -0.018113194033503532, 0.0023096271324902773, 0.01852899044752121, -0.039942581206560135, 0.019659440964460373, -0.017242616042494774, 0.002377843949943781, -0.04397062212228775, 0.01678783819079399, 0.007861178368330002, 0.0029966679867357016, 0.0018207397079095244, 0.00362198892980814, -0.02840418927371502, 0.02479194663465023, -0.011025140061974525, 0.0017720134928822517, -0.0666835755109787, 0.01855497807264328, 0.0029999164398759604, -0.023804426193237305, -0.021231677383184433, 0.006542318966239691, 0.004417852032929659, 0.004596515092998743, 0.0011475045466795564, 0.19999876618385315, 0.020205175504088402, 0.0006496841087937355, 0.04293112829327583, 0.008770735934376717, 0.0002828156284522265, 0.030145343393087387, 0.005710723344236612, -0.020205175504088402, -0.0025630039162933826, -0.018217142671346664, 0.0032338027376681566, -0.016450002789497375, -0.014189101755619049, 0.022998817265033722, -0.004758936353027821, -0.05249447748064995, -0.006418879143893719, -0.002928451169282198, 0.023986337706446648, 0.011993168853223324, -0.012246545404195786, -0.03160063549876213, -0.025558574125170708, 0.04701114445924759, -0.01868491619825363, -0.03167859837412834, 0.031366750597953796, 0.02010122686624527, 0.015852292999625206, -0.004836898297071457, -0.002377843949943781, 0.004401609767228365, 0.002995043760165572, -0.013864259235560894, 0.01673586294054985, 0.004365877248346806, 0.0031542163342237473, -0.007633788511157036, 0.013825277797877789, 0.016917774453759193, 0.01785331964492798, -0.021686455234885216, 0.007412896025925875, -0.015748342499136925, 0.021855374798178673, 0.009335961192846298, 0.014085152186453342, -0.0027562850154936314, 0.009433413855731487, -0.008595321327447891, -0.016995737329125404, -0.0021910597570240498, 0.03773365542292595, -0.00864079874008894, -0.01825612410902977, 0.02321971021592617, -0.011895716190338135, -0.0006387206958606839, 0.014981715939939022, -0.013149606995284557, 0.03536880388855934, 0.00555479945614934, 0.03594052419066429, 0.0006269451696425676, 0.020166195929050446, 0.01086271833628416, 0.025623541325330734, 0.019516510888934135, -0.0005327409598976374, -0.0060030813328921795, -0.0044081066735088825, -0.007672769483178854, 0.01895778253674507, 0.00013795636186841875, -0.03204242140054703, 8.207337668864056e-05, -0.00041864020749926567, 0.021140720695257187, 0.002402207115665078, -0.014111138880252838, -0.030795028433203697, -0.015306558459997177, -0.014098145999014378, -0.007835190743207932, -0.030483178794384003, 0.004349635448306799, -0.018593959510326385, -0.00942691694945097, -0.0036057468969374895, 0.0016014713328331709, -0.006574803497642279, -0.0106743099167943, -0.022881874814629555, -0.005061039235442877, -0.00022698339307680726, -0.014708848670125008, 0.012850752100348473, -0.00018373879720456898, -0.015826305374503136, -0.040644239634275436, 0.041189972311258316, 0.03890308365225792, -0.02099779061973095, -0.031314775347709656, -0.02015320211648941, 0.0021309638395905495, 0.014033176936209202, 0.011395459994673729, 0.00897863507270813, -0.013773303478956223, -0.043035075068473816, 0.0094009293243289, -0.0021423334255814552, 0.016151146963238716, 0.041215959936380386, 0.0028326227329671383, -0.03526485338807106, 0.040618252009153366, -0.006366904359310865, 0.030067380517721176, -0.007315443363040686, 0.014669867232441902, -0.0029300753958523273, -0.003165585920214653, 0.006730727385729551, -0.021283652633428574, 0.009355450980365276, -0.011285013519227505, -0.0533260740339756, -0.008510862477123737, -0.007445380091667175, 0.014669867232441902, -0.0002828156284522265, -0.007711750455200672, 0.01958147995173931, 0.011382466182112694, -0.010148066096007824, 0.0002061731938738376, -0.004875879269093275, 0.0028716039378196, 0.00603556539863348, -0.0052786837331950665, -0.007633788511157036, 0.021491551771759987, -0.010888705961406231, 0.00806907657533884, -0.015189615078270435, -0.02324569784104824, -0.03139273822307587, -0.02362251468002796, 0.022336140275001526, -0.004583521746098995, -0.014851778745651245, 0.0018061219016090035, 0.0011889219749718904, -0.03518689051270485, -0.04711509123444557, -0.018788864836096764, 0.01515063364058733, -0.018451029434800148, 0.034173384308815, 0.01457891147583723, -0.019282624125480652, -0.030535154044628143, -0.024155255407094955, -0.16153746843338013, 0.003985811956226826, 0.014098145999014378, -0.008062579669058323, 0.026468131691217422, 0.008595321327447891, 0.014643880538642406, 0.019282624125480652, -0.009517872706055641, -0.01828211173415184, 0.0011743040522560477, 0.005817921366542578, -0.03635632246732712, -0.0026945648714900017, -0.016501976177096367, 0.0061427634209394455, -0.03258815407752991, 0.005710723344236612, 0.03885111212730408, 0.00571722025051713, 0.0219203419983387, -0.034953005611896515, -0.004551037214696407, -0.010966667905449867, 0.0034433258697390556, -0.004622502718120813, 0.01422808226197958, -0.014007189311087132, -0.0022820155136287212, -0.024454111233353615, -0.0020351356361061335, -0.009121565148234367, 0.0011125840246677399, 0.0012530782260000706, 0.012311514467000961, -0.023882389068603516, 0.009732267819344997, 0.009998639114201069, 0.014422987587749958, 0.012727311812341213, 0.01038844883441925, 0.01573534868657589, 0.009771249257028103, -0.009667299687862396, 0.002103352453559637, 0.026922909542918205, 0.01852899044752121, -0.004986325744539499, 0.001033809850923717, 0.008056082762777805, -0.007594807539135218, -0.031236812472343445, -0.010271506384015083, -0.004583521746098995, -0.01598222926259041, -0.01250641979277134, -0.034849055111408234, 0.03175656124949455, -0.022141234949231148, -0.0011207051575183868, 0.00277739972807467, -0.01387725304812193, 0.004927854053676128, -0.002408703789114952, -0.02728673256933689, -0.011414949782192707, 0.012623362243175507, -0.019399568438529968, -0.003927340731024742, -0.006480599287897348, 0.01920466311275959, -0.012636356055736542, -0.0018873324152082205, -0.009024112485349178, 0.03339376300573349, -0.027676543220877647, -0.036720145493745804, -0.013318524695932865, -0.00022434405400417745, -0.012064633890986443, -0.003254917450249195, 0.041839659214019775, -0.011889219284057617, 0.001338349306024611, -0.00027022798894904554, 0.012428456917405128, 0.007958630099892616, -0.01700873114168644, 0.012974192388355732, -0.018866827711462975, -0.003911098465323448, -0.021959323436021805, -0.004531546961516142, 0.001645325100980699, -0.006704740226268768, 0.0038883595261722803, 0.018061218783259392, 0.008445893414318562, -0.0030924964230507612, 0.0020140206906944513, -0.0012132851406931877, -0.009154049679636955, -0.02718278393149376, 0.01082373782992363, 0.03547275438904762, 0.009582840837538242, -0.01982835866510868, 0.005593780428171158, 0.029963431879878044, -0.0023762197233736515, -0.0032061911188066006, 0.005213715136051178, -0.004473075270652771, 0.02980750799179077, -0.026455137878656387, 0.005421613808721304, -0.00450880778953433, -0.020556006580591202, -0.014656874351203442, -0.02107575349509716, 0.03027527965605259, -0.02639016881585121, -0.041137997061014175, -0.018710901960730553, 0.0069841044023633, -0.0008324077934958041, -0.11704709380865097, -0.016527963802218437, 0.0019945302046835423, 0.030795028433203697, 0.008972138166427612, 0.01855497807264328, -0.0206469614058733, 0.00866678636521101, -0.04028041660785675, 0.01755446568131447, -0.02590940333902836, -0.03666817396879196, 0.004008551128208637, -0.0008835704065859318, 0.006383146625012159, 0.0025094049051404, 0.017138667404651642, -0.002821253379806876, -0.01673586294054985, 0.018191155046224594, -0.016450002789497375, 0.002975553274154663, -0.0073739150539040565, -0.010368959046900272, -0.012512916699051857, 0.0028732281643897295, -0.026000358164310455, 0.018645934760570526, 0.004989574197679758, 0.01299368217587471, 0.03225031867623329, 0.006990601308643818, 0.010752271860837936, -0.008374428376555443, -0.004823904484510422, -0.019737403839826584, -0.03027527965605259, 0.023609520867466927, 0.01678783819079399, -0.006025820504873991, 0.015592418611049652, 0.014332031831145287, -0.0030421458650380373, -0.04272322729229927, -0.010154563002288342, -0.004898618441075087, -0.02918381057679653, 0.018671922385692596, 0.0005705038784071803, -0.01086271833628416, -0.01177227683365345, 0.001939307083375752, -0.01925663836300373, 0.004492565523833036, 0.024804940447211266, -0.019438549876213074, 0.01725560985505581, 0.008523855358362198, -0.003976067062467337, 0.014033176936209202, -0.005519066471606493, 0.01512464601546526, 0.002481793286278844, 0.013903240673244, 0.013286040164530277, -0.014942734502255917, 0.00847837794572115, 0.021816393360495567, -0.005372887942939997, -0.006305184680968523, -0.0018710902659222484, -0.015280570834875107, -0.03271809220314026, 0.009550356306135654, -0.01600821688771248, 0.0031867006327956915, -0.021036772057414055, -0.03360166400671005, -0.00022434405400417745, -0.001117456704378128, 0.003076254390180111, -0.017697395756840706, -0.005002567544579506, -0.005395626649260521, 0.015540444292128086, 0.030041394755244255, 0.02516876347362995, -0.019035745412111282, 0.02034810744225979, -0.03180853649973869, 0.0031720828264951706, 0.022089259698987007, 0.015306558459997177, -0.024584047496318817, 0.011343484744429588, -0.0031737070530653, 0.010992655530571938, 0.0309249646961689, 0.00803009606897831, -0.011330490931868553, -0.04261928051710129, 0.010979661718010902, -0.04807662591338158, 0.046153560280799866, 0.0008624556940048933, -0.04067022725939751, -0.005311167798936367, 0.007127034943550825, -0.024623028934001923, -0.014409993775188923, -0.013955214992165565, 0.017268603667616844, 0.00641563069075346, 0.021439576521515846, -0.014384006150066853, 0.014384006150066853, -0.004833649843931198, -0.01665790192782879, -0.002688067965209484, -0.0033296311739832163, -0.0024996597785502672, 0.005762698128819466, -0.005915374029427767, 0.007101047318428755, 0.012428456917405128, 0.014747830107808113, 0.004041035193949938, 0.019958296790719032, 0.0012547024525702, 0.032510194927453995, 0.02628622017800808, -0.029859483242034912, -0.016073185950517654, -0.011317497119307518, -0.020958809182047844, 0.017073698341846466, -0.005538557190448046, -0.029547633603215218, -0.0010191919282078743, 0.007679266389459372, 0.0061005339957773685, 0.03838333860039711, 0.008153535425662994, -0.00679569598287344, 0.015904268249869347, -0.025272712111473083, -0.015020697377622128, 0.0054183658212423325, -0.0052202120423316956, -0.0011604982428252697, 0.02590940333902836, 0.01209711842238903, 0.04506209120154381, 0.01728159748017788, 0.006058304570615292, -0.04134589806199074, -0.033003952354192734, -0.022440088912844658, 0.03840932622551918, 0.0035277847200632095, -0.012688331305980682, -0.03638231009244919, 0.028949923813343048, 0.0011613103561103344, 0.02663704939186573, -0.001668064040131867, 0.018386060371994972, 0.001338349306024611, 0.017463508993387222, 0.0013115498004481196, 0.015293564647436142, -0.006181744392961264, -0.007321940269321203, -0.0013960087671875954, -0.0007926146499812603, 0.010459914803504944, 0.0010736030526459217, 0.02438914217054844, 0.0045640310272574425, -0.02140059508383274, 0.0009818351827561855, 0.004593266639858484, -0.022622000426054, -0.014786810614168644, 0.009199527092278004, -0.011778772808611393, 0.014137126505374908, 0.018139181658625603, 0.0010874088620766997, 0.047946687787771225, 0.005528811831027269, 0.0036252373829483986, -0.01825612410902977, 0.012876739725470543, -0.0018451028736308217, -0.0005830915179103613, 0.006724230479449034, 0.0100376196205616, 0.01006360724568367, 0.012402470223605633, 0.01063532941043377, 0.017320578917860985, -0.013812284916639328, 0.004716706927865744, -0.004434094298630953, -0.012538903392851353, -0.003103866009041667, -0.0009729019948281348, -0.006581300403922796, -0.01603420451283455, -0.020231163129210472, 0.006204483564943075, -0.00025398589787073433, 0.011967181228101254, -0.0032679112628102303, 0.00679569598287344, -0.03770766779780388, -0.01129800733178854, -0.007822196930646896, -0.03705798089504242, -0.027052847668528557, 0.030223306268453598, 0.02384340763092041, 0.01730758510529995, 0.007068563252687454, -0.018944788724184036, 0.002389213303104043, 0.030223306268453598, 0.012395973317325115, 0.0031818279530853033, 0.02099779061973095, 0.007464870810508728, 0.01825612410902977, -0.006435121409595013, 0.015137639828026295, -0.02069893665611744, 0.011466925032436848, -0.022440088912844658, -0.0007532275631092489, 0.00923850852996111, 0.010680806823074818, 0.06491643935441971, -0.002122842939570546, -0.023453596979379654, -0.004872630815953016, -0.004297660663723946, 0.010641826316714287, 0.01830809935927391, 0.0250388253480196, -0.0017395291943103075, -0.013058651238679886, 0.03539479151368141, -0.0353168286383152, 0.0012668840354308486, 0.012610369361937046, -0.03633033484220505, 0.034277334809303284, 0.0007049072883091867, 0.001274192938581109, 0.005691233091056347, 0.007393405307084322, 0.016203122213482857, 0.016177134588360786, 0.008036592975258827, 0.0026133544743061066, -0.020231163129210472, -0.01787930727005005, 0.004385367967188358, 0.009706281125545502, -0.026494119316339493, -0.020841866731643677, 0.0025240229442715645, 0.0008210383239202201, -0.03027527965605259, -0.008250988088548183, -0.010265009477734566, 0.014422987587749958, -0.010459914803504944, 0.023921368643641472, 0.005970597267150879, -0.010914693586528301, -0.002140709199011326, -0.0010955298785120249, -0.013234065845608711, -0.014747830107808113, 0.012324508279561996, -0.0441005602478981, -0.008173026144504547, -0.01895778253674507, -0.020594986155629158]} +{"id": "test:10000002", "text": "\"Foil plaid lycra and spandex shortall with metallic slinky insets. Attached metallic elastic belt with O-ring. Headband included. Great hip hop or jazz dance costume. Made in the USA.\"", "vector_field": [-0.023279672488570213, -0.0021421685814857483, -0.021883688867092133, -0.02890348806977272, -0.011938975192606449, 0.0012497368734329939, -0.018785936757922173, -0.03839616850018501, -0.014544809237122536, -0.020434526726603508, 0.026111522689461708, -0.0012173301074653864, 0.02407737635076046, -0.007677904330193996, -0.0023648610804229975, 0.022269247099757195, 0.018533330410718918, -0.0003874267276842147, 0.0025792440865188837, -0.018533330410718918, -0.018200954422354698, 0.0026307625230401754, -0.0016186750726774335, -0.010782304219901562, -0.0021388446912169456, -0.033636532723903656, 0.016406118869781494, -0.019331036135554314, -0.0002644472988322377, 0.005633786786347628, 0.00893428921699524, 0.0009738640510477126, -0.016060447320342064, -0.02062065713107586, -0.008375896140933037, -0.0031858317088335752, 0.010715828277170658, -0.004271042067557573, 0.0216576736420393, 0.008316067978739738, -0.006797106005251408, 0.0209796242415905, 0.009260018356144428, 0.02494155615568161, -0.003945312928408384, -0.004054997116327286, -0.004706455860286951, -0.0018014822853729129, -0.00589636480435729, 0.010975082404911518, 0.0071461014449596405, 0.0330781415104866, 7.275936513906345e-05, -0.0037159728817641735, -0.0016494198935106397, -0.011892442591488361, 0.0014175871619954705, -0.0014516557566821575, 0.03664122149348259, -0.027600571513175964, -0.015063317492604256, 0.010237205773591995, -0.023505687713623047, -0.005490865092724562, 0.0017483020201325417, -0.01889229752123356, -0.009745288640260696, -0.017124053090810776, -0.013986416161060333, 0.014810711145401001, 0.01717723347246647, 0.0036827351432293653, 0.014558104798197746, -0.012324532493948936, 0.016073742881417274, 0.010203967802226543, -0.016964511945843697, 0.012018745765089989, -0.0009323169942945242, -0.003539812983945012, 0.023971015587449074, -0.013919941149652004, 0.00296812504529953, 0.028956668451428413, 0.020939739421010017, 0.009406263940036297, -0.00700650317594409, 0.028132373467087746, -0.009007412008941174, -0.01366733480244875, 0.0029548299498856068, 0.0009564142674207687, 0.025526538491249084, 0.022282540798187256, -0.011280869133770466, 0.010961787775158882, -0.023279672488570213, 0.023279672488570213, -0.007531658746302128, -0.0034866328351199627, 0.007418650668114424, 0.01195891760289669, -0.00867503508925438, -0.011427114717662334, -0.014212432317435741, -0.0004977342905476689, -0.012577138841152191, 0.0034966040402650833, 0.01789516769349575, 0.009505976922810078, -0.023332852870225906, 0.0043009561486542225, 0.02456929348409176, -0.05038832500576973, 0.010283738374710083, 0.017615972086787224, 0.0159806776791811, 0.0057866801507771015, -0.010529697872698307, -0.018520036712288857, 0.007378765381872654, 0.035710565745830536, 0.008149879053235054, -0.007717789616435766, 0.01907842978835106, 0.027441030368208885, -0.011520180851221085, -0.017509611323475838, 0.019184790551662445, 0.003536489326506853, 0.05187737196683884, -0.011759491637349129, 0.0008380050421692431, -0.003044571727514267, -0.009007412008941174, 0.01822754368185997, -0.007584839127957821, 0.00571688124909997, -0.011061500757932663, -0.014558104798197746, 0.018480150029063225, 0.016751792281866074, 0.014770825393497944, 0.004792873747646809, 0.01022391114383936, 0.028584405779838562, -0.012391007505357265, 0.005361238028854132, -0.006434815004467964, 0.00485270144417882, 0.02460918016731739, -0.027108652517199516, 0.017748922109603882, 0.007491773460060358, 0.013015876524150372, 0.021923575550317764, 0.00841578096151352, 0.03749210387468338, -0.005470922216773033, -0.024635769426822662, -0.008302772417664528, -0.004058321006596088, 0.019304445013403893, -0.012876277789473534, -0.0051950495690107346, 0.004187948070466518, 0.024037491530179977, -0.02479531057178974, 0.0013527736300602555, 0.016911331564188004, -0.0033835959620773792, 0.01669861190021038, -0.046692293137311935, -0.010024485178291798, -0.0012256394838914275, 0.011114681139588356, -0.003516546683385968, -3.087468940066174e-05, -0.01684485748410225, -0.02579244039952755, -0.0005014734924770892, -0.005364561919122934, 0.008901051245629787, 0.03916728496551514, -0.020727017894387245, -0.016898037865757942, -0.0038921325467526913, -0.02288082055747509, -0.0070264460518956184, -0.017124053090810776, 0.008887755684554577, 0.030206404626369476, -0.013826875947415829, -0.009073887020349503, -0.6560320854187012, 0.0011458690278232098, 0.00016951217548921704, -0.05429707467556, 0.02099291980266571, 0.001167473616078496, 0.01979636400938034, -0.00132285978179425, -0.007418650668114424, 0.010210615582764149, -0.0021139164455235004, 0.05025537312030792, 0.008322715759277344, -0.021803919225931168, -0.002285090507939458, -0.0034766613971441984, 0.012783212587237358, -0.02236231230199337, 0.01740325056016445, 0.003991845529526472, -0.010788951069116592, -0.012789859436452389, -0.0003766244917642325, 0.012317884713411331, 0.0012846364406868815, 0.020527591928839684, 0.016725201159715652, 0.004287661053240299, 0.027108652517199516, -0.007325585000216961, -0.0327058769762516, 0.02651037462055683, -0.020075559616088867, 0.015289333648979664, 0.04315580427646637, -0.006577737163752317, -0.034833088517189026, 0.021710854023694992, 0.010901959612965584, 0.016286464408040047, -0.03472673147916794, 0.023000475019216537, 0.002197010675445199, 0.010702533647418022, -0.017629265785217285, -0.00012516064452938735, 0.014691054821014404, -0.0016294773668050766, 0.016791677102446556, -0.00032157456735149026, 0.017429839819669724, -0.016991103067994118, 0.025672785937786102, -0.0038655423559248447, -0.0026457195635885, -0.01907842978835106, 0.024861786514520645, 0.02543347328901291, 0.010250501334667206, 0.01665872521698475, 0.0015472141094505787, 0.003132651560008526, -0.037944138050079346, -0.011473647318780422, -0.026909226551651955, 0.01687144674360752, -0.012424245476722717, -0.007239167112857103, 0.00641154870390892, -0.022601623088121414, -0.006607651244848967, 0.021750738844275475, -0.022588327527046204, 0.012231466360390186, 0.002092312090098858, 0.020673837512731552, 0.02478201501071453, 0.02818555384874344, -0.021059395745396614, 0.009147009812295437, -0.0029315634164959192, 0.012849687598645687, 0.0005018889787606895, -0.017748922109603882, 0.025752555578947067, -0.004882615525275469, -0.010808894410729408, -0.0036096121184527874, 0.016964511945843697, 0.006145647261291742, 0.01889229752123356, 0.014504924416542053, -0.001794834854081273, -0.027414439246058464, -0.030552076175808907, 0.01911831460893154, -0.020381346344947815, 0.01615351252257824, 0.023106835782527924, -0.02339932695031166, -0.00279362709261477, -0.003536489326506853, 0.004237804561853409, -0.011327401734888554, 0.008316067978739738, 0.011054852977395058, 0.004051673226058483, 0.019916018471121788, 0.02508780173957348, -0.02871735766530037, -0.0009364716825075448, -0.010795598849654198, -0.0015738041838631034, -0.010476517491042614, -0.01704428344964981, -0.03629554808139801, 0.032759059220552444, 0.012145048938691616, 0.022109705954790115, -0.031748633831739426, 0.011646484024822712, -0.004071616102010012, 0.016645431518554688, -0.009692108258605003, -0.01631305366754532, 0.032759059220552444, 0.005414418410509825, -0.009100477211177349, -0.03754528611898422, 0.008196412585675716, 0.014996841549873352, 0.009339788928627968, 0.02253514714539051, 0.004447201732546091, 0.005294762551784515, 0.002072369446977973, 0.03727938234806061, 0.01451821904629469, 0.0375186949968338, -0.04283672571182251, -0.021803919225931168, -0.008708272129297256, -0.01976977288722992, -0.017762217670679092, -0.014571399427950382, -0.012683499604463577, -0.03626895695924759, -0.001209851587191224, 0.006959970574826002, 0.006129028275609016, -0.02217618189752102, -0.018174365162849426, -0.018825823441147804, -0.01092854980379343, 0.00288336887024343, -0.02031487040221691, -0.012769917026162148, -0.0077842650935053825, -0.0031642273534089327, -0.009160305373370647, 0.032280437648296356, -0.009306550957262516, -0.010117550380527973, -0.00034920338657684624, 0.006049258168786764, 0.010556288063526154, 0.02168426290154457, -0.004403993021696806, -0.013441318646073341, -0.01409277692437172, -0.012690146453678608, -0.023612048476934433, 0.005191725678741932, -0.01632634922862053, -0.01023055799305439, 0.02323978766798973, 0.017110757529735565, -0.021950164809823036, 0.006953322794288397, -0.024861786514520645, 0.022136295214295387, 0.008096699602901936, -0.04469803348183632, 0.007365470286458731, 0.03262610733509064, 0.0031509322579950094, 0.01538239885121584, 0.013441318646073341, 0.004643304273486137, -0.018174365162849426, -0.03145614266395569, -0.0017715684371069074, -0.009452796541154385, 0.002436322160065174, -0.00841578096151352, -0.008920993655920029, -0.010263795964419842, -0.0175229050219059, 0.0065544708631932735, 0.011566713452339172, 0.022588327527046204, -0.012125106528401375, 0.008449018932878971, -0.008468961343169212, 0.010788951069116592, -0.03632213920354843, 0.02975437231361866, -0.0005629632505588233, 0.02510109730064869, 0.001721711945720017, 0.004796197637915611, -0.030179815366864204, 0.01804141327738762, -0.004227832891047001, 0.023160016164183617, 0.012444187887012959, -0.003669440047815442, 0.020846674218773842, -0.008801338262856007, 0.011068147607147694, 0.012617023661732674, -0.007564896252006292, 0.0194373968988657, 5.588085332419723e-05, -0.00438737403601408, -0.024702245369553566, -0.007511715870350599, 0.027972832322120667, 0.0021321973763406277, -0.016804972663521767, -0.006338425911962986, -0.010137492790818214, 0.04844724386930466, 0.028664177283644676, -0.002361537190154195, -0.0020374697633087635, 0.01667202077805996, -0.02422362193465233, 0.02253514714539051, -0.004108177497982979, 0.0278930626809597, 0.020381346344947815, 0.019198084250092506, 0.022136295214295387, 0.03634873032569885, -0.0003255215415265411, 0.03193476423621178, -0.011832614429295063, -0.016565660014748573, 0.017070872709155083, -0.012351122684776783, -0.007072978653013706, -0.0204877071082592, 0.0038954562041908503, 0.01410607248544693, -0.017443135380744934, 0.01503672730177641, 0.00014738834579475224, -0.0021670968271791935, 0.026098227128386497, -0.010948492214083672, -0.005035508889704943, 0.033796075731515884, 0.01802811771631241, 0.011786081828176975, -0.009838353842496872, -0.0017050930764526129, 0.0012480749282985926, -0.03358335420489311, 0.00765131413936615, -0.0003961516194976866, 0.0037159728817641735, 0.0006078340811654925, -0.00416800519451499, -0.0006759713287465274, 0.008276183158159256, -0.020567476749420166, 0.01737665943801403, 0.01350779365748167, 0.007571544032543898, -0.01580784097313881, -0.04387373849749565, -0.0012995933648198843, 0.025739260017871857, -0.011034910567104816, -0.02839827537536621, -0.022415492683649063, 0.018626397475600243, -0.0033320775255560875, 0.032280437648296356, 0.02664332464337349, 0.008495551533997059, 0.012145048938691616, 0.01700439862906933, -0.00047197507228702307, 0.017243709415197372, 0.016432709991931915, -0.029116209596395493, 0.03063184767961502, -0.021817214787006378, 0.0012447511544451118, 0.019211379811167717, -0.02805260382592678, -0.015728071331977844, 0.05759425461292267, -0.021272115409374237, 0.005962840281426907, -0.01753620058298111, 0.009240075014531612, -0.024183737114071846, 0.007079625967890024, -0.014824005775153637, -0.0306052565574646, 0.030153224244713783, 0.01632634922862053, 0.007564896252006292, -0.013707219623029232, -0.01927785575389862, 0.01203868817538023, -0.024622473865747452, 0.004164681304246187, -0.01719052903354168, -0.008901051245629787, -0.0018613102147355676, 0.07003844529390335, 0.01254390086978674, -0.006853609811514616, 0.021391771733760834, -0.016126923263072968, -0.0007777617429383099, -0.001563832862302661, -0.006478024180978537, 0.003278897376731038, -0.0063650161027908325, -0.015289333648979664, -0.014239022508263588, -0.016033858060836792, 0.003273911541327834, -0.007013150956481695, -0.019916018471121788, -0.003789095673710108, -0.008994116447865963, 0.01023055799305439, -0.006577737163752317, -0.006112409755587578, -0.016472594812512398, -0.0005226625362411141, 0.0043674311600625515, 0.008794690482318401, -0.0016469270922243595, 0.010948492214083672, 0.041879478842020035, 0.013614154420793056, -0.0022967238910496235, -0.001701769302599132, -0.016100332140922546, -0.01031032856553793, 0.0062287417240440845, 0.0006751404143869877, -0.005028861109167337, 0.010616115294396877, -0.017615972086787224, 0.014597989618778229, 0.014930366538465023, 0.022402197122573853, 0.013281777501106262, -0.004676541779190302, 0.004819463938474655, 0.03188158571720123, -0.006846962496638298, -0.015728071331977844, 0.03182840347290039, 0.022973885759711266, -0.02783988229930401, 0.025725966319441795, 0.018852412700653076, 0.006168913561850786, -0.012796507216989994, 0.015874316915869713, -0.009552509523928165, -0.01619339920580387, -0.015887612476944923, -0.0045502386055886745, -0.009632280096411705, 0.02370511367917061, -0.005131897982209921, 0.037571873515844345, 0.006394929718226194, -0.0035464605316519737, -0.0007644666475243866, -0.006803753320127726, 0.019344329833984375, -0.013826875947415829, 0.00826953537762165, -0.038582298904657364, -0.010110902599990368, -0.024981440976262093, -0.00575676653534174, -0.010177378542721272, -0.007053036242723465, 0.030897749587893486, -0.01056958269327879, 0.004440554417669773, -0.003995169419795275, -0.0017566115129739046, -0.002948182402178645, 0.0012040350120514631, -0.02617799863219261, 0.0007071316940709949, 0.011473647318780422, 0.01719052903354168, -0.007298994809389114, -0.04001816734671593, -0.007770969998091459, 0.013175416737794876, 0.0050055948086082935, 0.008881108835339546, 0.013826875947415829, -0.005431036930531263, 0.016898037865757942, 0.019663412123918533, 0.02456929348409176, -0.0015729733277112246, 0.013594212010502815, -0.018174365162849426, -0.035843513906002045, 0.008927641436457634, 0.017483020201325417, 0.026922522112727165, -0.004590123891830444, -0.006178884766995907, 0.0063184830360114574, -0.016804972663521767, 0.00816982239484787, 0.0013959825737401843, -0.010636057704687119, -0.005999401677399874, -0.013088999316096306, 0.02837168425321579, -0.012317884713411331, -0.01927785575389862, 0.013627449050545692, -0.012078573927283287, -0.02580573596060276, -0.00834930595010519, -0.01906513422727585, 0.016419414430856705, 0.023771589621901512, -0.015927497297525406, 0.02924915961921215, 0.014810711145401001, -0.01910501904785633, -0.01272338442504406, 0.017589380964636803, -0.0026656619738787413, 0.027813291177153587, -0.003003024496138096, -0.011546770110726357, -0.0015289332950487733, -0.009765231050550938, -0.021192345768213272, -0.0134213762357831, -0.009605689905583858, -0.03512558341026306, -0.018998658284544945, 0.016246579587459564, 0.022322427481412888, -0.0354180745780468, -0.0065012904815375805, -0.016472594812512398, -0.020727017894387245, -0.004244451876729727, -0.04350147768855095, 0.01926456019282341, -0.0070264460518956184, 0.0165390707552433, -0.010882017202675343, -0.012497368268668652, -0.0004379064484965056, 0.003456718986853957, -0.039619315415620804, -0.020607363432645798, 0.01228464674204588, 0.018307315185666084, 0.034460827708244324, 0.007864035665988922, 0.019224675372242928, 0.010203967802226543, 0.008183117024600506, 0.011068147607147694, 0.002482854761183262, -0.015781251713633537, -0.012204877100884914, 0.01998249441385269, 0.0009929757798090577, 0.007298994809389114, -0.00490255793556571, 0.0081764692440629, -0.007033093366771936, 0.005700262263417244, -0.0037060014437884092, -0.015728071331977844, -0.00013398940791375935, -0.007983691059052944, -0.002467897953465581, -0.012577138841152191, 0.0015588472597301006, 0.00010277714318362996, -0.03544466197490692, -0.015076612122356892, 0.03594987466931343, 0.006567765958607197, 0.01618010364472866, -0.008907699026167393, 0.022960590198636055, -0.006507938262075186, 0.01926456019282341, 0.03113706037402153, -0.0018397056264802814, 0.0003874267276842147, -0.011832614429295063, -0.01824083924293518, 0.0034966040402650833, 0.007431945763528347, -0.0015846064779907465, -0.01228464674204588, -0.034327875822782516, 0.007571544032543898, 0.004413964226841927, 0.02580573596060276, 0.0007195958169177175, -0.007398707792162895, 0.0028551167342811823, -0.021564608439803123, -0.0019660089164972305, -0.021737443283200264, -0.016738496720790863, -0.04552232846617699, -0.022947294637560844, 0.012164991348981857, -0.019703296944499016, 0.029887322336435318, -0.016073742881417274, -0.008395838551223278, 0.012224819511175156, -0.004503706004470587, 0.012444187887012959, -0.021285410970449448, 0.02494155615568161, 0.018506741151213646, 0.0012455821270123124, -0.020527591928839684, -0.008475609123706818, -0.018852412700653076, -0.026098227128386497, 0.006687421351671219, 0.029169389978051186, -0.016751792281866074, 0.022761164233088493, 0.024290097877383232, 0.003998493310064077, -0.02199004963040352, -0.03475331887602806, 0.026111522689461708, 0.016446005553007126, 1.9124650862067938e-05, -0.02357216365635395, -0.02099291980266571, 0.007930510677397251, 0.02423691749572754, -0.010097607970237732, 0.016219988465309143, -0.01701769232749939, -0.0029864057432860136, 0.00434748874977231, 0.01737665943801403, -0.006378311198204756, 0.015555234625935555, 0.013773695565760136, -0.012171639129519463, 0.0016726863104850054, -0.026377424597740173, -0.0017167262267321348, -0.0070264460518956184, 0.013919941149652004, 0.0016610531602054834, 0.01358756422996521, 0.01489048171788454, 0.017230413854122162, 0.0037060014437884092, 0.004653275478631258, 0.02697570249438286, 0.006218770053237677, 0.026284359395503998, -0.01926456019282341, 0.010237205773591995, 0.0035431368742138147, -0.015887612476944923, 0.014677760191261768, 0.012437540106475353, -0.010124198161065578, -0.007571544032543898, -0.005673672072589397, 0.01185920462012291, 0.0010436632437631488, -0.00485270144417882, 0.003722620429471135, -0.0003832720103673637, -0.02288082055747509, -0.005467598792165518, -0.004676541779190302, -0.021192345768213272, 0.0323336161673069, -0.029328929260373116, -0.02956824190914631, -0.01211845874786377, -0.00549751240760088, 0.0017682446632534266, -0.00511195557191968, 0.011240984313189983, 0.014970251359045506, 0.0157945454120636, -0.026018457487225533, 0.013840170577168465, 0.00511195557191968, 0.0005845677223987877, -0.02285422943532467, 0.018706167116761208, 0.005813270341604948, -0.03374289348721504, 0.004330869764089584, -0.0019543757662177086, -0.030365945771336555, -0.011267573572695255, 0.007285699713975191, -0.01891888864338398, 0.010203967802226543, -0.00032946851570159197, -0.012690146453678608, -0.013973121531307697, 0.00034754149965010583, 0.003822333412244916, -0.020434526726603508, 0.011546770110726357, -0.0035996409133076668, 0.012111810967326164, 0.015448874793946743, -0.010961787775158882, 0.003932017832994461, -0.00730564258992672, 0.01756279170513153, 0.0003155502490699291, -0.017296889796853065, 0.003391905454918742, -0.012537253089249134, 0.0015754661289975047, 0.015581824816763401, -0.021232230588793755, -0.003669440047815442, 0.013135531917214394, -0.014571399427950382, 0.011347344145178795, -0.02410396747291088, -0.003237350145354867, -0.006893495097756386, 0.00619217986240983, 0.005736823659390211, 0.033796075731515884, -0.009432854130864143, -0.016259873285889626, 0.006601003464311361, 0.006212122738361359, 0.003268925938755274, -0.013461261056363583, 0.01040339469909668, 0.060093726962804794, -0.008070109412074089, -0.05488206073641777, -0.01195891760289669, -0.01324853952974081, -0.007318937685340643, -0.005055451299995184, -0.02289411425590515, 0.020421231165528297, -0.009260018356144428, 0.02458258904516697, 0.010483164340257645, 0.007378765381872654, 0.026895930990576744, 0.006102438550442457, -0.003935341723263264, -0.008455665782094002, 0.028132373467087746, -0.006252008024603128, -0.018958773463964462, 0.0004495396569836885, -0.03132319077849388, -4.536424239631742e-05, 0.020075559616088867, 0.021591197699308395, 0.016459299251437187, 0.04453849419951439, -0.004752988461405039, -0.03142955154180527, -0.02754739113152027, 0.033290863037109375, 0.023266376927495003, -0.0031143708620220423, -0.006112409755587578, -0.04802180081605911, 0.005078717600554228, -0.024130556732416153, 0.017855282872915268, -0.0025991867296397686, -0.0003010087530128658, -0.017443135380744934, 0.011101385578513145, -0.0013411404797807336, -0.015435579232871532, 0.0015131453983485699, 0.008635149337351322, -0.005278144031763077, 0.0019161523086950183, 0.0011599950958043337, 0.001651081838645041, 0.01255719643086195, 0.0005363730597309768, -0.022654803469777107, -0.014651170000433922, -0.003056204877793789, -0.013102293945848942, -0.020195215940475464, 0.0306052565574646, -0.01963682286441326, -0.01306905597448349, 0.00667080283164978, -0.002683942671865225, -0.0029880674555897713, -0.0165390707552433, -0.02391783520579338, -0.007704494521021843, -0.03265269845724106, 0.016206692904233932, -0.007817503064870834, -0.018692871555685997, -0.017589380964636803, -0.005657053552567959, -0.009512624703347683, -0.02821214497089386, -0.014066186733543873, -0.0038190095219761133, -0.02975437231361866, -0.010975082404911518, 0.006069200579077005, -0.04094882309436798, -0.008555378764867783, -0.043581247329711914, -0.010642705485224724, -0.006168913561850786, 0.015355808660387993, 0.21038122475147247, 0.007591486442834139, 0.008974174037575722, 0.0427037738263607, -0.0027238279581069946, 0.014318793080747128, 0.004925824236124754, 0.005956192500889301, -0.021737443283200264, 0.00563711067661643, -0.004573504906147718, 0.008429075591266155, 0.005979458801448345, -0.0064481100998818874, 0.012045335955917835, -0.01958364248275757, -0.015714775770902634, -0.007624723948538303, -0.01503672730177641, -0.028690766543149948, -0.026018457487225533, 0.0003334154898766428, 0.004899234045296907, -0.023332852870225906, 0.013600858859717846, 0.0051418691873550415, -0.011260926723480225, 0.019503870978951454, -0.007717789616435766, 0.0030096720438450575, -6.621569627895951e-05, -0.007877330295741558, 0.012098516337573528, 0.00025530694983899593, 0.008681682869791985, -0.008468961343169212, 0.010203967802226543, -0.008189764805138111, 0.015129792504012585, 0.006853609811514616, 0.01825413480401039, 0.021724147722125053, 0.0037558579351752996, -0.005643758457154036, -0.005238258745521307, 0.020261690020561218, -0.0037525342777371407, -0.0017167262267321348, -0.020181920379400253, -0.010576230473816395, -0.018094593659043312, -0.03539148345589638, 0.036720991134643555, 0.029648011550307274, 0.026789572089910507, 0.02339932695031166, 0.03977885842323303, 0.003971903119236231, -0.006016020197421312, 0.02530052326619625, 0.019211379811167717, 0.004214537795633078, 0.009186895564198494, 0.00182641064748168, -0.0033968910574913025, 0.03360994532704353, -0.002660676371306181, -0.026523670181632042, 0.013135531917214394, -0.014079482294619083, 0.0102438535541296, 0.0040948824025690556, -0.006514585576951504, 0.009047296829521656, -0.027467619627714157, -0.038475941866636276, 0.01688474230468273, 0.012809802778065205, 0.013893350958824158, 0.025872211903333664, -0.006265303120017052, -0.030206404626369476, -0.005979458801448345, -0.0008841223316267133, 0.012145048938691616, -0.026723096147179604, 0.007398707792162895, -0.007963748648762703, 0.00420789048075676, 0.0010552963940426707, -0.0041015297174453735, 0.00038015597965568304, -0.011121327988803387, 0.006212122738361359, -0.010270443744957447, 0.003789095673710108, -0.0014217417920008302, 0.012630319222807884, -0.002315004589036107, 0.0038688660133630037, -0.013108941726386547, -0.012410949915647507, -0.0099646570160985, -0.0009913138346746564, 0.004487087018787861, -0.005414418410509825, -0.013042465783655643, -0.0020557506941258907, 0.01632634922862053, -0.031376373022794724, 0.027055472135543823, -0.032413385808467865, 0.0074718305841088295, -0.010788951069116592, 0.008468961343169212, -0.011413820087909698, 0.02978096343576908, -0.017935052514076233, 0.008369248360395432, -0.008050166070461273, -0.006188856437802315, -0.006617622449994087, 0.018440265208482742, -0.0006435646209865808, -0.005298086442053318, -0.014225727878510952, -0.007917216047644615, 0.0051418691873550415, -0.011925680562853813, -0.04435236379504204, 0.031030699610710144, 0.004487087018787861, -0.004184624180197716, -0.005952868610620499, -0.027999423444271088, -0.007817503064870834, 0.008409133180975914, 0.0028418218716979027, 0.010715828277170658, -0.0011433762265369296, -0.006331778131425381, -0.00407826341688633, 0.012922810390591621, -0.00486267264932394, 0.007485125679522753, -0.019477281719446182, -0.0153026282787323, -0.024688949808478355, -0.015781251713633537, -0.006291893310844898, -0.01700439862906933, -0.009958009235560894, 0.00425774697214365, -0.03555102273821831, 0.02096632868051529, -0.005926278419792652, -0.026523670181632042, -0.018719462677836418, -0.0315890908241272, 0.018001528456807137, 0.0022119677159935236, -1.6852543922141194e-05, 0.026523670181632042, -0.005118602886795998, -0.011141270399093628, -0.016087038442492485, -0.16985784471035004, 0.01117450837045908, -0.0006186363752931356, -0.025553129613399506, 0.0004856856248807162, -0.004234480671584606, 0.009991247206926346, 0.00963892787694931, -0.0273346696048975, 0.004523648414760828, 0.008136584423482418, 0.034141745418310165, -0.019384216517210007, -0.021803919225931168, -0.0034101861529052258, -0.023492394015192986, -0.04658593609929085, 0.015462169423699379, 0.032280437648296356, 0.007066330872476101, -0.0034500714391469955, -0.018613101914525032, 0.005055451299995184, 0.014677760191261768, 0.014185842126607895, -0.005876421928405762, 0.01322859711945057, 0.011706311255693436, -0.02047441154718399, 0.0021321973763406277, -0.004091558512300253, 0.0033486965112388134, 0.03868865966796875, 0.02169755846261978, 0.0030744855757802725, -0.0026457195635885, 0.008894403465092182, 0.019650116562843323, -0.01667202077805996, 0.035710565745830536, 0.008302772417664528, 0.025194162502884865, 0.013580916449427605, -0.00014146788453217596, 0.002423027064651251, 0.029993683099746704, 0.02164437808096409, -0.002507783006876707, 0.008196412585675716, -0.02030157670378685, 0.019304445013403893, -0.011227688752114773, 0.02632424421608448, -0.008143232204020023, 0.01805470883846283, 0.017443135380744934, 0.012470778077840805, 0.011832614429295063, 0.021524721756577492, -0.011400524526834488, -0.005052127409726381, -0.04456508159637451, 0.023678524419665337, -0.03142955154180527, -0.02479531057178974, -0.010210615582764149, -0.004733046051114798, 0.002911621006205678, -0.021923575550317764, 0.004610066302120686, 0.0036794112529605627, 0.014185842126607895, 0.00602599186822772, -0.006245360244065523, 0.02682945691049099, 0.010842131450772285, -0.04661252349615097, 0.0017117406241595745, -0.008621854707598686, -0.002625776920467615, 0.00174165447242558, 0.05727517232298851, -0.008841223083436489, -0.01685815118253231, 0.00627195043489337, 0.0209796242415905, -0.005713557358831167, 0.006913437973707914, 8.994947711471468e-05, 0.009326493367552757, 0.007252462208271027, -0.020594067871570587, -0.013973121531307697, -0.009791821241378784, -0.006388282403349876, 0.026005161926150322, 0.013348252512514591, -0.01720382459461689, -0.007112863939255476, -0.020873263478279114, 0.006109085865318775, -0.00981841143220663, 0.007618076633661985, 0.054243896156549454, 0.018453560769557953, 0.027733521535992622, 0.005849831737577915, 0.008369248360395432, 0.026204587891697884, -0.00494244322180748, -0.009991247206926346, 0.009499329142272472, 0.019809657707810402, 0.017496315762400627, -0.0012140063336119056, 0.01700439862906933, 0.027786701917648315, -0.04113495349884033, 0.03852912038564682, -0.01100832037627697, 0.025327112525701523, 0.001523947692476213, -0.018626397475600243, 0.030232995748519897, -0.03605623543262482, -0.0077244373969733715, -0.10210615396499634, -0.010875369422137737, 0.02322649210691452, 0.016406118869781494, -0.004713103175163269, 0.0159806776791811, -0.01314882654696703, 0.02839827537536621, 0.004320898558944464, 0.02011544443666935, 0.013354900293052197, -0.02201664075255394, -0.006119057070463896, 0.0008438216173090041, 0.015369104221463203, -0.004025083035230637, -0.0012588772224262357, -0.0165390707552433, -0.014730940572917461, 0.008262887597084045, 0.0026556907687336206, 0.0062320651486516, -0.01998249441385269, -0.027600571513175964, 0.0015979015734046698, -0.03589669615030289, -0.04344829544425011, 0.04105518385767937, 0.010941844433546066, 0.007478478364646435, -0.004433906637132168, -0.009645574726164341, 0.0019161523086950183, -0.0231999009847641, 0.006258655339479446, -0.005723528563976288, -0.026723096147179604, 0.018652986735105515, 0.04283672571182251, -0.034673549234867096, 0.011300811544060707, -0.013786990195512772, 0.0005214161355979741, -0.008927641436457634, 0.008615206927061081, -0.002875059377402067, -0.00980511587113142, 0.004713103175163269, 0.0031077233143150806, -0.0030911043286323547, -0.008708272129297256, -0.0019128285348415375, -0.025712670758366585, -0.018347200006246567, 0.00022310794156510383, -0.014385269023478031, 0.006245360244065523, 0.01322859711945057, 0.004759635776281357, -0.006079171784222126, 0.0006809569895267487, 0.0034101861529052258, 0.002428012667223811, -0.0022202772088348866, 0.01445174403488636, -0.007059683557599783, -0.001563832862302661, -0.0019360949518159032, 0.012238114140927792, -0.003902103751897812, -0.011247631162405014, 0.024875080212950706, -0.012975990772247314, 0.020727017894387245, -0.012995933182537556, 0.00589636480435729, -0.013999711722135544, -0.005357914138585329, 0.03299837186932564, -0.013786990195512772, -0.008050166070461273, -0.016738496720790863, -0.005893040914088488, -0.023266376927495003, -0.003278897376731038, 0.016618840396404266, 0.012550548650324345, 0.0025327112525701523, -0.0020341461058706045, -0.023439213633537292, -0.015129792504012585, 0.029807552695274353, 0.01926456019282341, -0.02341262251138687, -0.020421231165528297, -0.006285245530307293, 0.004141415003687143, -0.013334957882761955, -0.005746795330196619, 0.015169678255915642, -0.029887322336435318, -0.007079625967890024, -0.07381424307823181, 0.015369104221463203, 0.017124053090810776, -0.030817978084087372, -0.024516113102436066, 0.005610520485788584, 0.03169545158743858, 0.003938665147870779, 0.005334647838026285, 0.0032124218996614218, -0.027387849986553192, 0.005384504329413176, 0.00014188335626386106, -0.010017837397754192, -0.014903776347637177, -0.03337063267827034, 0.032280437648296356, 0.00032697568531148136, 0.01841367594897747, 0.003961931448429823, -0.0038954562041908503, 0.0175229050219059, 0.01487718615680933, 0.020009083673357964, -0.020859969779849052, 0.029328929260373116, -0.0014740912010893226, 0.016419414430856705, -0.023505687713623047, 0.004540267400443554, 0.028291914612054825, -0.025925392284989357, -0.010815541259944439, 0.000786071177572012, -0.010855427011847496, -0.043926920741796494, 0.025539834052324295, -0.0020707075018435717, 0.013693924993276596, -0.04725068807601929, -0.014544809237122536, -0.022123001515865326, -0.006109085865318775, -0.00446714460849762, -0.011054852977395058, 0.005942897405475378, 0.0043076034635305405, -0.003709325334057212, 0.006298540625721216, 0.008183117024600506, 0.03733256459236145, 0.01911831460893154, -0.008575322106480598, -0.020740313455462456, -0.0015671566361561418, -0.00014842701784800738, 0.008721567690372467, -0.02371840924024582, 0.003958608023822308, -0.011766139417886734, 0.03164227306842804, -0.002032484160736203, 0.01443844847381115, -0.015010137110948563, 0.02821214497089386, 0.022495262324810028, -0.001882914686575532, -0.00040570745477452874, -0.012610376812517643, -0.008920993655920029, -0.034514009952545166, -0.008914345875382423, -0.001594577799551189, 0.010423337109386921, 0.002592539181932807, 0.030817978084087372, 0.008003633469343185, -0.010263795964419842, -0.021218935027718544, 0.04435236379504204, 0.0021903631277382374, -0.011619893833994865, -0.028956668451428413, 0.011453704908490181, -0.011260926723480225, 0.0030611904803663492, -0.0107357706874609, 0.003282221034169197, -0.008562026545405388, 0.0024811928160488605, -0.025526538491249084, 0.0010012851562350988, -0.005268172360956669, -0.014411858282983303, 0.003423481248319149, 0.00912041962146759, 0.003902103751897812, -0.01056958269327879, -0.0012804816942662, 0.020048970356583595, 0.023505687713623047, -0.006900142878293991, -0.0029082971159368753, -0.0192512646317482, -0.010769008658826351, 0.003333739470690489, -0.008375896140933037, -0.03267928957939148, 0.0019460662733763456, 0.02940870076417923, 0.010483164340257645, -0.030552076175808907, -0.01177943404763937, -0.003126004012301564, -0.008382542990148067, 0.020328165963292122, -0.017310185357928276, -0.027600571513175964, -0.007883978076279163, -0.004510353319346905, 0.017283594235777855, 0.025061212480068207, 0.0010104255052283406, -0.004663246683776379, 0.044113051146268845, -0.00451700109988451, -0.0015447211917489767, -0.01636623404920101, 0.0006165590020827949, -0.004935795906931162, 0.0233860332518816, -0.007285699713975191, -0.00989818200469017, -0.00903400219976902, -0.018692871555685997, -0.0015438903355970979, 0.007059683557599783, 0.028956668451428413, -0.011327401734888554, 0.06546493619680405, 0.012942752800881863, -0.0008010281017050147, 0.0244363434612751, -0.016818266361951828, 0.0036760875955224037, -0.0032855449244379997, -0.0024762072134763002, -0.044299181550741196, 0.005068746395409107, 0.02100621536374092, -0.008741510100662708, -0.010742418467998505, -0.013720515184104443, -0.0313497819006443, -0.008236297406256199, 0.008741510100662708, 0.039672497659921646, -0.0244363434612751, 0.02096632868051529, 0.020540887489914894, 0.0057800328359007835, 0.020208509638905525, -0.011500237509608269, -0.005470922216773033, -0.020700428634881973, 0.025925392284989357, 0.008309420198202133, 0.0006414872477762401, -0.037917546927928925, -0.006348397117108107, -0.012776564806699753, -0.026377424597740173, -0.022960590198636055, -0.002394774928689003, 0.002241881564259529, -0.005746795330196619, 0.011234336532652378, -0.015874316915869713, 0.003971903119236231, -0.002436322160065174, 0.019822953268885612, -0.04509688541293144, -0.009731993079185486, -0.0022219389211386442, 3.3808955777203664e-05, -0.01023055799305439, -0.00037849409272894263, -0.017748922109603882]} +{"id": "test:10000003", "text": "\"How many backlinks per day for new site?\\nDiscussion in 'Black Hat SEO' started by Omoplata, Dec 3, 2010.\\n1) for a newly created site, what's the max # backlinks per day I should do to be safe?\\n2) how long do I have to let my site age before I can start making more blinks?\\nI did about 6000 forum profiles every 24 hours for 10 days for one of my sites which had a brand new domain.\\nThere is three backlinks for every of these forum profile so thats 18 000 backlinks every 24 hours and nothing happened in terms of being penalized or sandboxed. This is now maybe 3 months ago and the site is ranking on first page for a lot of my targeted keywords.\\nbuild more you can in starting but do manual submission and not spammy type means manual + relevant to the post.. then after 1 month you can make a big blast..\\nWow, dude, you built 18k backlinks a day on a brand new site? How quickly did you rank up? What kind of competition/searches did those keywords have?\"", "vector_field": [-0.010900961235165596, -0.007271698676049709, 0.0111446687951684, -0.0365692675113678, -0.0022230057511478662, -0.007745940238237381, -0.038071032613515854, -0.007910607382655144, -0.007100445218384266, -0.008707595989108086, 0.02626769430935383, 0.008220180869102478, -0.0035996227525174618, -0.0016532576410099864, 0.008595622144639492, 0.011164428666234016, 0.027479644864797592, 0.012290751561522484, 0.008938129991292953, -0.00025893899146467447, -0.017836740240454674, -0.01767865940928459, -0.03185320273041725, -0.0016474942676723003, -0.03261725604534149, 0.01583438739180565, 0.002624793443828821, -0.0090830372646451, -0.008266287855803967, -0.013739822432398796, 0.00869442243129015, -0.0008019287488423288, -0.00548012088984251, 0.015057158656418324, -0.031879547983407974, 0.023896489292383194, -0.013818862847983837, -0.014266757294535637, 0.01275181956589222, 0.0016804276965558529, 0.0025638665538281202, 0.028559859842061996, 0.010479413904249668, -0.0147014781832695, -0.00336579536087811, -0.0016549043357372284, 0.005226533394306898, -0.027664070948958397, -0.007904020138084888, 0.020405545830726624, -0.0021340856328606606, 0.01257397886365652, -0.027374258264899254, 0.03957279771566391, 0.014240410178899765, -0.003767583053559065, 0.007245352026075125, 0.040152423083782196, -0.0028882608748972416, 0.00314184813760221, 0.015887081623077393, -0.009649491868913174, -0.002794400556012988, -0.0039816503413021564, -0.02604374662041664, -0.004175957292318344, -0.014372143894433975, 0.03596329316496849, -0.005559161305427551, -0.0029376610182225704, 0.008180661126971245, 0.031774163246154785, 0.0017306511290371418, -0.00838484801352024, 0.013818862847983837, 0.006033402401953936, -0.0017158311093226075, -0.03825546056032181, -0.010980001650750637, -0.00592801533639431, 0.011717710644006729, 0.005908255465328693, -0.018587620928883553, 0.04007338359951973, 0.012587152421474457, -0.009155490435659885, -0.0036984228063374758, 0.011118322610855103, -0.015123025514185429, -0.0033213351853191853, 0.001985885202884674, 0.012554218992590904, 0.038861434906721115, -0.020142078399658203, -0.0187061820179224, 0.02708444371819496, -0.028138313442468643, 0.020063038915395737, -0.0052133603021502495, 0.003909196704626083, -0.006003762129694223, -0.0037313562352210283, -0.017718179151415825, -0.014372143894433975, 0.009649491868913174, 0.0050322264432907104, 0.0014795338502153754, -0.01507033221423626, 0.005341800395399332, -0.012047044932842255, -0.026636548340320587, 0.05061207711696625, 0.02101152203977108, -0.03219570964574814, -0.005542694590985775, -0.019825918599963188, -0.0077788736671209335, 0.005737001542001963, 0.012837446294724941, -0.006474710069596767, 0.02381744794547558, 0.0042648776434361935, 0.00966266542673111, 0.0005936248926445842, 0.038835085928440094, -0.013976942747831345, -0.01283085998147726, -0.0011263228952884674, 0.0051343198865652084, -0.008029167540371418, 0.0054834140464663506, 0.02035285346210003, 0.02099834755063057, -0.007581273093819618, -0.0028388607315719128, 0.04070570692420006, -0.00025235232897102833, -0.010103972628712654, -0.020405545830726624, -0.026122787967324257, 0.01741519197821617, 0.03649022802710533, 0.0187061820179224, -0.0008850856102071702, 0.005154080223292112, 0.03198493644595146, -0.00934650469571352, -0.0015388140454888344, 0.006198069546371698, -0.024015048518776894, 0.006672310642898083, 0.002252645790576935, -0.0028421541210263968, 0.028902368620038033, -0.005407667253166437, 0.01777087338268757, 0.007785459980368614, 0.025082090869545937, -0.01501763891428709, -0.011434483341872692, 0.004518465138971806, 0.024331210181117058, -0.002446952974423766, 0.024015048518776894, 0.004462478216737509, 0.022579152137041092, 0.027005404233932495, -0.01122370921075344, -0.021235467866063118, -0.016677483916282654, -0.01740201935172081, -0.0011082094861194491, -0.015755347907543182, 0.01611102931201458, 0.010914134792983532, 0.007344152312725782, 0.008174074813723564, 0.003905903548002243, -0.006033402401953936, -0.012402725405991077, -0.005088213365525007, -0.016901429742574692, 0.01753375120460987, 0.03388190269470215, 0.0023563860449939966, -0.004261584486812353, 0.016269108280539513, 0.00209291884675622, -0.0034547157119959593, -0.014095502905547619, 0.0013774402905255556, 0.012784752994775772, -0.03567348048090935, 0.0021423189900815487, -0.6314785480499268, -0.0055920942686498165, -0.0024222529027611017, 0.0051244399510324, 0.0002745823876466602, 0.007370498962700367, 0.008299221284687519, -0.0013091034488752484, -0.015399666503071785, 0.028928715735673904, -0.0007492352742701769, -0.003408608725294471, 0.014253583736717701, 0.005542694590985775, -0.009287224151194096, -0.003793929936364293, 0.04623851925134659, -0.018324153497815132, 0.0007809337112121284, -0.016045162454247475, -0.006201362702995539, -0.024660544469952583, -0.018521754071116447, 0.004963066428899765, -0.029429303482174873, 0.02383062243461609, 0.013568568974733353, -0.016901429742574692, -0.01831098087131977, 0.040415890514850616, -0.00548012088984251, 0.017955299466848373, 0.0077722868882119656, 0.0010769227519631386, 0.042576324194669724, -0.05543353036046028, 0.006013642065227032, 0.02460785023868084, 0.004014583770185709, 0.02950834296643734, -0.002914607524871826, -0.014833211898803711, 0.05335213989019394, 0.013028460554778576, -0.008615382015705109, -0.0037017161957919598, 0.01986543834209442, 0.00030051745125092566, 0.0053022801876068115, -0.016598442569375038, 0.009682425297796726, 0.010927308350801468, 0.0071136183105409145, -0.009036930277943611, -0.010663840919733047, -0.008733943104743958, 0.017560098320245743, 0.005163960158824921, 0.011151256039738655, -0.012060217559337616, 0.013792515732347965, 0.0181660745292902, -0.029719116166234016, -0.03670100122690201, -0.030746640637516975, 0.008714182302355766, -0.01857444830238819, 0.012587152421474457, 0.014187716878950596, 0.009287224151194096, 0.012185364961624146, 0.018745701760053635, -0.01673017628490925, -0.017942126840353012, 0.00104645942337811, 0.01217219140380621, 0.030114317312836647, 0.0015363439451903105, -0.01596612110733986, 0.013871556147933006, 0.019298983737826347, 0.0056711346842348576, -0.013087740167975426, -0.02008938603103161, 0.014253583736717701, 0.0015799807151779532, -0.030351439490914345, -0.008898609317839146, 0.0017586445901542902, -0.015426013618707657, 0.024410249665379524, 0.02990354411303997, 0.006741470657289028, -0.02667606994509697, -0.0022740524727851152, 0.0222893375903368, 0.02679462917149067, -0.017731351777911186, 0.007475886028259993, -0.020431892946362495, -0.019417542964220047, -0.019760051742196083, 0.007238765247166157, 0.026438947767019272, 0.017810393124818802, -0.005539400968700647, -0.0213935486972332, -0.017731351777911186, 0.02705809660255909, -0.009899785742163658, 0.034224409610033035, -0.0031072681304067373, 0.013239234685897827, -0.013212887570261955, -0.005305573809891939, -0.016045162454247475, -0.007910607382655144, -0.013120673596858978, 0.002028698567301035, -0.010617733933031559, 0.011836270801723003, 0.01753375120460987, -0.014293103478848934, -0.03222205862402916, 0.02265819162130356, 0.02858620695769787, -0.0037544097285717726, 0.00038120432873256505, 0.012060217559337616, -0.03135261312127113, -0.0199576523154974, 0.009972238913178444, 0.012936246581375599, -0.015821214765310287, 0.015742173418402672, 0.012376378290355206, 0.03245917707681656, -0.024199476465582848, 0.027110790833830833, -0.01767865940928459, -0.01180333737283945, -0.007074098568409681, 0.01741519197821617, -0.009465064853429794, -0.002140672178938985, -0.01372664887458086, -0.0013494468294084072, 0.002160432282835245, -0.011203949339687824, 0.02758503146469593, 0.020286986604332924, -0.0018492114031687379, -0.013232647441327572, 0.010966828092932701, -0.034092675894498825, -0.00877346284687519, -0.019694184884428978, -0.042207468301057816, -0.024555157870054245, 0.010545280762016773, -0.005717241670936346, 0.013028460554778576, 0.0014408370479941368, -0.014991291798651218, -0.010986588895320892, -0.017481058835983276, 0.00023382727522403002, 0.005921428557485342, 0.012712299823760986, -0.05048034340143204, -0.02475275658071041, -0.010472827591001987, -0.01651940308511257, 0.009774638339877129, -0.001692777732387185, -0.03754409775137901, -0.027031749486923218, 0.002152198925614357, 0.002372852759435773, 0.006902844645082951, 0.010189599357545376, 0.009227943606674671, -0.004699598997831345, 0.010084212757647038, 0.017349325120449066, -0.00029393075965344906, -0.007561512757092714, 0.019536104053258896, -0.02230251207947731, 0.030246051028370857, 0.0005895082140341401, 0.0059609487652778625, 0.012534459121525288, 0.023435421288013458, 0.01180333737283945, -0.02462102472782135, 0.0009377790847793221, 0.012817686423659325, -0.007732766680419445, 0.016216415911912918, 0.043630193918943405, -0.007080684881657362, 0.007554926443845034, -0.02592518739402294, -0.02964007668197155, -0.0074890595860779285, 0.005911548621952534, -0.002553986618295312, 0.01435897033661604, 0.0028240405954420567, 0.0008068687748163939, -0.016190068796277046, 0.016334975138306618, -0.026623375713825226, -0.00792378094047308, 0.014675131067633629, -0.017454711720347404, 0.020010344684123993, -0.01354222185909748, 0.028454473242163658, 0.014912252314388752, 0.030061624944210052, 0.024383902549743652, 0.0006475533591583371, -0.0015338739613071084, 0.0038235699757933617, 0.016677483916282654, 0.017046337947249413, 0.008160901255905628, -0.044130779802799225, 0.006211242638528347, -0.005674427840858698, -0.009280637837946415, -0.000929545727558434, 0.050190530717372894, 0.028164660558104515, 0.03340765833854675, -0.023106086999177933, 0.011493762955069542, -0.003948716912418604, 0.011572803370654583, 0.014688304625451565, 0.03904586285352707, -0.012547632679343224, 0.04268170893192291, 0.033697474747896194, 0.025753933936357498, -0.001684544375166297, -0.010650667361915112, 0.0013848502421751618, -0.004492118488997221, 0.013133847154676914, 0.005015759728848934, -0.01377934217453003, 0.0042352378368377686, -0.006296869833022356, 0.004106797277927399, 0.012863793410360813, 0.002036931924521923, -0.011157842352986336, 0.02177557721734047, 0.02268453873693943, 0.01634814962744713, 0.006543870083987713, 0.006362736690789461, 0.033697474747896194, -0.015136199072003365, -0.0062507628463208675, 0.019825918599963188, -0.01713855192065239, -0.029192183166742325, -0.0032439418137073517, 0.02705809660255909, -0.011500350199639797, 0.025174304842948914, -0.005424133967608213, 0.004330744501203299, -0.022710885852575302, 0.02447611652314663, 0.02758503146469593, -0.04468406364321709, -0.0345405712723732, 0.015649961307644844, 0.010525520890951157, -0.022078564390540123, -0.013390728272497654, -0.01634814962744713, -0.0064681232906877995, 0.02397552877664566, 0.027743112295866013, -0.021551629528403282, 0.024080915376544, -0.017718179151415825, -0.028691593557596207, 0.010255466215312481, -0.027901193127036095, 0.04060031846165657, -0.0051244399510324, 0.008523168973624706, -0.04976898059248924, 0.026056921109557152, -0.011553043499588966, -0.01688825711607933, -0.0345405712723732, 0.026452122256159782, 0.011776990257203579, -0.0004310161166358739, -0.012014111503958702, -0.0007965770782902837, -0.01763913966715336, -0.008556102402508259, -0.012033871375024319, -0.016782870516180992, 0.021867789328098297, 0.02679462917149067, 0.006527403369545937, -0.012191951274871826, -0.010933894664049149, 0.022605499252676964, -0.00023691478418186307, -0.0044394247233867645, -0.022328857332468033, -0.0018920248840004206, 0.03066759929060936, 0.051507867872714996, -0.013515874743461609, 0.008332154713571072, 0.030509518459439278, -0.02821735292673111, 0.012218298390507698, -0.022645018994808197, -0.008924956433475018, 0.03303880617022514, -0.0014161369763314724, -0.0016993643948808312, 0.01520206592977047, -0.02048458717763424, -0.011230295524001122, 0.03335496783256531, -0.02176240272819996, 0.01315360702574253, -0.023501288145780563, -0.003540342440828681, 0.007205831818282604, -0.010670427232980728, 0.03282803297042847, 0.0010308159980922937, 0.05013783648610115, 0.021815096959471703, -0.003196188248693943, 0.013884729705750942, 0.016058335080742836, 0.02410726249217987, -0.01377934217453003, -0.006718417629599571, 0.019114555791020393, 0.02149893529713154, 0.024673717096447945, -0.01282427366822958, 0.0051046800799667835, 0.003431662218645215, -0.005131026729941368, 0.02513478510081768, -0.018363675102591515, -0.0016664309659972787, 0.03132626786828041, -0.020642666146159172, -0.001742177875712514, 0.012758406810462475, -0.01919359713792801, -0.011500350199639797, 0.02603057399392128, -0.012600325979292393, -0.02708444371819496, 0.012659606523811817, 0.01449070405215025, -0.027874846011400223, -0.00139308359939605, 0.023132434114813805, -0.006517523434013128, 0.004087037406861782, -0.004485531710088253, -0.03778121620416641, -0.0057864016853272915, -0.012870379723608494, -0.028691593557596207, 0.03359208628535271, -0.0014614204410463572, -0.0191804226487875, -0.03530462458729744, -0.00477205216884613, 0.012356618419289589, -0.017705006524920464, -0.0029854143504053354, -0.014135023579001427, -0.0147014781832695, -0.017441539093852043, 0.016414016485214233, 0.02023429237306118, 0.0173888448625803, 0.012850619852542877, 0.013061393983662128, 0.0025325799360871315, 0.021736055612564087, -0.010966828092932701, -0.033302273601293564, -0.02771676518023014, -0.022197123616933823, 0.0063528562895953655, 0.02811196632683277, 0.0006895434344187379, 0.01185603067278862, -0.010815334506332874, -0.012501525692641735, -0.0012061864836141467, 0.017428364604711533, 0.0051046800799667835, -0.01983909122645855, 0.016031987965106964, 0.00047959291259758174, 0.004531638231128454, 0.02023429237306118, 0.027901193127036095, -0.008944716304540634, -0.004581038374453783, -0.029192183166742325, -0.007021404802799225, -0.023277340456843376, -0.00012411782518029213, 0.002249352401122451, 0.021841444075107574, -0.003988237120211124, -0.005536107812076807, -0.018231941387057304, 0.0016120908549055457, -0.0031665482092648745, 0.008095034398138523, -0.004758879076689482, -0.0009221357177011669, 0.038966819643974304, 0.012811100110411644, 0.025977879762649536, 0.024291690438985825, 0.01012373249977827, -0.0011880730744451284, -0.016018815338611603, 0.009188423864543438, 0.0037609965074807405, 0.01132909581065178, 0.0064450702629983425, -0.002435426227748394, -0.028428126126527786, 0.010677014477550983, 0.011236882768571377, 0.006102562416344881, 0.007212418597191572, -0.026201827451586723, -0.04268170893192291, -0.006645963992923498, -0.02798023261129856, -0.007008231710642576, -0.006777697708457708, -0.025543158873915672, -0.003718182910233736, -0.036911774426698685, -0.011098561808466911, 0.007752526551485062, -0.026649722829461098, 0.024897664785385132, -0.021591149270534515, -0.009017170406877995, -0.004600798711180687, 1.9850103853968903e-05, 0.03725428506731987, -0.009096209891140461, 0.004607385490089655, -0.029561037197709084, 0.00010003525676438585, -0.005002586171030998, -0.02871794067323208, -0.009497998282313347, 0.005621734540909529, 0.020286986604332924, 0.021854616701602936, 0.03978357091546059, -0.030746640637516975, -0.004511878360062838, 0.016717003658413887, 0.024278515949845314, -0.022249817848205566, -0.014780518598854542, 0.0004310161166358739, -0.007963300682604313, -0.002789460588246584, 0.0015174072468653321, 0.04315595328807831, 0.006494469940662384, 0.006629497278481722, 0.01354222185909748, 0.03385555371642113, 0.0053483871743083, -0.005075039807707071, -0.03456691652536392, -0.009241117164492607, 0.0021834857761859894, -0.009418957866728306, -0.007067511789500713, -0.009491411037743092, -0.033065151423215866, -0.013950596563518047, 0.02874428778886795, -0.0035304625052958727, 0.0042714644223451614, 5.0943883252330124e-05, 0.0061848959885537624, -0.019154075533151627, 0.006313336547464132, 0.014740997925400734, -0.0013659135438501835, -0.008450714871287346, -0.019430717453360558, -0.03222205862402916, 0.013515874743461609, 0.00966266542673111, 0.029982583597302437, 0.014187716878950596, 0.004027757328003645, 0.011651843786239624, -0.021604321897029877, -0.001332980114966631, 0.0005870381719432771, -0.023672541603446007, -0.0024123729672282934, -0.0012506465427577496, 0.011427896097302437, -0.031010106205940247, -0.009247704409062862, 0.0027565271593630314, -0.01051234733313322, -0.011658430099487305, -0.009899785742163658, 0.02798023261129856, -0.006491176784038544, -0.02603057399392128, 0.02476593106985092, -0.002152198925614357, 0.010808748193085194, 0.019522931426763535, 0.017204418778419495, 0.03772852569818497, 0.027795804664492607, -0.003370735328644514, 0.006201362702995539, 0.0031237348448485136, -0.0052067735232412815, 0.021973177790641785, 0.026135960593819618, -0.03577886521816254, 0.006883084308356047, 0.0071333786472678185, 0.000987179228104651, -0.0030018810648471117, -0.03282803297042847, 0.03509385138750076, 0.000812220445368439, -0.012600325979292393, -0.014964945614337921, -0.0034119021147489548, -0.005266053602099419, -0.011217121966183186, -0.0018179246690124273, 0.022513285279273987, 0.035752519965171814, -0.01327216811478138, -0.017849912866950035, 0.01417454332113266, 0.012191951274871826, 0.0014861205127090216, 0.005183720029890537, 0.004867559298872948, -0.017072685062885284, -0.034198060631752014, 0.011836270801723003, 0.03211667016148567, -0.017217591404914856, 0.024423424154520035, -0.004646905232220888, 0.00731121888384223, 0.00805551465600729, 0.01970735751092434, -0.0190618634223938, -0.025991054251790047, -0.020550454035401344, 0.02884967438876629, -0.00070148182567209, 0.0025161132216453552, -0.013752995990216732, -0.014200890436768532, -0.03464595600962639, -0.020985174924135208, -0.015254759229719639, -0.008022581227123737, -0.03620041534304619, -0.016295455396175385, 0.037833910435438156, 0.012646432965993881, -0.007587859872728586, 0.016690656542778015, -0.0027729938738048077, -0.016414016485214233, 0.013173367828130722, 0.008068687282502651, 0.021090561524033546, -0.03712255135178566, -0.0034415421541780233, -0.004360384773463011, -0.0007990470621734858, 0.015623614192008972, -0.0006706067360937595, -0.015913428738713264, 0.022724058479070663, 0.026083268225193024, -0.00583580182865262, 0.006296869833022356, 0.01051234733313322, 0.008398021571338177, -0.0024140195455402136, 0.0024288396816700697, -0.009919545613229275, -0.014398491010069847, -0.01057162694633007, 0.00992613285779953, -0.023751581087708473, -0.0188247412443161, 0.0021851323544979095, -0.0030776280909776688, 0.01082192175090313, -0.03480403870344162, 0.03000893071293831, 0.0049235462211072445, 0.008127967827022076, -0.00037976348539814353, -0.009241117164492607, -0.00650764349848032, -0.017573272809386253, -0.009616558440029621, 0.014398491010069847, -0.006869911216199398, 0.009702185168862343, -0.015373320318758488, 0.026649722829461098, 0.025991054251790047, -0.01879839599132538, -0.012547632679343224, -0.014266757294535637, 0.01341048814356327, -0.023132434114813805, -0.029034102335572243, -0.05092823877930641, -0.006086095701903105, -0.020682187750935555, 0.03567348048090935, -0.016031987965106964, -0.03801833838224411, 0.0026478469371795654, 0.01803434081375599, -0.025714414194226265, 0.0029952945187687874, 0.007864500395953655, -0.028138313442468643, 0.0036918362602591515, -0.02937660925090313, 0.019351676106452942, -0.004067277070134878, -0.006751351058483124, 0.0298771969974041, -0.018627142533659935, -0.036015987396240234, 0.002791107166558504, -0.02500305138528347, -0.02626769430935383, -0.016954123973846436, 0.010103972628712654, 0.017849912866950035, 0.017059510573744774, 0.0034217822831124067, 0.002213125815615058, 0.03567348048090935, -0.023027045652270317, 0.023870142176747322, -0.03335496783256531, -0.0014902371913194656, -0.014451184310019016, 0.015254759229719639, 0.018916955217719078, -0.012705712579190731, -0.033697474747896194, 0.032406482845544815, 0.019944477826356888, -0.0035666893236339092, -0.02758503146469593, 0.015584093518555164, -0.010611147619783878, 0.00186073814984411, 0.00508491974323988, -0.014345796778798103, 0.0249767042696476, 0.00040405188337899745, 0.004241824150085449, -0.044262513518333435, 1.4781444406253286e-05, 0.02010255865752697, -0.004017876926809549, -0.019522931426763535, -0.019878610968589783, 0.013739822432398796, -0.003718182910233736, -0.011618910357356071, -0.0042352378368377686, -0.021841444075107574, 0.004070570692420006, -0.0036951296497136354, 0.02900775521993637, -0.0035535157658159733, 0.0053483871743083, 0.030377784743905067, 0.01596612110733986, -0.017296630889177322, -0.003909196704626083, 0.003675369545817375, -0.008668076246976852, 0.01038061361759901, -0.0005643964395858347, -0.03129992261528969, 0.033802859485149384, -0.024133609607815742, 0.031879547983407974, -0.015913428738713264, -0.0021225588861852884, -0.007080684881657362, -0.02848082035779953, -0.03546270355582237, 0.01030157320201397, 0.04926839470863342, -0.010196186602115631, -0.0182846337556839, 0.010202772915363312, -0.031378962099552155, 0.0016433775890618563, -0.010360853746533394, -0.024146782234311104, -0.03749140352010727, 0.01597929559648037, 0.01583438739180565, 0.011447655968368053, -0.023435421288013458, 0.0190618634223938, 0.009899785742163658, -0.02099834755063057, -0.0036490228958427906, 0.1888533979654312, -0.0005318747134879231, -0.01686191000044346, 0.040389545261859894, -0.0015478706918656826, 0.004907079506665468, 0.0024700064677745104, 0.014029636047780514, -0.004531638231128454, -0.001269583241082728, -0.014464357867836952, -0.02937660925090313, -0.013792515732347965, -0.005776521749794483, 0.02924487553536892, -0.00775911333039403, -0.012475178577005863, -0.025569505989551544, 0.010835094377398491, -0.025714414194226265, -0.0035172891803085804, 0.005730414763092995, -0.036411188542842865, 0.010156665928661823, 0.026320388540625572, 0.02463419735431671, -0.014675131067633629, -0.02874428778886795, 0.009761465713381767, 0.020326506346464157, -0.0008826156263239682, -0.023040220141410828, -0.010071039199829102, -0.002051752060651779, -0.020392373204231262, -0.007587859872728586, 0.030246051028370857, -0.02307973988354206, 0.005842388607561588, -0.002091272035613656, 0.0032192417420446873, 0.006086095701903105, -0.03907220810651779, -0.01625593565404415, 0.0013535635080188513, 0.013219473883509636, 0.007153138518333435, -0.010268639773130417, -0.021051041781902313, 0.01998399756848812, -0.01674335077404976, -0.006576803512871265, 0.0280856192111969, -0.006402256432920694, 0.01211949810385704, -0.01528110634535551, 0.010848267935216427, 0.003392142243683338, -0.000875205616466701, 0.021604321897029877, -0.008141141384840012, 0.031905896961688995, -0.012455418705940247, 0.04009972885251045, -0.006026815623044968, 0.0033410952892154455, 0.00881956983357668, 0.0032637016847729683, 0.02755868434906006, -0.00972194503992796, 0.014727825298905373, -0.010808748193085194, -0.008852503262460232, -0.012191951274871826, 0.009465064853429794, -0.019417542964220047, -0.0021126787178218365, 0.012389551848173141, 0.02937660925090313, 0.01986543834209442, -0.007660313043743372, -0.01725711114704609, 0.003971770405769348, 0.018719354644417763, 0.011157842352986336, -0.04017877206206322, 0.027874846011400223, -0.01128298882395029, 0.009781225584447384, 0.0021851323544979095, -0.004347211215645075, 0.011546456255018711, -0.014016463421285152, -0.005430720746517181, -0.015043986029922962, -0.013924249447882175, 0.03198493644595146, 0.009023756720125675, -0.01700681820511818, -0.020207945257425308, -0.017757698893547058, 0.0474504679441452, 0.03620041534304619, 0.005760055035352707, -0.01520206592977047, -0.022328857332468033, 0.002017171820625663, 0.01662478968501091, 0.0013617968652397394, -0.020181598141789436, 0.007607619743794203, -0.015136199072003365, 0.027795804664492607, -0.000637673307210207, 0.0055097611621022224, 0.013897902332246304, -0.009254290722310543, -0.022974353283643723, -0.010756054893136024, -0.02319830097258091, -0.015294279903173447, -0.006718417629599571, 0.028955060988664627, -0.013436835259199142, -0.011783577501773834, -0.007429779041558504, -0.04067935794591904, -0.01224464550614357, -0.004301104694604874, -0.04060031846165657, 0.019154075533151627, -0.00033386252471245825, -0.01507033221423626, 0.011045868508517742, 0.002908020978793502, -0.004357091151177883, -0.012093150988221169, -0.02603057399392128, -0.007192658726125956, -0.004804985597729683, 0.006790870800614357, 0.002845447277650237, 0.011632083915174007, -0.018732529133558273, 0.02202587015926838, -0.03580521419644356, 0.009873438626527786, 0.013963769190013409, -0.012007524259388447, 0.009649491868913174, -0.017823565751314163, 0.024397077038884163, -0.007475886028259993, -0.018113380298018456, -0.01262008585035801, 0.005585507955402136, -0.04937378317117691, -0.02334320731461048, 0.00042607609066180885, -0.01647988334298134, -0.015360146760940552, -0.0006940718158148229, 0.00811479426920414, -0.006346269976347685, -0.04191765561699867, -0.007001644931733608, -0.16524672508239746, 0.006817217450588942, 0.032933417707681656, -0.0058950819075107574, 0.008398021571338177, 0.016572095453739166, 0.04110090807080269, 0.009096209891140461, 0.00044418947072699666, -0.010545280762016773, 0.009695598855614662, 0.018534928560256958, -0.013357794843614101, -0.050822850316762924, 0.00010682777792681009, -0.01815290004014969, 0.015162546187639236, 0.024700064212083817, 0.0381237268447876, -0.012890140525996685, 0.036015987396240234, -0.004031050484627485, 0.003955303691327572, -0.022473765537142754, 0.013285340741276741, -0.012099738232791424, -0.0003733826451934874, -0.006198069546371698, 0.021880963817238808, -0.01867983490228653, -0.017586445435881615, 0.014833211898803711, 0.017099030315876007, 0.0010365793714299798, -0.0009040223667398095, -0.0012457065749913454, 0.014477530494332314, -0.00280592730268836, -0.02359350025653839, -0.004126557148993015, 0.021195948123931885, 0.02410726249217987, -0.021354028955101967, -0.004330744501203299, -0.009208183735609055, 0.0181660745292902, 0.004110090434551239, -0.0044328379444777966, -0.022882139310240746, -0.027505991980433464, 0.004100210499018431, 0.0035996227525174618, 0.03000893071293831, -0.006471416912972927, -0.0023712061811238527, 0.00993271917104721, 0.00966925173997879, 0.022605499252676964, 0.004436131566762924, 0.021973177790641785, -0.018969649448990822, -0.0016079741762951016, -0.014530224725604057, -0.025068918243050575, -0.03185320273041725, -0.020919308066368103, -0.012132671661674976, 0.01674335077404976, -0.02255280502140522, 0.010090799070894718, 0.002494706539437175, -0.010281813330948353, 0.014345796778798103, -0.006418723147362471, 0.004133143927901983, -0.009893199428915977, -0.010578214190900326, -0.006613030564039946, 0.019773224368691444, 0.03577886521816254, -0.0068435645662248135, 0.03288072347640991, -0.014688304625451565, 0.04086378589272499, -0.020286986604332924, 0.00864172913134098, 0.000997059280052781, -0.0007751703378744423, -0.017309805378317833, -0.018969649448990822, 0.014688304625451565, -0.00498941307887435, 0.005842388607561588, -0.004492118488997221, 0.014820038340985775, -0.007034578360617161, 0.0068369777873158455, -0.009563865140080452, 0.0003311866894364357, -0.011302749626338482, 0.004301104694604874, -0.002964007668197155, -0.04974263533949852, 0.018890608102083206, 0.038861434906721115, -0.009814159013330936, 0.0026840735226869583, 0.00850340910255909, 0.04004703834652901, -0.003434955608099699, 0.016954123973846436, 0.012837446294724941, 0.026860496029257774, -0.004623852204531431, -0.01597929559648037, 0.017243938520550728, 0.01802116632461548, -0.025622200220823288, -0.01488590519875288, -0.0009171956917271018, 0.02666289545595646, -0.01327216811478138, -0.031247228384017944, 0.010426720604300499, -0.02086661383509636, -0.021354028955101967, -0.08536341786384583, 0.019140902906656265, 0.0009715358610264957, 0.008747115731239319, -0.01282427366822958, 0.021749230101704597, 0.01854810118675232, -0.014095502905547619, 0.004666665568947792, 0.03417171537876129, -0.010413547046482563, -0.020827094092965126, -0.0033427421003580093, -0.007541752886027098, 0.02552998624742031, 0.012415898963809013, 0.03490942344069481, -0.012903313152492046, -0.022526457905769348, 0.015254759229719639, 0.018324153497815132, -0.014411663636565208, -0.0025095264427363873, -0.014503877609968185, 0.007192658726125956, -0.03522558510303497, -0.01983909122645855, 0.019430717453360558, 0.011362029239535332, -0.020076211541891098, -0.013351207599043846, -0.009965652599930763, -0.01257397886365652, -0.012527872808277607, -0.001341213472187519, 0.0016359675209969282, -0.00477205216884613, -0.010630907490849495, 0.011151256039738655, -0.029692770913243294, -0.00877346284687519, 0.016677483916282654, 0.0060630422085523605, -0.010446480475366116, -0.02319830097258091, -0.012725473381578922, 0.001691131037659943, 0.013515874743461609, -0.030193358659744263, -0.027005404233932495, -0.004472358152270317, -0.003876263275742531, -0.029692770913243294, -0.003019994590431452, 0.011250055395066738, -0.012936246581375599, -0.004834625869989395, -0.0006010349025018513, -0.0018409781623631716, 0.034619610756635666, -0.008358501829206944, 0.039098553359508514, 0.0021274988539516926, -0.010657254606485367, 0.035857904702425, -0.00302822794765234, 0.002504586474969983, 0.010663840919733047, -0.011427896097302437, 0.016308628022670746, -0.000870265590492636, 0.022460591048002243, 0.0016071508871391416, 0.008075274527072906, -0.013357794843614101, 0.005839094985276461, -0.020155252888798714, -0.02784849889576435, 0.007370498962700367, -0.009497998282313347, -0.012936246581375599, -0.005908255465328693, -0.0013272167416289449, -0.006652550771832466, 0.008279461413621902, -0.009359677322208881, 0.003196188248693943, 0.0015404606238007545, -0.020550454035401344, -0.03403998166322708, 0.007008231710642576, 0.013950596563518047, 0.009135730564594269, 0.026452122256159782, 0.020906133577227592, 0.02848082035779953, -0.004630438517779112, -0.017481058835983276, 0.010334506630897522, 0.009313571266829967, -0.02140672132372856, -0.0010225826408714056, -0.041311681270599365, 0.002033638535067439, 0.017757698893547058, -0.0187061820179224, 0.006303456146270037, -0.0021785455755889416, 0.003304868470877409, 0.013832035474479198, -0.01715172454714775, 0.016809217631816864, 0.002372852759435773, 0.006659137085080147, -0.054959289729595184, -0.015426013618707657, -0.0071333786472678185, -0.002376146148890257, 0.012126084417104721, -0.008780049160122871, 0.02268453873693943, -0.021577976644039154, 0.010597974061965942, 0.011342269368469715, -0.008134554140269756, 0.012468592263758183, 0.0042582908645272255, 0.0024090795777738094, -0.01740201935172081, 0.004297811072319746, 0.009761465713381767, -0.0011847796849906445, 0.003843330079689622, -0.024462943896651268, -0.0033443886786699295, -0.0026494935154914856, -0.022104911506175995, -0.03815007209777832, -0.004719358868896961, 0.026465294882655144, -0.001053046085871756, 0.027374258264899254, -0.024067742750048637, -0.021433068439364433, -0.0008529755286872387, -0.017494231462478638, -0.005878615193068981, 0.008233354426920414, -0.012969180010259151, 0.0017290044343098998, -0.015215239487588406, -0.021103734150528908, 0.027137137949466705, -0.004870852455496788, -0.016598442569375038, -0.021288162097334862, -0.025688067078590393, -0.011203949339687824, 0.015267932787537575, 0.00383344991132617, 0.04017877206206322, -0.03375016897916794, 0.03567348048090935, 0.007265112362802029, 0.01572900079190731, -0.03714889660477638, 0.0013617968652397394, 0.01997082494199276, 0.00659985700622201, 0.0024601262994110584, -0.0007710536592639983, -0.027374258264899254, 0.019825918599963188, 0.0065405769273638725, -0.0031632548198103905, 0.03272264450788498, 0.006672310642898083, -0.030509518459439278, 0.006784284487366676, -0.0016376142157241702, -0.022381551563739777, 0.012014111503958702, 0.002791107166558504, -0.031220881268382072, 0.00881956983357668, -0.0014951771590858698, 0.02140672132372856, 0.0005487530725076795, 0.007074098568409681, 0.019009169191122055, 0.01647988334298134, 0.006260642781853676, -0.022987525910139084, -0.006883084308356047, 0.0014647138305008411, -0.004841212648898363, -0.01016983948647976, -0.008457302115857601, 0.00358974258415401, 0.014780518598854542, -0.001106562907807529, -0.007877673953771591, 0.023646194487810135, -0.00454151863232255, -0.008134554140269756, -0.020708533003926277, -0.024067742750048637, 0.014148197136819363, 0.019904958084225655, -0.028638901188969612, -0.015215239487588406, 0.03027239814400673, 0.008760289289057255, 0.019641490653157234, 0.006435189861804247, 0.011210535652935505, 0.00547682773321867, 0.02037919871509075, -0.004370264708995819, -0.003612796077504754, -0.024001875892281532, 0.04526368901133537, -0.00699505815282464, -0.008325568400323391, 0.03541001304984093, -0.04420982301235199, 0.0013560334919020534, -0.04160149395465851, 0.014530224725604057, -0.004001410212367773, 0.03309150040149689, -0.012876966968178749, 0.014912252314388752, -0.013752995990216732, -0.027242524549365044, -0.00013667368330061436, -0.004646905232220888, -0.035884253680706024, 0.0051935999654233456, 0.019285809248685837, 0.02060314640402794, 0.057646654546260834, 0.028059272095561028, -0.030852027237415314, 0.018785221502184868, -0.005065159872174263, 0.01754692569375038, 0.0038663833402097225, -0.018890608102083206, 0.02231568470597267, -0.014411663636565208, 0.018666662275791168, -0.03093106672167778, 0.02488449029624462, -0.020273812115192413, -0.03169512376189232, 0.009320157580077648, -0.0073968456126749516, -0.0027450004126876593, -0.016677483916282654, 0.0005084096337668598, 0.0458960123360157, -0.018732529133558273, 0.007976474240422249, 0.013094327412545681, -0.019641490653157234, 0.0008043987327255309, 0.032933417707681656, -0.013568568974733353, -0.03066759929060936, -0.03567348048090935, -0.001506703905761242, 0.02190731093287468, -0.0019298982806503773, -0.004739118739962578, -0.01172429695725441, 0.008174074813723564, -0.008924956433475018, 0.015360146760940552, 0.018916955217719078, 0.0280856192111969, -0.008332154713571072, 0.030904719606041908, -0.02267136611044407, -0.02153845503926277, -0.035383664071559906, -0.02798023261129856, 0.005279227159917355, -0.01700681820511818, -0.03612137213349342]} +{"id": "test:10000004", "text": "\"The Denver Board of Education opened the 2017-18 school year with an update on projects that include new construction, upgrades, heat mitigation and quality learning environments.\\nWe are excited that Denver students will be the beneficiaries of a four year, $572 million General Obligation Bond. Since the passage of the bond, our construction team has worked to schedule the projects over the four-year term of the bond.\\nDenver voters on Tuesday approved bond and mill funding measures for students in Denver Public Schools, agreeing to invest $572 million in bond funding to build and improve schools and $56.6 million in operating dollars to support proven initiatives, such as early literacy.\\nDenver voters say yes to bond and mill levy funding support for DPS students and schools. Click to learn more about the details of the voter-approved bond measure.\\nDenver voters on Nov. 8 approved bond and mill funding measures for DPS students and schools. Learn more about what\u2019s included in the mill levy measure.\"", "vector_field": [-0.004989261738955975, 0.017017124220728874, -0.02820737287402153, -0.04743606224656105, -0.0292403195053339, 0.005754039157181978, -0.008535047993063927, -0.0021751460153609514, -0.037503886967897415, -0.008965441957116127, 0.01109754852950573, 0.023201556876301765, -0.0063135516829788685, -0.010097709484398365, -0.016791995614767075, 0.008601262234151363, 0.017692511901259422, -0.0008007814758457243, 0.016500651836395264, -0.0118126654997468, -0.03861629217863083, -0.006104975938796997, -0.011481592431664467, -0.01052810437977314, 0.0007093227468430996, -0.006786985322833061, 0.032921843230724335, -0.024353690445423126, -0.004916425794363022, 0.04457559436559677, -0.005965925753116608, 0.011382270604372025, -0.03782171756029129, -0.0076875025406479836, -0.008283432573080063, -0.005336888134479523, -0.006340037565678358, 0.01459367386996746, 0.022843999788165092, -0.000673318631015718, 0.02276454120874405, -0.019467059522867203, 0.010581075213849545, 0.0037047008518129587, -0.00683995708823204, -0.01015730295330286, -8.033680205699056e-05, -0.015030689537525177, -0.03368993103504181, 0.0005562017322517931, 0.015043932013213634, 0.03851034864783287, -0.03697417303919792, -0.010005009360611439, -0.007641152013093233, -0.028074944391846657, 0.0031617418862879276, 0.028339801356196404, -0.005277295131236315, -0.04452262446284294, -0.008455590344965458, -0.020725134760141373, -0.01683172397315502, -0.002426760969683528, -0.03130621090531349, 0.001560178934596479, 0.010137438774108887, 0.022102396935224533, -0.009375971741974354, 0.006356590893119574, 0.03925194963812828, 0.0014145070454105735, 0.030405694618821144, -0.02751874178647995, 0.02276454120874405, 0.008356268517673016, -0.011269706301391125, 0.004244348965585232, 0.020010018721222878, 0.00417482340708375, 0.020168934017419815, -0.02591635100543499, -0.03244509920477867, 0.030882438644766808, -0.004681364633142948, -0.003138566855341196, 0.0007982984534464777, -0.0026353367138653994, -0.014659888111054897, -0.007402780000120401, 0.001454235753044486, 0.014752588234841824, 0.001639636349864304, 0.011236598715186119, -0.01590472087264061, 0.02992895059287548, -0.00759480195119977, 0.04187404364347458, 0.002431727247312665, -0.006926035508513451, 0.008005332201719284, 0.0021420386619865894, -0.005211080424487591, -0.022791028022766113, -0.02357235923409462, -0.014699616469442844, 0.014739345759153366, 0.012911825440824032, 0.01709658093750477, -0.007952360436320305, -0.01513663213700056, 0.025373393669724464, -0.015388247556984425, -0.03117378242313862, -0.009130978025496006, 0.014779074117541313, -0.004644946660846472, 0.014898260124027729, 0.020976750180125237, 0.010773098096251488, 0.012037794105708599, 0.016739023849368095, 0.02855168841779232, -0.004813793580979109, 0.02992895059287548, -0.0017050231108441949, -0.0026105062570422888, -0.012607239186763763, 0.017083339393138885, 0.009561371989548206, -0.01808979921042919, 0.02391667477786541, 0.007422644179314375, -0.0019218756351619959, -0.024406660348176956, 0.011753072030842304, 0.0009327966254204512, 0.0346699059009552, -0.05262727662920952, -0.025929594412446022, -0.0005673753912560642, 0.001873870030976832, -0.0255720354616642, -0.011792800389230251, -0.002307574963197112, 0.023201556876301765, 0.00743588712066412, 0.013600456528365612, 0.016686052083969116, -0.026300394907593727, 0.01843411475419998, 0.012772775255143642, 0.004184755962342024, 0.02884303219616413, 0.018381142988801003, -0.010938634164631367, 0.007555073592811823, 0.005803700070828199, -0.006287065800279379, 0.005356752313673496, 0.010250003077089787, -0.011865636333823204, 0.026406338438391685, 0.004668121691793203, 0.006624759640544653, 0.009481915272772312, -0.010879040695726871, -0.023532630875706673, 0.014487730339169502, -0.00743588712066412, 0.005648096092045307, 0.017467383295297623, -0.030246779322624207, 0.020923778414726257, 0.02334722876548767, 0.01338857039809227, -0.015043932013213634, 0.0010230138432234526, -0.012494673952460289, -0.004903182853013277, -0.02751874178647995, -0.0017364750383421779, 0.0057474179193377495, 0.046667974442243576, -0.00906476378440857, -0.006806849502027035, 0.02088405005633831, 0.025982566177845, 0.02042054943740368, -0.0033041031565517187, 0.003982801456004381, 0.018447356298565865, 0.00944218598306179, -0.0003784571890719235, -0.6432870030403137, 0.0024234503507614136, -0.002848878502845764, -0.0018987004877999425, -0.026114994660019875, -0.014024228788912296, 0.01888437196612358, 0.017639540135860443, -0.0253469068557024, 0.0009675592300482094, 0.015666348859667778, -0.002628715243190527, -0.011130656115710735, -0.025360150262713432, 0.01871221512556076, 0.004135095048695803, 1.1141363756905776e-05, -0.009693801403045654, -0.017983855679631233, -0.0032229903154075146, -0.010422160848975182, 0.012640345841646194, -0.009283271618187428, 0.0028422570321708918, 0.020235149189829826, 0.024247746914625168, -0.02396964654326439, -0.02007623389363289, -0.012547645717859268, -0.0026436136104166508, -0.020327849313616753, 0.004860143642872572, -0.002999516436830163, 0.02665795385837555, 0.033213187009096146, 0.030564608052372932, -0.01602390594780445, 0.02253941260278225, -0.023135343566536903, 0.03022029437124729, -0.020182177424430847, 0.005432899110019207, 0.01562662050127983, -0.002378755481913686, -0.01722901128232479, 0.023069128394126892, 0.024327203631401062, 0.0071776509284973145, 0.016580108553171158, -0.017811698839068413, -0.005085272714495659, 0.0007660188712179661, -0.003216368844732642, -0.007468994706869125, -0.018381142988801003, -0.014858531765639782, 0.012779396958649158, -0.0026369921397417784, 0.0038636154495179653, -0.01962597481906414, 0.007157786283642054, 0.00014184384781401604, 0.0002961029240395874, 0.0034961251076310873, -0.025439606979489326, -0.019665703177452087, -0.019665703177452087, 0.02735982835292816, 0.0005851705791428685, -0.018460599705576897, -0.02449936233460903, 0.008117896504700184, -0.011945093981921673, 0.011375649832189083, 0.04966086894273758, -0.018063312396407127, 0.03649742901325226, -0.012070901691913605, -0.008733691647648811, 0.02129458077251911, -0.014726102352142334, -0.01427584420889616, -0.024790706112980843, 0.009667315520346165, 0.026525525376200676, -0.04733011871576309, 0.01230265200138092, -0.0071776509284973145, 0.000962593185249716, -0.0004012184217572212, 0.006535370368510485, 0.0030955274123698473, 0.00215197098441422, -0.02894897572696209, 0.0052806055173277855, 0.020579462870955467, -0.013997742906212807, -0.0005272328853607178, 0.019480302929878235, -0.023029400035738945, -0.015600133687257767, -0.011203492060303688, 0.016275521367788315, -0.005191216245293617, 0.04033786803483963, 0.036020681262016296, -0.005942750722169876, 0.0001497068296885118, 0.033319130539894104, -0.026565253734588623, -0.010349324904382229, 0.003352108644321561, -0.03625905513763428, 0.0005888951127417386, -0.019083015620708466, -0.022459955886006355, 0.0055785709992051125, 0.005611678119748831, 0.01750711165368557, 0.025148263201117516, 0.008270190097391605, -0.0030839398968964815, 0.013653428293764591, -0.019639218226075172, -0.012891961261630058, 0.019440574571490288, 0.00045770767610520124, -0.016897937282919884, -0.017983855679631233, -0.00017660646699368954, -0.0035722716711461544, 0.015255819074809551, 0.013825586065649986, 0.012183465994894505, 0.020552977919578552, 0.014183144085109234, -0.002444969955831766, -0.013918286189436913, -0.011508078314363956, -0.01671253703534603, -0.009587857872247696, 0.004052327014505863, 0.0042211739346385, -0.0004974363837391138, -0.01683172397315502, -0.011653750203549862, -0.022645356133580208, -0.002292676828801632, -0.011382270604372025, -0.0021436940878629684, -0.0033322442322969437, -0.0010577765060588717, -0.013328976929187775, -0.008442347869277, -0.012461567297577858, -0.0025773991364985704, 0.013759370893239975, 0.00818411074578762, 0.002011265140026808, -0.006704217288643122, 0.016619836911559105, -0.0179706122726202, 0.0011281294282525778, -0.018791671842336655, 0.004840279463678598, -0.025677978992462158, 0.0032445101533085108, -0.0013424988137558103, -0.009574615396559238, -0.03225969895720482, 0.023784244433045387, -0.014580430462956429, 0.024631790816783905, -0.000584342866204679, 0.012362245470285416, -0.007641152013093233, -0.01681848056614399, -0.02253941260278225, -0.011680236086249352, -0.0004142544057685882, 0.022380497306585312, -0.0009857682744041085, -0.01556040532886982, 0.01430233009159565, 0.040417324751615524, 0.014951231889426708, 0.02260562777519226, 0.01762629672884941, -0.02236725576221943, -0.0018109662923961878, 0.016328493133187294, 0.01001825276762247, -0.006912793032824993, 0.003426599781960249, -0.0007589836022816598, 5.276467345538549e-05, 0.0006774570210836828, -0.023784244433045387, 0.006273822858929634, 0.02505556307733059, 0.02900194749236107, -0.01938760280609131, 0.01387855689972639, 0.006638002581894398, -0.01050161849707365, -0.018725458532571793, 0.023492900654673576, -0.014739345759153366, 0.012137115933001041, -0.008011952973902225, 0.009647451341152191, -0.04995221272110939, -0.03798063099384308, -0.008680719882249832, -0.001938429195433855, 0.004694607574492693, -0.006641313433647156, 0.020725134760141373, -0.016924424096941948, -0.015494191087782383, 0.006124840583652258, -0.010051359422504902, 0.02781008556485176, 0.00603876169770956, -0.02436693198978901, -0.006091732997447252, -0.002289365977048874, 0.019480302929878235, 0.01893734373152256, -0.009620965458452702, -0.00042873882921412587, 0.02610175311565399, 0.0355704240500927, 0.009561371989548206, -0.015825264155864716, -0.00045605230843648314, 0.017189281061291695, 0.004264213144779205, 0.030140835791826248, 0.007442508824169636, 0.0262076947838068, 0.011355784721672535, 0.023996131494641304, -0.003337210277095437, 0.009197193197906017, -0.01648740842938423, 0.0029581324197351933, 0.009058142080903053, 0.0016669498290866613, 0.011819286271929741, -0.010494996793568134, 0.007396158762276173, 0.005863293074071407, -0.023757759481668472, 0.010316217318177223, -0.014010986313223839, 0.019705431535840034, 0.024870162829756737, 0.008369511924684048, 0.017480624839663506, 0.019122743979096413, 0.018447356298565865, 0.01013081707060337, -0.010965120047330856, 0.024340447038412094, 0.008170868270099163, 0.012766153551638126, -0.016964152455329895, -0.022327525541186333, -0.012183465994894505, 0.017838183790445328, -0.00516141951084137, 0.0007353947148658335, -0.025850137695670128, 0.011594157665967941, 0.0025575347244739532, -0.03628554195165634, 0.016580108553171158, 0.010342703200876713, 0.0342196486890316, -0.013434920459985733, -0.012666831724345684, -0.0016371533274650574, 0.01830168440937996, -0.004373467061668634, -0.016805237159132957, 0.012554267421364784, -0.00021602478227578104, -0.018672486767172813, 0.005717621184885502, -0.006068557966500521, -0.002848878502845764, -0.013640184886753559, -0.0002956890966743231, 0.00609504384920001, -0.0463501438498497, -0.0060188970528542995, 0.019321387633681297, -0.03816603124141693, 0.002847223076969385, 0.0021354174241423607, 0.006138083059340715, -0.010600940324366093, -0.026803625747561455, 0.022817512974143028, 0.006452602334320545, 0.01487177424132824, -0.032868873327970505, -0.0046515678986907005, -0.0286046601831913, 0.01934787444770336, -0.007972224615514278, -0.013719642534852028, -0.0007428438402712345, -0.011057820171117783, 0.005273984279483557, 0.0031319453846663237, 0.0001942981471074745, 0.013097226619720459, -0.007541830651462078, -0.011547807604074478, 0.00901179201900959, -0.01573256216943264, 0.0035159895196557045, 0.08470157533884048, 0.026459310203790665, -0.0015146564692258835, 0.009554751217365265, 0.009309757500886917, 0.01578553393483162, -0.017718996852636337, -0.021188637241721153, -0.0051084477454423904, 0.007548451889306307, -0.0038834798615425825, -0.007958982139825821, -0.008230460807681084, -0.0019632596522569656, -0.0011438552755862474, -0.006184433586895466, 0.004674742929637432, -0.013315734453499317, -0.019877590239048004, 0.0017911019967868924, -0.013322355225682259, 0.008866120129823685, 0.0355704240500927, 0.025889866054058075, 0.014977717772126198, -0.005581881385296583, 0.020129205659031868, 0.028869517147541046, -0.005965925753116608, 0.009296514093875885, -0.010362567380070686, -0.000257822684943676, 0.01705685257911682, 0.020725134760141373, -0.019374359399080276, -0.006217540707439184, -0.0006054487312212586, -0.03353101760149002, 0.03268347308039665, 0.027200913056731224, 0.006575098726898432, 0.010991604998707771, -0.019122743979096413, 0.004413195885717869, 0.0003064489283133298, -0.011282948777079582, -0.00978650152683258, 0.015997420996427536, 0.016222549602389336, -0.00027499705902300775, 0.014606916345655918, -0.031385667622089386, -0.045052338391542435, -0.0006613172008655965, -0.006217540707439184, -0.008131139911711216, 0.002196665620431304, -0.028922488912940025, -0.01029635313898325, -0.00336866220459342, -0.014262601733207703, -0.031332697719335556, 0.02381073124706745, -0.003110425779595971, -0.005389859434217215, 0.004042394459247589, -0.0039000334218144417, 0.025532307103276253, -0.02357235923409462, 0.005154798272997141, -0.0057474179193377495, -0.009793123230338097, -0.0385633185505867, -0.0034100462216883898, 0.015772292390465736, 0.009713665582239628, 0.007906010374426842, 0.01548094768077135, 0.014063958078622818, 0.012872097082436085, -0.022910213097929955, 0.010382432490587234, -0.0037576723843812943, -0.020566221326589584, -0.019705431535840034, 0.017771968618035316, -0.004287388175725937, 0.0018407627940177917, -0.018751943483948708, 0.02031460590660572, 0.010177167132496834, -0.007502101827412844, 0.004605217836797237, -0.012858853675425053, -0.004701228812336922, 0.0077868239022791386, 0.025704465806484222, 0.019612731412053108, -0.006985628977417946, -0.014024228788912296, 0.012362245470285416, 0.03157106786966324, -0.02048676274716854, -0.0017646162305027246, 0.030008407309651375, 0.016169577836990356, -0.0013731230283156037, 0.018447356298565865, -0.014037472195923328, 0.003446464193984866, 0.014567187987267971, -0.04627068713307381, -0.013150197453796864, 0.0034100462216883898, 0.01836789958178997, -0.005740796215832233, 0.010369189083576202, 0.015043932013213634, 0.026459310203790665, -0.006330105476081371, 0.027147941291332245, -0.026300394907593727, 0.0316770114004612, 0.030458666384220123, -0.00044322325265966356, -0.0009427288314327598, 0.009740151464939117, -0.01132929977029562, -0.016977395862340927, -0.011700100265443325, -0.019890833646059036, 0.01955975964665413, -0.006538680754601955, -0.017718996852636337, -0.009005170315504074, -0.018381142988801003, -0.0158385057002306, -0.01018378883600235, -0.026300394907593727, -0.004966086708009243, -0.015427975915372372, -0.004694607574492693, 0.021784568205475807, -0.0351201668381691, 0.052441876381635666, -0.025426363572478294, -0.010236760601401329, -0.022910213097929955, 0.0024813879281282425, 0.004188066348433495, -0.003890101332217455, 0.016791995614767075, -0.016328493133187294, 0.009693801403045654, -0.004297320265322924, -0.0050521655939519405, -0.0015734218759462237, -0.018579786643385887, 0.026062022894620895, 0.006972386036068201, 0.0468004010617733, -0.013600456528365612, 0.008965441957116127, -0.011653750203549862, 0.008627748116850853, -0.0016106674447655678, 0.002948200097307563, -0.011839151382446289, -0.01518960390239954, 0.005068719387054443, 0.03141215443611145, 0.01553391944617033, -0.015573647804558277, -0.03117378242313862, 0.01109754852950573, 0.00529384845867753, 0.012978040613234043, -0.011395514011383057, -0.016063636168837547, -0.026273909956216812, -0.0027892854996025562, 0.0035159895196557045, -0.0017977234674617648, 0.019493546336889267, -0.056838519871234894, -0.009170707315206528, 0.009124357253313065, 0.020473521202802658, 0.037212543189525604, -0.018526814877986908, 0.02849871665239334, -0.01900355890393257, 0.021042965352535248, 0.015282304026186466, 0.02191699668765068, -0.01358721312135458, -0.008680719882249832, -0.056785546243190765, 0.001014737063087523, 0.015666348859667778, 0.0031766402535140514, 0.0033454871736466885, 0.009720287285745144, 0.0021916995756328106, -0.01301776897162199, 0.007455751765519381, 0.013785856775939465, -0.02310885675251484, -0.006704217288643122, -0.033160217106342316, 0.006349969655275345, -0.027306856587529182, 0.0030226914677768946, -0.005025679711252451, -0.004803861491382122, -0.009501779451966286, -0.03366344794631004, 0.03154458478093147, -0.010826068930327892, 0.006412873510271311, 0.015441219322383404, 0.006538680754601955, 0.008687341585755348, -0.007521966006606817, -0.004995883442461491, -0.0013474648585543036, 0.0029614430386573076, -0.025307178497314453, 0.0033322442322969437, 0.015467705205082893, -0.00759480195119977, 0.030564608052372932, 0.010865798220038414, -0.0296905767172575, -0.025611765682697296, 0.016778752207756042, 0.031200267374515533, -0.009031656198203564, -0.02386370301246643, 0.02849871665239334, 0.009309757500886917, -0.025267450138926506, 0.015255819074809551, 0.011362406425178051, -0.023373715579509735, 0.007382915820926428, -0.0019433953566476703, 0.004985951352864504, -0.0018854576628655195, -0.01573256216943264, -0.018871130421757698, 0.013852071017026901, 0.002368823392316699, 0.019467059522867203, 0.014726102352142334, -0.018977073952555656, -0.0020841010846197605, 0.010859176516532898, -0.01830168440937996, -0.0024416593369096518, -0.0218640249222517, 0.024287475273013115, -0.017666026949882507, 0.025386635214090347, 0.0014625125331804156, 0.005555395968258381, -0.0023026089183986187, 0.008833013474941254, -0.0013416711008176208, 0.020804593339562416, -0.018288442865014076, -0.0020029882434755564, -0.0019483614014461637, -0.015096903778612614, 0.022857241332530975, 0.013315734453499317, -0.022393740713596344, -0.03048515133559704, -0.022910213097929955, -0.005403102375566959, 0.004353602882474661, 0.01144186407327652, -0.0355704240500927, 0.013514377176761627, -0.00975339487195015, -0.02449936233460903, -0.006677731405943632, -0.006780363619327545, 0.03517313674092293, -0.017321711406111717, -0.02820737287402153, 0.0057441070675849915, -0.0023572358768433332, 0.024750975891947746, 0.0026237491983920336, 0.021771324798464775, -0.0011943439021706581, 0.01969218999147415, -0.04137081280350685, 0.02562500722706318, 0.003602068405598402, 0.012262923642992973, -0.008846255950629711, -0.00798546802252531, -0.021241609007120132, -0.018050070852041245, -0.013600456528365612, -0.012527781538665295, -0.02522772178053856, -0.021135665476322174, 0.002446625381708145, 0.0025293936487287283, -0.012057659216225147, -0.010713504627346992, 0.0010213585337623954, -4.0840892324922606e-05, 0.00986595917493105, 0.01004473865032196, -0.032868873327970505, 0.03742443025112152, 0.002709828084334731, -0.008740312419831753, 0.040867581963539124, -0.010541346855461597, -0.012368867173790932, -0.014183144085109234, 0.012653589248657227, -0.014818803407251835, -0.022685084491968155, -0.010581075213849545, -0.009223678149282932, 0.012799261137843132, -0.021082693710923195, -0.01843411475419998, -0.034352079033851624, -0.01819574274122715, 0.0047111609019339085, 0.016725780442357063, -0.013746128417551517, -0.010759854689240456, 0.03787468746304512, 0.009627587161958218, 0.017983855679631233, -0.002857155166566372, -0.03268347308039665, -0.04054975509643555, -0.01144186407327652, -0.018500328063964844, -0.010236760601401329, -0.040761642158031464, -0.00032134720822796226, 0.031491611152887344, 0.005141555331647396, -0.01659335196018219, -0.009276649914681911, -0.002350614406168461, -0.033319130539894104, -0.008395997807383537, 0.003658350557088852, 0.020500006154179573, 0.02455233223736286, 0.019546518102288246, -0.0053468202240765095, 0.025783922523260117, -0.0047177826054394245, 0.03437856212258339, -0.03334561735391617, -0.0014947921736165881, 0.020235149189829826, -0.03535853698849678, 0.01207752339541912, 0.01847384311258793, -0.013315734453499317, 0.020910536870360374, 0.023360472172498703, 0.010951876640319824, 0.02569122239947319, 0.010759854689240456, 0.008700584061443806, -0.020513249561190605, 0.0011173695093020797, 0.018116284161806107, -0.009932173416018486, 0.01738792471587658, 0.024102075025439262, -0.007992088794708252, 0.00679360656067729, 0.009667315520346165, -0.0008665821515023708, 0.00697900727391243, -0.018156012520194054, 0.01779845543205738, -0.02031460590660572, 0.03061757981777191, -0.01207752339541912, -0.012004687450826168, -0.007978846319019794, 0.006035450845956802, 0.0007482237415388227, 0.00815100409090519, -0.01432881597429514, 0.01848708651959896, 0.014951231889426708, 0.010203653015196323, -0.002999516436830163, 0.019864346832036972, -0.03758334368467331, -0.01900355890393257, -0.011057820171117783, -0.005611678119748831, 0.02243346907198429, -0.013719642534852028, -0.018288442865014076, -0.008276810869574547, -9.347623563371599e-05, -0.00031120810308493674, 0.002105620689690113, -0.030988382175564766, 0.02614148147404194, -0.011905365623533726, -0.02889600396156311, -0.012713181786239147, -0.034299105405807495, 0.0024251057766377926, -0.024353690445423126, -0.015083661302924156, -0.007912632077932358, -0.023492900654673576, -0.01050161849707365, -0.0068200924433767796, -0.015917964279651642, -0.04285401850938797, -0.006075179670006037, -0.01558689121156931, -0.0008856187923811376, -0.007204136811196804, 0.20997940003871918, -0.00563816400244832, -0.016010664403438568, 0.04441668093204498, -0.024009374901652336, -0.002426760969683528, 0.006919414270669222, 0.00017919296806212515, -0.029823007062077522, 0.010766476392745972, -0.012395353056490421, 0.00301938084885478, -0.01745413988828659, -0.00489325076341629, 0.01612984947860241, 0.003615311114117503, -0.011395514011383057, -0.011753072030842304, -0.009667315520346165, 0.013123712502419949, -0.013276005163788795, 0.010700262151658535, -0.004509206861257553, -0.0034034247510135174, -0.00927002914249897, 0.012514539062976837, 0.0038834798615425825, 0.009760015644133091, 7.894008740549907e-05, 0.03753037378191948, -0.001168685732409358, 0.011110791936516762, 0.002291021402925253, -0.004515828099101782, -0.0019930561538785696, -0.018010340631008148, 0.017997099086642265, -0.016116607934236526, 0.011322678066790104, 0.05268025025725365, 0.024737734347581863, 0.01347464881837368, -0.0088727418333292, -0.028763575479388237, 0.006098354700952768, 0.006644624285399914, -0.01358721312135458, -0.005181283690035343, -0.00018043449381366372, -0.009799744933843613, -0.01683172397315502, 0.0007225656299851835, 0.02791602909564972, 0.016924424096941948, -0.0017166106263175607, -0.02128133736550808, 0.012031173333525658, -0.014951231889426708, 0.002582365181297064, -0.01253440324217081, -0.015057175420224667, 0.025664737448096275, 0.009501779451966286, 0.03541151061654091, 0.0021238296758383512, 0.006866442505270243, 0.0039198980666697025, 0.047065261751413345, 0.016275521367788315, 0.014368544332683086, -0.006899550091475248, 0.002216530032455921, -0.01883140206336975, -0.021387280896306038, -0.008958820253610611, -0.011991444043815136, 0.037450917065143585, 0.012620481662452221, -0.00818411074578762, 0.005754039157181978, 0.0012158636236563325, -0.0013756060507148504, -0.010084467008709908, -0.006469155661761761, -0.02105620875954628, -0.0346699059009552, 0.01974516175687313, -0.01962597481906414, 0.0012042759917676449, -0.023612087592482567, 0.021387280896306038, 0.0012663521338254213, -0.025320421904325485, 0.004390020854771137, 0.018513571470975876, 0.0027147941291332245, 0.01344816293567419, 0.028816547244787216, -0.01550743356347084, 0.022446712478995323, -0.01390504278242588, 0.05731526389718056, 0.017639540135860443, 0.020221905782818794, 0.023651815950870514, 0.00486345449462533, -0.01699063740670681, 0.021440252661705017, -0.0034067356027662754, -0.007886146195232868, 0.008833013474941254, -0.02488340623676777, 0.0016487408429384232, 0.0020211972296237946, -0.014421516098082066, 0.008031818084418774, -0.0079192528501153, -0.023757759481668472, 0.01095849834382534, -0.018513571470975876, -0.006697595585137606, -0.018871130421757698, -0.01201130822300911, -0.006826714146882296, 0.007707366719841957, 0.01155442837625742, 0.0031733294017612934, 0.011428620666265488, 0.00516141951084137, -0.017295224592089653, 0.010739990510046482, -0.026843354105949402, 0.009402457624673843, -0.03882817551493645, -0.008548290468752384, -0.0014848599676042795, 0.0020526491571217775, 0.02226131223142147, -0.026220938190817833, -0.0001244625455001369, 0.003953005187213421, -0.005257430486381054, 0.013772614300251007, -0.0004208758473396301, 0.002519461326301098, -0.03180944174528122, 0.017017124220728874, -0.010879040695726871, -0.009144221432507038, -0.008826391771435738, -0.03604716807603836, 0.010574454441666603, 0.034007761627435684, -0.056838519871234894, 0.014050714671611786, -0.010144059546291828, -0.006594963371753693, -0.025082049891352654, -0.007932496257126331, 0.019665703177452087, -0.020433790981769562, 0.0071114362217485905, -0.011978201568126678, -0.03231267258524895, -0.0210032369941473, 0.012653589248657227, -0.16739022731781006, 0.008448968641459942, 0.02226131223142147, -0.00547924917191267, -0.006333415862172842, 0.0076808808371424675, 0.02722739800810814, -0.014077200554311275, -0.014063958078622818, -0.003549096640199423, 0.015110147185623646, -0.006022207904607058, -0.01612984947860241, 0.0031898831948637962, 0.0024946308694779873, 0.019361117854714394, -0.010170545428991318, 0.0053435093723237514, 0.008515183813869953, 0.001738130347803235, 0.03925194963812828, -0.01824871450662613, -0.00118358398322016, -0.021824296563863754, 0.009899066761136055, -0.007475615944713354, -0.0012572476407513022, 0.02196996845304966, 0.004476099740713835, -0.01814277097582817, -0.021466737613081932, 0.02655201032757759, 0.015096903778612614, 0.023188315331935883, 0.011806043796241283, -0.006108286790549755, -0.0037245650310069323, -0.009872580878436565, 0.006760499440133572, 0.02065892145037651, 0.020447034388780594, 0.007171029224991798, 0.022499684244394302, 0.0054494524374604225, -0.017255496233701706, 0.01779845543205738, 0.031200267374515533, -0.014805560000240803, 0.004635014571249485, -0.00016181166574824601, 0.026790382340550423, -0.030167322605848312, -0.0072835939936339855, -0.004184755962342024, -0.003352108644321561, 0.011375649832189083, -0.006959143094718456, -0.03451099246740341, -0.006508884485810995, 0.0002022645785473287, -0.0158385057002306, 0.001246487838216126, -0.0012456601252779365, -0.0006745601422153413, 0.012872097082436085, -0.023426687344908714, -0.003602068405598402, -0.008369511924684048, -0.03461693599820137, -0.004363534972071648, -0.011978201568126678, -0.019122743979096413, 0.011428620666265488, 0.03411370515823364, 0.012315895408391953, 0.02426098845899105, 0.020341090857982635, -0.01393152866512537, -0.002277778461575508, -0.002014575991779566, -0.004747578874230385, 0.04902520775794983, -0.002019542036578059, 0.013395191170275211, -0.013706399127840996, 0.012958175502717495, -0.006740635260939598, 0.01516311801970005, 0.04595285654067993, 0.00775371678173542, 0.014077200554311275, -0.01773224025964737, 0.005393170285969973, 0.010448646731674671, -0.0006385559681802988, 0.013600456528365612, -0.004267523996531963, -0.010938634164631367, 0.009230299852788448, -0.0007349808583967388, -0.023254528641700745, 0.004830346908420324, -0.014209629967808723, 0.021837538108229637, -0.0019135987386107445, -0.002522772178053856, 0.008713826537132263, 0.03771577402949333, 0.0023555804509669542, -0.0010503273224458098, -0.02155943773686886, 0.006260579917579889, 0.0128787187859416, -0.02438017539680004, 0.018566543236374855, 0.014236115850508213, 0.003027657512575388, 0.00013708468759432435, 0.008819770067930222, 0.011289570480585098, 0.05273322016000748, 0.015123389661312103, -0.03445802256464958, 0.019718674942851067, -0.008753555826842785, -0.007952360436320305, -0.10122871398925781, -0.010250003077089787, 0.00673732440918684, 0.03694768622517586, -0.00497601879760623, 0.01553391944617033, 0.02426098845899105, 0.02019541896879673, -0.0008351302822120488, 0.01093201246112585, -0.014792317524552345, -0.011547807604074478, 0.014527459628880024, -0.00038818243774585426, 0.021135665476322174, -0.014024228788912296, -0.0015593512216582894, -0.027651172131299973, -0.04695931822061539, 0.018778430297970772, 0.018910858780145645, -0.006985628977417946, -0.0007171857287175953, -0.0028737089596688747, 0.016222549602389336, -0.027306856587529182, -0.03186241164803505, 0.03043217957019806, 0.009130978025496006, 0.025042319670319557, 0.019030043855309486, 0.013083983212709427, 0.014209629967808723, -0.005081962328404188, -0.009971902705729008, -0.009601101279258728, -0.030802981927990913, -0.024353690445423126, -0.009091249667108059, -0.006591652520000935, 0.012170223519206047, 0.01602390594780445, 0.006296997889876366, -0.044151823967695236, -0.001313529908657074, 0.0016280488343909383, 0.003416667692363262, -0.009607722982764244, -0.007521966006606817, -0.018500328063964844, -0.022685084491968155, -0.0006737324292771518, -0.014712859876453876, 0.0035656504333019257, -0.0085549121722579, -0.007091572042554617, 0.0036715934984385967, -0.0012423493899405003, -0.022221583873033524, -0.009932173416018486, -0.014262601733207703, 0.017030367627739906, -0.0027627996169030666, 0.016434436663985252, 0.007846416905522346, -0.006939278449863195, -0.011031334288418293, 0.009687179699540138, 0.005922886077314615, -0.04616474360227585, 0.0007035289891064167, 0.03297481685876846, -0.0063764555379748344, -0.009130978025496006, -0.015772292390465736, 0.002784319221973419, -0.029134375974535942, -0.010349324904382229, 0.0013590523740276694, -0.004840279463678598, -0.04113244265317917, -0.01018378883600235, 0.01705685257911682, 0.0007229794864542782, 0.01398450043052435, 0.015639862045645714, -0.01824871450662613, 0.001239038654603064, 0.01756008341908455, -0.01264696754515171, -0.011547807604074478, 0.01899031549692154, -0.00034410844091326, -0.008482076227664948, -0.008991927839815617, 0.03914600610733032, 0.01093201246112585, -0.010428782552480698, 0.017864668741822243, 0.009336243383586407, -0.02596932277083397, -0.015176361426711082, -0.0550374835729599, 0.008621126413345337, -0.00020050576131325215, 0.009051521308720112, -0.012997904792428017, -0.0030706969555467367, 0.018871130421757698, -0.01652713678777218, -0.009402457624673843, 0.00013180822134017944, 0.001524588675238192, 0.007462373003363609, 0.010720126330852509, -0.02386370301246643, -0.022963184863328934, -0.02071189321577549, 0.034007761627435684, 0.009634207934141159, 0.008885984309017658, 0.029611120000481606, -0.033769391477108, -0.0033570746891200542, 0.028737088665366173, 0.020513249561190605, -0.038139548152685165, 0.02453909069299698, -0.018050070852041245, 0.02157268114387989, 0.016447680070996284, -0.024671519175171852, -0.0021734905894845724, -0.04629717022180557, 2.2567242922377773e-05, 0.020672164857387543, -0.025360150262713432, -0.023069128394126892, 0.004535692743957043, 0.002999516436830163, 0.023559115827083588, -0.009892445057630539, 0.008250325918197632, -0.01567959226667881, -0.0034563965164124966, -0.03225969895720482, 0.029955435544252396, -0.01848708651959896, -0.012653589248657227, 0.005088583566248417, 0.014474487863481045, 0.003340521128848195, 0.018394386395812035, 0.012779396958649158, -0.004211241379380226, -0.0013333943206816912, 0.0086674764752388, 0.008501940406858921, -0.016341736540198326, 0.005297159310430288, 0.005710999947041273, -0.01619606465101242, 0.029717063531279564, 0.01075323298573494, 0.04309239238500595, -0.02541312202811241, 0.026220938190817833, -0.029955435544252396, -0.007819931022822857, -0.008647612296044827, -0.002829014090821147, 7.956085028126836e-05, -0.018036827445030212, -0.008402618579566479, 0.01974516175687313, 0.025717707350850105, -0.014143414795398712, 0.011574292555451393, 0.025598522275686264, -0.01333559863269329, -0.024578819051384926, 0.014368544332683086, 0.019983533769845963, -0.002183422911912203, 0.007674259599298239, 0.013395191170275211, 0.00762128783389926, 0.021612409502267838, -0.022062668576836586, -0.015653105452656746, 0.004992572590708733, 0.028922488912940025, -0.016977395862340927, 0.008078168146312237, 0.011408756487071514, -0.0014873429900035262, 0.019467059522867203, 0.00695252139121294, 0.03472287952899933, -0.0024217949248850346, -0.00815100409090519, 0.029372747987508774, -0.0077470955438911915, 0.007468994706869125, -0.00789276696741581, -0.016460921615362167, -0.04709174484014511, -0.0014409928116947412, -0.012574131600558758, -0.01916247420012951, -0.006124840583652258, 0.03837791830301285, 0.0023307499941438437, 0.007184272166341543, 0.0008223011973313987, 0.01979813352227211, -0.013203169219195843, 0.010773098096251488, -0.008674098178744316, -0.022102396935224533, -0.013640184886753559, 0.026009051129221916, 0.011243220418691635, -0.0033140352461487055, 0.01864599995315075, 0.016050392761826515, 0.03416667878627777, -0.01358721312135458, -0.00529384845867753, 0.005466006230562925, -0.011157141998410225, 0.01516311801970005, -0.01184577215462923, -0.002430071821436286, 0.0026816867757588625, -0.02196996845304966, 0.015878234058618546, -0.030908923596143723, -0.026194453239440918, 0.009905687533318996, 0.020844321697950363, 0.06139407679438591, 0.04409885033965111, 0.0036550399381667376, 0.006985628977417946, 0.005773903802037239, 0.02443314716219902, -0.012183465994894505, 0.014355301856994629, 0.01189874392002821, -0.008064924739301205, 0.025029078125953674, -0.012368867173790932, -0.0039099655114114285, -0.026009051129221916, -0.026591738685965538, -0.002587331226095557, 0.004628392867743969, 0.01955975964665413, -0.02831331640481949, 0.012070901691913605, 0.018871130421757698, 0.0023622019216418266, 0.029054919257760048, 0.03403424844145775, -0.030352722853422165, -0.010607561096549034, 0.019083015620708466, -0.026922812685370445, 0.006134772673249245, -0.008409240283071995, -0.011931851506233215, 0.00041446133400313556, -0.02506880648434162, -0.009402457624673843, 0.009415701031684875, 0.004896561615169048, -0.018328171223402023, -0.017811698839068413, -0.012441703118383884, -0.020685406401753426, -0.009667315520346165, 0.014063958078622818, -0.009667315520346165, -0.04449613764882088, -0.0025740882847458124, 0.01427584420889616, -0.034405048936605453, -0.023267772048711777, -0.0033752836752682924]} +{"id": "test:10000005", "text": "\"BANGALORE CY JUNCTION SBC to GONDIA JUNCTION G train timings, routes, stops, and complete info.\\nAs of now, 1 trains run between from BANGALORE CY JUNCTION (YPR) to GONDIA JUNCTION (G).\\nThe fastest train from BANGALORE CY JUNCTION (YPR) to GONDIA JUNCTION (G) is YPR KRBA WAINGANGA EXP (12251) that departs at 23:40 and arrives to at 21:15. It takes approximately 21:35 hours.\"", "vector_field": [0.010673399083316326, -0.0002009706513490528, 0.038127005100250244, 0.0026261291932314634, -0.024346159771084785, -0.0011154715903103352, -0.008842708542943, -0.016091162338852882, -0.008504942990839481, -0.03639764338731766, -0.005525848362594843, 0.008376591838896275, -0.0006020675646141171, -0.014834674075245857, -0.0020688155200332403, -5.409530422184616e-05, 0.01009244192391634, 0.010909835807979107, 0.018698714673519135, 0.001013297471217811, 0.00618786970153451, 0.0005991120706312358, 0.002947006607428193, -0.0038302643224596977, 0.0057825506664812565, 0.03701913356781006, 0.0010335633996874094, 0.003951860126107931, -7.63667412684299e-05, 0.01523999311029911, -0.0055731358006596565, 0.019414778798818588, -0.01199744176119566, 0.010045154951512814, 0.002648083958774805, -0.01557775866240263, 0.003826886648312211, 0.011666431091725826, 0.012956696562469006, 0.002119480399414897, -0.009558772668242455, -0.004222072660923004, -0.0037424450274556875, 0.013044515624642372, -0.002452179789543152, 0.02604849822819233, 0.003853907808661461, -0.0205091405659914, -0.022508712485432625, 0.02363009564578533, -0.010450473986566067, -0.01798265241086483, -0.029804455116391182, -0.024129988625645638, -0.0015334567287936807, -0.02315722405910492, 0.005184704903513193, 0.020968500524759293, 0.007363294716924429, -0.005586646497249603, 0.012510845437645912, -0.0105180274695158, -0.021279245615005493, 0.012740526348352432, 0.010153240524232388, -0.009970846585929394, -0.0006987529923208058, -0.023184245452284813, 0.021130628883838654, -0.03147977218031883, 0.012841856107115746, 0.015226482413709164, 0.013760578818619251, -0.00639728456735611, 0.041558705270290375, -0.021900733932852745, -0.02834530733525753, 0.013064781203866005, -0.016253290697932243, 0.013706536032259464, 0.021617012098431587, -0.025508074089884758, -0.003119267290458083, 0.0012184901861473918, 0.007093081716448069, 0.0009026791085489094, 0.013645738363265991, 0.013645738363265991, -0.024048924446105957, -0.007370050065219402, 0.04288274794816971, 0.022589776664972305, 0.0217791385948658, 0.011335420422255993, 0.020009245723485947, -0.011024676263332367, -0.024940626695752144, 0.009025102481245995, -0.007140369154512882, -0.014713078737258911, -0.00019537641492206603, 0.01949584297835827, 0.027696795761585236, 0.0003858974378090352, -0.013875419273972511, -0.008856219239532948, 0.008876485750079155, -0.013625472784042358, 0.017847545444965363, -0.011551590636372566, 0.0019269539043307304, 0.02102254331111908, -0.01926616206765175, 6.301443499978632e-05, -0.0008798799244686961, -0.030804241076111794, 0.021765628829598427, -0.001487858360633254, -0.012341962195932865, -0.021360309794545174, 0.013861908577382565, 0.02354903146624565, 0.007626751903444529, 0.001986062852665782, 0.037586577236652374, 0.012963451445102692, -0.007829410955309868, -0.008754889480769634, 0.0025214217603206635, 0.0009786763694137335, -0.0007641951087862253, 0.013334994204342365, -0.005552869755774736, 0.025967435911297798, -0.022562755271792412, 0.014550951309502125, -0.01652350276708603, 0.022562755271792412, -0.014659035950899124, -0.02064424566924572, 0.00514417327940464, 0.019022969529032707, 0.007221432868391275, -0.01542914193123579, 0.00035022091469727457, 0.04253147169947624, 0.00028119003400206566, 0.015550737269222736, 0.0051036411896348, 0.007288985885679722, 0.042693596333265305, -0.009139942936599255, 0.01601010002195835, -0.0038032429292798042, 0.013193132355809212, -0.010680154897272587, 0.006100050639361143, 0.022279033437371254, -0.000267257186351344, -0.02194126695394516, -0.006684385240077972, 0.016239779070019722, 0.00298078334890306, -0.0047658756375312805, 0.016645098105072975, 0.02704828605055809, -0.007221432868391275, -0.019225629046559334, 0.005556247662752867, -0.02285999059677124, 0.016266800463199615, 0.02430562674999237, -0.029074881225824356, 0.0038201313000172377, 0.013625472784042358, 0.00472534354776144, 0.0313987098634243, -0.02088743820786476, -0.02522435039281845, -0.004063322674483061, 0.0027460360433906317, -0.01070042047649622, 0.0324525386095047, 0.012943185865879059, -0.015861481428146362, -0.011889356188476086, 0.02080637402832508, -0.0026075521018356085, -0.010794995352625847, -0.0185230765491724, -0.009842495433986187, 0.02056318148970604, 0.006400662008672953, 0.009666857309639454, -0.623002290725708, -0.028048072010278702, 0.00026683497708290815, -0.020914459601044655, 0.017698928713798523, -0.0011821803636848927, 0.02506222203373909, -0.00554273696616292, 0.01775297150015831, 0.01875275745987892, 0.0034283229615539312, 0.02796700783073902, 0.012213611975312233, -0.01379435509443283, -0.01138270739465952, -0.026602433994412422, 0.01942828856408596, -0.006765448953956366, 0.024575840681791306, 0.0006662430241703987, -0.02612956240773201, -0.003289839019998908, 0.0026058631483465433, 0.03620849549770355, 0.0003966637304984033, -0.010646378621459007, 0.029993603006005287, -0.021846693009138107, -0.007288985885679722, 0.031804028898477554, -0.02522435039281845, 0.02476498857140541, 0.028318285942077637, 0.003446900052949786, 0.04844912886619568, -0.011369196698069572, -0.009727654978632927, -0.001299553900025785, 0.011713718064129353, 0.04334210976958275, -0.016509993001818657, 0.00588388042524457, -0.008788666687905788, -0.007072816137224436, -0.01085579302161932, -0.015334567986428738, 0.022751905024051666, -0.011835313402116299, -0.01226089894771576, -0.022346585988998413, -0.0008967682369984686, 0.022968074306845665, -0.015361588448286057, 0.0015343012055382133, -0.02796700783073902, 0.006400662008672953, 0.03815402463078499, 0.010619357228279114, 0.006907310802489519, -0.008856219239532948, 0.010990899056196213, 0.016239779070019722, -0.023819245398044586, -0.005076620262116194, -0.0057859281077980995, 0.010200527496635914, -0.012976962141692638, -0.012024462223052979, 0.01614520512521267, 0.001521634985692799, 0.029345093294978142, -0.0025872860569506884, -0.022495202720165253, 0.023373393341898918, -0.01028159074485302, -0.006792470347136259, 0.015604780055582523, 0.001948908669874072, -0.0062250238843262196, 0.014091589488089085, -0.018104247748851776, -0.0016508303815498948, -0.01798265241086483, -0.008396858349442482, 0.03283083811402321, -0.012402760796248913, 0.009099410846829414, -0.011524569243192673, 0.007619996555149555, 0.001969174714758992, -0.009437176398932934, -0.0112003143876791, -0.008349570445716381, -0.014226695522665977, -0.005171194672584534, -0.0037086685188114643, -0.006910688243806362, -0.0008131711510941386, 0.018698714673519135, -0.038883600383996964, -0.0017749592661857605, -0.023211266845464706, -0.001739493920467794, -0.017320631071925163, 0.01195690967142582, 0.015077865682542324, -0.02939913608133793, -0.007444358430802822, 0.02492711693048477, -0.0059548113495111465, -0.016131695359945297, -0.01551020611077547, -0.021157650277018547, 0.0005252258270047605, -0.00546505069360137, -0.030290838330984116, 0.026697009801864624, -0.01463201455771923, 0.012308185920119286, -0.018374459818005562, -0.00603587506338954, -0.028372326865792274, 0.011740739457309246, -0.00603587506338954, -0.010031644254922867, 0.004928003065288067, 0.024656902998685837, 0.0008625694317743182, 0.00998435728251934, -0.003384413430467248, 0.0010065421229228377, -0.013713291846215725, 0.006508747115731239, -0.013713291846215725, 0.029426157474517822, 0.004127498250454664, 0.015658821910619736, -0.0032290411181747913, 0.008525208570063114, -0.03845125809311867, -0.039126791059970856, -0.007424092385917902, 8.43886737129651e-05, 0.0010825394419953227, 0.005529226269572973, -0.014929248951375484, -0.020671267062425613, -0.007951007224619389, -0.008606272749602795, -0.02154945768415928, -0.02475147880613804, -0.008667070418596268, -0.00287607591599226, -0.0023829378187656403, -0.022643819451332092, -0.03304700553417206, -0.021887224167585373, -0.0026784827932715416, -0.016726162284612656, -0.04755742475390434, 0.028534455224871635, 0.018725736066699028, -0.013645738363265991, -0.012997228652238846, 0.02391381934285164, -0.003617471782490611, 0.004215317312628031, 0.03126360476016998, -0.005833215545862913, 0.00011473482300061733, -0.00149039167445153, -0.015023822896182537, -0.00268692709505558, 0.021657543256878853, 0.0060527632012963295, -0.02269786223769188, 0.00466792332008481, -0.000715219066478312, -0.009808719158172607, -0.007964517921209335, 0.0031243336852639914, 0.0019185098353773355, -0.03329019621014595, -0.012537866830825806, 0.033263176679611206, 0.00949121918529272, 0.0023880042135715485, -0.008403613232076168, -0.02453530766069889, 0.012321696616709232, -0.009045368060469627, 0.021914245560765266, -0.03588423877954483, 0.002496089320629835, 0.010031644254922867, -0.01505084428936243, -0.00501244468614459, 0.003071980085223913, -0.016496481373906136, 0.005698109045624733, 0.03885657712817192, -0.0018323794938623905, 0.020225416868925095, -0.006248667370527983, -0.008450900204479694, -0.012659462168812752, 0.011112495325505733, 0.000508337514474988, 0.037748705595731735, 0.02277892641723156, 0.009997867979109287, -0.013666004873812199, 0.003759333398193121, 0.007099837064743042, 0.02796700783073902, 0.007653773296624422, -0.014253716915845871, 0.015307546593248844, -0.0044382428750395775, 0.013031004928052425, 0.0008093712967820466, 0.0037559557240456343, 0.025386478751897812, 0.0006434438400901854, -0.008707602508366108, 0.014456376433372498, 0.003051714040338993, 0.05706891044974327, 0.00520159350708127, -0.019995735958218575, 0.00040531897684559226, 0.04644955322146416, 0.004867205396294594, -0.01722605526447296, 0.023049138486385345, 0.015523716807365417, 0.020157862454652786, -0.01195690967142582, 0.047800615429878235, -0.008971059694886208, 0.002423469675704837, 0.004303136374801397, 0.03861338645219803, -0.017874566838145256, -0.007390315644443035, 0.006255422718822956, 0.03269572928547859, 0.020455097779631615, 0.00474223168566823, 0.03153381496667862, -0.020482119172811508, -0.003164865542203188, -0.043315086513757706, -0.010342389345169067, 0.017942119389772415, -0.0013654183130711317, 0.010653133504092693, 0.004640902392566204, 0.023711159825325012, 0.01195690967142582, -0.019374245777726173, 0.004417976830154657, 0.02842636965215206, -0.008119889535009861, -0.002928429516032338, 0.018023183569312096, 0.023657117038965225, -0.02154945768415928, -0.01505084428936243, 0.00316993216983974, 0.00481991795822978, 0.004674678668379784, 0.013699781149625778, 0.0011408040300011635, 0.00535358814522624, -0.0010496072936803102, 0.004360556602478027, 0.0056879762560129166, -0.007025528699159622, 0.01563180238008499, -0.004816540516912937, -0.0229275431483984, 0.0014583038864657283, 0.015185950323939323, -0.007363294716924429, -0.01496978010982275, -0.004279492888599634, -0.00563731137663126, -0.03010168857872486, 0.014456376433372498, -0.016361376270651817, 0.023589564487338066, 0.036748919636011124, 0.013139089569449425, 0.004928003065288067, -0.010680154897272587, 0.00018724892288446426, 0.0040194131433963776, 0.017347652465105057, -0.006127071566879749, -0.01584797166287899, 0.036451686173677444, -0.007829410955309868, -0.00880217645317316, 0.01519946102052927, -0.007613241206854582, -0.019833607599139214, -0.02780488133430481, 0.0190499909222126, -0.026089031249284744, -0.016104673966765404, -0.0023981372360140085, -0.021292757242918015, 0.010153240524232388, -0.007484890054911375, 0.021090097725391388, -0.010849038138985634, -0.004357178695499897, 0.015456163324415684, -0.003316860180348158, -0.015942545607686043, -0.006312842946499586, -0.008282017894089222, -0.013132334686815739, 0.06695869565010071, -0.0035195196978747845, -0.008417123928666115, -0.005522470921278, -0.027994029223918915, -0.008504942990839481, -0.03085828386247158, -0.017239566892385483, 0.01150430366396904, 0.022035840898752213, 0.0025737753603607416, -0.018374459818005562, 0.01606414094567299, -0.021846693009138107, 0.013152600266039371, 0.003816753625869751, -0.01791509799659252, -0.027237433940172195, 0.029912540689110756, 0.016361376270651817, -0.006495236419141293, -0.0027916342951357365, 0.013280951417982578, 0.03726232424378395, 0.023373393341898918, 0.018482545390725136, 0.03769466280937195, 0.004282870329916477, 0.021454883739352226, -0.017766481265425682, -0.012125791981816292, 0.013875419273972511, 0.004424732178449631, 0.01101116556674242, -0.013409302569925785, -0.01676669530570507, 0.007072816137224436, -0.0077821239829063416, -0.016807226464152336, 0.0232382882386446, 0.03161488100886345, 0.018685204908251762, -0.0031817539129406214, -0.007106592413038015, 0.016212759539484978, -0.013591695576906204, -0.011558345519006252, -0.011443505063652992, 0.008538719266653061, -0.017793502658605576, 0.029669348150491714, 0.01750977896153927, -0.00046907225623726845, -0.0020755708683282137, 0.015604780055582523, 0.00575215183198452, -0.017955631017684937, -0.023130202665925026, -0.03229041025042534, -0.034695304930210114, 0.011774515733122826, -0.014199674129486084, 0.015645312145352364, 0.011862334795296192, -0.013699781149625778, -0.012990472838282585, -0.030047645792365074, -0.029209986329078674, -0.011659675277769566, -0.025724243372678757, -0.01463201455771923, -0.039721257984638214, -0.00590414647012949, 0.006316220387816429, 0.013078291900455952, 0.024724457412958145, 0.028750624507665634, 0.006100050639361143, -0.01759084314107895, 0.03212828189134598, -0.020900947973132133, -0.02285999059677124, -0.007214677520096302, -0.0028338551055639982, -0.0324525386095047, -0.0036681366618722677, 0.01614520512521267, 0.004870582837611437, -0.006008853670209646, 0.006924198940396309, -0.023373393341898918, -0.007363294716924429, 0.01896892674267292, -0.02453530766069889, 0.009389889426529408, 0.003185131587088108, 0.02049562893807888, 0.009416910819709301, 0.008052336983382702, 0.0030061155557632446, 0.01950935274362564, -0.0034722324926406145, -0.0037052908446639776, -0.020995521917939186, 0.028696583583950996, 0.015226482413709164, 0.010193771682679653, 0.007167390547692776, -0.02353552170097828, 0.0028693205676972866, -0.020536160096526146, -0.05620422959327698, 0.013632227666676044, 0.008437389507889748, -0.015226482413709164, 0.025008181110024452, 0.0212522242218256, 0.012760791927576065, -0.0103626549243927, -0.013456589542329311, 0.011517814360558987, -0.009862761944532394, 0.012713504955172539, -0.005073242355138063, -0.02248169295489788, 0.017253076657652855, -0.021535947918891907, -0.008133400231599808, 0.011031431145966053, 0.01055180374532938, 0.020076800137758255, 0.01013972982764244, 0.009349357336759567, -0.02049562893807888, -0.029669348150491714, -0.002401514910161495, -0.021468395367264748, 0.022224990651011467, 0.021833181381225586, -0.022130414843559265, -0.03501955792307854, 0.016401907429099083, 0.022887010127305984, -0.024102967232465744, -0.016874779015779495, -0.0229275431483984, -0.004715210758149624, -0.01903648115694523, 0.020387543365359306, -0.004049811977893114, 0.010788239538669586, 0.02772381715476513, -0.03566806763410568, -0.03061509318649769, -0.01819882169365883, -0.021995309740304947, -0.0007570176385343075, -0.0095249954611063, -0.003641115501523018, 0.029885519295930862, 0.0425044484436512, 0.004539572633802891, 0.00027021265123039484, 0.01949584297835827, 0.016482971608638763, -0.006951220333576202, -0.0037660887464880943, -0.014564462006092072, -0.03991040587425232, 0.01568584330379963, -0.004884093534201384, -0.007714570965617895, 0.004451753105968237, 0.008106379769742489, -0.01752329058945179, 0.03718125820159912, 0.004029545933008194, -0.004637524485588074, 0.018482545390725136, -0.018644671887159348, -0.030290838330984116, 0.009633081033825874, 0.007728081662207842, 0.004735476803034544, -0.030371900647878647, -0.013814621604979038, 0.04601721465587616, -0.006768826860934496, 0.02253573387861252, -0.022076373919844627, 0.025413500145077705, -0.013044515624642372, 0.012997228652238846, 0.00013616183423437178, 0.00979520846158266, -0.007363294716924429, -0.0102072823792696, -0.02780488133430481, 0.0009254782926291227, -0.00960605964064598, -0.0068600233644247055, 0.01002488937228918, -0.02591339312493801, 0.0065898108296096325, -0.013172866776585579, 0.015442652627825737, -0.04439593851566315, -0.02300860732793808, 0.00037787549081258476, -0.013476856052875519, -0.020211905241012573, -0.0380999818444252, 0.01661807857453823, -0.015523716807365417, -0.014267227612435818, -0.0033725916873663664, 0.00553935905918479, 0.0305610504001379, -0.0028034562710672617, -0.006674252450466156, 0.006518879905343056, -0.006880289409309626, 0.02522435039281845, 0.01865818351507187, 0.030966369435191154, 0.019793076440691948, 0.013483610935509205, 0.0069174435921013355, 0.004657790530472994, -0.011220579966902733, -0.023184245452284813, 0.016199247911572456, 0.01195690967142582, -0.017158502712845802, -0.0028912753332406282, 0.0027274589519947767, 0.030344879254698753, -0.008896751329302788, -0.015739886090159416, 0.0038707961793988943, 0.011598877608776093, -0.005424519069492817, -0.012348718009889126, 0.004178163129836321, 0.00011642365279840305, 0.009369623847305775, -0.009957335889339447, 0.009275048971176147, -0.00990329310297966, -0.01705041714012623, -0.01028159074485302, 0.03426296263933182, -0.02530541457235813, 0.012456802651286125, 0.025791797786951065, -0.0018830443732440472, -0.015604780055582523, 0.023265309631824493, -0.01652350276708603, -0.012781058438122272, 0.0012446669861674309, 0.021806159988045692, -0.009248027577996254, 0.00523874768987298, 0.0010935168247669935, 0.008173932321369648, -0.008822442963719368, -0.009862761944532394, 0.012179834768176079, 0.02666998840868473, -0.0016846070066094398, -0.01667211949825287, 0.0022782303858548403, 0.00569473160430789, 0.00603587506338954, 0.017009885981678963, -0.012578398920595646, -0.013301216997206211, -0.03031785786151886, -0.002448802115395665, -0.002205610740929842, 0.0029216741677373648, -0.010463984683156013, -0.0195769052952528, -0.013942972756922245, -0.03174998611211777, -0.021306267008185387, 0.012308185920119286, 0.010011378675699234, -0.036992110311985016, -0.003526275046169758, -0.013409302569925785, -0.023494988679885864, 0.02627817913889885, -0.021130628883838654, -0.0039991470985114574, -0.003332059597596526, 0.006221645977348089, -0.008687336929142475, 0.002556887222453952, 0.025724243372678757, 0.038370195776224136, -0.014915738254785538, 0.01584797166287899, -0.013584940694272518, -0.03815402463078499, 0.02964232675731182, 0.003995769657194614, -0.02088743820786476, 0.006714784074574709, 0.0029706503264606, 0.00555962510406971, 0.0032155304215848446, -0.008788666687905788, 0.03761360049247742, 0.017077438533306122, -0.012037972919642925, 0.0053637209348380566, -0.04077508673071861, -0.007403826341032982, 0.009666857309639454, 0.0008857908542267978, 0.04426082968711853, -0.008376591838896275, 0.030344879254698753, -0.002310318173840642, -0.011146271601319313, 0.019779564812779427, -0.01844201236963272, 0.009923559613525867, 0.0026700387243181467, -0.008998081088066101, 0.007788879331201315, -0.0051306625828146935, -0.013382281176745892, -0.011781271547079086, -0.04731423407793045, 0.039883386343717575, -0.019468821585178375, -0.011970420368015766, 0.02583232894539833, 0.000919567421078682, -0.015969567000865936, 0.03207423910498619, -0.00236267177388072, -0.013787600211799145, -0.0006020675646141171, -0.03126360476016998, -0.038370195776224136, -0.013753823935985565, -0.0018222464714199305, 0.03499253839254379, 0.01722605526447296, -0.020306481048464775, -0.0037627110723406076, -0.02492711693048477, -0.03845125809311867, 0.0036816473584622145, 0.01668563112616539, 0.02826424315571785, 0.0038606631569564342, 0.025616157799959183, 0.021833181381225586, 0.021198181435465813, 0.019549885764718056, 0.013058026321232319, 0.0025315547827631235, 0.04447700083255768, 0.017888076603412628, -0.002337339334189892, 0.008471166715025902, 0.011747494339942932, -0.02384626679122448, 0.005735263228416443, -0.0022680973634123802, 0.008018559776246548, 0.01112600602209568, 0.0364246629178524, -0.004569971468299627, 0.00563393346965313, 0.008748134598135948, -0.029615305364131927, 0.029966581612825394, -0.004154519643634558, -0.00282878871075809, -0.04550381004810333, -0.02363009564578533, 0.006231779232621193, 0.004684811923652887, 0.010153240524232388, -0.019860628992319107, -0.025967435911297798, -0.009822229854762554, -0.012963451445102692, -0.02223850041627884, -0.02080637402832508, 0.004036301281303167, -0.013625472784042358, 0.021049564704298973, 0.03220934793353081, 0.0024977780412882566, 0.019752543419599533, 0.009950581006705761, 0.002391381887719035, 0.01398350391536951, -0.003874173853546381, -0.012308185920119286, -0.014456376433372498, 0.015185950323939323, -0.02948020026087761, 0.017185524106025696, -0.01119355857372284, 0.02277892641723156, 0.012159569188952446, -0.009923559613525867, -0.01791509799659252, -0.0043842000886797905, -0.018482545390725136, 0.01911754533648491, 0.022792436182498932, -0.018185310065746307, 0.007329517975449562, -0.003901195013895631, 0.0042862482368946075, 0.014915738254785538, 0.007498400751501322, -0.016023609787225723, -0.04069402441382408, -0.006721539422869682, -0.002506222343072295, -0.021806159988045692, -0.009585793130099773, -0.004846939351409674, 0.01398350391536951, -0.005870369728654623, -0.022360095754265785, 0.18406885862350464, -0.006512124557048082, 0.014956270344555378, 0.018644671887159348, -0.009545261971652508, -0.0047692530788481236, 0.004191673826426268, 0.015820950269699097, 0.000560691230930388, 0.01798265241086483, -0.026913179084658623, -0.0028676316142082214, -0.0229275431483984, -0.0033117937855422497, -0.0058973911218345165, 0.00033375484053976834, -0.045287638902664185, -0.025656690821051598, -0.03123658150434494, -0.00497191259637475, 0.009254783391952515, -0.017401693388819695, -0.020076800137758255, -0.008045581169426441, 0.013301216997206211, -0.001997884828597307, -0.014483397826552391, -0.010605846531689167, 0.04180189594626427, 0.01798265241086483, -0.010187016800045967, 0.016239779070019722, -0.018401481211185455, 0.0014245272614061832, -0.020468607544898987, 0.005529226269572973, 0.03023679554462433, -0.04263955354690552, -0.013287707231938839, -0.004900981672108173, -0.01903648115694523, 0.013875419273972511, -0.009079145267605782, -0.010572069324553013, -0.02406243607401848, -0.018617650493979454, -0.0212522242218256, -0.014348291791975498, -0.019104033708572388, 0.011788026429712772, -0.05142146721482277, -0.0005556247779168189, 0.02652137167751789, 0.02422456443309784, 0.020387543365359306, -0.02141435258090496, 0.008640049025416374, -0.001008230959996581, 0.00523874768987298, 0.010788239538669586, -0.03510062396526337, 0.03215530514717102, -0.022495202720165253, 0.008248240686953068, 0.005478561390191317, 0.0026548393070697784, 0.01775297150015831, 0.02141435258090496, 0.02049562893807888, -0.0031733098439872265, -0.002394759561866522, 0.01913105510175228, 0.03002062439918518, -0.01496978010982275, -0.03329019621014595, 0.00043044029735028744, 0.06079784408211708, 0.0032290411181747913, 0.010355900041759014, 0.006768826860934496, 0.0017850922886282206, 0.01638839766383171, 0.018090736120939255, -0.021076586097478867, 0.014672546647489071, -0.02317073382437229, 0.03145275264978409, -0.009531751275062561, -0.004259226843714714, -0.015537227503955364, 0.02734551951289177, 0.005471806041896343, -0.0027105705812573433, -0.017023395746946335, -0.021468395367264748, 6.285610288614407e-05, -0.0056677102111279964, 0.017577333375811577, 0.009964091703295708, 0.005549492314457893, -0.006657363846898079, 0.009092655032873154, 0.008200953714549541, -0.0010783172911033034, -0.031128495931625366, 0.01097063347697258, -0.015321057289838791, 0.003779599443078041, -0.011402973905205727, -0.025724243372678757, 0.0023221399169415236, 0.007288985885679722, 0.010376165620982647, -0.021684564650058746, 0.025710733607411385, -0.014483397826552391, 0.016550524160265923, -0.026710519567131996, 0.008309039287269115, -0.00532994419336319, 0.01501031219959259, -0.028453391045331955, -0.015645312145352364, 0.008640049025416374, -0.02033350244164467, -0.024656902998685837, -0.0006907310453243554, 0.007072816137224436, -0.018604140728712082, -0.04261253401637077, 0.010295101441442966, -0.019401267170906067, 0.004049811977893114, -0.0056710876524448395, -0.008667070418596268, 0.018995948135852814, -0.0024082702584564686, -4.646918750950135e-05, -0.009653346613049507, -0.03639764338731766, 0.022724883630871773, -0.004488907754421234, -0.0008537030662409961, -0.009173719212412834, 0.006545901298522949, 0.006427683401852846, 0.022724883630871773, 0.006566167343407869, -0.03323615714907646, -0.007930740714073181, -0.012308185920119286, -0.02788594551384449, -0.0020401054061949253, -0.0026784827932715416, 0.04199104383587837, 0.005265769083052874, -0.02003626711666584, -0.041720833629369736, -3.728617957676761e-05, -0.01791509799659252, -0.026629455387592316, -0.020927969366312027, 0.009072389453649521, -0.00895754899829626, 0.01850956678390503, 0.0014633703976869583, -0.1708824783563614, 0.019982224330306053, 0.020995521917939186, 0.0012320007663220167, 0.013051270507276058, 0.016969354823231697, 0.04974614828824997, -0.024265095591545105, -0.014577971771359444, 0.0010090753203257918, 0.01599658839404583, 0.007951007224619389, -0.048557210713624954, -0.015064354985952377, -0.017036907374858856, 0.014672546647489071, -0.026116052642464638, 0.03131764382123947, 0.02223850041627884, -0.005414385814219713, 0.008856219239532948, -0.02535945735871792, 0.029426157474517822, -0.016847757622599602, -0.02186020277440548, -0.0031209560111165047, -0.05098912492394447, 0.017847545444965363, -0.013064781203866005, -0.021603500470519066, -0.031966157257556915, 0.006133826915174723, 0.00523874768987298, -0.02415701001882553, -0.007133613806217909, 0.011781271547079086, -0.008268507197499275, 0.014375312253832817, -0.01519946102052927, 0.03353339061141014, 0.023873286321759224, -0.00010064365778816864, 0.00031581101939082146, 0.040504876524209976, -0.008477921597659588, 0.03847828134894371, -0.018387969583272934, -0.014348291791975498, 0.014145632274448872, 0.013334994204342365, 0.01557775866240263, 0.004282870329916477, 0.013902440667152405, -0.01607765257358551, 0.003630982479080558, 0.0002328473055968061, 0.023292329162359238, 0.01622626930475235, 0.02126573584973812, 0.004009279888123274, -0.007153879851102829, -0.012051483616232872, 0.013139089569449425, -0.004965157248079777, -0.0023170735221356153, -0.01561829075217247, -0.015712864696979523, 0.0026362622156739235, -0.01750977896153927, 0.01667211949825287, -0.006900555454194546, 0.0005425363196991384, 0.02560264803469181, 0.0007350628147833049, 0.020441586151719093, 0.0003360769769642502, -0.004728721454739571, -0.009727654978632927, 0.015550737269222736, 0.012085260823369026, -9.431054786546156e-05, 0.053069762885570526, -0.015645312145352364, 0.010828771628439426, -0.011639409698545933, 0.0008503253920935094, 0.02575126476585865, 0.00037871990934945643, -0.02614307403564453, 0.01440233364701271, 0.02848041243851185, -0.01444286573678255, -0.0031395331025123596, -0.014483397826552391, 0.012348718009889126, -0.020927969366312027, 0.003033136948943138, -0.012672972865402699, 0.008268507197499275, -0.007998294197022915, 0.01705041714012623, 0.0019168209983035922, -0.034695304930210114, -0.0058399708941578865, 0.043990619480609894, 0.011673185974359512, 0.02780488133430481, 0.003749200375750661, 0.014726589433848858, -0.016780205070972443, 0.03331721946597099, 0.010376165620982647, -0.0024217807222157717, 0.022062862291932106, 0.002462312811985612, 0.010261325165629387, 0.0017986028688028455, -0.013625472784042358, 0.017158502712845802, -0.010031644254922867, 0.04299083352088928, -0.013362015597522259, -0.008221219293773174, -0.001896554953418672, -0.025399988517165184, 0.014848184771835804, -0.08657613396644592, -0.0018458900740370154, 0.022508712485432625, 0.009477708488702774, 0.003502631327137351, 0.006775582209229469, -0.008349570445716381, 0.041396576911211014, -0.00028520100750029087, 0.02049562893807888, -0.013490366749465466, -0.022197969257831573, -0.00020002068777102977, 0.01192313339561224, 0.05455593392252922, -0.008065847679972649, 0.009876271709799767, -0.02210339345037937, 0.015834461897611618, 0.0020130842458456755, 0.02079286240041256, -0.02246818132698536, 0.000671731773763895, 0.0003586650709621608, -0.017266588285565376, -0.0207253098487854, -0.033560410141944885, 0.01653701439499855, 0.003955237567424774, 0.010227548889815807, -0.0017817146144807339, -0.020144352689385414, 0.0182123314589262, 0.009437176398932934, -0.029453178867697716, -0.019563395529985428, -0.06485103815793991, -0.031668923795223236, 0.038127005100250244, -0.044449981302022934, 0.003850530134513974, -0.011713718064129353, 0.020982012152671814, -0.002443735720589757, -0.012308185920119286, -0.010599090717732906, 0.01421318482607603, 0.02977743372321129, -0.004164652433246374, -0.03388466686010361, -0.008160421624779701, -0.0028591875452548265, 0.008160421624779701, -0.042180195450782776, 0.038343172520399094, 0.0015250126598402858, -0.0008013493497855961, 0.03315509110689163, -0.015348078683018684, -0.016334354877471924, -0.014740100130438805, 0.010646378621459007, 0.00258221966214478, -0.01039643120020628, 0.033695515245199203, 0.019401267170906067, -0.03442509099841118, 0.006873534061014652, -0.00556300301104784, -0.006029119715094566, -0.0062047578394412994, 0.008768400177359581, -0.01063286792486906, -0.011416484601795673, -0.03450615331530571, 0.00576903996989131, -0.0034283229615539312, -0.02567020058631897, 0.0022309431806206703, 0.005174572113901377, -0.023103181272745132, -0.038829557597637177, -0.005167816765606403, -0.0012176457094028592, 0.007613241206854582, 0.003691780148074031, 0.002816966734826565, -0.006812736392021179, 0.0010411631083115935, -0.013666004873812199, 0.011720473878085613, 0.02140084095299244, -0.0052590137347579, -0.02552158385515213, 0.008450900204479694, 0.014699568040668964, -0.008106379769742489, -0.010720686987042427, -0.028318285942077637, 0.020157862454652786, -0.04726019129157066, -0.003769466420635581, -0.07906422019004822, -0.008862975053489208, 0.031425729393959045, 0.01675318367779255, 0.022589776664972305, -0.008525208570063114, 0.03718125820159912, 0.01131515484303236, 0.005593401845544577, 0.007863188162446022, -0.012632440775632858, 0.013902440667152405, -0.02956126257777214, -0.020752331241965294, -0.009011591784656048, 0.005356965586543083, 0.03153381496667862, -0.008032070472836494, 0.017469247803092003, 0.01806371472775936, -0.018874352797865868, -0.002063749125227332, 0.02612956240773201, 0.01043696328997612, -0.033479347825050354, 0.02788594551384449, 0.006984997075051069, 0.02689966931939125, -0.008322549052536488, -0.02530541457235813, 0.02154945768415928, -0.013942972756922245, -0.009248027577996254, -0.004900981672108173, 0.002045172033831477, -0.0033725916873663664, 0.0021127250511199236, 0.029074881225824356, 0.011396218091249466, 0.03701913356781006, -0.03204721957445145, -0.01157861202955246, -0.012254143133759499, -0.009856006130576134, 0.007545688189566135, -0.010565314441919327, -0.0023170735221356153, 0.012524356134235859, 0.009342602454125881, 0.012976962141692638, -0.01191637758165598, 0.005755529273301363, -0.0033523256424814463, -0.022670840844511986, 0.015672333538532257, -0.015874993056058884, -0.012450047768652439, -0.0004008857940789312, 0.02688615769147873, -0.011153026483952999, 0.033398281782865524, -0.002705504186451435, 0.02589988149702549, 0.004259226843714714, 0.03261466696858406, -0.03018275275826454, -0.020252438262104988, -0.02126573584973812, -0.00584672624245286, -0.03510062396526337, -0.008173932321369648, -0.00035634293453767896, 0.028507433831691742, 0.023427436128258705, -0.023251798003911972, 0.011470526456832886, 0.02284647896885872, 0.011254356242716312, -0.029534241184592247, -0.01142323948442936, 0.038802534341812134, -0.009045368060469627, -0.023954350501298904, -0.0031716208904981613, -0.000537047628313303, -0.0005754684680141509, 0.03337126225233078, -0.022197969257831573, 0.008822442963719368, 0.03399275243282318, -0.0182123314589262, -0.01903648115694523, 0.0020840149372816086, 0.0193607360124588, -0.006657363846898079, 0.03129062429070473, -0.008592762053012848, -0.015942545607686043, 0.0038707961793988943, 0.030452964827418327, 0.0011509369360283017, 0.021454883739352226, -0.019766055047512054, -0.01790158823132515, -0.011436750181019306, 0.018860843032598495, -0.0008558140834793448, -0.05174572020769119, -0.028075093403458595, -0.019333714619278908, 0.005606912542134523, -0.019063502550125122, 0.004147764295339584, 0.0012936430284753442, -0.006471592932939529, 0.005910901818424463, 0.03566806763410568, -0.017658395692706108, -0.005042843520641327, 0.019536374136805534, -0.01478063128888607, 0.02383275516331196, -0.029372114688158035, 0.006295954808592796, 0.04380146786570549, 0.007059305440634489, 0.012382494285702705, -0.018955416977405548, -0.006512124557048082, -0.011477282270789146, 0.007316007278859615, 0.013341749086976051, -0.027386050671339035, -0.013105313293635845, -0.023886797949671745, 0.004451753105968237, -0.020238926634192467, 0.041937001049518585, -0.0229275431483984, 0.07938847690820694, 0.01859062910079956, -0.007532177492976189, 0.032803814858198166, -0.013888929970562458, 0.005282657220959663, -0.005289412569254637, 0.007761858403682709, -0.02765626460313797, -0.06474295258522034, 0.009774942882359028, -0.026480838656425476, -0.016496481373906136, -0.01584797166287899, -0.010794995352625847, -0.0066472310572862625, 0.0011897800723090768, 0.007978028617799282, 0.008795421570539474, 0.0009491218952462077, 0.023589564487338066, -0.00872786808758974, 0.011024676263332367, 0.0016525192186236382, -0.003630982479080558, -0.01249057985842228, 0.013368770480155945, 0.030128709971904755, 0.004016035236418247, -0.03931593894958496, 0.011659675277769566, -0.007234943564981222, -0.0025197328068315983, -0.0006438660784624517, -0.011781271547079086, -0.00246737920679152, -0.015375099144876003, -0.03961317241191864, 0.006677629891782999, 0.019333714619278908, -0.00158496608491987, 0.020171374082565308, -0.011889356188476086, -0.021617012098431587, 0.007586219813674688, -0.021144138649106026, -0.033263176679611206, -0.013888929970562458, -0.007005262654274702]} +{"id": "test:10000006", "text": "\"I thought I was going to finish the 3rd season of the Wire tonight.\\nBut there was a commentary on episode 11, so I had to re-watch Middle Ground with the commentary. Hopefully I can finish the season next weekend.\"", "vector_field": [-0.006273483391851187, -0.02776634320616722, -0.005765725392848253, -0.02039049193263054, 0.007275637239217758, 0.01178533025085926, -0.015513342805206776, -0.029717203229665756, -0.0396585687994957, -0.0033772585447877645, 0.039311155676841736, 0.029877549037337303, 0.011538132093846798, -0.0206978190690279, -0.004402796272188425, -0.006701068952679634, 0.0207379050552845, 0.004703442100435495, 0.0018706872360780835, -0.034126680344343185, 0.0010631182231009007, 0.011745243333280087, 0.012593734078109264, -0.03548961132764816, 0.026069363579154015, 5.992045043967664e-05, 0.002233132952824235, -0.020711179822683334, -0.008197618648409843, -0.00908619537949562, 0.011885545216500759, 0.01193231251090765, 0.003497517202049494, -0.006567448377609253, -0.007349128369241953, -0.005535230040550232, 0.002425212413072586, -0.03025168552994728, 0.017290495336055756, -0.015660325065255165, 0.014724981039762497, -0.005251286551356316, -0.004653334617614746, -0.005211200099438429, -0.02811375819146633, 0.02013661153614521, 0.012974552810192108, -0.023169798776507378, -0.02240816131234169, 0.028995653614401817, 0.012927785515785217, 0.010990288108587265, -0.006807965692132711, -0.03407323360443115, 0.010890072211623192, -0.003587710903957486, 0.015326273627579212, 0.04086115583777428, 0.016555583104491234, -0.02035040594637394, -0.00023007782874628901, -0.030812891200184822, -0.01687627099454403, 0.00899934209883213, 0.010061625391244888, 0.009794384241104126, 0.009707530960440636, -0.016996530815958977, -0.023624107241630554, -0.011377787217497826, 0.015179291367530823, 0.00758296437561512, -0.00036891791387461126, 0.007689861115068197, 0.0176913570612669, 0.0025371196679770947, 0.00015658655320294201, 0.017156874760985374, 0.011818734928965569, 0.0007724936585873365, 0.022207729518413544, -0.019308164715766907, -0.020283594727516174, 0.03599736839532852, 0.031347375363111496, 0.003106677206233144, 0.011878863908350468, 0.013856448233127594, -0.0006438838900066912, 0.0018974114209413528, 0.010034901089966297, 0.024612899869680405, 0.020083164796233177, 0.02052411250770092, 0.006487276405096054, 0.023370228707790375, -0.05040166154503822, 0.025080570951104164, -0.028247378766536713, 0.0032837241888046265, 0.009507100097835064, -0.00869869627058506, 0.021218938753008842, -0.005685552954673767, -0.023076264187693596, -0.010589426383376122, 0.02373100444674492, 0.02194049023091793, 0.024385744705796242, -0.0075963265262544155, -0.002134587848559022, 0.030438754707574844, 0.015312911942601204, -0.01523273903876543, -0.0021312471944838762, -0.03714650496840477, 0.0004309261857997626, -0.031347375363111496, -0.007041801232844591, -0.020965060219168663, 0.000989626976661384, 0.014457739889621735, 0.006373698823153973, -0.02145945467054844, -0.005104303825646639, 0.008444816805422306, -0.015259463340044022, -0.006751176901161671, -0.004716804251074791, 0.018920665606856346, 0.018626701086759567, 0.014417653903365135, 0.011451278813183308, 0.015259463340044022, -0.017170237377285957, 0.01928144134581089, -0.0217667818069458, 0.018653424456715584, -0.011524769477546215, -0.03666546940803528, 0.004195684101432562, 0.015112481079995632, 0.0005850073648616672, -0.010061625391244888, -0.00766981765627861, 0.02454608865082264, 0.006657642312347889, 0.006343634333461523, 0.0029363108333200216, -0.03687926381826401, 0.0011700147297233343, -0.0002261109766550362, 0.017798252403736115, -0.011812053620815277, -0.008144170977175236, 0.0024452554062008858, -0.012680587358772755, 0.022849109023809433, -0.018827131018042564, 0.002767615020275116, 0.019481871277093887, -0.010589426383376122, -0.004095469135791063, -0.012734035961329937, 0.010081668384373188, 0.03033185750246048, 0.01704997755587101, 0.005424993112683296, -0.022608591243624687, 0.0036211160477250814, 0.010843304917216301, 0.0021312471944838762, -0.007803438231348991, 0.006089755333960056, 0.015179291367530823, 0.014751705341041088, 0.011725200340151787, -0.010596107691526413, -0.018947388976812363, -0.00912628136575222, -0.008551713079214096, -0.0025671841576695442, 0.023931434378027916, 0.012493518181145191, -0.009814427234232426, -0.022461609914898872, 0.022809023037552834, -0.037039607763290405, -0.0018189093098044395, -0.0067077502608299255, 0.002124566351994872, 0.01747756451368332, 0.023624107241630554, -0.004977364558726549, -0.6396681666374207, -0.015566790476441383, 0.008104084990918636, 0.0037113099824637175, 0.04564477130770683, 0.005264648701995611, -0.013161621056497097, -0.009567229077219963, -0.01171851996332407, 0.021740058436989784, 0.005007429048418999, 0.0070952498354017735, 0.0018155687721446157, -0.021058594807982445, -0.012500199489295483, -0.000989626976661384, 0.006179949268698692, -0.006627577822655439, -0.021218938753008842, -0.01171851996332407, -0.035195644944906235, 0.009714212268590927, -0.04289218783378601, 0.011404511518776417, -0.0042457920499145985, -0.009620677679777145, 0.015045670792460442, -0.01920126937329769, -0.0238646250218153, -0.0034140043426305056, -0.0455111488699913, 0.017464201897382736, 0.004152257461100817, 0.0018138985615223646, 0.055799927562475204, -0.0108833909034729, 0.0006171597633510828, 0.012961190193891525, 0.00446626590564847, 0.019896095618605614, 0.00021880360145587474, -0.016261616721749306, 0.022782297804951668, -0.008311196230351925, -0.01988273300230503, 0.009801065549254417, 0.013295241631567478, 0.00916636735200882, -0.006881456822156906, -0.03401978313922882, 0.005050855688750744, 0.010769814252853394, -0.006273483391851187, 0.031293924897909164, 0.011945674195885658, 0.023837901651859283, 0.03757409006357193, 0.026443500071763992, 0.01223295833915472, -0.021860316395759583, -0.020978420972824097, 0.004636631812900305, -0.012660544365644455, 0.0017270451644435525, -0.02613617293536663, 0.00016358074208255857, -0.026617208495736122, -0.002757593523710966, 0.01364265475422144, -0.0006463892641477287, 0.004907213617116213, 0.018199114128947258, -0.0270715169608593, -0.00671443110331893, 0.008431455120444298, 0.019000837579369545, 0.0002912509662564844, -0.006995034404098988, 0.00736917182803154, 0.022675402462482452, 0.00021859482512809336, -0.03246978670358658, -0.025949105620384216, 0.00873878225684166, 0.030518926680088043, -0.008571756072342396, -0.013081449083983898, 0.004018637351691723, 0.015219377353787422, -0.0029262893367558718, -0.0009336734074167907, 0.012914422899484634, 0.005612061824649572, 0.008237704634666443, 0.02421203814446926, -0.0053515019826591015, 0.007115292828530073, 0.030358582735061646, 0.027525827288627625, -0.01961549185216427, -0.001745418063364923, 0.0009278274956159294, 0.01045580580830574, -0.0207779910415411, 0.002826073905453086, 0.0015249442076310515, -0.006624237168580294, -0.008598480373620987, 0.02163316309452057, 0.0034140043426305056, -0.003838249482214451, 0.012246320955455303, -0.020791353657841682, -0.02497367560863495, -0.02021678537130356, -0.0185999758541584, 0.009199772961437702, 0.029209446161985397, -0.011090503074228764, -0.00907951407134533, 0.002380115445703268, 0.019374975934624672, -0.0039017191156744957, -0.011377787217497826, -0.018319373950362206, 0.017210323363542557, 0.020083164796233177, 0.020684456452727318, -0.0176111850887537, 0.009607315063476562, -0.0061331819742918015, -0.009279944933950901, 0.012820889241993427, -0.00022277045354712754, 0.00446626590564847, 0.005598699674010277, 0.0434533916413784, -0.026977982372045517, 0.013362051919102669, -0.026897810399532318, 0.003540943842381239, 0.01696980558335781, 0.004269175697118044, -0.012346535921096802, -0.0026005893014371395, -0.03217582032084465, -0.02918272092938423, -0.011711838655173779, -0.033137887716293335, -0.0019742432050406933, 0.004803657531738281, 0.01509911846369505, -0.009346755221486092, 0.015593514777719975, 0.0027008047327399254, -0.013509034179151058, 0.011163994669914246, -0.008331239223480225, -0.046286147087812424, -0.007088568527251482, -0.007516154088079929, 0.009119600057601929, -0.017170237377285957, 0.004776933696120977, 0.039177536964416504, -0.003347194055095315, 0.004813679028302431, 0.011985760182142258, -0.0010322185698896646, -0.015139205381274223, 0.01735730469226837, -0.015860754996538162, -0.013963344506919384, -0.015673687681555748, -0.007382533513009548, 0.014818515628576279, 0.010288779623806477, -0.0028043605852872133, -0.01683618500828743, 0.0011474662460386753, -0.0016769375652074814, 0.03444736823439598, -0.005608721170574427, -0.004710123408585787, 0.027659447863698006, 0.00880559254437685, 0.042651668190956116, 0.006995034404098988, 0.0052780103869736195, 0.01722368411719799, 0.01966894045472145, -0.0022164303809404373, -0.009694168344140053, 0.0016418620944023132, 0.01520601473748684, -0.003151773940771818, 0.008939213119447231, 0.017424115911126137, 0.0028344253078103065, 0.034206852316856384, 0.015914203599095345, -0.0023149754852056503, 0.00028978948830626905, -0.009934686124324799, -0.007890291512012482, -0.03591719642281532, 0.010809900239109993, -0.023383591324090958, 0.013722827658057213, -0.01216614805161953, 0.01038899552077055, -0.0330042690038681, 0.0008743793005123734, -0.006306888535618782, 0.006861413829028606, 0.020884886384010315, -0.022515058517456055, 0.032496511936187744, -0.0031751575879752636, 0.02048402652144432, -0.0035643272567540407, 0.008665290661156178, 0.0068146465346217155, -0.0023684236221015453, -0.03190857917070389, 0.007055163383483887, 0.028006860986351967, 0.03137409687042236, 0.006370358169078827, -0.01914782077074051, 0.004636631812900305, 0.01790514960885048, 0.02287583239376545, 0.011304295621812344, 0.028087032958865166, -0.03792150318622589, 0.02497367560863495, -0.007162059657275677, 0.04273184388875961, -0.03482150658965111, 0.0001478177000535652, 0.023143073543906212, 0.016034461557865143, -0.0237443670630455, 0.02433229610323906, -0.0056554884649813175, 0.02592238038778305, 0.0057456823997199535, -0.017878426238894463, 0.0016869590617716312, 0.004984045401215553, 0.0019441785989329219, -0.02463962323963642, -0.03222927078604698, 0.005581997334957123, -0.0108633479103446, -0.014030154794454575, -2.7350450181984343e-05, 0.025174105539917946, 0.025160744786262512, 0.008585118688642979, 0.014016792178153992, 0.016983168199658394, -0.001939167850650847, -0.02064437046647072, 0.0027459017001092434, -0.0238646250218153, 0.007175421807914972, -0.0031734872609376907, -0.012065933085978031, 0.004125533625483513, -0.011324338614940643, 0.007349128369241953, -0.012526923790574074, 0.016609029844403267, -0.007576283533126116, 0.021793507039546967, 0.006153224967420101, 0.00023592372599523515, 0.004332645330578089, -0.0011533121578395367, -0.006016263738274574, 0.026630569249391556, -0.0038248873315751553, -0.026510311290621758, -0.008418092504143715, 0.012085976079106331, 0.003480814630165696, 0.0037079693283885717, 0.020283594727516174, -0.024091780185699463, 0.03148099407553673, -0.00017057494551409036, 0.011905588209629059, -0.011905588209629059, -0.013101492077112198, 0.033218059688806534, -0.007716584950685501, -0.0039952537044882774, 0.002672410337254405, 0.0057991305366158485, 0.01718359813094139, 0.01936161331832409, -0.014350843615829945, 0.03161461651325226, 0.019802561029791832, -0.010308823548257351, -0.029743928462266922, -0.005318096838891506, -0.007970464415848255, -0.008591799065470695, -0.006280164234340191, -0.023249970749020576, 0.0065641081891953945, -0.015219377353787422, -0.0018306011334061623, 0.010228650644421577, -0.015967652201652527, 0.013682741671800613, -0.001267724670469761, 0.0035142197739332914, -0.023877987638115883, -0.0269913449883461, -0.010956882499158382, 0.11395157873630524, 0.015459894202649593, -0.0036077541299164295, 0.038028400391340256, 0.008378006517887115, -0.008538351394236088, 0.009293307550251484, -0.024960312992334366, 0.023423677310347557, -0.018640061840415, 0.02099178358912468, 0.0029864185489714146, -0.00020523276180028915, 0.003450749907642603, 0.035062026232481, -0.008925850503146648, -0.014658170752227306, -0.015954289585351944, -0.018586615100502968, -0.030652547255158424, 0.0008618523715995252, -0.01360256876796484, -0.012947828508913517, 0.004392774775624275, -0.004506351891905069, -0.00305823958478868, 0.01953531987965107, -0.006644280161708593, 0.018626701086759567, -0.028033584356307983, -0.0029363108333200216, 0.012827569618821144, -0.008671971969306469, 0.00880559254437685, -0.011691795662045479, 0.03102668561041355, 0.03709305450320244, 0.014524550177156925, 0.02159307524561882, -0.012373260222375393, -0.012032527476549149, 0.006213354412466288, 0.007997187785804272, -0.014537912793457508, 0.026924535632133484, -0.007235551252961159, -0.007729947101324797, 0.02330341935157776, 0.0035075386986136436, -0.009092876687645912, 0.030866339802742004, 0.011444597505033016, -0.028568066656589508, 0.011771967634558678, 0.018346097320318222, 0.01523273903876543, -0.012767440639436245, -0.020978420972824097, -0.008919170126318932, 0.06157233566045761, -0.00010992375609930605, -0.0050642178393900394, -0.0006100611644797027, 0.016528857871890068, -0.01221291534602642, -0.021967213600873947, -0.007235551252961159, -0.006189970765262842, -0.013148259371519089, -0.026256432756781578, -0.00252208742313087, -0.006313569378107786, 0.0019341569859534502, 0.006306888535618782, 0.006620896980166435, 0.019468510523438454, 0.004599886480718851, -0.006874775979667902, -0.02961030788719654, -0.010382314212620258, 0.008571756072342396, -0.017170237377285957, -0.003064920660108328, 0.022007299587130547, 0.0009428597986698151, 0.014257309027016163, 0.0022698785178363323, 0.0025488114915788174, -0.0046767182648181915, -0.0045497785322368145, 0.011137270368635654, 0.003624456701800227, -0.012587052769958973, -0.024626262485980988, -0.0014172126539051533, -0.003044877666980028, 0.0045497785322368145, 0.013696103356778622, 0.020470663905143738, 0.011838777922093868, 0.017838340252637863, -0.014163775369524956, -0.011391148902475834, -0.002821063157171011, 0.01484523992985487, -0.02677755244076252, -0.01332196593284607, -0.00271583697758615, -0.02227454073727131, 0.019548682495951653, 0.020377129316329956, -0.010121754370629787, 0.012767440639436245, 0.019588768482208252, 0.025267640128731728, 0.013362051919102669, 0.006560767535120249, 0.032362889498472214, -0.023584021255373955, -0.032843925058841705, 0.007395895663648844, -0.041582707315683365, 0.03364564850926399, -0.001230143941938877, -0.021900402382016182, 0.012413346208631992, 0.005789109040051699, -0.01924135535955429, -0.001942508271895349, -0.0025321089196950197, 0.010181883350014687, 0.029797375202178955, -0.006544065196067095, -0.023717641830444336, -0.011163994669914246, -0.010181883350014687, 0.012346535921096802, -0.009032746776938438, -0.011604942381381989, -0.016689203679561615, -0.004723485559225082, -0.018052132800221443, -0.004148917272686958, -0.019521957263350487, -0.005037493538111448, -0.018132304772734642, -0.013615931384265423, -0.018586615100502968, -0.0012735705822706223, 0.010870029218494892, -0.014644809067249298, -0.016889633610844612, 0.016315065324306488, 0.013448905199766159, -0.010849986225366592, -0.033271510154008865, 0.0022097493056207895, -0.005284691695123911, 0.007322404533624649, 0.03701288253068924, 0.01996290497481823, 0.004499671049416065, 0.008190938271582127, 0.03423357754945755, 0.0037981632631272078, -0.016341788694262505, 0.024065054953098297, -0.006433828268200159, -0.01628834195435047, 0.03869650140404701, -0.0038048443384468555, 0.0006104787462390959, 0.00025868098600767553, 0.01523273903876543, 0.008585118688642979, 0.04313270375132561, 0.010923477821052074, 0.02475988306105137, -0.005231243558228016, 0.004005275201052427, -0.015152567066252232, 0.02202066220343113, 0.0037547366227954626, 0.005294713191688061, -0.013235112652182579, -0.006981672253459692, 0.0061331819742918015, -0.0018990816315636039, -0.007469387259334326, -0.011103864759206772, 0.018693510442972183, -0.00533479917794466, 0.015246101655066013, -0.020924974232912064, 0.001357918488793075, -0.01936161331832409, -0.007790076546370983, -0.020083164796233177, 0.01370946504175663, -0.011077141389250755, 0.018519803881645203, -0.002714166883379221, -0.031641338020563126, 0.01523273903876543, 0.0013036351883783937, 0.04091460257768631, -0.03915081173181534, -0.0021963873878121376, -0.008565075695514679, -0.02184695564210415, -0.012433389201760292, -0.019214630126953125, 0.004325964488089085, -0.013208388350903988, 0.028995653614401817, 0.008177575655281544, 0.00448630889877677, 0.008484902791678905, 0.012720673345029354, -0.03589047119021416, 0.0048738084733486176, -0.011037055402994156, 0.030224962159991264, 0.02107195556163788, 0.037680987268686295, 0.017971960827708244, 0.006988353095948696, -0.026496948674321175, -0.008337920531630516, -0.005197838414460421, -0.011163994669914246, 0.015152567066252232, 0.022648677229881287, -0.009266583248972893, 0.015312911942601204, -0.015499980188906193, 0.01636851392686367, -0.008565075695514679, -0.04438873752951622, 0.01032218523323536, -0.0011182366870343685, 0.008551713079214096, -0.020363766700029373, -0.01054265908896923, -0.02188704162836075, 0.042437877506017685, -0.014390929602086544, -0.016341788694262505, 6.978540477575734e-05, -0.0031718171667307615, 0.004643313121050596, 0.01974911242723465, -0.014150412753224373, 0.015820669010281563, -0.009333393536508083, -0.005592018831521273, -0.019815923646092415, -0.03359219804406166, 0.001215946744196117, 0.0006326096481643617, -0.001022196956910193, 0.02562841586768627, -0.030064616352319717, -0.013375413604080677, 0.022247817367315292, -0.0012560328468680382, -0.01602110080420971, -0.01782497763633728, -0.020924974232912064, 0.02570858784019947, -0.01576722227036953, 0.005972837097942829, -0.0291292741894722, 0.0009436949039809406, -0.00448630889877677, 0.006063031032681465, 0.0004305086040403694, -0.04326632618904114, 0.012420027516782284, -0.0018773683113977313, 0.0039117406122386456, 0.009921323508024216, -0.01683618500828743, 0.005969496909528971, -0.01043576281517744, -0.002471979707479477, -0.027605999261140823, -0.001672761864028871, 0.001556679024361074, -0.05296717584133148, -0.0499473512172699, -0.029209446161985397, 0.0007140346569940448, 0.02664393186569214, 0.008117446675896645, -0.008498265407979488, 0.01060278806835413, 0.03885684907436371, -0.0010881720809265971, 0.009179729968309402, -0.023022815585136414, 0.02091161161661148, -0.026296518743038177, 0.04318615049123764, -0.0063670179806649685, -0.038455985486507416, 0.0334051288664341, -0.007509473245590925, -0.008712057955563068, -0.01354243978857994, 0.006737814750522375, -0.011097184382379055, -0.02728530950844288, -0.0042825378477573395, 0.019334889948368073, -0.009968090802431107, -0.01231981161981821, -0.015780583024024963, -0.023249970749020576, 0.011645028367638588, -0.03869650140404701, 0.02091161161661148, 0.02673746645450592, 0.0008668631198816001, 0.008959256112575531, -0.0001803876948542893, 0.0021162149496376514, 0.009513781405985355, -0.027365483343601227, 0.027445655316114426, 0.004128874279558659, 0.03372582048177719, 0.004706782754510641, -0.014243947342038155, -0.008070679381489754, -0.008170894347131252, -0.00925322063267231, 0.04647321626543999, -0.019562043249607086, 0.0035643272567540407, 0.003918421920388937, 0.0270314309746027, 0.024612899869680405, 0.014337481930851936, 0.006584151182323694, -0.015112481079995632, 0.014497826807200909, -0.023597383871674538, -0.040460292249917984, -0.0237844530493021, -0.006767879240214825, 0.016862910240888596, -0.00921313464641571, -0.011711838655173779, -0.033993061631917953, -0.012593734078109264, -0.021352559328079224, -0.018746959045529366, -0.015526704490184784, 0.018706873059272766, 0.05008096992969513, -0.004356028977781534, -0.00875882524996996, 0.01778489165008068, 0.020751267671585083, -0.014711619354784489, 0.0022982729133218527, -0.005047515034675598, -0.014898687601089478, 0.006767879240214825, -0.005281351041048765, 0.015299549326300621, -0.013007957488298416, 0.021098680794239044, 0.017490925267338753, 0.014751705341041088, -0.004265835043042898, 0.007168740965425968, -0.01747756451368332, -0.011090503074228764, -0.020109888166189194, -0.007135335821658373, -0.008244385942816734, -0.0009846162283793092, -0.0065273623913526535, -0.029449962079524994, 0.009781021624803543, 0.017557736486196518, 0.0038115254137665033, -0.015352997928857803, 0.00446626590564847, 0.005929410457611084, 0.001748758601024747, -0.004830381833016872, -0.0037179910577833652, -0.006463892757892609, 0.00427251635119319, -0.005962815601378679, 0.021566351875662804, 0.02021678537130356, -0.02489350363612175, 0.0038983786944299936, 0.02664393186569214, 0.003008131869137287, 0.01815902814269066, 0.00434934813529253, -0.03102668561041355, 0.005885983817279339, 0.009453651495277882, -0.006303547881543636, 0.01692971959710121, 0.0071286545135080814, 0.013362051919102669, -0.0176913570612669, 0.00189574109390378, -0.02425212413072586, 0.0017888447036966681, -0.02270212583243847, 0.014791791327297688, 0.0035710083320736885, 0.004382753279060125, 0.004616588819772005, 0.0009512110846117139, -0.027686171233654022, -0.005107644479721785, -0.022849109023809433, -0.015780583024024963, -0.04259822145104408, -0.015419808216392994, -0.013789637945592403, 0.019054286181926727, 0.0021011827047914267, 0.001091512618586421, 0.010181883350014687, -0.016355151310563087, 0.03198875114321709, 0.2330341935157776, -0.010529297403991222, -0.015580153092741966, -0.001506571308709681, -0.0064739142544567585, 0.018693510442972183, 0.020083164796233177, 0.015433169901371002, 0.004008615389466286, 0.0006710255402140319, -0.0006242583622224629, -0.0074359821155667305, -0.038242191076278687, -0.009654082357883453, 0.019562043249607086, -0.022287903353571892, -0.03182840719819069, -0.02733875811100006, -0.022247817367315292, -0.01042908150702715, 0.02961030788719654, 0.022434884682297707, -0.011237485334277153, -0.011798691935837269, 0.026189621537923813, -0.004987386055290699, 0.01473834365606308, 0.032416339963674545, 0.011217442341148853, 0.02429221011698246, -0.03770771250128746, -0.007449343800544739, 0.014390929602086544, -0.012085976079106331, -0.017931872978806496, -0.004576502833515406, -0.002411850495263934, 0.012760759331285954, 0.002921278588473797, 0.011030374094843864, 0.007155378814786673, 0.02827410213649273, -0.03556978330016136, -0.001312821637839079, 0.005795789882540703, 0.017370667308568954, -0.0026857724878937006, -0.013856448233127594, -0.008504945784807205, 0.003484155051410198, 0.002183025237172842, -0.0044428822584450245, 0.0333249568939209, 0.035062026232481, 0.0047301664017140865, -0.004152257461100817, 0.013722827658057213, -0.009687487967312336, -0.03238961473107338, 0.029369790107011795, -0.0217667818069458, 0.022862471640110016, -0.01611463539302349, 0.03693271055817604, -0.007248913403600454, 0.014511188492178917, 0.01330192293971777, 0.02798013761639595, 0.01191226951777935, -0.017063340172171593, 0.020764628425240517, -0.005966156255453825, -0.012119380757212639, -0.00544169545173645, -0.054730966687202454, -0.008524988777935505, 0.03156116604804993, 0.005879302974790335, 0.012206234969198704, 0.029583582654595375, -0.011945674195885658, 0.005358182825148106, 0.0071286545135080814, -0.00448630889877677, -0.0038081847596913576, -0.05505165457725525, 0.030572375282645226, 0.020510749891400337, 0.00010527835547691211, -0.007375852670520544, 0.006310229189693928, -0.016261616721749306, -0.018412908539175987, 0.004108830820769072, 0.0024519364815205336, 0.004893851466476917, 0.004786955192685127, 0.008197618648409843, -0.003490836126729846, 0.0012635490857064724, -0.00295134331099689, -0.03204220160841942, -0.0017103427089750767, -0.005585337523370981, -0.011150632053613663, 0.024345658719539642, 0.021820230409502983, 0.025975828990340233, -0.00018748629372566938, -0.012627138756215572, 0.007028439547866583, -0.010442443192005157, 0.02514738216996193, 0.016127996146678925, -3.760686740861274e-05, 0.001411366742104292, -0.02467970922589302, -0.0217266958206892, -0.041368912905454636, -0.01619480736553669, -0.007977144792675972, -0.022087471559643745, -0.016862910240888596, -0.007429300807416439, -0.01902756281197071, -0.022047385573387146, -0.01056270208209753, -0.0017086723819375038, -0.004042020533233881, -0.027352120727300644, 0.0010063295485451818, 0.0017855041660368443, 0.008965936489403248, -0.019856009632349014, -0.004122192971408367, -0.0005507670575752854, 0.0024085098411887884, -0.008177575655281544, -0.012246320955455303, 0.007342447526752949, 0.009467014111578465, -0.0006735309143550694, -0.02605600096285343, -0.013909895904362202, 0.031053408980369568, -0.010682960972189903, 0.004085447173565626, -0.008177575655281544, -0.03543616086244583, -0.013682741671800613, 0.005388247314840555, -0.011471321806311607, 0.0012293087784200907, -0.014243947342038155, 0.013629293069243431, -0.0031918601598590612, -0.019121095538139343, -0.032790474593639374, -0.02475988306105137, 0.02545470930635929, -0.018640061840415, -0.013121535070240498, 0.029316341504454613, -0.006066371686756611, -0.019722389057278633, -0.010161840356886387, -0.17039288580417633, 0.022809023037552834, 0.03447409346699715, -0.002032702090218663, 0.042999085038900375, 0.015246101655066013, 0.043800804764032364, -0.0009603974758647382, -0.02116549015045166, 0.007923697121441364, -0.01473834365606308, 0.026015914976596832, -0.022608591243624687, 0.0031116879545152187, -0.00252208742313087, 0.016889633610844612, -0.03885684907436371, 0.005789109040051699, 0.023837901651859283, 0.00330376741476357, 0.032924097031354904, -0.019722389057278633, 0.008050636388361454, -0.018880579620599747, 0.010081668384373188, 0.02141936868429184, -0.030091341584920883, 0.011711838655173779, -0.019682303071022034, -0.006022945046424866, -0.032924097031354904, 0.010823261924088001, 0.00437941262498498, 0.004716804251074791, 0.011751924641430378, -0.0023450402077287436, -0.008063998073339462, 0.012406664900481701, -0.02454608865082264, 0.017103426158428192, -0.016729289665818214, 0.0005286361556500196, 0.004596545826643705, 0.009914642199873924, -0.02570858784019947, 0.00041839925688691437, 0.004416157957166433, 4.3035201088059694e-05, -0.0008221837342716753, -0.014751705341041088, 0.005491803400218487, -0.016742650419473648, 0.013128216378390789, -0.025013761594891548, 0.011083821766078472, -0.0064371684566140175, -0.01957540586590767, -0.0022481651976704597, -0.008992660790681839, 0.009066152386367321, -0.017317218706011772, -0.008237704634666443, -0.020831439644098282, -0.017237046733498573, -0.011264209635555744, 0.003360555972903967, -0.011638347059488297, -0.013375413604080677, -0.00738921482115984, 0.005391587968915701, -0.011504726484417915, -0.011123908683657646, 0.010836624540388584, -0.0009737595682963729, 0.01045580580830574, 0.004025318194180727, -0.022889195010066032, 0.0020661072339862585, 0.010529297403991222, 0.01654222048819065, 0.01324179396033287, 0.03629133477807045, -0.03364564850926399, 0.009072833694517612, -0.00784352421760559, 0.008043955080211163, 0.02236807532608509, 0.0010981936939060688, 0.022301264107227325, -0.0207379050552845, 0.0238245390355587, 0.011150632053613663, -0.00923317763954401, -0.01075645163655281, -0.004740187898278236, 0.009580591693520546, 0.02039049193263054, -0.01966894045472145, -0.006737814750522375, -0.00908619537949562, 0.015312911942601204, 0.0031384120229631662, -0.02485341764986515, 0.030224962159991264, 0.005675531458109617, 0.025214191526174545, 0.005034152884036303, 0.010876710526645184, 0.011498046107590199, -0.01974911242723465, -0.015379722230136395, -0.016475409269332886, 0.0067244525998830795, 0.017504287883639336, 0.010943520814180374, 0.018199114128947258, 0.0217667818069458, -0.02425212413072586, 0.00026076880749315023, 0.01377627532929182, 0.044789597392082214, -0.011598261073231697, -0.0186801478266716, 0.0006196651374921203, -0.0018823790596798062, -0.029637031257152557, -0.10021539032459259, -0.01751765049993992, 0.006086414679884911, 0.02095169760286808, 0.020831439644098282, 0.00795042049139738, -0.010382314212620258, 0.004422839265316725, -0.019054286181926727, 0.02904910035431385, -0.007823481224477291, -0.015446532517671585, -0.001261043711565435, -0.010783175937831402, 0.02120557613670826, -0.0033705777022987604, 0.010208607651293278, -0.0013904885854572058, 0.00589266512542963, 0.032790474593639374, -0.01953531987965107, 0.01590084098279476, -0.01970902644097805, -0.02471979707479477, 0.004025318194180727, -0.014765067026019096, -0.015219377353787422, 0.011357744224369526, -0.007963783107697964, -0.02729867212474346, -0.00760300736874342, -0.021873679012060165, 0.0456714928150177, -0.015580153092741966, -0.012727354653179646, -0.04302580654621124, -0.006410444620996714, -0.023557297885417938, 0.01982928439974785, -0.041662879288196564, -0.000800887995865196, 0.011310976929962635, 0.006230056751519442, -0.012647182680666447, -0.006547405384480953, -0.014230585657060146, -0.026496948674321175, 0.023036178201436996, 0.010950201191008091, -0.002470309380441904, -0.012987914495170116, 0.004633291624486446, -0.003238627454265952, -0.009573910385370255, 0.022728851065039635, -0.02112540416419506, 0.01495213620364666, -0.020016353577375412, -0.028514618054032326, -0.0011182366870343685, -0.015793945640325546, -0.03222927078604698, -0.01197239849716425, 0.02159307524561882, -0.007101930677890778, -0.0186801478266716, -0.005424993112683296, -0.005408290773630142, -0.006961629260331392, -0.005181135609745979, 0.0030281750950962305, 0.005070898681879044, -0.008378006517887115, 0.011157313361763954, -0.015032308176159859, 0.013789637945592403, 0.001945848809555173, -0.01607454940676689, 0.021406007930636406, 0.0017771528800949454, -0.008992660790681839, -0.016395237296819687, 0.0032269356306642294, -0.027953412383794785, 0.034260302782058716, 0.011584899388253689, -0.0028544683009386063, 0.008378006517887115, 0.01047584880143404, -0.005648807622492313, -0.009961409494280815, 0.010201926343142986, 0.021579714491963387, -0.014457739889621735, -0.000847655173856765, -0.0025120656937360764, -0.006861413829028606, -0.009360117837786674, -0.003440728411078453, 0.0033839396201074123, -0.021392645314335823, -0.009440289810299873, -0.05392924323678017, 0.01527282502502203, 0.0032737026922404766, -0.017544373869895935, -0.0005712277488783002, -0.017985321581363678, 0.0008284472278319299, -0.008030593395233154, -0.012633820064365864, 0.009467014111578465, 0.004997407551854849, 0.012754078954458237, 0.0005891830078326166, -0.014230585657060146, -0.009760978631675243, -0.002152960514649749, 0.00113827979657799, -0.017931872978806496, 0.008832315914332867, -0.003641159273684025, -0.014163775369524956, 0.028434446081519127, -0.0017871744930744171, 0.02763272263109684, 0.005712277255952358, 0.01195235550403595, -0.002906246343627572, 0.01165170967578888, -0.015032308176159859, -0.024238761514425278, 0.01687627099454403, 0.002769285347312689, 0.0010873370338231325, 0.007162059657275677, 0.013762913644313812, -0.018840493634343147, 0.015526704490184784, 0.03645167872309685, 0.015446532517671585, 0.016127996146678925, -0.02891547977924347, -0.026550397276878357, 0.011745243333280087, 0.026884447783231735, 0.008144170977175236, -0.004038680344820023, -0.03581029921770096, 0.01501894649118185, -0.003728012554347515, 0.013789637945592403, 0.006968310102820396, 0.01017520297318697, -0.02124566212296486, -0.008184256963431835, -0.02832755073904991, -0.029503410682082176, 0.01970902644097805, -0.021392645314335823, 0.010549340397119522, -0.012740716338157654, 0.018493080511689186, 0.01070968434214592, 0.015593514777719975, 0.012480156496167183, 0.028808584436774254, 0.022675402462482452, 0.013963344506919384, 0.005027472041547298, -0.006493957247585058, -0.005538570694625378, -0.036184437572956085, -0.03690598905086517, 0.0006321920664049685, 0.051951657980680466, -0.016862910240888596, 0.002420201664790511, 0.018653424456715584, -0.017851701006293297, -0.0009370138868689537, 0.003450749907642603, 0.01984264701604843, -0.0027893283404409885, -0.012974552810192108, 0.020978420972824097, 0.008043955080211163, 0.02454608865082264, -0.014110326766967773, -0.01388317160308361, 0.012653863057494164, 0.009620677679777145, -0.002420201664790511, 0.0036277971230447292, 0.0041923439130187035, 0.019161183387041092, 0.007409257814288139, 0.0010247023310512304, 0.004596545826643705, 0.012807526625692844, -0.0022197708021849394, 0.04722149297595024, -0.009366798214614391, -0.0030181535985320807, -0.02518746815621853, 0.0013295242097228765, -0.02112540416419506, 0.01197907980531454, -0.027499103918671608, -0.01692971959710121, -0.019348250702023506, -0.0003941805334761739, -0.02287583239376545, -0.0061498843133449554, 0.025975828990340233, 0.006657642312347889, 0.006373698823153973, 0.019081009551882744, -0.0044261799193918705, -0.02584220841526985, -0.03225599229335785, -0.009847831912338734, 0.007569602690637112, 0.022261178120970726, -0.004322623834013939, -0.017597822472453117, 0.00901938509196043, 0.006210013758391142, 0.033191338181495667, -4.115616320632398e-05, -0.0038248873315751553, -0.0007044306839816272, 0.011852140538394451, -0.01032218523323536, -0.0044428822584450245, -0.009313350543379784, -0.027873240411281586, 0.01484523992985487, -0.022207729518413544, 0.054517172276973724, -0.03404650837182999, 0.03591719642281532, 0.002054415410384536, -0.011571536771953106, 0.003016483271494508, -0.010228650644421577, 0.011177356354892254, -0.01704997755587101, 0.026844361796975136, 0.004195684101432562, -0.016034461557865143, 0.005528549198061228, -0.019094372168183327, 0.020724542438983917, -0.021312473341822624, -0.04190339520573616, 0.024626262485980988, 0.012333174236118793, 0.020510749891400337, 0.003198541235178709, -0.0057991305366158485, 0.01516592875123024, -0.01714351214468479, 0.019121095538139343, -0.007749990094453096, 0.004148917272686958, -0.0028811923693865538, 0.019000837579369545, 0.0045698219910264015, 0.006103117484599352, -0.03607754036784172, 0.0028795222751796246, 0.004312602337449789, -0.013936620205640793, -0.004703442100435495, -0.0007002550410106778, -0.019521957263350487, 0.009994815103709698, -0.02343703992664814, 0.019735751673579216, 0.024105140939354897, -0.007055163383483887, -0.02120557613670826, -0.024826692417263985, -0.029877549037337303, 0.018012046813964844, -0.011344382539391518, -0.018533166497945786, 0.0018656764877960086, -0.02129911072552204]} +{"id": "test:10000007", "text": "\"The rich get richer and the poor get poorer eh?\\nOr is it the rich think different and play by a different set of rules?\\nDo the rich take responsibility and action?\\nPoor people believe 'Life happens to me.' Rich people are committed to be rich.\\nPoor people WANT to be rich. Rich people think big.\\nPoor people think small. Rich people focus on opportunities.\\nPoor people focus on obstacles. Rich people are willing to promote themselves and their value.\\nPoor people think negatively about selling and promotion.\\nPoor people are closed to new ideas..\\nDo You think rich or poor?\"", "vector_field": [-0.013108582235872746, -0.03062605671584606, 0.029025208204984665, -0.03367029130458832, -0.035271137952804565, 0.003562542609870434, 0.01098286546766758, 0.01997123286128044, -0.04219939932227135, -0.036609552800655365, 0.0034378862474113703, 0.010720431804656982, 0.007682757452130318, -0.002240530913695693, -0.0016779378056526184, 0.014171440154314041, 0.030547326430678368, -0.019984355196356773, -0.007781170308589935, -0.02035176195204258, -0.015483610332012177, 0.023579701781272888, -0.004054606426507235, -0.003182013053447008, -0.006688788533210754, -0.0033296323381364346, 0.024734411388635635, -0.016257790848612785, 0.017189431935548782, -0.023947108536958694, 0.0019502132199704647, 0.0016098689520731568, -0.014184561558067799, 0.004044765140861273, -0.016874510794878006, -0.008194503374397755, -0.003910267725586891, 0.002670266665518284, 0.01087789237499237, -0.008817784488201141, 0.01467006467282772, 0.0036871987394988537, -0.01218350138515234, -0.013147946447134018, -0.023632187396287918, 0.024170177057385445, -0.008883393369615078, -0.02214943617582321, -0.02023366652429104, 0.012564031407237053, 0.03117716684937477, 0.014656942337751389, -0.013948370702564716, -0.006790481507778168, 0.007715561427175999, -0.0003409592609386891, -0.013817153871059418, 0.024904992431402206, -0.008909637108445168, 0.004973125644028187, 0.004471220541745424, 0.0010800801683217287, -0.01257715281099081, 0.0033033888321369886, -0.018304776400327682, -0.0022306896280497313, -0.005025612656027079, 0.004553230945020914, 0.025416739284992218, 0.0046614850871264935, 0.04219939932227135, 0.024996845051646233, 0.012675565667450428, -0.0028621715027838945, 0.0037626484408974648, 0.00702667236328125, -0.015378636308014393, 0.0017566680908203125, 0.008896514773368835, 0.025915363803505898, 0.012819903902709484, -0.011862020008265972, -0.03807918354868889, 0.029156425967812538, -0.009172070771455765, 0.01390900555998087, -0.014368264935910702, 0.04403643682599068, -0.011022230610251427, -0.007085720077157021, -0.0002388560096733272, 0.005396300461143255, 0.01322011649608612, 0.021152185276150703, 0.019721919670701027, 0.021834515035152435, -0.0029409017879515886, 0.0126033965498209, 0.010293976403772831, -0.0061343964189291, 0.012747734785079956, 0.007374397478997707, -0.02225440926849842, -0.004448257386684418, 0.0003754037315957248, 0.014656942337751389, 0.00582603644579649, 0.00404148455709219, 0.006518206093460321, -0.02587599866092205, -0.021283403038978577, 0.02129652537405491, 0.01297080423682928, -0.033775266259908676, 0.023002346977591515, 0.0007614688365720212, 0.005845719017088413, -0.02944510243833065, -0.020837265998125076, -0.013672814704477787, 0.002376668620854616, 0.0002538229455240071, 0.017937367781996727, -0.010490802116692066, 0.00553407846018672, -0.013056094758212566, -0.02604658156633377, -0.017425622791051865, -0.017018849030137062, 0.02414393424987793, 0.004382648970931768, 0.025285523384809494, -0.0014975144295021892, -0.001609048922546208, -0.006203285418450832, 0.005002649500966072, -0.027293143793940544, 0.017202552407979965, -0.041018445044755936, -0.01728128269314766, 0.015024350956082344, -0.0020486258435994387, -0.006478840950876474, 0.006915137637406588, 0.003706881310790777, 0.020876631140708923, 0.009696939028799534, 0.01907895691692829, 0.010064346715807915, -0.013475989922881126, 0.028789017349481583, 0.010956622660160065, 0.03088849037885666, 0.0021896841935813427, -0.0021470387000590563, 0.03028489090502262, -0.012478739954531193, 0.013272603042423725, -0.029313884675502777, -0.021480228751897812, 0.010851648636162281, 0.012649321928620338, -0.005320850759744644, 0.02218880131840706, -0.0003965214709751308, 0.027870498597621918, 0.03519240766763687, 0.03445759415626526, 0.02999621443450451, 0.0065608518198132515, -0.012931439094245434, 0.011822654865682125, -0.0202992744743824, 0.01343662478029728, 0.014132075011730194, 0.013974614441394806, 0.023002346977591515, 0.010622018948197365, -0.0026571450289338827, -0.011219056323170662, -0.0031098437029868364, 0.019157687202095985, 0.014880011789500713, 0.018081707879900932, -0.014565090648829937, -0.010438315570354462, 0.021230915561318398, 0.0001815510622691363, 0.023120442405343056, -0.01657271198928356, -0.010307097807526588, 0.028789017349481583, -0.01477503776550293, 0.0029343408532440662, -0.6436982750892639, -0.033092934638261795, -0.020364884287118912, -0.007879583165049553, 0.0040152412839233875, 0.017425622791051865, -0.022398747503757477, 0.030862245708703995, -0.0316757932305336, 0.013830275274813175, -0.0023274621926248074, -0.010930378921329975, -0.006081909406930208, -0.022897372022271156, -0.0015844457084313035, -0.016467737033963203, 0.018764035776257515, -0.022936737164855003, -0.01851472444832325, -0.0108188446611166, -0.0026620656717568636, 0.004795982502400875, -0.01494562067091465, -0.009775669313967228, -0.012636200524866581, 0.013502232730388641, -0.0020896312780678272, -0.006416513118892908, 0.008056726306676865, 0.05626586452126503, 0.006298417691141367, 0.00967725645750761, 0.012413131073117256, 0.026689544320106506, 0.0486290343105793, 0.017045093700289726, 0.014092709869146347, 0.013961492106318474, -0.0008594715618528426, 0.02018117904663086, -0.01700572855770588, -0.011973554268479347, 0.01611345261335373, -0.01218350138515234, 0.011593025177717209, 0.008358525112271309, 0.002183123491704464, 0.003775770077481866, 0.0011194453109055758, 0.016021599993109703, 0.002944182138890028, 0.017635568976402283, -0.041569557040929794, -0.005238839890807867, 0.03466754034161568, 0.00017806561663746834, 0.003575664246454835, -0.020259909331798553, -0.00582603644579649, 0.003956193570047617, -0.019787529483437538, -0.00017304247012361884, -0.012386888265609741, -0.009723181836307049, -0.005602967459708452, 0.015470487996935844, -0.010812283493578434, 0.03206944465637207, 0.009401700459420681, -0.019918745383620262, 0.0032558226957917213, 0.013830275274813175, -0.016034722328186035, -0.00480910437181592, 0.004773019813001156, 0.0008315879385918379, 0.0169532410800457, 0.01846223697066307, -0.026794519275426865, 0.0151686891913414, -0.024734411388635635, -0.010812283493578434, -0.02292361669242382, -0.012275354005396366, 0.013029851950705051, 0.0031836533453315496, -0.022621816024184227, -0.012951121665537357, 0.004149738699197769, -0.019275782629847527, -0.02426202967762947, 0.02024678885936737, -0.0058719622902572155, -0.008266673423349857, -0.003236140124499798, 0.024327637627720833, 0.00102103257086128, 0.005504554603248835, 0.004966564942151308, -0.005917888134717941, -0.0008668525260873139, -0.003221378196030855, -0.0028998965863138437, 0.014079587534070015, 0.02721441350877285, 0.01728128269314766, 0.014289534650743008, 0.002015821635723114, 0.03317166492342949, -0.029025208204984665, -0.008804663084447384, -0.00917863193899393, 0.01789800263941288, -0.006751116365194321, -0.00034752010833472013, -0.017084458842873573, 0.04078225418925285, -0.009546039626002312, -0.0009931488893926144, -0.05469125881791115, 0.006367306690663099, 0.01452572550624609, 0.007144767325371504, -0.018107950687408447, 0.02325165830552578, 0.03902394697070122, -0.01923641748726368, -0.012846147641539574, 0.004310479387640953, -0.002274975413456559, -0.0004535188782028854, -0.023579701781272888, 0.012229427695274353, -0.009198314510285854, -0.007033233065158129, -0.006682227365672588, 0.019275782629847527, -0.009408261626958847, -0.00393323041498661, -0.028526583686470985, 0.010713870637118816, 0.007886143401265144, 0.02554795704782009, -0.020548587664961815, -0.009972495026886463, -0.03519240766763687, -0.02079790085554123, -0.03246309608221054, -0.0227530337870121, -0.01220974512398243, -0.011665194295346737, -0.022372504696249962, -0.03401145711541176, 0.014066466130316257, 0.0019551338627934456, -0.00733503233641386, -0.0024718008935451508, -0.0037462464533746243, -0.01427641324698925, -0.022726790979504585, 0.001157990307547152, 0.016323398798704147, -0.035769764333963394, -0.010746675543487072, -0.0035920662339776754, 0.005983497016131878, -0.029156425967812538, 0.02045673504471779, 0.006862651091068983, -0.012472178786993027, 0.008194503374397755, -0.0036871987394988537, -0.0002991338260471821, 0.02815917506814003, 0.00020430902077350765, 0.023343510925769806, -0.020876631140708923, 0.015575462020933628, 0.006370587274432182, -0.016297155991196632, 0.011114083230495453, -0.023579701781272888, -0.02246435545384884, 0.0058686817064881325, 0.018947739154100418, -0.020824143663048744, 0.004064447712153196, 0.013738423585891724, -0.01594286970794201, 0.020430492237210274, 0.004520426969975233, 0.014433873817324638, 0.01104191318154335, 0.0009587044478394091, -0.005156829487532377, 0.013607206754386425, 0.028631556779146194, 0.014368264935910702, -0.007748365867882967, 0.009854399599134922, -0.009605087339878082, -0.004389209672808647, -0.014880011789500713, -0.02174266241490841, 0.020758535712957382, 0.0030704785604029894, 0.007649953011423349, -0.023395996540784836, 0.01879027858376503, 0.02441949024796486, 0.0044580986723303795, -0.01817355863749981, 0.0086996890604496, 0.0007893524598330259, -0.0031574098393321037, 0.045637283474206924, 0.017583083361387253, 0.022385627031326294, -0.004087410867214203, 0.0038577807135879993, 0.011802972294390202, -0.00011348223051754758, -0.012833026237785816, -0.006144237704575062, 0.012557470239698887, 0.01427641324698925, -0.0090342927724123, 0.0037429658696055412, 0.004841908812522888, -0.012314719147980213, -0.009185192175209522, -0.004034923855215311, 0.02313356287777424, 0.015627948567271233, -0.002598097315058112, -0.005766988731920719, 0.03117716684937477, -0.01745186559855938, 0.02525927871465683, -0.00021363772975746542, 0.014840646646916866, 0.012531226500868797, -0.018383506685495377, -0.004887834656983614, 0.010871331207454205, -0.004648363683372736, 0.021795149892568588, 0.02397335320711136, -0.02035176195204258, 0.027686795219779015, 0.010169319808483124, 0.024353882297873497, -0.011264982633292675, -0.007308789063245058, -0.012281914241611958, -0.03128214180469513, 0.00437608826905489, -0.014617578126490116, 0.017136944457888603, 0.004999368917196989, -0.0011210854863747954, 0.016493981704115868, -0.009972495026886463, -0.01162582915276289, 0.010851648636162281, 0.009211435914039612, 0.0009373817010782659, -0.01934139057993889, 0.010477679781615734, -0.008548789657652378, -0.012721491977572441, -0.0040185218676924706, 0.013974614441394806, -0.0004555691557470709, 0.0029851875733584166, 0.010726992972195148, -0.0057046604342758656, -0.018081707879900932, 0.02242499217391014, 0.008778419345617294, -0.016323398798704147, -0.01823916845023632, 0.020102450624108315, 0.02448509819805622, -0.013751544989645481, -0.01867218315601349, -0.03411642834544182, 0.017753664404153824, -0.014013979583978653, 0.01779302954673767, -0.006409951951354742, 0.0074531277641654015, -0.0009513234836049378, 0.0027818011585623026, 0.019761284813284874, 0.03314542397856712, 0.016480859369039536, 0.013922126963734627, 0.025233035907149315, -0.021283403038978577, 0.010307097807526588, -0.0075252968817949295, -0.010129954665899277, 0.009637891314923763, 0.029313884675502777, 0.010326780378818512, 0.003457568818703294, 0.0033345529809594154, -0.020378004759550095, -0.0015631229616701603, 0.00785990059375763, -0.03183325380086899, 0.0019370914669707417, 0.020889751613140106, 0.014079587534070015, -0.023054832592606544, -0.009382017888128757, 0.01861969754099846, 0.015326149761676788, 0.011619267985224724, 0.005622650031000376, -0.0068364073522388935, -0.036110926419496536, 0.00814201682806015, 0.10203436762094498, 0.014342022128403187, -0.020771656185388565, 0.037003204226493835, 0.009368896484375, -0.006462438963353634, -0.003716722596436739, -0.04007368162274361, 0.016979483887553215, -0.015404880046844482, 0.026479598134756088, -0.011061595752835274, 0.025246158242225647, -0.007964873686432838, 0.024393247440457344, 0.009158949367702007, -0.022070705890655518, -0.012504983693361282, 0.0042219082824885845, -0.0042153471149504185, -0.015299906022846699, -0.010858209803700447, 0.02264806069433689, 0.0042186276987195015, -0.020758535712957382, 0.027345629408955574, 0.025692295283079147, 0.02270054630935192, 0.01728128269314766, -0.02136213332414627, -0.004530268255621195, 0.018947739154100418, 0.008817784488201141, 0.02577102556824684, -0.010136515833437443, 0.00443513598293066, -0.0004896035534329712, 0.010825405828654766, 0.037659287452697754, -0.010418632999062538, 0.003434605896472931, 0.00828635599464178, 0.02141462080180645, -0.020535465329885483, -0.0037626484408974648, 0.00025935866869986057, -0.007544979453086853, 0.022621816024184227, -0.0029343408532440662, -0.01734689250588417, 0.011730803176760674, -0.00961820874363184, -0.02235938236117363, 0.008988366462290287, 0.004530268255621195, 0.0029851875733584166, -0.018580332398414612, -0.011219056323170662, -0.012196623720228672, -0.0173993781208992, -0.010018420405685902, -0.023448484018445015, 0.01427641324698925, -0.009631330147385597, 0.008824345655739307, -0.019643189385533333, -0.008929319679737091, -0.005504554603248835, -0.01187514141201973, -0.009447626769542694, 0.02532488852739334, -0.027791768312454224, -0.01467006467282772, -0.01750435307621956, 0.010248050093650818, 0.009126144461333752, 0.0072825453244149685, 0.014171440154314041, -0.012045723386108875, -0.0061343964189291, 0.022543085739016533, -0.05198818817734718, -0.0031328066252171993, -0.009113023057579994, 0.009237679652869701, 0.004769739229232073, -0.002086350927129388, 0.007125085219740868, -0.007610587868839502, 0.00953291729092598, -0.006590375676751137, -0.004054606426507235, 0.025338008999824524, -0.0024537586141377687, -0.015641070902347565, -0.009257362224161625, 0.004937041085213423, -0.008430694229900837, 0.02409144677221775, 0.0022733351215720177, -0.002281536115333438, -0.03385399654507637, -0.011199373751878738, 0.00366751616820693, 0.006974185351282358, -0.012288475409150124, 0.009519795887172222, -0.000645423773676157, -0.016205303370952606, -0.011474929749965668, 0.04605717957019806, -0.015614827163517475, 0.008994927629828453, 0.002214287407696247, -0.005192914046347141, 0.018422871828079224, 0.012301596812903881, 0.03500870615243912, 0.012747734785079956, 0.006744555663317442, -0.039627544581890106, -0.03322415426373482, 0.007663074880838394, 0.0034182036761194468, -0.026256529614329338, -0.010497362352907658, -0.005648893304169178, -0.03857780992984772, -0.027450604364275932, -0.005504554603248835, 0.014381387270987034, 0.0036445532459765673, -3.98520496673882e-05, -0.023343510925769806, -0.003956193570047617, -0.003319791052490473, 0.011310908012092113, -0.022385627031326294, -0.020719170570373535, -0.0011325670639052987, 0.00930984877049923, -0.0245375856757164, -0.0035559816751629114, -0.02487874962389469, 0.02865779958665371, -0.03558605909347534, -0.017202552407979965, 0.014630699530243874, -0.02023366652429104, 0.010667945258319378, 0.020942239090800285, -0.002639102516695857, -0.04450881853699684, 0.010864770039916039, 0.0202992744743824, -0.027056952938437462, -0.004776299931108952, -0.04020490124821663, 0.008496303111314774, 0.019000226631760597, 0.021060334518551826, 0.0036576748825609684, 0.012675565667450428, -0.0018682025838643312, 0.0111534483730793, -0.011980115436017513, -0.005560321733355522, -0.03406394273042679, 0.0007741805166006088, 0.03143960237503052, 0.0013958212221041322, 0.007092280779033899, 0.007997678592801094, -0.012314719147980213, -0.0008824345422908664, 0.02687324956059456, -0.012071967124938965, -0.010392389260232449, -0.043196648359298706, -0.048707764595746994, -0.0019731761422008276, 0.020706048235297203, -0.009329531341791153, 0.016375886276364326, -0.034877486526966095, -0.009027731604874134, 0.008824345655739307, -0.016375886276364326, -0.0001052811712725088, 0.0078074135817587376, 0.009683817625045776, -0.0036937594413757324, 0.014722551219165325, 0.006711751222610474, -0.0034149233251810074, -0.011146887205541134, -0.01201948057860136, -0.01707133650779724, -0.001613149419426918, 0.006829846650362015, 0.009217997081577778, 0.028631556779146194, 0.017268162220716476, 0.02057483047246933, 0.014381387270987034, 0.0074531277641654015, -0.02035176195204258, -0.02035176195204258, 0.009099901653826237, -0.01863281987607479, -0.00034403466270305216, -0.0086996890604496, -0.02264806069433689, -0.010536727495491505, 0.0035034948959946632, 0.0058325971476733685, -0.0104842409491539, 0.0011965353041887283, -0.029287641867995262, -0.03340785577893257, -0.0028162456583231688, -0.007295667193830013, 0.009795351885259151, 0.010891013778746128, 0.013128264807164669, 0.00872593279927969, -0.008588154800236225, 0.006209846120327711, 0.00783365685492754, 0.017911124974489212, 0.004087410867214203, 0.04030987247824669, -0.0014786520041525364, -0.012157258577644825, 0.010280855000019073, -0.003739685518667102, -0.017373135313391685, 0.0011957152746617794, -0.012669004499912262, 0.031807009130716324, -0.01455196924507618, 0.003956193570047617, -0.008483181707561016, -0.006577253807336092, -0.010687627829611301, 0.021545836701989174, 0.01952509395778179, -0.021493351086974144, -0.02252996526658535, -0.046424586325883865, -0.015614827163517475, 0.011560221202671528, -0.009014610201120377, 0.010438315570354462, 0.014460117556154728, -0.022949859499931335, -0.0026587850879877806, 7.760257722111419e-05, -0.010956622660160065, 0.01011683326214552, 0.0025128063280135393, -0.006514925975352526, 0.0015376996016129851, 0.023212293162941933, 0.013056094758212566, 0.00770900072529912, -0.004569633398205042, -0.028421610593795776, -0.01143556460738182, 0.027424359694123268, -0.03277801349759102, -0.01745186559855938, -0.012052284553647041, -0.011921067722141743, -0.00567513657733798, 0.005727623589336872, -0.00557672418653965, -0.021230915561318398, -0.004202225711196661, 0.004093971569091082, 0.00967725645750761, 0.012065405957400799, -0.023579701781272888, 0.0024029118940234184, -0.001981377135962248, -0.006961063481867313, 0.013738423585891724, 2.8344929887680337e-05, 0.02352721430361271, 0.003451008116826415, -0.014355143532156944, -0.01655958965420723, 0.00197973707690835, 0.017386257648468018, -0.004874712787568569, -0.005744025576859713, -0.006849529221653938, 0.004727093502879143, -0.013462867587804794, 0.0013318528654053807, 0.012957681901752949, -0.0005958073306828737, -0.010523606091737747, 0.018107950687408447, -0.020036840811371803, -0.00845037680119276, 0.014617578126490116, -0.01840974949300289, -0.023553457111120224, -0.01198667660355568, 0.0018977263243868947, 0.0012785459402948618, -0.014748794957995415, -0.013830275274813175, 0.003283706260845065, 0.023894622921943665, 0.012898634187877178, -0.020758535712957382, -0.0018747634021565318, 0.021440863609313965, -0.005442226305603981, 0.0008684927597641945, 0.037895478308200836, 0.004008680582046509, 0.01684826798737049, 0.0065214866772294044, 0.021480228751897812, 0.004776299931108952, -0.008404451422393322, 0.00044490775326266885, -0.002931060502305627, 0.028789017349481583, -0.007407201454043388, -0.0038118548691272736, -0.011389638297259808, 0.015693556517362595, -0.007013550493866205, 0.024065203964710236, 0.013200433924794197, -0.016100330278277397, -0.011717680841684341, 0.029969971626996994, 0.004067728295922279, 0.005819475278258324, 0.010536727495491505, -0.0010054504964500666, -0.010503923520445824, 0.0021240757778286934, -0.028474096208810806, -0.015050593763589859, -0.004526987671852112, 0.045532312244176865, -0.004996088799089193, -0.012032601982355118, -0.02469504624605179, -0.0027785208076238632, -0.04356405511498451, -0.009408261626958847, 0.017478108406066895, 0.030311135575175285, 0.029156425967812538, -0.0025308486074209213, -0.010366145521402359, 0.030363621190190315, 0.003677357453852892, 0.019262660294771194, -0.025167427957057953, 0.00808296911418438, -0.019708799198269844, -0.011048474349081516, -0.011455247178673744, 0.0063607459887862206, -0.014604455791413784, 0.005694819148629904, 0.03495621681213379, 0.002545610535889864, -0.015352393500506878, 0.02419642172753811, -0.005415983032435179, 0.019367635250091553, -0.0047402153722941875, 0.03912891820073128, 0.03574351966381073, -0.006744555663317442, 0.02894647791981697, -0.029418859630823135, -0.002399631543084979, 0.013541597872972488, 0.009270483627915382, -0.030416108667850494, 0.009992177598178387, -0.005268363747745752, -0.011619267985224724, 0.01322011649608612, 0.0019879380706697702, -0.009336091578006744, -0.001171932090073824, 0.02731938660144806, 0.020824143663048744, -0.00576042803004384, -0.0026620656717568636, 0.032987963408231735, 0.02582351304590702, -0.01884276606142521, 0.0060786292888224125, 0.01616593822836876, 0.0303373783826828, -0.025508591905236244, 0.002740795724093914, -0.01301672961562872, 0.0003553111164364964, -0.019997475668787956, -0.019485728815197945, -0.02436700277030468, 0.0004904236411675811, -0.003956193570047617, 0.009303287602961063, -0.006311539560556412, 0.015864139422774315, 0.022005096077919006, 0.003045875346288085, 0.0013548159040510654, -0.027922984212636948, -0.025075575336813927, -0.03511367738246918, -0.015378636308014393, 0.0173993781208992, 0.01571980118751526, -0.012669004499912262, -0.02436700277030468, 0.011238738894462585, -0.029208911582827568, -0.007059476338326931, -0.007899265736341476, 0.009723181836307049, 0.01756996102631092, 0.21152186393737793, 0.008037043735384941, 0.016926998272538185, 0.03367029130458832, 0.012150697410106659, 0.026978222653269768, 0.04183199256658554, 0.0013088899431750178, -0.01380403246730566, 0.005812914576381445, -0.013699058443307877, 0.004530268255621195, -0.01951197348535061, 0.008463499136269093, -0.017989855259656906, -0.01839662902057171, -0.021926365792751312, -0.021545836701989174, 0.023159807547926903, -0.023186050355434418, 0.0101496372371912, -0.019826894626021385, -0.014197682961821556, -0.019695676863193512, 0.0216376893222332, 0.032043199986219406, -0.03088849037885666, -0.019603824242949486, 0.03506119176745415, -0.004123495426028967, -0.014538847841322422, -0.013462867587804794, 0.0036248706746846437, 0.013502232730388641, -0.015221175737679005, -0.0005876062787137926, -0.005212596617639065, -0.01673017255961895, 0.02035176195204258, 0.001038254820741713, 0.05340533331036568, 0.029130181297659874, -0.013712179847061634, -0.014250170439481735, 0.03534986823797226, 0.03369653597474098, -0.0070791589096188545, 0.0006167200626805425, -0.022070705890655518, -0.001423704787157476, -0.018094828352332115, -0.007794291712343693, 0.04626712575554848, 0.041910719126462936, -0.015050593763589859, 0.009395139291882515, 0.01722879707813263, -0.00018749684386420995, -0.018921496346592903, -0.00350021431222558, 0.019446363672614098, 0.054008930921554565, -0.008056726306676865, 0.007157889194786549, 0.007328471168875694, 0.02292361669242382, -0.018750913441181183, -0.007420323323458433, -0.006882333662360907, -0.007597466465085745, 0.013974614441394806, -0.022779276594519615, -0.00808296911418438, -0.014709429815411568, -0.0029294202104210854, -0.020154936239123344, 0.05563602223992348, -0.00018052593804895878, 0.021401498466730118, 0.04038860276341438, -0.02040424942970276, 0.012439374811947346, -0.020063085481524467, -0.014683186076581478, -0.021677054464817047, -0.026689544320106506, 0.021047212183475494, -0.010891013778746128, -0.02419642172753811, -0.0020043402910232544, 0.02755557745695114, -0.021427741274237633, -0.015037472359836102, -0.016074087470769882, 0.0072497413493692875, -0.005064977332949638, 0.029969971626996994, 0.009506674483418465, -0.026190919801592827, -0.005953973159193993, -0.016795780509710312, 0.046818237751722336, -0.006468999665230513, 0.03280425816774368, 0.005773549433797598, -0.015509853139519691, 0.01879027858376503, 0.027503089979290962, 0.04477125406265259, -0.0027309544384479523, -0.005232279188930988, -0.013266041874885559, 0.0009980695322155952, -0.011304347775876522, 0.009834717027842999, -0.00961820874363184, 0.00814201682806015, -0.013607206754386425, 0.023343510925769806, -0.010005299001932144, -0.023225415498018265, -0.00998561643064022, -0.0058686817064881325, -0.007702440023422241, -0.016756415367126465, -0.0016016679583117366, -0.009473869577050209, 0.0005314290174283087, 0.003962754271924496, -0.02716192603111267, 0.0158378966152668, -0.031807009130716324, -0.010956622660160065, -0.015260540880262852, -0.012997047044336796, -0.001074339495971799, 0.022280652076005936, -0.02090287394821644, 0.02581039071083069, -0.005091221071779728, 0.00018062845629174262, 0.017215674743056297, -0.0062820157036185265, -0.00661005824804306, 0.008050165139138699, -0.014066466130316257, 0.006367306690663099, -0.017648691311478615, -0.001290027517825365, 0.008187943138182163, -0.049941204488277435, -0.012170379981398582, 0.005255242343991995, -0.029261399060487747, 0.005478311330080032, -0.005311009474098682, -0.01935451291501522, -0.002297938335686922, 0.014342022128403187, 0.01779302954673767, -0.024288272485136986, 0.020509222522377968, 0.02001059800386429, -0.01544424518942833, -0.00565873458981514, 0.00379217229783535, -0.16533346474170685, 0.023015467450022697, 0.026020338758826256, -0.01997123286128044, 0.021703297272324562, -0.015024350956082344, 0.024078326299786568, -0.009736304171383381, -0.02304171212017536, 0.016979483887553215, 0.01100910920649767, -0.01377778872847557, -0.018882131204009056, 0.004487622529268265, -0.00911958422511816, -0.010339902713894844, 0.005064977332949638, 0.030809760093688965, 0.017911124974489212, 0.010864770039916039, 0.0024504780303686857, -0.026794519275426865, 0.014158317819237709, 0.023632187396287918, -0.004454818554222584, 0.03162330389022827, -0.018317898735404015, 0.01755683869123459, -0.0034214842598885298, -0.04030987247824669, 0.0014155037933960557, -0.008043603971600533, 0.031649548560380936, 0.01722879707813263, -0.008765297941863537, 0.013475989922881126, -0.0086996890604496, 0.007066037505865097, -0.02860531397163868, -0.004530268255621195, 0.021060334518551826, 0.019197052344679832, 0.029812511056661606, 0.0034838123247027397, -0.007630270440131426, 0.03023240529000759, 0.033775266259908676, 0.0034838123247027397, -0.006291856989264488, -0.0004297357809264213, -0.010766358114778996, -0.01589038223028183, 0.019157687202095985, 0.009828155860304832, 0.031754523515701294, -0.014788160100579262, 0.013567841611802578, 0.02169017679989338, -0.011278104037046432, -0.0004260452988091856, 0.0028031240217387676, -0.02285800687968731, -0.0013326730113476515, -0.012203183956444263, -0.027135683223605156, -0.0072825453244149685, 0.0018928056815639138, -0.01928890496492386, -0.01400085724890232, 0.004884554073214531, 0.0006339423125609756, -0.008174821734428406, 0.018554089590907097, -0.0023504251148551702, 0.00900804903358221, 0.016651442274451256, -0.03128214180469513, -0.04303918778896332, -0.027083195745944977, 0.017832394689321518, -0.010129954665899277, 0.003775770077481866, -0.011225617490708828, -0.010307097807526588, -0.01667768508195877, 0.0031360872089862823, 0.0070397937670350075, -0.01011683326214552, 0.002762118587270379, -0.035271137952804565, 0.012833026237785816, -0.024721289053559303, -0.036058440804481506, -0.010648262687027454, 0.008680006489157677, 0.019315147772431374, 0.02125716023147106, 0.02158520184457302, 0.004198945127427578, 0.005274924915283918, -0.00557672418653965, 0.007066037505865097, -0.034431349486112595, 0.014722551219165325, 0.00655101053416729, -0.012583713978528976, 0.027686795219779015, -0.011173130944371223, 0.03251558169722557, -0.005019051488488913, -0.023776527494192123, 0.00404148455709219, 0.006708471104502678, 0.018331019207835197, -0.02776552550494671, 0.016808902844786644, 0.0023356631863862276, 0.002616139594465494, 0.015339271165430546, -0.009847838431596756, 0.06765550374984741, -0.014342022128403187, -0.024957479909062386, 0.0220838263630867, 0.008594715967774391, -0.010740114375948906, -0.11746548861265182, -0.01634964346885681, -0.0018222766229882836, -0.005209316499531269, 0.027739280834794044, 0.0042612734250724316, -0.004795982502400875, -0.008397890254855156, -0.004887834656983614, 0.022503720596432686, 0.008306038565933704, -0.015575462020933628, -0.012104771099984646, -0.016152817755937576, 0.024380125105381012, -0.012780538760125637, 0.009080219082534313, -0.0028966160025447607, -0.026190919801592827, 0.02649271860718727, 0.005543919745832682, -0.023330388590693474, 0.005940851289778948, -0.020889751613140106, -0.02141462080180645, -0.0029589440673589706, -0.028395365923643112, -0.0032197379041463137, -0.004185823258012533, -0.02218880131840706, -0.024996845051646233, -0.014591334387660027, -0.001794392941519618, -0.033827751874923706, 0.01123217772692442, -0.002885134657844901, -0.03653082251548767, -0.017386257648468018, 0.014499482698738575, -0.024012716487050056, 0.006856089923530817, 0.025967851281166077, 0.008850589394569397, -0.043249133974313736, -0.008102651685476303, -0.012892073951661587, -0.03994246572256088, -0.007577783893793821, 0.006573973223567009, 0.0009931488893926144, -0.018331019207835197, -2.4231583665823564e-05, -0.033381614834070206, -0.014591334387660027, 0.013935249298810959, -0.007873021997511387, -0.009408261626958847, 0.016205303370952606, 0.010549849830567837, 0.031807009130716324, -0.011973554268479347, 0.004727093502879143, -0.011356834322214127, 0.009172070771455765, 0.033486586064100266, -0.025915363803505898, -0.008102651685476303, -0.002524287672713399, -0.004720532801002264, -0.014263291843235493, -0.004917358513921499, 0.031098436564207077, 0.007479371037334204, 0.027581820264458656, -0.014565090648829937, -0.01007090788334608, -0.022595573216676712, -0.046319615095853806, 0.03277801349759102, -0.010333341546356678, -0.009434504434466362, -0.011488051153719425, 0.01131746917963028, -0.032200660556554794, 0.01896086148917675, 0.0158378966152668, 0.0040185218676924706, 0.01684826798737049, 0.0036576748825609684, -0.004107092972844839, -0.0047369347885251045, 0.037790507078170776, 0.02805420197546482, -0.02537737414240837, 0.0029851875733584166, 0.005360215902328491, 0.00350021431222558, -0.0339064821600914, -0.001768149551935494, 0.0020273032132536173, -0.017596203833818436, -0.038787756115198135, -0.01911832205951214, 0.012176941148936749, -0.014932498335838318, -0.02592848613858223, -0.019656311720609665, 0.002274975413456559, 0.003713442012667656, -0.011160008609294891, -0.01556234061717987, 0.006974185351282358, -0.013390698470175266, 0.03650457784533501, 0.0010907415999099612, -0.009539478458464146, -0.015851017087697983, -0.002422594465315342, -0.004668045789003372, 0.007938630878925323, -0.002607938600704074, -0.008404451422393322, -0.017819274216890335, -0.0076958793215453625, 0.018435994163155556, 0.00037683892878703773, 0.012951121665537357, 0.0029113779310137033, -0.02532488852739334, 0.02934012934565544, -0.025889120995998383, -0.019643189385533333, 0.006810164079070091, -0.022110071033239365, 0.014880011789500713, 0.009605087339878082, 0.008994927629828453, -0.005330692045390606, -0.006206565536558628, 0.014092709869146347, 0.014040222391486168, 0.03863029554486275, -0.0249837227165699, -0.018973983824253082, 0.0009947891812771559, -0.02665017917752266, -0.005192914046347141, 0.007433445192873478, -0.037396855652332306, 0.007925508543848991, 0.01934139057993889, -0.016323398798704147, 0.030599812045693398, 0.027135683223605156, -0.03474627062678337, -0.011606146581470966, -0.021178429946303368, -0.029366372153162956, -0.011619267985224724, 0.01566731370985508, 0.005235559772700071, -0.028185419738292694, 0.020430492237210274, -0.00922455731779337, 0.013272603042423725, -0.016585832461714745, 0.0034280449617654085, 0.001171112060546875, 0.008240429684519768, 0.010438315570354462, -0.011337151750922203, -0.026584571227431297, 0.0032410607673227787, 0.005645612720400095, 0.006737994961440563, 0.031807009130716324, 0.022766156122088432, 0.027739280834794044, 0.0009111383114941418, -0.008837467059493065, -0.0040119607001543045, 0.012288475409150124, -0.022766156122088432, 0.008844028227031231, -0.03506119176745415, 0.018554089590907097, 0.022621816024184227, 0.01343662478029728, -0.03317166492342949, -0.005064977332949638, 0.0001703771122265607, 0.005169951356947422, 0.0014679905725643039, 0.012413131073117256, -0.032594311982393265, -0.006029422860592604, 0.021401498466730118, 0.0018534406553953886, -0.03241060674190521, 0.011848898604512215, -0.005976935848593712, 0.02141462080180645, 0.032489337027072906, -0.009395139291882515, -0.011186252348124981, -0.012734613381326199, -0.02431451715528965, -0.005806353874504566, 0.011048474349081516, -0.022005096077919006, 0.019000226631760597, 0.00578339071944356, 0.008653763681650162, -0.009906886145472527, -0.012176941148936749, -0.020679805427789688, -0.00959196500480175, 0.005734184291213751, -0.015457366593182087, -0.02090287394821644, -0.0216376893222332, 0.008378207683563232, 0.00040021195309236646, 0.0008898155065253377, -0.003447727533057332, 0.011842337436974049, 0.003214817261323333, 0.011763607151806355, -0.009460748173296452, -0.017241917550563812, -0.007105402648448944, -0.011114083230495453, 0.0018665622919797897, -0.0027112720999866724, -0.010215246118605137, 0.0024553986731916666, -0.004048045724630356, -0.04317040368914604, 0.013554719276726246, 0.03472002595663071, -0.018134193494915962, 0.04608342424035072, 0.01756996102631092, 0.0012482020538300276, 0.022661181166768074, 0.001440927037037909, 0.006065507419407368, 0.010307097807526588, -0.0014811123255640268, -0.0022503721993416548, -0.012275354005396366, 0.015851017087697983, -0.014394508674740791, 0.010182442143559456, -0.0365833081305027, 0.00785990059375763, 0.015483610332012177, -0.004618839826434851, 0.01657271198928356, -0.008292916230857372, 0.025679172948002815, 0.026256529614329338, 0.0021601603366434574, 0.02894647791981697, 0.010943500325083733, 0.007420323323458433, -0.019603824242949486, 0.006829846650362015, 0.01104191318154335, 0.010838527232408524, -0.03157081827521324, -0.010661384090781212, 0.006465719547122717, -0.008607837371528149, -0.006888894364237785, 0.007348153740167618, -0.006619899533689022, 0.0021683615632355213, -0.025023087859153748, -0.022556208074092865, -0.006790481507778168, -0.002722753444686532, 0.00702667236328125, -0.009946251288056374, -0.052670516073703766, 0.03154457360506058, -0.0034149233251810074, -0.01657271198928356, -0.01661207713186741, -0.025167427957057953]} +{"id": "test:10000008", "text": "\"Biomedics 1 Day Extra are daily replacement disposable contact lenses by CooperVision Hydron. Buy one box of 90 lenses.\\nBiomedics 1 Day Extra contacts give you all the convenience of a daily disposable lens with no need for solutions, cases or cleaning and are perfect for the occasional wear. These lenses have greater comfort handling with superior ease of insertion and removal.\\nBiomedic 1 Day Extra are also marketed under various other brand names including Clear Choice 1-day, Ascend 1-day, easyvision CLARISION SPHERE, Clearsight 1 Day and ProView Daily Disposable.\"", "vector_field": [0.006330564152449369, 0.01616111770272255, 0.01666403003036976, -0.03767756000161171, 0.009480555541813374, 0.020782465115189552, 0.010826182551681995, -0.040423184633255005, 0.0030429523903876543, -0.025104781612753868, 0.0032825148664414883, 0.014339764602482319, -0.02450672537088394, -0.006333962548524141, -0.009949485771358013, 0.012701905332505703, 0.010853366926312447, 0.009385409764945507, 0.006857261992990971, -0.030827095732092857, -0.009732010774314404, 0.01737082377076149, -0.010731036774814129, -0.00026143735158257186, -0.019396061077713966, -0.028135841712355614, -0.004543190822005272, 0.0023718378506600857, -0.027184387668967247, -0.02173391729593277, 0.009392205625772476, -0.005593187641352415, -0.010316475294530392, 0.004291735123842955, 0.010880551300942898, 0.03495912253856659, 0.008420363999903202, 0.006867455784231424, 0.027605745941400528, -0.010316475294530392, 0.020741688087582588, -0.0023616435937583447, 0.006782504729926586, 0.005708721466362476, -0.014774714596569538, 0.01222617831081152, -0.008726187981665134, -0.025553325191140175, -0.013034913688898087, -0.007258231285959482, 0.003571348963305354, -0.005671342834830284, -0.016052380204200745, -0.007346580736339092, 0.005046101752668619, -0.014910636469721794, 0.015685392543673515, 0.002570623764768243, 0.014353356324136257, -0.02587953768670559, -0.01436694897711277, -0.02895137295126915, -0.017398007214069366, 0.0008185048936866224, -0.002844166476279497, 0.008168908767402172, -0.022712556645274162, -0.0032485343981534243, -0.004590763244777918, -0.022005761042237282, 0.011342685669660568, -0.0026334875728935003, 0.02215527556836605, -0.0240989588201046, 0.027972735464572906, -0.028543606400489807, -0.01440772507339716, 0.0047028991393744946, 0.011240744031965733, 0.020931977778673172, 0.0030446513555943966, -0.005973769351840019, -0.024112552404403687, 0.019885379821062088, -0.004648530390113592, 0.03854745998978615, 0.014937820844352245, 0.0374600850045681, -0.004889791831374168, -0.017058202996850014, 0.01369413547217846, -0.0020269358064979315, 0.012565983459353447, 0.018784411251544952, -0.008780557662248611, 0.032702818512916565, 0.003221350023522973, 0.05252023786306381, 0.0003899260482285172, -0.018145577982068062, -0.003639309899881482, 0.01394559070467949, -0.015563062392175198, -0.019042663276195526, -0.024207696318626404, 0.0065344469621777534, 0.01650092378258705, -0.01054074615240097, 0.01650092378258705, 0.005001927260309458, -0.015766944736242294, 0.021897023543715477, 0.021108677610754967, -0.012817438691854477, 0.013870833441615105, -0.003985910676419735, -0.0068368734791874886, 0.004471831955015659, -0.008270850405097008, -0.023840708658099174, 0.008345606736838818, -0.0016692912904545665, 0.013789280317723751, -0.014244618825614452, 0.006031536031514406, 0.02851642295718193, -0.036563001573085785, -0.01047278568148613, 0.0001986796414712444, 0.003758241655305028, 0.00861065462231636, 0.01572616770863533, 0.031425151973962784, 0.015631023794412613, -0.011553364805877209, 0.018199946731328964, -0.0025757206603884697, 0.01443490944802761, -0.028434868901968002, -0.0030327581334859133, 0.019423244521021843, 0.029250402003526688, -0.0030956221744418144, 0.021856248378753662, -0.009602884761989117, 0.026042643934488297, 0.013483456335961819, 0.022277604788541794, 0.011091230437159538, -0.0014492676127701998, 0.022413527593016624, -0.009025217033922672, 0.0075300754979252815, -0.006330564152449369, 0.015603838488459587, -0.002460187068209052, 0.0029529042076319456, 0.003082029987126589, -0.031996022909879684, -2.1675832613254897e-05, -0.018811596557497978, -0.031207676976919174, 0.015984419733285904, -0.015100927092134953, 0.04613190516829491, 0.012932972051203251, -0.015250441618263721, -0.005977167282253504, -0.0046043554320931435, 0.004937364254146814, -0.015100927092134953, 0.039281439036130905, -0.010812589898705482, 0.03286592289805412, 0.011730063706636429, 0.016990242525935173, -0.0015078840078786016, -0.006364544853568077, -0.03308340162038803, -0.025988275185227394, -1.2955059901287314e-05, 0.00293931202031672, 0.009752399288117886, 0.011383462697267532, -0.003469407558441162, 0.00620143860578537, 0.018757227808237076, 0.008128131739795208, 0.010635891929268837, -0.01025531068444252, -0.0008325218805111945, 0.024180512875318527, 0.018770819529891014, -0.004648530390113592, -0.6419864892959595, -0.03311058506369591, -0.015209664590656757, -0.026273710653185844, -0.006439301650971174, 0.003924746066331863, 0.014462093822658062, 0.005294159520417452, -0.005192217882722616, -0.0009930796222761273, 0.01075822114944458, 0.0017584901070222259, -0.01815917156636715, 0.016963057219982147, -0.00961647741496563, -0.010214533656835556, -0.009065993130207062, -0.019491206854581833, 0.028434868901968002, 0.0021118870936334133, -0.0391727015376091, -0.006422311533242464, -0.029467877000570297, 0.016568884253501892, 0.024710608646273613, 0.003503388026729226, 0.016487330198287964, -0.002366740722209215, 0.003887367434799671, 0.01580772176384926, -0.006867455784231424, 0.024071775376796722, 0.004108240362256765, -0.027619337663054466, 0.04159211367368698, -0.010866958647966385, -0.004903384018689394, -0.009568904526531696, 0.004247560631483793, 0.0034626114647835493, -0.032213497906923294, -0.002664069877937436, 0.0005466610309667885, -0.021557219326496124, -0.028190210461616516, 0.01916499249637127, -0.0025655266363173723, -0.0024839735124260187, -0.01265433244407177, 0.015930050984025, 0.016351409256458282, -0.002492468571290374, -0.0021560618188232183, -0.020687319338321686, -0.01014657225459814, 0.005504838656634092, 0.04501734673976898, 0.0005611027590930462, 0.03240378946065903, -0.0363183431327343, -0.02794555015861988, 0.014339764602482319, -0.017832959070801735, -0.012464041821658611, -0.016813544556498528, 0.03101738542318344, -0.016419369727373123, 0.0016854320419952273, 0.04080376774072647, -0.0109349200502038, 0.0007964175892993808, 0.021706733852624893, 0.0116485096514225, 0.013707727193832397, 0.009956281632184982, 0.0024822743143886328, 0.035774655640125275, -0.005637362599372864, -0.014951413497328758, 0.011091230437159538, 0.025485362857580185, -0.014067920856177807, -0.007468910422176123, -0.012266955338418484, 0.008515509776771069, 0.008175704628229141, -0.015399955213069916, 2.4635850422782823e-05, 0.004899986088275909, -0.006337360478937626, 0.013660155236721039, 0.023405758664011955, 0.02165236510336399, -0.027619337663054466, -0.023120321333408356, 0.021258190274238586, -0.021856248378753662, 0.007176678162068129, 0.052493054419755936, 0.016188303008675575, 0.000594233744777739, -0.01176404394209385, 0.02458827942609787, 0.014489278197288513, -0.004573773127049208, -0.0018264510435983539, -0.03626397252082825, 0.0013430786784738302, 0.03286592289805412, -0.0330018475651741, 0.0065650297328829765, -0.020551396533846855, -0.005837847478687763, 0.0013830057578161359, -0.009263080544769764, -0.028570791706442833, 0.011485403403639793, 0.012532002292573452, -0.01715334877371788, -0.0260970126837492, 0.007224251050502062, -0.009507739916443825, -0.006745126098394394, -0.007536871358752251, 0.008868906646966934, 0.013096078298985958, -0.0038397947791963816, 0.009745603427290916, 0.003758241655305028, -0.029331954196095467, -0.0036087273620069027, -0.008522305637598038, 0.004855811130255461, -0.010710649192333221, 0.01787373423576355, 0.012932972051203251, -0.011940741911530495, 0.006660175044089556, -0.014502870850265026, -0.005980565212666988, -0.00793104525655508, -0.0007760293083265424, 0.01722130924463272, -0.005630566272884607, -0.014747530221939087, -0.011743655428290367, -0.029359139502048492, 0.0031669812742620707, 0.014733938500285149, 0.007400949485599995, 0.0030633406713604927, -0.013232001103460789, -0.003951930440962315, 0.02437080442905426, 0.00045448896707966924, -0.02365041710436344, -0.007339784875512123, -0.011356278322637081, -0.03672610595822334, -0.00017414998728781939, 0.004203386139124632, 0.024343619123101234, -0.005260178819298744, -0.007625220809131861, 0.0015843401197344065, -0.00589561415836215, 0.010595114901661873, 0.01419025007635355, -0.02029314450919628, -0.041429005563259125, 0.005946584977209568, -0.008943663910031319, -0.03425232693552971, 0.02715720236301422, 0.00580726470798254, -0.022603819146752357, -0.010126184672117233, 0.01247083768248558, 0.021747510880231857, -0.017914511263370514, -0.0035747468937188387, -0.017112571746110916, -0.033137768507003784, 0.015331994742155075, 0.015114519745111465, 0.006249011028558016, 0.006303379777818918, -0.024058183655142784, 0.015848498791456223, 0.020632950589060783, 0.0008100097766146064, 0.01830868422985077, -0.012484430335462093, 0.01836305297911167, 0.0014959907857701182, -0.02745623141527176, 0.017778590321540833, 0.01132909394800663, 0.0009964776691049337, -0.0015554566634818912, 0.07144057005643845, -0.018743636086583138, 0.010302882641553879, -0.015957236289978027, 0.01412228960543871, -0.024792160838842392, 0.009969874285161495, -0.008440752513706684, 0.0014934423379600048, 0.0260970126837492, 0.012926176190376282, -0.008522305637598038, 0.0129057876765728, 0.013170835562050343, 0.0047606658190488815, 0.028978556394577026, 0.01779218204319477, 0.016704807057976723, -0.019844602793455124, 0.0004054296587128192, 0.0011196569539606571, -0.006731533911079168, 0.009922301396727562, -0.03294747695326805, -0.009908709675073624, 0.010697056539356709, 0.01921936124563217, 0.02072809636592865, 0.013123263604938984, -0.023704785853624344, 0.014163065701723099, 0.009161138907074928, 0.02294362336397171, 0.018757227808237076, 0.014652385376393795, 0.010785405524075031, 0.016528107225894928, -0.01182520855218172, 0.018254315480589867, 0.005555809009820223, 0.006405321415513754, 0.016677621752023697, 0.03161544352769852, 0.0004058544000145048, -0.016555292531847954, 0.016759175807237625, 0.023201875388622284, 0.009514535777270794, 0.01694946549832821, 0.016065973788499832, -0.029848458245396614, -0.016419369727373123, -0.00034596381010487676, -0.02201935462653637, -0.0016115244943648577, -0.0032825148664414883, 0.025281481444835663, 0.017398007214069366, 0.016269855201244354, 0.028978556394577026, -0.002854360733181238, 0.01873004250228405, 0.0018213540315628052, 0.008427159860730171, 0.006395127158612013, -0.001203758642077446, 0.018634898588061333, -0.01837664656341076, -0.02509118989109993, -0.029930010437965393, -0.009670846164226532, -0.009888321161270142, 0.01693587377667427, -0.025852352380752563, 0.026898952201008797, 0.01815917156636715, -0.00557279959321022, 0.008488325402140617, 0.009487351402640343, 0.016840727999806404, -0.020034894347190857, -0.03365427255630493, -0.004597559571266174, 0.010737833566963673, 0.005148043390363455, 0.0016123739769682288, -0.029196033254265785, -0.002162857912480831, -0.0034201357048004866, 0.024846529588103294, -0.007999005727469921, 0.008005802519619465, -0.01394559070467949, 0.004502414260059595, 0.005046101752668619, -0.011607733555138111, 0.025702837854623795, -0.008746576495468616, 0.015196072869002819, -0.006670369300991297, -0.0023786339443176985, 0.00858347024768591, 0.016854319721460342, -0.03868338465690613, 0.034279514104127884, -0.01851256750524044, -0.02193780057132244, 0.0029257198330014944, 0.003571348963305354, -0.031479522585868835, -0.024697016924619675, -0.005593187641352415, -0.02416692115366459, 0.01068346481770277, 0.022834885865449905, 0.0009633467416279018, -0.03781348466873169, -0.0037446494679898024, 0.019341692328453064, -0.024629054591059685, -0.0023293623235076666, -0.023895077407360077, 0.018961111083626747, -0.0016268156468868256, 0.054994016885757446, 0.015549469739198685, -0.01355141680687666, 0.039417363703250885, -0.0037888239603489637, -0.0033929513301700354, -0.016568884253501892, -0.008488325402140617, 0.007197066675871611, 0.007115513551980257, -0.003958726301789284, -0.010615503415465355, -0.00793104525655508, -0.002050722250714898, 0.00045109092025086284, 0.006102894898504019, -0.02129896730184555, -0.020429067313671112, -0.0037990182172507048, -0.007699977606534958, -0.009140750393271446, 0.020768871530890465, 0.026491185650229454, 0.0017143154982477427, 0.013415494933724403, -0.01765625923871994, 0.005545615218579769, 0.023392165079712868, 0.02452031709253788, -0.021122269332408905, -0.003066738834604621, 0.010370844043791294, -0.013130059465765953, 0.007135901600122452, -0.011043657548725605, -0.004254356492310762, 0.018621305003762245, -0.01859412156045437, 0.011335889808833599, -0.0029868846759200096, 0.0038397947791963816, 0.03604649752378464, 0.020999940112233162, -0.007482502609491348, -0.016065973788499832, -0.0126475365832448, -0.0023599446285516024, 0.005729109514504671, 0.02366400882601738, -0.03830280154943466, 0.030120301991701126, 0.0021543626207858324, 0.006636388599872589, -0.025363033637404442, 0.0036596981808543205, 0.0037038729060441256, -0.009596088901162148, -0.013986367732286453, -0.004403871018439531, -0.01928732357919216, 0.011220356449484825, -0.018906742334365845, 0.015291217714548111, 0.008420363999903202, 0.0014739035395905375, -0.03218631446361542, -0.020605765283107758, -0.0083659952506423, -0.0046349382027983665, -0.016392186284065247, -0.012117440812289715, 0.006364544853568077, -0.0035509606823325157, 0.019695088267326355, 0.008603858761489391, 0.006170855835080147, 0.013347534462809563, 0.017479561269283295, 0.012328119948506355, 0.0034897958394140005, 0.011505791917443275, -7.815511344233528e-05, -0.003024263074621558, -0.005718915723264217, -0.006330564152449369, 0.0008108593174256384, 0.000989681575447321, -0.029413508251309395, -0.02279410883784294, 0.037133872509002686, 0.0036087273620069027, -0.008984440006315708, -0.0006592213758267462, -0.01140385027974844, 0.0024890704080462456, -0.01987178809940815, 0.03694358468055725, 0.012667925097048283, -0.018471792340278625, -0.0017143154982477427, -0.013374718837440014, -0.03373582661151886, -0.016528107225894928, -0.007740754168480635, 0.015386363491415977, 0.00014441706298384815, 0.022209644317626953, 0.021842654794454575, -0.0032162528950721025, 0.005718915723264217, -0.0015851896023377776, -0.014543646946549416, 0.024697016924619675, 0.0035985333379358053, -0.011132006533443928, 0.014339764602482319, -0.02166595682501793, -0.005661148577928543, 0.008624247275292873, -0.022930031642317772, 0.016528107225894928, -0.02681739814579487, 0.011716471053659916, -0.019531982019543648, -0.010343659669160843, 0.009908709675073624, 0.006062118336558342, -0.033572718501091, -0.009201915003359318, 0.015984419733285904, 0.0038737752474844456, 0.027714483439922333, 0.00814852025359869, -0.008440752513706684, -0.045941613614559174, -0.026681477203965187, 0.00789706502109766, 0.021978577598929405, -0.0065242531709373, -0.003219650825485587, -0.0010041233617812395, -0.0031041172333061695, -0.0030514474492520094, 0.018566936254501343, 0.002929117763414979, -0.016854319721460342, -0.0049645486287772655, 0.002295381622388959, -0.01579413004219532, -0.007863083854317665, -0.005977167282253504, -0.01973586529493332, -0.01175724808126688, -0.013021321967244148, -0.01594364270567894, -0.03389893099665642, -0.00763201666995883, -0.017955288290977478, 0.03324650600552559, 0.0464853011071682, 0.031914472579956055, -0.009704826399683952, -0.0007263328297995031, 0.03286592289805412, 0.013313554227352142, 0.026545554399490356, -0.029984379187226295, -0.010377639904618263, -0.008481528609991074, 0.008128131739795208, -0.01564461551606655, 0.009888321161270142, 0.017629075795412064, -0.00857667438685894, 0.0064528938382864, 0.02687176689505577, -0.007727161981165409, -0.017071794718503952, 0.015141704119741917, -0.03900959715247154, -0.007951433770358562, 0.0028917391318827868, 0.01311646681278944, 0.013252388685941696, -0.017384415492415428, -0.0048184324987232685, -0.001828150125220418, 0.022264013066887856, -0.02230479009449482, -0.015345586463809013, 0.025322256609797478, -0.01118637528270483, 0.03623678907752037, 0.008467936888337135, 0.010493173263967037, 0.006085904780775309, -0.01329316571354866, -0.022467896342277527, -0.010832978412508965, 0.029005741700530052, 0.0035305724013596773, 0.0020558193791657686, 0.009596088901162148, -0.022114498540759087, -0.0023055758792907, 0.010839774273335934, 0.0027813026681542397, -0.015902867540717125, -0.01708538830280304, -0.018186355009675026, 7.661537529202178e-05, -0.01136307418346405, -0.04610472172498703, 0.006789300590753555, 0.005603381898254156, 0.006785902660340071, -0.007217454724013805, 0.03302903100848198, 0.006374739110469818, -0.029930010437965393, -0.0032927088905125856, -0.0147611228749156, 0.012538799084722996, 0.009963078424334526, 0.0330018475651741, 0.036563001573085785, -0.015576654113829136, -0.009195119142532349, -0.008651431649923325, 0.005086878314614296, -0.011730063706636429, 0.016405778005719185, 0.04072221368551254, -0.04439210519194603, -0.0053077517077326775, 0.025784391909837723, -0.0040029012598097324, 0.0007127406424842775, -0.04246201366186142, 0.016133934259414673, 0.0003912003303412348, 0.0075096869841217995, -0.0010219630785286427, -0.00011287892266409472, 0.006364544853568077, 0.007278619799762964, -0.010241718031466007, -0.0022053334396332502, -0.0034048445522785187, 0.010336863808333874, 0.002342954510822892, 0.00407765805721283, -0.02551254816353321, 0.0002939311962109059, -0.00763201666995883, -0.023677602410316467, -0.03482320159673691, -0.007108717225492001, 0.009677642025053501, -0.0037888239603489637, -0.011865985579788685, 0.0039995028637349606, -0.010051427409052849, 0.012450449168682098, -0.003985910676419735, -0.009249487891793251, 0.011770839802920818, -0.020130038261413574, -0.007652405183762312, 0.0261513814330101, 0.015427139587700367, -0.002672565169632435, 0.013531029224395752, 0.014557239599525928, -0.013666951097548008, -0.009942689910531044, -0.031696997582912445, -0.00034129148116335273, -0.0010533949825912714, -0.01176404394209385, 0.02802710421383381, 0.02195139229297638, 0.00208300375379622, -0.027401862666010857, 0.00081977917579934, -0.005844643339514732, -0.025825168937444687, 0.011682490818202496, -0.004607753828167915, -0.03854745998978615, -0.00903880875557661, 0.010336863808333874, -0.0167319905012846, 0.0035815429873764515, -0.019273729994893074, 0.01014657225459814, -0.0013447776436805725, -0.009412594139575958, -0.005654352717101574, 0.0029580011032521725, 0.010887347161769867, 0.018988294526934624, -0.03610086813569069, 0.020184407010674477, -0.0002901083789765835, -0.03139796853065491, 0.015454323962330818, 0.01247763354331255, -0.026110604405403137, 0.007523279171437025, -0.020891202613711357, -0.010017447173595428, 0.01824072375893593, -0.018485384061932564, 0.007958229631185532, -0.00012041834270348772, 0.015671798959374428, 0.011981518939137459, -0.050427038222551346, 0.013429087586700916, -0.015087335370481014, -0.009351429529488087, -0.00861065462231636, -0.012939768843352795, -0.00790386088192463, -0.014693161472678185, -0.020755279809236526, 0.012178605422377586, -0.0312892310321331, -0.009344633668661118, -0.017982471734285355, 0.01858052983880043, 0.011247540824115276, -0.025648469105362892, -0.020632950589060783, 0.014652385376393795, -0.0334639810025692, 0.05167752131819725, 0.012599963694810867, -0.0005428382428362966, 0.013028117828071117, -0.009534924291074276, 0.0386018306016922, 0.02458827942609787, -0.03773193061351776, 0.0022851875983178616, -0.02114945277571678, 0.014815491624176502, 0.01372811570763588, -0.012090256437659264, 0.003459213301539421, 0.05039985477924347, -0.01161452941596508, 0.0031363987363874912, 0.01179802417755127, -0.01879800483584404, -0.03240378946065903, -0.0036766885314136744, 0.010567930527031422, 0.0006048526265658438, -0.0029580011032521725, 0.01966790482401848, 0.03697076812386513, 0.0261513814330101, 0.008107743225991726, 0.0078019192442297935, -0.026355264708399773, 0.006711145862936974, 0.014924229122698307, 0.020497027784585953, 0.006891242228448391, -0.016568884253501892, -0.04017852619290352, 0.03379019349813461, 0.008032986894249916, -0.021244598552584648, 0.01033006701618433, 0.039199888706207275, -0.026246527209877968, -0.019627127796411514, 0.021761102601885796, -0.011594140902161598, 0.009154342114925385, 0.01047278568148613, -0.015766944736242294, -0.005820856895297766, -0.00010358738654758781, 0.025702837854623795, 0.0266678836196661, -0.00897084828466177, -0.010221329517662525, -0.008590266108512878, -0.005219402257353067, 0.026994096115231514, -0.0037276591174304485, 0.0021713529713451862, -0.005905808415263891, -0.044500842690467834, -0.012097052298486233, -0.007190270349383354, 0.022413527593016624, -0.004512608516961336, 0.022318381816148758, -0.023283427581191063, 0.002665769075974822, 0.01658247597515583, 0.007258231285959482, -0.017425192520022392, 0.013463067822158337, -0.00452280230820179, 0.013225204311311245, -0.017846550792455673, -0.002981787547469139, -0.0031550880521535873, 0.024819346144795418, 0.003853386966511607, 0.03743290156126022, -0.034496989101171494, 0.004043677821755409, -0.0034965919330716133, -0.03359990194439888, -0.01526403333991766, 0.0022580032236874104, -0.01071744505316019, 0.019831011071801186, -0.004781054332852364, 0.015318402089178562, -0.04047755151987076, 0.035693101584911346, 0.02138052135705948, -0.02938632294535637, 0.0010601910762488842, 0.012980544939637184, 0.023691194131970406, 0.007210658863186836, 0.029712535440921783, 0.1908344030380249, 0.004736879374831915, 0.0014832481974735856, 0.041700851172208786, 0.0038397947791963816, 0.009881525300443172, 0.00398251274600625, -0.00876016914844513, 0.019137809053063393, 0.028679529204964638, 0.019613536074757576, -0.015603838488459587, -0.023609640076756477, -0.007917452603578568, 0.03588339313864708, -0.01723490096628666, -0.023609640076756477, -0.014135881327092648, -0.01458442397415638, -0.020497027784585953, -0.01394559070467949, -0.0028917391318827868, 0.008739780634641647, -0.0026114003267139196, 0.010629095137119293, -0.0015987817896530032, 0.02558050863444805, -0.006877650041133165, 0.018199946731328964, 0.014557239599525928, -0.00471649132668972, -0.041700851172208786, -0.012110644951462746, 0.011233948171138763, -0.0055999839678406715, -0.0017754803411662579, -0.003605329431593418, -0.002093197777867317, -0.002810186007991433, 0.008413568139076233, -0.02693972736597061, 0.02789118140935898, 0.00512765534222126, 0.0017618881538510323, 0.015196072869002819, 0.036780476570129395, 0.013170835562050343, -0.019844602793455124, -0.020211592316627502, 0.007332988549023867, -0.02558050863444805, -0.0036427080631256104, 0.00013815191050525755, 0.001281913835555315, 0.010411620140075684, -0.008862110786139965, 0.016392186284065247, -0.0031296026427298784, 0.002016741782426834, 0.002774506574496627, -0.03101738542318344, 0.03593776002526283, -0.004403871018439531, 0.009548516012728214, -0.004159211181104183, 0.016895096749067307, 0.0005279717734083533, -0.02230479009449482, 0.00514464545994997, -0.004872801713645458, 0.027714483439922333, 0.014013552106916904, -0.010452397167682648, 0.0029274187982082367, -0.011009677313268185, -0.009025217033922672, -0.01094171591103077, 0.010595114901661873, 0.024778569117188454, 0.04096687212586403, -0.0129057876765728, 0.01118637528270483, -0.020401883870363235, -0.008766965009272099, -0.04281540960073471, -0.03316495195031166, 0.021543627604842186, -0.024343619123101234, -0.013796077109873295, 0.01735723204910755, 0.006949008908122778, -0.023908669129014015, -0.014054328203201294, -0.0008732984424568713, 0.015495100989937782, -0.007958229631185532, -0.024928083643317223, 0.010370844043791294, -0.024465948343276978, -0.001495141303166747, -0.01779218204319477, 0.05341732129454613, 0.005759692285209894, -0.012069867923855782, -0.011458219029009342, -0.01140385027974844, -0.026790214702486992, 0.0218698401004076, -0.003928143996745348, -0.0073261926881968975, 0.0006044278852641582, -0.005946584977209568, 0.01047278568148613, -0.003955328371375799, -0.00878735352307558, 0.028462054207921028, 0.0017448979197070003, -0.039553284645080566, 0.015889273956418037, 0.003510184120386839, 0.04759986326098442, -0.00582425482571125, 0.030392145738005638, 0.004991733003407717, -0.002842467511072755, -0.017425192520022392, 0.00305654457770288, 0.0019080041674897075, 0.00720386253669858, -0.0346057265996933, 0.00821648072451353, 0.0028900401666760445, -0.006293185520917177, -0.032267868518829346, -0.0017618881538510323, 0.0022342167794704437, -0.0007972671301104128, -0.00437668664380908, -0.016691213473677635, -0.007285416126251221, -0.0010491474531590939, -0.012532002292573452, 0.009412594139575958, 0.007496094796806574, 0.013911610469222069, -0.022780517116189003, 0.03264844790101051, -0.002441497752442956, -0.006599009968340397, 0.024533910676836967, 0.0047028991393744946, -0.012525206431746483, -0.009874728508293629, -0.013068893924355507, 0.004495617933571339, -0.00015078840078786016, -0.04759986326098442, 0.00537911057472229, 0.009840748272836208, 0.0033368836157023907, -0.03759600967168808, -0.019994117319583893, 0.02903292514383793, -8.797759801382199e-05, -0.016813544556498528, -0.021407704800367355, -0.17332765460014343, 0.014217434450984001, 0.002696351381018758, 0.021108677610754967, 0.011662102304399014, -0.022345567122101784, -0.00990191288292408, 0.004740277770906687, -0.018199946731328964, 0.003192466450855136, 0.02608341909945011, 0.01864849030971527, -0.026464002206921577, -0.031071754172444344, 0.007448522374033928, 0.025485362857580185, -0.017098980024456978, 0.03610086813569069, -0.0031635831110179424, 0.004781054332852364, 0.03550281003117561, 0.004104842431843281, 0.010173756629228592, -0.027320310473442078, -0.0022749933414161205, -0.015671798959374428, 0.0068266792222857475, 0.016256263479590416, 0.015182480216026306, -0.002042227191850543, 0.010418416932225227, -0.01458442397415638, 0.030582435429096222, 0.009922301396727562, 0.014570832252502441, -0.026708660647273064, -0.00772036612033844, 0.01480189897119999, 0.001648903009481728, 0.007761142682284117, 0.003503388026729226, -0.005430081393569708, -0.0012598264729604125, 0.015753353014588356, -0.01032327115535736, 0.019885379821062088, 0.023623233661055565, -0.02408536709845066, -0.007883472368121147, -0.01616111770272255, 0.03343679755926132, -0.02717079594731331, -0.004696102812886238, -0.00343202892690897, 0.03079991042613983, 0.016636844724416733, 0.01938246749341488, 0.040559105575084686, -0.017330046743154526, -0.023337796330451965, 0.0031482919584959745, -0.016922282055020332, -0.007564055733382702, -0.025852352380752563, -0.011818412691354752, 0.01629704050719738, 0.016133934259414673, -0.013102875091135502, -0.011743655428290367, 0.03311058506369591, -0.004315521568059921, -0.007795123383402824, 0.012878603301942348, 0.013028117828071117, 0.02802710421383381, 0.013204816728830338, -0.013007729314267635, 0.00046850592480041087, 0.0054640620946884155, -0.002066013403236866, -0.031479522585868835, 0.025702837854623795, -0.001836645184084773, 0.019708681851625443, -0.009059197269380093, 0.010099000297486782, -0.0017134658992290497, 0.013843649066984653, 0.012994137592613697, 0.006058720406144857, 0.003253631293773651, -0.03430669754743576, 0.0007055198075249791, -0.005654352717101574, -0.021394113078713417, 0.018417421728372574, -0.00432231742888689, -0.012171809561550617, 0.027551377192139626, -0.014027143828570843, -0.0017236600397154689, 0.0056475563906133175, -0.026994096115231514, 0.029930010437965393, 0.02472420036792755, -0.0021730519365519285, 0.028788266703486443, 0.008019394241273403, 0.014312580227851868, -0.02658633142709732, -0.015780536457896233, -0.025634877383708954, 0.022331973537802696, 0.011322297155857086, -0.007591240108013153, 0.014285395853221416, -0.031914472579956055, -0.0005071587511338294, 0.00582425482571125, -0.023392165079712868, 0.028543606400489807, -0.006731533911079168, -0.03645426407456398, -0.014774714596569538, -0.01886596530675888, -0.0029172245413064957, -0.09362303465604782, -0.018811596557497978, 0.01333394180983305, 0.009915505535900593, 0.005368916783481836, 0.003000476863235235, -0.005746100097894669, 0.011057249270379543, -9.99238618533127e-05, 0.023256244137883186, -0.009365021251142025, -0.0010457494063302875, -0.008236869238317013, -0.018811596557497978, 0.00983395241200924, -0.029250402003526688, 0.003168680239468813, -0.02129896730184555, -0.02530866488814354, 0.0010618901578709483, 0.0038024161476641893, 0.00326212658546865, -0.01419025007635355, -0.02623293362557888, -0.00969123374670744, -0.04325035959482193, -0.015753353014588356, 0.020972754806280136, 0.03604649752378464, 0.007659201044589281, 8.87209243956022e-05, -0.004196589812636375, -0.008134927600622177, -0.0037446494679898024, 0.009405798278748989, 0.005698527209460735, -0.005481052212417126, -0.0055999839678406715, 0.0055897897109389305, -0.060294970870018005, 0.0044208611361682415, 0.018784411251544952, -0.004903384018689394, -0.02302517555654049, -0.0018740236992016435, 0.006785902660340071, -0.002067712601274252, -0.0013371320674195886, -0.024629054591059685, -0.011811616830527782, -0.010302882641553879, -0.0057359058409929276, -0.015603838488459587, -0.00427134707570076, -0.007135901600122452, 0.00839317962527275, -0.0013609183952212334, 0.00503930589184165, -0.015100927092134953, -0.018471792340278625, -0.007496094796806574, -0.004295133054256439, -0.040640659630298615, 0.014652385376393795, 0.019994117319583893, -0.014326171949505806, -0.012430061586201191, 0.0015503596514463425, -0.015549469739198685, 0.0006689907750114799, 0.008386383764445782, 0.03514941409230232, 0.008053374476730824, 0.01708538830280304, -0.007869880646467209, 0.0009293662733398378, 0.01244365330785513, -0.024425173178315163, -0.02121741510927677, 0.00695240730419755, -0.028652343899011612, 0.0013040010817348957, 0.012797050178050995, -0.0312348622828722, -0.030636804178357124, 0.02193780057132244, 0.020850425586104393, 0.00482183089479804, 0.034415435045957565, -0.05860954150557518, -0.017098980024456978, 0.01694946549832821, 0.019110623747110367, -0.01843101531267166, -0.02216886729001999, 0.0037718338426202536, 0.0010542445816099644, -0.02745623141527176, 0.024792160838842392, 0.00043027789797633886, -0.03979794308543205, -0.012239770963788033, -0.06453573703765869, 0.018498975783586502, -0.008087355643510818, 0.0019504798110574484, 0.006344156339764595, -0.010622299276292324, 0.009099973365664482, -0.01414947398006916, -0.013823261484503746, 0.008726187981665134, -0.020252369344234467, -0.004142221063375473, -0.00833201501518488, 0.006755320355296135, -0.025145558640360832, -0.01436694897711277, 0.023188281804323196, 0.010017447173595428, 0.03987949714064598, -0.00333178648725152, -0.010880551300942898, 0.006602408364415169, -0.00695240730419755, 0.013483456335961819, -0.024928083643317223, 0.006908232346177101, -0.01482908334583044, 0.010894143022596836, 0.004495617933571339, -0.013164039701223373, 0.015848498791456223, -0.008474732749164104, 0.010826182551681995, 0.008692207746207714, 0.008549490012228489, -0.0066227964125573635, -0.003945134114474058, -0.00688104797154665, -0.015563062392175198, -0.021856248378753662, -0.04129308462142944, -0.01973586529493332, 0.020931977778673172, -0.006028138101100922, -0.02908729389309883, -0.008495121262967587, -0.02131255902349949, -0.004267948679625988, 0.018757227808237076, -0.0021050909999758005, 0.024860123172402382, -0.006422311533242464, -0.007611628621816635, -0.022467896342277527, -0.015671798959374428, -0.030392145738005638, 0.004132026806473732, -0.020333921536803246, 0.011179579421877861, -0.006299981847405434, 0.03966202214360237, 0.003673290368169546, -0.011349481530487537, -0.000666017469484359, 0.04232609272003174, 0.013585397973656654, 0.004301929380744696, 0.007747550494968891, 0.0079650254920125, -0.017561115324497223, -0.015834905207157135, 0.0032094568014144897, 0.01458442397415638, 0.023324204608798027, 0.020972754806280136, -0.01497859787195921, 0.022182460874319077, -0.00012424115266185254, 0.011818412691354752, -0.0016930776182562113, 0.01458442397415638, 0.012729089707136154, -0.013857241719961166, 0.02223682962357998, -0.011254336684942245, 0.019396061077713966, -0.01950479857623577, 0.011111618019640446, 0.008909682743251324, 0.0010296086547896266, -0.021067900583148003, 0.02979408949613571, -0.007129105739295483, -0.0042067840695381165, -0.003907755948603153, 0.00416600750759244, 0.005633964203298092, -0.01993974857032299, -0.028271762654185295, 0.0210135318338871, 0.00360193126834929, 0.02473779208958149, -0.015902867540717125, -0.020741688087582588, -0.03332806006073952, 0.01200190745294094, -0.0033063010778278112, -0.023691194131970406, -0.016636844724416733, 0.020211592316627502, 0.014353356324136257, -0.003066738834604621, 0.025906721130013466, 0.01550869271159172, -0.003809212241321802, 0.007835899479687214, 0.028897004202008247, -0.030500883236527443, 0.004876199644058943, 0.035421255975961685, 0.009174730628728867, -0.004128628876060247, 0.011478607542812824, -0.02008926309645176, 0.032566897571086884, -0.0025927110109478235, 0.021190229803323746, -0.0072378432378172874, 0.025675654411315918, 0.01412228960543871, 0.009990262798964977, -0.024710608646273613, -0.01526403333991766, -0.010404824279248714, -0.01482908334583044, -0.020401883870363235, 0.0044786278158426285, 0.02794555015861988, -0.006972795352339745, 0.09150265157222748, 0.027836812660098076, -0.015631023794412613, 0.02895137295126915, -0.04197269305586815, 0.012498022057116032, -0.016487330198287964, 0.01183200441300869, -0.02710283361375332, 0.005627168342471123, 0.0021730519365519285, -0.005943186581134796, 0.007849492132663727, 0.017900919541716576, -0.034143589437007904, -0.01728926971554756, 0.01786014251410961, -0.006921824533492327, -0.008318422362208366, 0.014217434450984001, 0.02287566289305687, -0.024982452392578125, 0.01311646681278944, -0.029277585446834564, -0.00011903788254130632, 0.007951433770358562, 0.01140385027974844, -0.023297019302845, -0.027632929384708405, -0.02344653382897377, -0.0006604956579394639, 0.025485362857580185, -0.02259022556245327, -0.02737467922270298, -0.00623541884124279, -0.00011532127246027812, -0.010459193028509617, 0.024357210844755173, -0.01737082377076149, 0.01923295482993126, -0.002076207660138607, 0.05303674191236496, -0.007285416126251221, -0.023827115073800087, -0.012851418927311897, -0.022141683846712112, -0.03558436408638954, -0.01265433244407177, -0.009331041015684605]} +{"id": "test:10000009", "text": "\"Sysco Corp. has terminated its planned $3.5 billion takeover of US Foods, it announced Monday, after a federal judge blocked the combination. The company is opting instead to add $3 billion to its stock-buyback program.\\nWith the deal breaking up, Sysco will pay a $300 million termination fee to US Foods and a $12.5 million fee to Performance Food Group, which had agreed to buy some US Foods facilities. Sysco, based in Houston, plans to make the share repurchases over the next two years.\\nRosemont, Illinois-based US Foods operates a major distribution center in Fishers. Sysco has a large warehouse at 4000 W. 62nd St., in Indianapolis.\\nSysco had fought for more than a year to gain government approval for the transaction, which antitrust regulators said would hurt competition and lead to higher prices. Sysco and US Foods dominate a market known as broadline foodservice, which supplies school cafeterias, restaurants and hotels. Sysco had argued that the acquisition would bring $1 billion in savings, letting it offer lower prices to customers.\\nInvestors have responded with relief to the deal\u2019s demise, reflecting concerns about the company undertaking an ambitious merger. Sysco shares rose 3.1 percent the day the transaction was halted by U.S. District Judge Amit Mehta, and the stock climbed again Monday morning after the merger was withdrawn.\\nMehta blocked the merger on June 23 when he granted a Federal Trade Commission request to delay the transaction. The FTC had sued the companies in February, saying the deal would give Sysco an oversized share of an industry where it\u2019s already the biggest player.\\nIn arguments before Mehta in May, the two sides clashed over the scope of the market in which the companies compete. Sysco and US Foods argued that the commission was relying on a \u201ctortured\u201d analysis, ignoring the variety of distribution channels available to customers.\\nSysco said on Monday that it weighed embarking on an appeal but decided against it.\\nShares of Sysco rose as much as 1.6 percent to $39 in early trading. The stock had slid 3.3 percent this year through the end of last week.\"", "vector_field": [-0.017086388543248177, -0.025730885565280914, 0.0101032555103302, -0.002652442315593362, -0.027351729571819305, 0.001396288862451911, -0.008806581608951092, -0.01705937460064888, 0.0014334331499412656, -0.026230646297335625, 0.016883783042430878, 0.003663780866190791, -0.0035016967449337244, -0.013371956534683704, 0.009792594239115715, 0.010326121933758259, 0.0157356858253479, 0.004342508967965841, -0.00018403324065729976, -0.03800877183675766, -0.015141377225518227, 0.021300580352544785, -0.03371353819966316, -0.0009969874517992139, -0.015181898139417171, 0.012068528681993484, 0.0052170888520777225, -0.021152004599571228, 0.011150050908327103, 0.03085004910826683, 0.007739526219666004, -0.00773277273401618, -0.018977371975779533, 0.022367635741829872, -0.024974491447210312, -0.029175177216529846, 0.006679224781692028, -0.008806581608951092, 0.03279506042599678, -0.017491599544882774, 0.021800341084599495, 0.014371476136147976, 0.010474699549376965, 0.0163164883852005, -0.020598215982317924, -0.03238984942436218, 0.0015651267021894455, -0.010900170542299747, -0.006456358823925257, 0.028013573959469795, 0.023583268746733665, 0.024906957522034645, -0.032065682113170624, -0.00680078798905015, 0.017653683200478554, -0.00954271387308836, 0.028715938329696655, 0.007894856855273247, 0.027338221669197083, -0.00126543955411762, -0.0023519110400229692, -0.0015786337899044156, -0.03560452163219452, 0.01988234370946884, 0.00871878582984209, -0.03301117196679115, -0.010832635685801506, -0.02285388857126236, -0.011487726122140884, 0.005166437476873398, 0.012730373069643974, 0.03484812751412392, 0.01323013287037611, -0.00453498400747776, 0.037495505064725876, -0.009157763794064522, -0.007692251820117235, 0.023623788729310036, 0.014101335778832436, -0.00557840196415782, 0.01217658445239067, -0.03457798808813095, -0.018315527588129044, 0.019733766093850136, 0.01764017716050148, 0.014655123464763165, -0.005585155449807644, 0.023326635360717773, -0.024974491447210312, 0.02674391306936741, 0.03101213276386261, 0.046680282801389694, -0.006250376813113689, -0.013169351033866405, -0.006314535159617662, 0.013277406804263592, -0.04451915994286537, 0.036712098866701126, -0.0079691456630826, -0.028229685500264168, -0.004187178332358599, 0.003670534584671259, -0.0019720259588211775, -0.009617002680897713, -0.018977371975779533, 0.0011177064152434468, 0.030796021223068237, -0.002139175310730934, 0.012878949753940105, -0.022286593914031982, -0.01868021860718727, -0.01445251889526844, -0.0025612700264900923, -0.016140896826982498, 0.010028967633843422, -0.004085875581949949, -0.005963352043181658, 0.018126429989933968, 0.006456358823925257, -0.03798175975680351, 0.006233492866158485, 0.019409596920013428, 0.007536920718848705, -0.007746279705315828, 0.04881439357995987, 0.0027115356642752886, -0.006412460934370756, -0.01862618885934353, -0.004659924190491438, 0.0068446858786046505, 0.01610037498176098, 0.015398010611534119, 0.03425382077693939, 0.009441412054002285, -0.03468604385852814, 0.010069488547742367, -0.01264933031052351, 0.019801301881670952, -0.010657044127583504, -0.02000390738248825, 0.019679738208651543, 0.0069561186246573925, -0.01179838739335537, -0.013182858005166054, -0.03295714408159256, 0.0016250641783699393, 0.02231360785663128, 0.026514293625950813, -0.0029850529972463846, -0.022948438301682472, 0.009927664883434772, -0.02293493039906025, -0.01850462704896927, -0.0010248456383123994, 0.03193061053752899, 0.0062233624048531055, 0.013034280389547348, -0.012872196733951569, -0.006534024141728878, -0.028661910444498062, -0.012743880040943623, -0.012980252504348755, 0.0022945061791688204, -0.005527750588953495, -0.030823035165667534, 6.72713213134557e-05, 0.004251336678862572, -0.008164997212588787, -0.01445251889526844, -0.0046869381330907345, 0.018815288320183754, -0.010879909619688988, -0.005142800509929657, 0.006044394336640835, -0.001600582618266344, 0.0173025019466877, 0.0168972909450531, 0.012500753626227379, -0.024569282308220863, -0.013588069006800652, -0.005176567938178778, -0.004224322736263275, 0.03676612675189972, 0.010177544318139553, 0.009596742689609528, -0.012703358195722103, 0.021516693755984306, -0.004507970064878464, 0.014682138338685036, 0.0012857001274824142, -0.0083473427221179, 0.003321040188893676, -0.00435601593926549, -0.024312647059559822, -0.6310482621192932, -0.012804660946130753, -0.012865442782640457, -0.027635376900434494, -0.010062734596431255, 0.03582063317298889, 0.030796021223068237, -0.0025849072262644768, 0.01500630658119917, -0.013534041121602058, -0.002743614837527275, 0.0161138828843832, -0.0020851471927016973, -0.023704832419753075, -0.004001456778496504, 0.00844864547252655, -0.006189594976603985, -0.0202065110206604, -0.02492046356201172, 0.03484812751412392, -0.020301060751080513, 0.02363729663193226, -0.002868554787710309, -0.01198748592287302, 0.04460020363330841, 0.011062255129218102, 0.009306341409683228, -0.032822076231241226, -0.0009471802623011172, -0.019733766093850136, -0.013925744220614433, 0.0032366211526095867, -0.003808981506153941, -0.0032383096404373646, 0.056081175804138184, 0.023326635360717773, -0.022880902513861656, -0.010744839906692505, 0.01962571032345295, 0.02852684073150158, -0.017194444313645363, -0.021300580352544785, 0.03600973263382912, -0.006537401117384434, 0.004548491444438696, 0.0037718371022492647, 0.01256828848272562, 0.0108866635710001, 0.013405723497271538, -0.007773293647915125, 0.022745832800865173, 0.02120603248476982, -0.029580388218164444, -0.009488685987889767, 0.0074963998049497604, 0.021057454869151115, 0.017910316586494446, -0.03603674843907356, 0.016262460500001907, 0.0045586214400827885, 0.022421663627028465, 0.0006424279417842627, 0.002463344018906355, -0.015911277383565903, -0.031066160649061203, 0.0256363358348608, 0.0018808534368872643, -0.0054534622468054295, 0.04781487584114075, 0.010299107991158962, 0.013925744220614433, 0.012385943904519081, 0.01235892903059721, 0.0052339727990329266, 0.006280767265707254, 0.02066574990749359, -0.005375796463340521, -0.002071640221402049, 0.025123069062829018, 0.022880902513861656, -0.009441412054002285, -0.024245113134384155, 0.0009176336461678147, -0.018599174916744232, 0.0051866983994841576, -0.0005415473715402186, -0.006250376813113689, 0.011001473292708397, 0.0038157349918037653, -0.0400078147649765, -0.019139457494020462, 0.02922920510172844, 0.002059821505099535, -0.007820568047463894, 0.022300101816654205, 0.005294754635542631, -0.03203866630792618, -0.006280767265707254, 0.02558230794966221, -0.029634416103363037, -0.006895337253808975, -0.02836475521326065, 0.01070431899279356, 0.008036680519580841, 0.025042027235031128, 0.03333534300327301, -0.009623756632208824, 0.019369076937437057, 0.026987038552761078, -0.01329091377556324, 0.019760780036449432, -0.003596245776861906, -0.017383543774485588, -0.007314054761081934, -0.008408124558627605, -0.014736166223883629, 0.017586149275302887, 0.0027723172679543495, -0.02062522992491722, -0.01668117754161358, 0.0003819956036750227, -0.0066826012916862965, 0.026365716010332108, 0.017451077699661255, -0.0045113470405340195, 0.01764017716050148, 0.011278367601335049, -0.017694205045700073, -0.0025123069062829018, -0.02906712144613266, 0.004514723550528288, -0.029904557392001152, 0.017734726890921593, -0.022462185472249985, -0.0009142568451352417, 0.022556735202670097, 0.010110009461641312, -0.0013456374872475863, 0.02479890175163746, -0.035010214895009995, 0.0035759853199124336, 0.002409315900877118, 0.022799860686063766, -0.011980732902884483, 0.00034569547278806567, -0.02836475521326065, 0.0045586214400827885, -0.011062255129218102, -0.013250392861664295, 0.0012375813676044345, 0.01472265925258398, -0.0018031880026683211, 0.00281621515750885, -0.026635857298970222, -0.009400890208780766, -0.008502673357725143, -0.005179944913834333, -0.043249499052762985, -0.02409653551876545, -0.02596050500869751, -0.002473474247381091, 0.011393177323043346, -0.044249020516872406, -0.004822008311748505, -0.006297651212662458, -0.004828762263059616, 0.03908933699131012, -0.019477132707834244, -0.0108866635710001, -0.0031015509739518166, 0.02297545224428177, -0.003808981506153941, 0.028391769155859947, 0.009117242880165577, 0.004774733912199736, 0.019490638747811317, 0.0018859185511246324, 0.03736043721437454, 0.007881349883973598, -0.006496879737824202, 0.030201710760593414, -0.0019095558673143387, -0.015411517582833767, 0.009711552411317825, 0.040683165192604065, -0.008576962165534496, 0.012615563347935677, 0.025082549080252647, -0.03390263766050339, 0.008543194271624088, 0.013182858005166054, 0.005190074909478426, 0.038657110184431076, 0.0007838296587578952, -0.009866883046925068, -0.021665271371603012, 0.0054365782998502254, -0.01323013287037611, 0.006996640004217625, 0.002063198247924447, 0.01784278266131878, 0.022880902513861656, -0.009036201052367687, 0.014641616493463516, -0.013182858005166054, -0.014709152281284332, 0.025285154581069946, -0.037495505064725876, 0.013594822026789188, 0.028337741270661354, 0.020503666251897812, -0.01768069714307785, 0.01639753021299839, -0.0062402463518083096, 0.020935891196131706, 0.02806760184466839, 0.006135567091405392, -0.011285120621323586, 0.0026085444260388613, 0.013810934498906136, 0.010738085955381393, -0.013115323148667812, 0.014047307893633842, -0.030958104878664017, 0.003609752980992198, -0.018977371975779533, 0.01188618317246437, 0.010305861011147499, 0.011258106678724289, 0.0030492113437503576, 0.009758826345205307, 0.013858209364116192, -0.00938063021749258, 0.031147204339504242, 0.009042954072356224, -0.01538450364023447, 0.010069488547742367, -0.002130733570083976, 0.0314713716506958, 0.00119537184946239, 0.0023468458093702793, 0.019315049052238464, 0.04481631517410278, -0.015370996668934822, -0.01379067450761795, -0.029121149331331253, 0.010481452569365501, 0.010994719341397285, -0.022705310955643654, -0.0008256169967353344, -0.01066379714757204, 0.00439991382881999, -0.019990399479866028, -0.011507987044751644, 0.019693244248628616, -0.008907884359359741, 0.012791153974831104, -0.0026878982316702604, 0.041574627161026, 0.034632015973329544, -0.008685017935931683, -0.008658003993332386, 0.023488719016313553, 0.019922863692045212, -0.011041994206607342, 0.0025038651656359434, 0.0026473773177713156, -0.007043914403766394, 0.003084667259827256, -0.010548987425863743, -0.03101213276386261, -0.0035793620627373457, -0.018126429989933968, 0.03220074996352196, 0.00730730127543211, 0.017316007986664772, -0.005321768578141928, 0.005075265187770128, 0.005456838756799698, -0.019814807921648026, -0.03257894888520241, -0.018491119146347046, 0.016735205426812172, 0.02421809919178486, -0.0041162665002048016, 0.031039146706461906, -0.005125916562974453, -0.0006546687218360603, 0.005396056920289993, 0.018410077318549156, -0.008948405273258686, -0.013371956534683704, -0.01341247744858265, 0.035172298550605774, -0.026392729952931404, 0.006983132567256689, 0.018247993662953377, -0.007921870797872543, -0.017748232930898666, -0.007550427690148354, -0.003148825606331229, 0.008725538849830627, -0.008124476298689842, 0.01500630658119917, 0.07126307487487793, -0.0029107641894370317, 0.014249913394451141, -0.01996338553726673, -0.001999039901420474, -0.014493039809167385, -0.004177047871053219, -0.014209392480552197, -0.03201165422797203, 0.017032360658049583, -0.0003361983399372548, -0.007908363826572895, -0.012102295644581318, -0.029337262734770775, 0.01254802756011486, 0.0077530331909656525, 0.009151010774075985, -0.005520997103303671, -0.047382649034261703, -0.004707198590040207, 0.09725058823823929, 0.03908933699131012, 0.013689371757209301, 0.018612682819366455, 0.008583715185523033, 0.0027267311234027147, -0.013236885890364647, -0.019841821864247322, 0.012129309587180614, -0.0006664873217232525, 0.005595285911113024, -0.003657027380540967, 0.0018234485760331154, -0.021233046427369118, 0.014222899451851845, 0.0008906195871531963, 0.010339628905057907, -0.02968844398856163, 0.0028432291001081467, -0.017397049814462662, 0.0009370499756187201, -0.011210831813514233, 0.01884230226278305, 0.02844579890370369, 0.013466505333781242, 0.024650324136018753, 0.036874182522296906, 0.05883660912513733, 0.012656084261834621, -0.02820267155766487, -0.02198943868279457, -0.01554658729583025, -0.006588052026927471, 0.026689885184168816, -0.0268249548971653, -0.01095419842749834, 0.006598182488232851, -0.012980252504348755, 0.006628573406487703, 0.009144256822764874, 0.013648850843310356, 0.0017863042885437608, 0.0008695148280821741, -0.022516213357448578, -0.011271613650023937, -0.006412460934370756, -0.019315049052238464, 0.03249790519475937, -0.016492079943418503, 0.012784400954842567, 0.016505585983395576, -0.019490638747811317, -0.03754953294992447, 0.0019737142138183117, 0.015668150037527084, -0.021043946966528893, -0.0018960488960146904, -0.010218065232038498, -0.029040107503533363, -0.017113402485847473, -0.00975207332521677, -0.020030921325087547, 0.0015862314030528069, 0.0037481999024748802, -0.006122059654444456, -0.021584227681159973, -0.008333835750818253, -0.014506546780467033, -0.007617963012307882, 0.005055004730820656, 0.02646026574075222, -0.02537970244884491, -0.009657524526119232, 0.011757866479456425, 0.03333534300327301, -0.007915117777884007, -0.004507970064878464, 0.015262939967215061, -0.00906996801495552, 0.008948405273258686, 0.004852399230003357, -0.002858424559235573, -0.017451077699661255, -0.018558654934167862, -0.009860129095613956, 0.013885223306715488, 0.007178984582424164, 0.0010839388705790043, -0.02240815758705139, 0.019098935648798943, 0.02728419378399849, 0.01863969676196575, 0.024704352021217346, -0.022583749145269394, -0.003653650637716055, -0.026622349396348, 0.009819608181715012, 0.002541009336709976, 0.018369555473327637, 0.004862529691308737, -0.004470825660973787, 0.018653204664587975, -0.035388410091400146, -0.0020935891661792994, 0.0013211560435593128, -0.0166946854442358, 0.01986883580684662, -0.004676808137446642, 0.007509906776249409, 0.023826394230127335, 0.01461460255086422, -0.020395610481500626, -0.008313574828207493, -0.0055648949928581715, 0.007388343568891287, 0.00455524493008852, 0.010893416590988636, 0.007124956697225571, -0.0027689405251294374, -0.020098455250263214, 0.0028314103838056326, -0.009367123246192932, 0.002856736071407795, -0.018437091261148453, -0.015519573353230953, -0.006550908088684082, 0.018747752532362938, -0.03293013200163841, -0.03911634907126427, -0.01341923139989376, -0.03522632643580437, 0.007550427690148354, -0.0030188204254955053, 0.01399328000843525, -0.005936338100582361, -0.02550126612186432, -0.022786354646086693, 0.010481452569365501, -0.012737126089632511, -0.03317325562238693, -0.018896330147981644, -0.026946518570184708, 0.009765580296516418, -0.01441199705004692, 0.004315495025366545, -0.030444838106632233, -0.002333338838070631, -0.014790194109082222, -0.015965305268764496, 0.01884230226278305, -0.002326585352420807, -0.017815768718719482, -0.010110009461641312, 0.030363796278834343, 0.0031724628061056137, -0.04319547116756439, 0.013898730278015137, -0.01014377735555172, 0.044249020516872406, 0.04484332725405693, 0.04497839882969856, -0.0029884297400712967, 0.007104696240276098, 0.026109082624316216, 0.002849982585757971, -0.012534520588815212, -0.0066353268921375275, 0.018869316205382347, 0.02120603248476982, 0.023029480129480362, 0.014952278696000576, 0.00842838454991579, -0.004713952075690031, 0.009927664883434772, 0.0062402463518083096, 0.029958585277199745, 0.0015414893859997392, -0.029256219044327736, -0.03349742665886879, 0.006098422687500715, 0.025325674563646317, -0.03176852688193321, -0.010076241567730904, 0.02311052195727825, -0.03325429931282997, -0.03474007174372673, 0.02103044092655182, 0.004666677676141262, 0.012345422059297562, 0.007064174860715866, 0.025433730334043503, -0.007266780361533165, 0.012581795454025269, -0.018153443932533264, -0.020395610481500626, 0.001061989925801754, -0.0008838660432957113, -0.014290434308350086, -0.008414877578616142, 0.00807720236480236, 0.0013220002874732018, 0.007590949069708586, -0.0060477713122963905, -0.013311174698174, -0.0020243655890226364, 0.0215707216411829, -0.03736043721437454, -0.018558654934167862, 0.01492526475340128, -0.012824921868741512, 0.007699005305767059, -0.0006821048445999622, 0.004612649790942669, -0.01900438591837883, -0.01934206299483776, 0.011859169229865074, -0.0007585039711557329, -0.00412977347150445, -0.0001899425551528111, -0.029634416103363037, 0.016492079943418503, -0.011568767949938774, 0.004764603450894356, 0.007557181641459465, 0.03157942742109299, 0.024812407791614532, 0.0024481485597789288, -0.021935410797595978, -0.007415357511490583, 0.024812407791614532, -0.003408835968002677, 0.004207438789308071, 0.04492437094449997, -0.010609769262373447, -0.006216608919203281, -0.011332395486533642, 0.007617963012307882, -0.01742406375706196, -0.029013093560934067, 0.005274493712931871, 0.007915117777884007, 0.01676221936941147, -0.021179018542170525, -0.0024869812186807394, -0.014884743839502335, 0.021665271371603012, -0.0204091165214777, 0.012473738752305508, -0.002257361775264144, -0.029202191159129143, -0.001801499631255865, 0.0318765826523304, -0.006125436630100012, 0.018896330147981644, 0.024407196789979935, -0.018085908144712448, -0.02090887725353241, -0.02860788255929947, 0.001603115233592689, 0.02868892438709736, -0.005125916562974453, 0.008671510964632034, 0.002080082194879651, 0.04079122096300125, -0.01210904959589243, -0.014979292638599873, -0.03533438220620155, 0.010305861011147499, -0.027770446613430977, 0.013534041121602058, 0.0006225894903764129, -0.014749673195183277, 0.009677784517407417, -0.00239074369892478, -0.0009429592755623162, -0.00576750049367547, -0.026527799665927887, -0.0322277657687664, -0.023677818477153778, -0.013182858005166054, 0.005098902620375156, 0.010069488547742367, -0.009812855161726475, -0.002194891916587949, -0.0020733284763991833, -0.006311158183962107, -0.015087348408997059, -0.011609289795160294, 0.028256699442863464, 0.0055480110459029675, -0.006692731752991676, -0.00890113040804863, 0.02012546919286251, 0.002738549606874585, 0.008212272077798843, -0.008489166386425495, -0.014466025866568089, 0.027378743514418602, 0.004126396495848894, 0.011359409429132938, -0.02240815758705139, 0.021476171910762787, -0.011778127402067184, 0.015046827495098114, -0.04195282608270645, -0.010657044127583504, 0.01565464399755001, -0.011271613650023937, -0.014898250810801983, -1.3823597328155302e-05, 0.011014980264008045, 0.016910796985030174, 0.024231605231761932, -0.010278847068548203, 0.004231076221913099, 0.02463681623339653, 0.007287040818482637, -0.013486766256392002, 0.004396537318825722, 0.004643040243536234, -0.0054534622468054295, -0.00047570059541612864, 0.02182735502719879, -0.009130749851465225, 0.020611722022294998, 0.005159683991223574, -0.011892937123775482, -0.021975932642817497, -0.002071640221402049, 0.014938771724700928, 0.012696605175733566, 0.03444291651248932, -0.006983132567256689, 0.015019813552498817, -0.019774287939071655, -4.561259629554115e-05, 0.0103936567902565, 0.009407644160091877, 0.007854335941374302, -0.025703871622681618, -0.013405723497271538, 0.013858209364116192, -0.0014275239082053304, 0.0080569414421916, -0.0074221109971404076, -0.011305381543934345, -0.02339416928589344, -0.010150530375540257, -0.03001261316239834, -0.012264380231499672, 0.0028786850161850452, 0.017869796603918076, -0.016032841056585312, -0.0400078147649765, -0.028013573959469795, 0.0015549964737147093, -0.03157942742109299, 0.005210335366427898, -0.01876126043498516, 0.0032957145012915134, 0.01387171633541584, 0.007381590083241463, -0.023745352402329445, 0.021341102197766304, -0.019531160593032837, 0.020098455250263214, 0.0026912749744951725, 0.006712992209941149, 0.015681657940149307, 0.0009252313175238669, -0.01239945087581873, -0.0018808534368872643, -0.04840918257832527, -0.005932961590588093, 0.01558710914105177, -0.007381590083241463, 0.032687004655599594, 1.8084114344674163e-05, -0.0017913694027811289, 0.005412940867245197, 0.003337923903018236, -0.007503153290599585, 0.00346792908385396, 0.015924785286188126, 0.01988234370946884, -0.017586149275302887, -0.008441891521215439, 0.024852929636836052, -0.0028938804753124714, -0.00917127076536417, 0.008164997212588787, -0.011251353658735752, -0.016843263059854507, 0.001399665605276823, 0.0055007366463541985, 0.0038866468239575624, 0.0029124526772648096, 0.0024076274130493402, 0.02600102685391903, 0.004724082536995411, 0.0069898865185678005, -0.020517174154520035, 0.035010214895009995, -0.022421663627028465, -0.02377236634492874, 0.0034814360551536083, -0.023907437920570374, 0.00468018464744091, -0.0075436742044985294, 0.00656441505998373, 0.016032841056585312, -0.020152483135461807, 0.01080562174320221, 0.004747719969600439, -0.018275007605552673, 0.018653204664587975, -0.0012206975370645523, -0.03627987205982208, 0.02451525256037712, -0.03047185204923153, -0.012845182791352272, -0.0025004884228110313, -0.012190091423690319, -0.0035084502305835485, -0.013068048283457756, -0.011710592545568943, 0.003071160288527608, -0.028878023847937584, -0.017815768718719482, -0.010346382856369019, -0.02223256602883339, -0.04219594970345497, -0.006209855433553457, 0.005635806825011969, -0.01751861348748207, 0.017707711085677147, 0.19623008370399475, 0.029499346390366554, 0.010562494397163391, 0.03846801072359085, 0.00043686790741048753, 0.013817688450217247, 0.009887143969535828, 0.02716263011097908, -0.01888282410800457, -0.012304901145398617, -0.02363729663193226, 0.016748713329434395, -0.023259099572896957, -0.0008914637728594244, 0.021881382912397385, -0.0166946854442358, -0.012102295644581318, -0.03482111543416977, -0.02936427667737007, 0.002476850990206003, 0.03482111543416977, -0.014844221994280815, -0.004744342993944883, -0.018666710704565048, 0.008786320686340332, -0.00173396454192698, -0.0030981742311269045, 0.029985599219799042, 0.008793074637651443, 0.021003426983952522, -0.013736645691096783, 0.00031509360997006297, 0.023178057745099068, 0.006338172126561403, 0.006250376813113689, -0.002253985032439232, -0.0059194546192884445, -6.0359525377862155e-05, 0.018437091261148453, -0.019369076937437057, 0.003663780866190791, -0.01788330264389515, 0.011771373450756073, -0.0215707216411829, -0.01677572727203369, 0.0025528280530124903, -0.029283232986927032, 0.0048321387730538845, -0.024366676807403564, 0.004224322736263275, -0.03600973263382912, 0.008394616656005383, 0.017694205045700073, 0.028148643672466278, 0.012892456725239754, 0.028337741270661354, 0.010548987425863743, 0.003555724862962961, -0.005264363717287779, -0.008043434470891953, -0.021057454869151115, 0.027608362957835197, -0.013750153593719006, 0.01377041358500719, -0.03444291651248932, -0.0018268253188580275, 0.024893449619412422, 0.008151490241289139, 0.011960471980273724, -0.008144737221300602, 0.010636783204972744, 0.021543707698583603, -0.020301060751080513, -0.009083474986255169, -0.015316967852413654, -0.0065070101991295815, 0.011102776043117046, 0.01826149970293045, 0.025028521195054054, 0.02906712144613266, -0.0019247513264417648, 0.023164549842476845, -0.015316967852413654, 0.010724578984081745, 0.009684538468718529, -0.02153019979596138, 0.014357969164848328, 0.007699005305767059, -0.0015102544566616416, -0.027176138013601303, 0.018788274377584457, 0.0019028023816645145, -0.011676824651658535, -0.019152963533997536, 0.010528727434575558, -0.016505585983395576, 0.012730373069643974, 0.013203118927776814, 0.00028997898334637284, 0.01623544655740261, -0.02608206868171692, 0.05624325945973396, -0.011291874572634697, -0.010670551098883152, -0.008441891521215439, -0.04203386604785919, -0.0006107708904892206, 0.029904557392001152, 0.0010214688954874873, -0.013068048283457756, 0.005274493712931871, -0.013547548092901707, -0.00967103149741888, -0.015141377225518227, 0.00033556518610566854, 0.005787760950624943, 0.006226739380508661, -0.01142694428563118, 0.004865906201303005, 0.0105962622910738, 0.05559492111206055, -0.004143280442804098, 0.0056290533393621445, 0.017383543774485588, -0.0008788009290583432, -0.005044874269515276, 0.017748232930898666, -0.0205847080796957, -0.004075745120644569, -0.0272031519562006, 0.010744839906692505, 0.019517652690410614, -0.014101335778832436, -0.020895369350910187, -0.009373876266181469, -0.01304103434085846, -0.0018859185511246324, 0.010792113840579987, -0.026757419109344482, 0.010488206520676613, -0.01123784575611353, 0.026055054739117622, -0.0029158294200897217, -0.012791153974831104, 0.020652243867516518, -0.01105550117790699, 0.004734212998300791, 0.0008095774101093411, -0.038522038608789444, -0.012554781511425972, 0.005851919297128916, -0.0003822066355496645, -0.012318408116698265, 0.007043914403766394, -0.009779087267816067, -0.044330060482025146, -0.039683643728494644, -0.01838306337594986, 0.0018605929799377918, 0.013439491391181946, -0.05213712155818939, -0.01461460255086422, 0.03093109093606472, 0.021678777411580086, -0.04468124359846115, -0.003998079802840948, -0.1710529923439026, 0.017775246873497963, -0.007138463668525219, 0.015357489697635174, 0.02922920510172844, -0.015154884196817875, 0.022043468430638313, -0.012237366288900375, -0.010440931655466557, -0.0032872725278139114, 0.009535960853099823, 0.001000364194624126, -0.024339662864804268, -0.061213843524456024, 0.006041017826646566, -0.008205519057810307, -0.03333534300327301, 0.010758346877992153, 0.0031133696902543306, 0.014695645309984684, 0.019693244248628616, -0.00642934488132596, 0.018329035490751266, -0.03063393570482731, 0.02211100235581398, 0.030093654990196228, -0.0041635408997535706, -0.005250856745988131, -0.006547531113028526, 0.01970675215125084, -0.00562229985371232, -0.0034172777086496353, -0.012797907926142216, 0.01414185669273138, -0.003998079802840948, -0.026946518570184708, 0.004150033928453922, 0.004713952075690031, 0.018491119146347046, 0.05105656012892723, 0.01601933315396309, -0.004112889524549246, 0.007962392643094063, 0.009745319373905659, -0.005615546368062496, -0.01312883011996746, 0.006658964324742556, 0.0010543923126533628, 0.0010763411410152912, -0.011278367601335049, 0.014641616493463516, 0.003913660999387503, -0.0055885324254632, 0.02351573295891285, -0.008016420528292656, 0.0008762683719396591, -0.0170998964458704, -0.014911757782101631, -0.005922831129282713, -0.009603495709598064, 0.01312207616865635, 0.007516660261899233, 0.013696124777197838, -0.008685017935931683, -0.01546554546803236, -0.03220074996352196, -0.0046869381330907345, -0.02239464968442917, -0.018112922087311745, -0.011865923181176186, 0.017288994044065475, 0.006669094320386648, 0.013486766256392002, 0.01492526475340128, -0.007347822654992342, 0.003680664813145995, 0.008516180329024792, -0.003334547160193324, -7.249474583659321e-05, -0.0009969874517992139, 0.003663780866190791, 0.008867363445460796, -0.022286593914031982, -0.009015940129756927, -0.02744627743959427, 0.004794994369149208, 0.01788330264389515, 0.007057421375066042, 0.015168391168117523, 0.0019129326101392508, 0.0217733271420002, 0.007408604025840759, -0.0028060846962034702, 0.015600616112351418, -0.008975419215857983, -0.007030407432466745, 0.01734302192926407, 0.013081555254757404, -0.019531160593032837, 0.004585635382682085, -0.01664065755903721, 0.011656563729047775, -0.015519573353230953, 0.016046347096562386, 0.00469031510874629, 0.012487245723605156, -0.017275488004088402, 0.01441199705004692, 0.007178984582424164, -0.020220018923282623, -0.03565854951739311, 0.014884743839502335, 0.022921424359083176, 0.0030357041396200657, 0.00017685763305053115, 0.014668631367385387, 0.015830235555768013, -0.007766540162265301, 0.01627596653997898, -0.011102776043117046, 0.06823749840259552, 0.011541754007339478, -0.013547548092901707, 0.02315104380249977, -0.01884230226278305, -0.038035787642002106, -0.09082124382257462, -0.015235926024615765, 0.007503153290599585, 0.029634416103363037, -0.02377236634492874, -0.0037684603594243526, -0.001332974643446505, 0.02782447449862957, -0.029958585277199745, 0.03341638296842575, 0.029418304562568665, -0.03698223829269409, -0.017667191103100777, -0.010846142657101154, 0.06418538838624954, 0.022664790973067284, 0.00533189857378602, -0.017464585602283478, 0.005315015092492104, 0.02202996052801609, 0.009988445788621902, -0.014587588608264923, -0.013169351033866405, -0.02571737952530384, -0.010555741377174854, -0.010454438626766205, -0.021935410797595978, 0.009414397180080414, -0.005109032616019249, 0.025406716391444206, 0.03047185204923153, -0.011919951066374779, 0.033929649740457535, -0.006858192849904299, -0.004656547214835882, -0.005514243617653847, -0.0423850491642952, -0.02537970244884491, 0.01581672765314579, -0.0423850491642952, -0.0003465396584942937, -0.0003437960403971374, 0.002777382265776396, -0.02745978534221649, -0.019693244248628616, -0.010913677513599396, 0.0033142867032438517, -0.02554178796708584, 0.023529240861535072, 0.012696605175733566, 0.0068446858786046505, -0.022138016298413277, -0.007469385862350464, 0.0011430321028456092, 0.027311207726597786, 0.030201710760593414, 0.0013642096891999245, -0.012885703705251217, -0.03144435957074165, 0.003964312374591827, -0.00199228641577065, 0.0034409151412546635, -0.011568767949938774, 0.017829274758696556, -0.019355569034814835, -0.01577620767056942, -0.016046347096562386, -0.02385340817272663, 0.028553854674100876, -0.015911277383565903, -0.0010822504991665483, -0.0037110554985702038, -0.017613163217902184, 0.002730107866227627, -0.007955638691782951, -0.0055648949928581715, -0.03514528274536133, -0.012291394174098969, 0.03544243797659874, -0.015249432995915413, -0.016991838812828064, -0.029256219044327736, 0.009508946910500526, -0.0063044046983122826, 0.006358432583510876, 0.004295234568417072, -0.0008142204605974257, -0.0017407180275768042, 0.020557694137096405, -0.02161124348640442, 0.026095574721693993, 0.028472812846302986, 0.030282754451036453, 0.009839869104325771, -0.01605985499918461, 0.03879218176007271, -0.021084468811750412, 0.006851439364254475, 0.0018386440351605415, 0.0021661894861608744, -0.009718305431306362, 0.027108602225780487, -0.05710770934820175, 0.02493397146463394, 0.00403860118240118, -0.0009497128194198012, 0.0015769454184919596, -0.00453498400747776, -0.016140896826982498, -0.019531160593032837, -0.01480370108038187, 0.0030829787719994783, -0.014817208051681519, 0.017775246873497963, -0.012723619118332863, -0.016343502327799797, -0.014101335778832436, -0.013750153593719006, 0.04203386604785919, 0.007131710182875395, 0.038522038608789444, 0.01718093827366829, 0.002208398887887597, -0.022462185472249985, 0.006841308902949095, 0.015100855380296707, 0.014357969164848328, 0.012804660946130753, -0.020220018923282623, 0.025784913450479507, 0.004014963749796152, -0.0215707216411829, 0.006142320577055216, -0.0225297212600708, -0.03063393570482731, 0.01978779397904873, 0.004548491444438696, -0.005831658840179443, -0.02591998502612114, 0.002468409016728401, 0.0017727972008287907, -0.025730885565280914, -0.035253338515758514, -0.018288513645529747, -0.010488206520676613, -0.006841308902949095, -0.005004353355616331, -0.004443811718374491, -0.0007732772501185536, 0.0075436742044985294, 0.0037785908207297325, 0.0035118269734084606, -0.008164997212588787, 0.038197871297597885, -0.020503666251897812, -0.015249432995915413, -0.008327081799507141, -0.01581672765314579, -0.004220945760607719, -0.01788330264389515, -0.006348302587866783, -0.02307000197470188, 0.0014975916128605604, 0.007266780361533165, 0.0007133398321457207, 0.01764017716050148, 0.017950838431715965, -0.0023012596648186445, -0.02790551632642746, 0.001396288862451911, 0.004990846384316683, -0.02691950462758541, -0.019598696380853653, 0.020787313580513, 0.03006664104759693, -0.020800821483135223, 0.003653650637716055, -0.0034341614227741957, 0.0015279824147000909, -0.008300067856907845, -0.019477132707834244, 0.039521560072898865, 0.00964401662349701, -0.011217585764825344, -0.007746279705315828, 0.0006850595236755908, 0.022799860686063766, -0.008982173167169094, -0.014898250810801983, -0.0007137619541026652, -0.011352655477821827, 0.010501713491976261, -0.02016599103808403, 0.02132759429514408, -0.00717223109677434, -0.008029927499592304, 0.011062255129218102, 0.039359476417303085, 0.020071441307663918, -0.013783920556306839, -0.01358131505548954, -0.0047038220800459385, 0.0013127141864970326, 0.009103735908865929, -0.010859649628400803, -0.016302980482578278, -0.035712577402591705, 0.0017660437151789665, -0.016289474442601204, -0.03211970999836922, -0.024231605231761932, 0.010981212370097637, 0.005288001149892807, 0.011953718960285187, 0.023029480129480362, 0.02165176346898079, -0.0073748365975916386, 0.009515699930489063, 0.0036907950416207314, -0.02868892438709736, -0.014574081636965275, 0.026987038552761078, 0.014938771724700928, 0.025987518951296806, 0.016708191484212875, 0.013101816177368164, 0.02120603248476982, 0.01217658445239067, 0.0029580388218164444, -0.023488719016313553, 0.006145697087049484, 0.009900650940835476, -0.01179838739335537, 0.013162597082555294, -0.007915117777884007, -0.015533080324530602, -0.017815768718719482, -0.002868554787710309, -0.0010982900857925415, 0.00199228641577065, 0.008597222156822681, 0.036252859979867935, 0.01986883580684662, -0.004808501340448856, 0.001140499603934586, -0.00411964301019907, 0.010528727434575558, 0.006672471296042204, -0.004197308328002691, -0.020571202039718628, -0.022502707317471504, 0.026851968839764595, -0.012865442782640457, 0.008907884359359741, -0.008766060695052147, -0.006297651212662458, 0.0006027510389685631, 0.016505585983395576, 0.02323208563029766, -0.020422624424099922, 0.02124655246734619, -0.0014460959937423468, -0.02762186899781227, 0.01705937460064888, -0.0047848643735051155, -0.015019813552498817, -0.024312647059559822, -0.004524854011833668, -0.009313094429671764, 0.0038123582489788532, -0.022880902513861656, 0.008036680519580841, 0.025987518951296806, -0.006179464515298605, -0.018288513645529747, 0.006588052026927471, 0.005804644897580147, -0.015060334466397762, 0.02144915796816349, 0.012534520588815212, -0.0034814360551536083, 0.004156787414103746, 0.014236406423151493, 0.0032197374384850264, -0.02575789950788021, 0.005730356089770794, 0.0018217602046206594, -0.0019416350405663252, -0.010960952378809452, 0.0074491254054009914]} +{"id": "test:10000010", "text": "\"Pencarian FILM Untuk \\\"Peace Breaker 2017\\\"\\nyuk mampir ke channel say..\\nEdges East provides the l..\\nA corrupt cop makes one w..\\nPeace Breaker 2017 ~ \u7834\ufffd..\\nN\u00e1o Lo\u1ea1n - Peace Break..\\nPlease subscribe and hit ..\\nuploaded in HD at http://..\\nI cannot believe I manage..\"", "vector_field": [-0.007715163752436638, -0.01570870913565159, -0.0044905696995556355, -0.02485555224120617, -0.014025160111486912, 0.008596707135438919, -0.018147867172956467, -0.01805507391691208, -0.021819861605763435, -0.017206670716404915, 0.014396335929632187, 0.032371871173381805, 0.010373050346970558, -0.006180747412145138, -0.0006205603131093085, -0.013799803331494331, 0.019645828753709793, -0.02274780161678791, 0.012441032566130161, -0.008424375206232071, -0.0031433990225195885, 0.013892597518861294, 0.002568407217040658, 0.007814586162567139, 0.012003575451672077, 0.016543855890631676, 0.01635826751589775, -0.023397359997034073, -0.011957177892327309, -0.019924210384488106, 0.018558813259005547, 0.002177346497774124, -0.015377301722764969, -0.01647757552564144, -0.023437129333615303, -0.01106237806379795, 0.0027755368500947952, -0.010956327430903912, 0.018611837178468704, -0.012951400130987167, 0.036693423986434937, -0.02356969192624092, -0.005116929765790701, -0.016636651009321213, -0.01497961301356554, 0.003963632043451071, 0.005087103229016066, -0.009829542599618435, -0.02105099707841873, 0.03250443562865257, 0.0007108688005246222, -0.004066368564963341, -0.0013786546187475324, -0.013289435766637325, -0.008391235023736954, -0.007801329717040062, 0.01643780618906021, 0.007967033423483372, 0.03406867757439613, -0.02151496708393097, 0.01606662943959236, 0.006720941513776779, -0.01611965522170067, 5.675351349054836e-05, -0.012460917234420776, 0.0012311784084886312, 0.006382905878126621, -0.0034400084987282753, -0.02018933743238449, -0.029932713136076927, 0.03107275441288948, 0.006545295473188162, 0.011261222884058952, -0.0027755368500947952, 0.0278647318482399, -0.024126457050442696, -0.0017945709405466914, -0.016464319080114365, -0.006392848212271929, 0.013746777549386024, 0.03597758337855339, -0.01067131757736206, -0.02374202385544777, 0.01565568521618843, 0.010810508392751217, 0.020587025210261345, -0.019195115193724632, -0.005325716454535723, -0.017657384276390076, -0.008106224238872528, -0.0034267522860318422, 0.016954801976680756, 0.0278647318482399, -0.00042150873923674226, -0.008868461474776268, 0.024046918377280235, -0.027175404131412506, 0.02403366193175316, -0.01565568521618843, -0.01776343584060669, 0.008464144542813301, 0.008145993575453758, 0.004208873491734266, -0.006525411270558834, -0.041227076202631, -0.02382156252861023, 0.009219753555953503, -0.0221247561275959, 0.033485401421785355, 0.01842625066637993, -0.019672341644763947, 0.004063054453581572, 0.007748304400593042, -0.014078184962272644, -0.0018591954139992595, -0.012520570307970047, 0.02390109933912754, -0.00974337663501501, -0.015934066846966743, -0.03931817039847374, 0.0243518128991127, 0.02683074027299881, 0.005743289832025766, -0.018558813259005547, 0.016663162037730217, -0.004384519532322884, 0.004652959294617176, -0.004633075091987848, 0.01623896136879921, 0.004464057274162769, 0.028607085347175598, -0.012865234166383743, 0.027917757630348206, 0.010326653718948364, -0.019234884530305862, 0.024471120908856392, -0.010770739987492561, 0.004689414519816637, -0.024325301870703697, -0.05625971406698227, 0.011102147400379181, 0.016623394563794136, -0.007436781655997038, -0.034678466618061066, 0.01287849061191082, 0.01386608462780714, 0.002747367136180401, 0.002005014568567276, 0.01751156523823738, 0.029614562168717384, 0.010081412270665169, 0.013315948657691479, 0.0039106071926653385, 0.012706158682703972, -0.016185935586690903, 0.012991169467568398, 0.012600108049809933, 0.012593479827046394, -0.00925289373844862, 0.002599890809506178, 0.021501710638403893, -0.003950376063585281, -0.00010273628868162632, -0.015602659434080124, 0.02505439706146717, 0.014250516891479492, 0.0008260328904725611, -0.01606662943959236, -0.023105721920728683, -0.014369823969900608, 0.0035825136583298445, 0.02604861930012703, -0.02130286581814289, 0.021236583590507507, -0.012069856747984886, 0.009862683713436127, 0.013057450763881207, 0.01574847847223282, -0.01594732329249382, -0.010591779835522175, -0.00366205140016973, -8.445709681836888e-05, 0.01883719488978386, 0.02278757095336914, -0.022071730345487595, 0.008570194244384766, 0.004384519532322884, -0.02929641120135784, -0.0011541261337697506, -0.003272647736594081, 0.009484878741204739, 0.024060174822807312, -0.0009155128500424325, -0.018187636509537697, -0.6295149326324463, -0.023556435480713844, 1.0194660717388615e-05, 0.005899051204323769, 0.0248953215777874, 0.03889396786689758, 0.010160950012505054, 0.022906877100467682, -0.03902653232216835, -0.02583651803433895, -0.01083702128380537, 0.01018083468079567, 0.010691202245652676, -0.0010870161931961775, 0.0037913003470748663, -0.007224680855870247, 0.005696892738342285, -0.03889396786689758, 0.0130044249817729, -0.011592630296945572, -0.02036166936159134, 0.04796127602458, -0.011970434337854385, 0.014992869459092617, -0.000831003999337554, -0.009736748412251472, 0.0005766488029621542, -0.011605885811150074, -0.011930665001273155, 0.030515991151332855, -0.015562890097498894, 0.027758682146668434, 0.011917409487068653, 0.0315764956176281, 0.062410637736320496, -0.008537054061889648, -0.014780769124627113, 0.0025336092803627253, 0.021541479974985123, 0.04215501621365547, -0.01195054966956377, -0.009405340999364853, 0.017259696498513222, -0.016000347211956978, 0.0002259784087073058, 0.011407041922211647, 0.04356018453836441, -0.028209395706653595, -0.0059156217612326145, 0.0015029323985800147, 0.011698679998517036, 0.0012858606642112136, 0.0060249860398471355, -0.0020033575128763914, -0.012248816899955273, 0.008875089697539806, 0.026022106409072876, -0.05090417340397835, 0.024988116696476936, 0.02018933743238449, -0.007947148755192757, 0.015523121692240238, -0.014117954298853874, -0.021077508106827736, -0.02645956538617611, 0.028076833114027977, -0.023012928664684296, -0.020997971296310425, 0.030940191820263863, -0.017538078129291534, 0.0009900794830173254, 0.0059156217612326145, -0.01664990559220314, 0.006518783047795296, -0.011122031137347221, 0.025173703208565712, 0.03189464658498764, -0.0113341324031353, 0.012215675786137581, 0.0002910170878749341, 0.001935419044457376, -0.0034897197037935257, 0.006648031994700432, 0.019844673573970795, 0.015801504254341125, -0.0051268720999360085, -0.007821214385330677, -0.0020712960977107286, 0.02229708805680275, -0.0033869834151118994, 0.018041817471385002, 0.000630088266916573, 0.005640553310513496, -0.01780320331454277, -0.005816199351102114, 0.0007597514195367694, 0.0005994330858811736, -0.017339233309030533, 0.025160448625683784, -0.015231482684612274, 0.009803029708564281, -0.005584214348345995, -0.004391147755086422, -0.005030763801187277, 0.012361494824290276, -0.008676244877278805, 0.013905853033065796, -0.026234207674860954, 0.04570770636200905, -0.00457342155277729, 0.01891673356294632, -0.021567991003394127, 0.005875852890312672, -0.02823590859770775, -0.019261397421360016, -0.03465195372700691, 0.01721992716193199, -0.004656273405998945, 0.031046243384480476, -0.03406867757439613, -0.008967883884906769, -0.011625770479440689, -0.0014035102212801576, -0.02233685739338398, 0.028660109266638756, 0.03971586003899574, -0.010379678569734097, 0.0023413931485265493, -0.030728092417120934, -0.0026794285513460636, -0.02022910676896572, -0.019380703568458557, 0.026963304728269577, -0.015682196244597435, -0.0017846287228167057, -0.0009130273247137666, 0.013428626582026482, -0.0005986045580357313, 0.0030274062883108854, -0.03629573434591293, -0.0032958462834358215, 0.0010489043779671192, 0.0044242884032428265, 0.004126021638512611, -0.015881041064858437, -0.008351465687155724, -0.03738275170326233, -0.015443583950400352, -0.02671143412590027, 0.016795726493000984, -0.008185761980712414, 0.018187636509537697, -0.023635974153876305, 0.00212266412563622, -0.013905853033065796, -0.007450037635862827, -0.01706085167825222, -0.019513266161084175, -0.023715510964393616, -0.024948347359895706, 0.03507615625858307, 0.0047391252592206, -0.027148891240358353, -0.003860895987600088, 0.02448437735438347, 0.004821977112442255, 0.023344336077570915, 0.013024309650063515, 0.00847077276557684, -0.01870463229715824, 0.018929990008473396, -0.006899901665747166, -0.008464144542813301, 0.002485555363819003, 0.0045104543678462505, 0.010909930802881718, -0.010472472757101059, 0.008662988431751728, 0.017193414270877838, -0.013945622369647026, 0.007947148755192757, 0.016040116548538208, -0.02394086867570877, -0.011572745628654957, 0.003493033815175295, 0.029773637652397156, 0.006634775549173355, 0.02333107963204384, -0.04666215926408768, 0.002455728594213724, -0.002964438870549202, 0.02110402099788189, 0.016225704923272133, -0.00682036392390728, 0.0077748168259859085, 0.019712110981345177, -0.013256294652819633, 0.02163427323102951, 0.04316249489784241, 0.010830393061041832, 0.0371706523001194, 0.0034134958405047655, 0.007960405200719833, -0.016954801976680756, 0.01805507391691208, -0.0300917886197567, -0.001377826207317412, -0.03465195372700691, 0.026764459908008575, 0.021024484187364578, 0.02032190002501011, -0.04194291681051254, -0.019062552601099014, 0.006137664429843426, 0.00793389230966568, 0.0020215848926454782, -0.01903603971004486, 0.02757309377193451, -0.020799126476049423, -0.003004207741469145, -0.0185853261500597, -0.022615239024162292, -0.0228671096265316, -0.0033919543493539095, -0.014277029782533646, 0.008941370993852615, 0.024471120908856392, 0.024365069344639778, 0.014886818826198578, -0.020997971296310425, -0.011214825324714184, 0.027811706066131592, 0.010843649506568909, 0.02929641120135784, -0.00660494901239872, -0.005733347497880459, 0.022283831611275673, 0.005836084019392729, 0.017935767769813538, 0.0011002725223079324, 0.009286034852266312, 0.03481103107333183, 0.0025236671790480614, -0.0005062247510068119, 0.004026599694043398, 0.006541981361806393, 0.010989468544721603, 0.017113877460360527, -0.0030655181035399437, 0.018969757482409477, -0.023105721920728683, 0.028156369924545288, -0.015377301722764969, 0.014316798187792301, 0.03234535828232765, -0.018929990008473396, -0.004560165572911501, 0.012215675786137581, 0.05008228123188019, -0.0113341324031353, 0.00467615807428956, 0.0010149350855499506, -0.006485642399638891, 0.02254895679652691, -0.013256294652819633, 0.017962278798222542, -0.0007829499081708491, -0.014992869459092617, -0.021316122263669968, -0.018717888742685318, 0.00815924908965826, -0.0156159158796072, 0.022562213242053986, -0.011897524818778038, -0.002777193672955036, -0.027599606662988663, -0.008583450689911842, 0.0013612557668238878, 0.026327000930905342, -0.008769039064645767, 0.004026599694043398, -0.01961931586265564, 0.04451463744044304, 0.02905779890716076, -0.019022783264517784, -0.008059827610850334, -0.016371523961424828, 0.005899051204323769, -0.032159771770238876, 0.00016187179426196963, -0.017630871385335922, 0.03178859502077103, 0.01635826751589775, 0.0011334131704643369, -0.015284508466720581, -0.017405515536665916, 0.02167404256761074, 0.013090590946376324, 0.0021060938015580177, -0.0024540715385228395, -0.0065817502327263355, 0.007907380349934101, -0.029561538249254227, -0.00919324066489935, 0.005103673320263624, -0.001227864297106862, -0.004984366707503796, 0.007039092481136322, -0.005763174034655094, -0.003267676802352071, -0.02855405956506729, 0.011049121618270874, 0.0003104872885160148, -0.0067507680505514145, -0.018572069704532623, -0.014833793975412846, 0.015178457833826542, -0.002048097550868988, 0.012441032566130161, -0.009710236452519894, -0.02855405956506729, -0.007741676177829504, -0.030675066635012627, 0.0008078054524958134, 0.10843649506568909, 0.007648881990462542, 0.018863707780838013, 0.02122332900762558, 0.009822914376854897, -0.013958878815174103, -0.024365069344639778, -0.014820537529885769, -0.0036686796229332685, -0.022270575165748596, 0.0036554234102368355, -0.020494231954216957, -0.0060051013715565205, -0.004371263086795807, 0.019831417128443718, -0.005156698636710644, -0.010532126761972904, -0.02431204542517662, 0.005925563629716635, -0.009915708564221859, 0.0006768995663151145, -0.0018840509001165628, 0.019632572308182716, 0.021289609372615814, -0.004450800828635693, -0.008563566952943802, 0.027599606662988663, 0.01106237806379795, 0.013163500465452671, 0.01211625337600708, -0.013627471402287483, 0.006154234986752272, 0.0006184889934957027, 0.00975000485777855, -0.01723318360745907, 0.027003072202205658, 0.031178805977106094, -0.001015763613395393, 0.03634876012802124, -0.011963806115090847, 0.012785696424543858, 0.022641751915216446, 0.004374577198177576, -0.007224680855870247, 0.005763174034655094, -0.013342460617423058, -0.0016495801974087954, 0.012845349498093128, 0.0027921071741729975, -0.03353842720389366, 0.03730321303009987, 0.0004697699623648077, -0.04446161165833473, 0.0056836362928152084, 0.022111499682068825, -0.012242188677191734, 0.0053389729000627995, 0.008358093909919262, 0.0023148804903030396, -0.0006826992030255497, 0.011307619512081146, -0.01735248975455761, 0.01234161015599966, -0.0006271884776651859, -0.009816286154091358, -0.015483352355659008, 0.0027705656830221415, 0.002213801257312298, 7.990024460013956e-05, 0.014820537529885769, -0.021276352927088737, -0.012580224312841892, -0.033034685999155045, -0.010810508392751217, 0.00016342525486834347, -0.01211625337600708, 0.027095867320895195, -0.0046794721856713295, -0.010817136615514755, 0.010929815471172333, -0.021395660936832428, -0.011586002074182034, 0.013653983362019062, -0.009822914376854897, 0.003986830823123455, 0.021978937089443207, -0.007025836501270533, -0.014144466258585453, -0.006309996359050274, 0.009637326002120972, 0.020202593877911568, 0.01067131757736206, 0.018929990008473396, -0.009113702923059464, -0.027334479615092278, 0.022575469687581062, 0.029614562168717384, 0.008510541170835495, 0.005885794758796692, 0.007788073271512985, 0.007695279084146023, -0.028129857033491135, -0.018108099699020386, 0.0077085355296730995, 0.0007303389720618725, 0.0030920307617634535, 0.018969757482409477, 0.004351378884166479, -0.004169104620814323, -0.016345011070370674, 0.008066455833613873, -0.033193763345479965, 0.003986830823123455, 0.015430327504873276, 0.01664990559220314, 0.006134350318461657, 0.006280169822275639, 0.010538754053413868, 0.0030207782983779907, -0.00017098549869842827, 0.0020182710140943527, -0.015364046208560467, 0.011539604514837265, -0.01024048775434494, -0.012686274014413357, 0.012235560454428196, 0.0047888364642858505, -0.02288036420941353, 0.03353842720389366, 0.012785696424543858, -0.02749355509877205, 0.0011789817363023758, -0.01706085167825222, -0.007165027316659689, -0.03322027623653412, -0.007860982790589333, 0.013839571736752987, 0.019261397421360016, -0.022734545171260834, -0.021316122263669968, -0.028951747342944145, -0.0226682648062706, -0.013368973508477211, 0.005879167001694441, -0.006860132794827223, -0.021130533888936043, -0.001622239127755165, 0.002082895254716277, -0.00010133816249435768, -0.0007377956644631922, -0.013110475614666939, -0.00042130163637921214, -0.014383080415427685, 0.009902452118694782, -0.012288585305213928, -0.0367729626595974, -0.026565615087747574, -0.028129857033491135, 0.0037780441343784332, 0.02345038577914238, 0.027758682146668434, -0.007436781655997038, -0.018227405846118927, 0.023516668006777763, -0.004331494215875864, 0.004006715025752783, -0.016915032640099525, -0.01702108234167099, -0.02048097550868988, 0.0004080453363712877, 0.012997796759009361, -0.0007038264302536845, 0.006704370956867933, -0.0094782505184412, 0.007052348926663399, 0.0028020492754876614, -0.000387332373065874, 0.007728419732302427, 0.007184911984950304, -0.025173703208565712, -0.014144466258585453, 0.006681172642856836, -0.006548609584569931, -0.018770914524793625, -0.005507990717887878, -0.01912883296608925, 0.0045270249247550964, -0.0038907225243747234, 0.011658911593258381, 0.004066368564963341, 0.0027490241918712854, -0.007443409413099289, 0.01619919203221798, 0.010028387419879436, 0.010492357425391674, -0.005494734272360802, -0.018519043922424316, -0.006117780227214098, 0.013680496253073215, 0.02460368350148201, -0.005713463295251131, 0.0020862093660980463, -0.003589141881093383, 0.013693752698600292, -0.009928965009748936, 0.03886745497584343, -0.017895998433232307, -0.022429650649428368, -0.008769039064645767, 0.0002748609986156225, 0.001948675373569131, -0.024086687713861465, -0.010565266944468021, -0.028819184750318527, 0.010724342428147793, -0.008676244877278805, -0.016543855890631676, -0.007191540207713842, 0.024868808686733246, -0.019155345857143402, -0.0042121876031160355, -0.008092967793345451, 0.0016984627582132816, 0.012262072414159775, 0.039291657507419586, 0.017630871385335922, -0.0002796249755192548, -0.01611965522170067, 0.048730138689279556, 0.0008931428892537951, -0.0027125694323331118, 0.010730970650911331, 0.012374751269817352, 0.0017829716671258211, -0.005408568307757378, -0.031841620802879333, -0.0032428211998194456, -0.027148891240358353, -0.029614562168717384, 0.015059150755405426, 0.0007166684372350574, 0.01182461529970169, -0.018638350069522858, 0.004351378884166479, -0.014621693640947342, -0.012414520606398582, -0.03255746141076088, 0.04056426137685776, -0.02567744255065918, -0.008172505535185337, -0.022482676431536674, 0.00919324066489935, -0.016384780406951904, 0.020202593877911568, 0.010227231308817863, -0.016954801976680756, -0.012858605943620205, 0.0041459063068032265, -0.010094668716192245, 0.012798952870070934, -0.002646287903189659, 0.016782470047473907, -0.018147867172956467, -0.01139378547668457, 0.02868662215769291, 0.021117277443408966, -0.01801530458033085, 0.0027440530247986317, 0.007237936835736036, 0.02749355509877205, -0.021316122263669968, 0.009398712776601315, -0.016451062634587288, -0.012315098196268082, 0.02427227608859539, 0.002023241948336363, -0.006309996359050274, -0.013839571736752987, -0.016623394563794136, 0.012235560454428196, 0.018598582595586777, 0.007158399093896151, -0.008928114548325539, -0.016132911667227745, -0.01535078976303339, 0.006479014176875353, -0.020984714850783348, -0.013653983362019062, 0.030834142118692398, -0.03428077697753906, -0.005395311862230301, 0.006266913376748562, -0.01801530458033085, 0.005299204029142857, 0.0032577344682067633, -0.007715163752436638, -0.012553711421787739, 0.01768389716744423, -0.030887167900800705, -0.0024275591131299734, 0.014608437195420265, 0.011274478398263454, -0.004218815825879574, 0.002985980361700058, 0.01903603971004486, -0.014011903665959835, 0.026353513821959496, -0.026817483827471733, 0.010041643865406513, 0.009372200816869736, -0.02427227608859539, -0.0004391147813294083, 0.0023927611764520407, -0.0020812381990253925, -0.03078111633658409, 0.0045270249247550964, -0.011347388848662376, -0.02538580447435379, -0.02958805114030838, 0.008411118760704994, -0.03165603056550026, 0.023543179035186768, 0.02855405956506729, -0.026737947016954422, 0.00908719003200531, -0.028421496972441673, 0.023357592523097992, 0.013117103837430477, -0.01116842869669199, 0.008484029211103916, 0.015854528173804283, -0.0017150331987068057, 0.0017382316291332245, -0.017988791689276695, -0.024669965729117393, -0.016596881672739983, -0.022933389991521835, 0.013680496253073215, -0.016663162037730217, 0.03202720731496811, 0.03091367892920971, 0.034148216247558594, 0.005027449689805508, 0.020348412916064262, 0.009398712776601315, -0.03367098793387413, -0.005686950404196978, -0.017736922949552536, 0.015191714279353619, -0.02077261358499527, -0.029084311798214912, 0.024338558316230774, 0.021899398416280746, -0.011639026924967766, -0.02204521931707859, -0.020454462617635727, -0.046874258667230606, 0.009405340999364853, -0.01999049261212349, 0.03714413940906525, 0.01332920417189598, -0.0017498309025540948, -0.004072996787726879, 0.031019730493426323, 0.0015319306403398514, 0.02122332900762558, -0.0019171916646882892, 0.0001794778072508052, 0.01014106534421444, -0.020494231954216957, -0.0014035102212801576, 0.022058473899960518, -0.030197840183973312, 0.0076223695650696754, 0.018147867172956467, 0.011698679998517036, -0.01887696422636509, 0.032902125269174576, -0.01747179590165615, -0.01874440163373947, 0.025743724778294563, -0.011579373851418495, 0.005329030565917492, -0.01582801528275013, 0.0006582579226233065, -0.04422299936413765, 0.016888519749045372, 0.018280431628227234, 0.016212448477745056, -0.0032212797086685896, 0.006860132794827223, 0.0024010464549064636, -0.0014192520175129175, -0.007628997787833214, -0.02460368350148201, -0.02007003128528595, -0.020679820328950882, -0.02018933743238449, 0.019924210384488106, 0.009504763409495354, 0.0038343833293765783, 0.013408741913735867, 0.010412819683551788, 0.013408741913735867, 0.0036951922811567783, -0.005507990717887878, -0.009531276300549507, -0.007695279084146023, 0.01061166450381279, -0.01611965522170067, 0.028527546674013138, -0.029773637652397156, 0.023344336077570915, 0.003332301042973995, -0.011804730631411076, -0.018969757482409477, 0.011314247734844685, -0.012759183533489704, 0.020096542313694954, 0.02518695965409279, -0.050798121839761734, -0.014992869459092617, -0.0055510737001895905, -0.028739647939801216, -0.019751880317926407, -0.015072407200932503, 0.009882567450404167, -0.03515569493174553, 0.014740999788045883, -0.01234161015599966, -0.004875002428889275, -0.04777568578720093, 0.006498898379504681, -0.014409592375159264, -0.001466477639041841, 0.006114466115832329, 0.22843247652053833, -0.01627873070538044, -0.004185675177723169, 0.03486405685544014, 0.027414018288254738, 0.018147867172956467, 0.006479014176875353, 0.02176683582365513, -0.004507140256464481, 0.013024309650063515, -0.0008600021246820688, -0.005246178712695837, -0.01788274198770523, -0.009286034852266312, -0.004709298722445965, -0.009431853890419006, -0.05700206756591797, -0.036985062062740326, -0.0046198186464607716, -0.01686200685799122, 0.0019818160217255354, 0.014886818826198578, 0.012639877386391163, -0.008967883884906769, 0.0036388528533279896, -0.014356567524373531, -0.011904153041541576, -0.006203946191817522, 0.0300917886197567, 0.020215850323438644, -0.011194940656423569, -0.013249666430056095, 0.014634949155151844, -0.0033604707568883896, -0.02352992258965969, 0.009524648077785969, -0.007476550526916981, -0.011566117405891418, 0.01999049261212349, 0.029985738918185234, -0.013349088840186596, -0.00048509755288250744, 0.003019121242687106, -0.0008011773461475968, 0.011102147400379181, -0.003077117493376136, -0.005246178712695837, -0.013813059777021408, -0.00809959601610899, 0.021978937089443207, -0.03428077697753906, -0.022694777697324753, 0.021501710638403893, 0.0024507574271410704, -0.010054899379611015, 0.005793001037091017, 0.005494734272360802, 0.01731272041797638, -0.005736661609262228, -0.019261397421360016, 0.020626794546842575, 0.01768389716744423, -0.030675066635012627, 0.01686200685799122, -0.015509865246713161, 0.023914355784654617, -0.015629172325134277, -0.014634949155151844, 0.022138012573122978, -0.023397359997034073, 0.013720265589654446, -0.022442907094955444, -0.026433052495121956, -0.001125956536270678, -0.021488454192876816, -0.027705656364560127, 0.026976559311151505, 0.00962407048791647, 0.02105099707841873, 0.05175257474184036, -0.002594919642433524, -0.006250343285501003, 0.014502386562526226, -0.02868662215769291, 0.0003051019157283008, -0.014064928516745567, 0.006899901665747166, -0.009783145971596241, -0.0008376321638934314, -0.004705984611064196, 0.0023049383889883757, -0.008901601657271385, -0.01570870913565159, 0.02431204542517662, -0.0008189904619939625, 0.01578824780881405, 0.012361494824290276, 5.701242480427027e-05, -0.030197840183973312, -0.016000347211956978, -0.01281883753836155, -0.0005410225130617619, 0.02665840834379196, 0.013216526247560978, -0.02818288281559944, 0.007357243914157152, 0.01569545269012451, 0.02720191702246666, 0.022389881312847137, -0.03576548397541046, 0.005786372814327478, -0.03409519046545029, 0.015364046208560467, -0.014780769124627113, 0.0006984410574659705, 0.009233009070158005, 0.008351465687155724, -0.026804229244589806, 0.008523797616362572, 0.0005290090339258313, -0.012315098196268082, -0.036242708563804626, -0.014064928516745567, 0.00875578261911869, 0.002803706331178546, -0.012268700636923313, -0.0031997382175177336, 0.012825465761125088, -0.004165790509432554, -0.03189464658498764, 0.01553637720644474, -0.0039536901749670506, 0.030569016933441162, -0.009577672928571701, 0.008974512107670307, 0.010803880169987679, 0.021369148045778275, -0.030012251809239388, 0.009484878741204739, 0.00737712811678648, 0.007032464258372784, -0.012328354641795158, 0.0026015478651970625, 0.003870838088914752, 0.021170303225517273, -0.0011450124438852072, 0.004116079770028591, 0.0017216613050550222, -0.028156369924545288, -0.0018906790064647794, -0.026353513821959496, -0.026857253164052963, -0.01776343584060669, 0.00045692792627960443, 0.03611014783382416, 0.0018409679178148508, -0.039742372930049896, -0.04167779162526131, -0.02360946126282215, 0.03107275441288948, -0.02097145840525627, -0.0009933935943990946, 0.015483352355659008, -0.009895823895931244, -0.004855118226259947, -0.01343525480479002, -0.1686200648546219, 0.006548609584569931, 0.03404216468334198, -0.004682786297053099, -0.014475873671472073, 0.02010979875922203, 0.007682023104280233, -0.005650495644658804, -0.023065952584147453, -0.008888346143066883, 0.008404490537941456, 0.0023397360928356647, -0.03040994144976139, -0.0167294442653656, -0.010598408058285713, 0.01676921360194683, 7.25989302736707e-05, 0.03276956081390381, 0.011420298367738724, 0.021289609372615814, 0.05450988560914993, 0.0047888364642858505, -0.005623983219265938, 0.004311610013246536, 0.014727743342518806, 0.02048097550868988, -0.018731145188212395, 0.01332920417189598, -0.012275328859686852, -0.024497633799910545, -0.01586778461933136, -0.0012204076629132032, 0.027467042207717896, 0.01977839134633541, -0.01619919203221798, 0.005908993538469076, -0.015138688497245312, -0.022734545171260834, -0.0035261744633316994, 0.01211625337600708, 0.02188614383339882, 0.028580572456121445, 0.024259019643068314, 0.014237260445952415, 0.02555813640356064, 0.0256376750767231, 0.018943244591355324, -0.006787222810089588, 0.0226682648062706, 0.005643867421895266, -0.003493033815175295, -0.03984842076897621, 0.020136311650276184, 0.002656230004504323, -0.003348871599882841, 0.030118301510810852, -0.003857581876218319, -0.0010837020818144083, 0.014144466258585453, -0.020759357139468193, -0.034148216247558594, 0.00046231329906731844, -0.0066414037719368935, -0.0338035523891449, -0.022933389991521835, -0.01271278690546751, -0.006512154825031757, 0.0068535045720636845, -0.004633075091987848, 0.014210748486220837, -0.016291987150907516, 0.006999323610216379, 0.02089192159473896, -0.005004251375794411, 0.009272778406739235, -0.03557989373803139, -0.01842625066637993, 0.01821414940059185, -0.009962105192244053, 0.03242489695549011, 0.0003751117328647524, 0.03897350654006004, 0.0003589556145016104, 0.021090764552354813, 0.025452086701989174, 0.021316122263669968, 0.013070707209408283, 0.008775667287409306, -0.006840248126536608, -0.012308469973504543, 0.023834818974137306, -0.0020116427913308144, 0.009856055490672588, -0.011188313364982605, 0.006512154825031757, 0.010969583876430988, 0.005587528459727764, 0.002144205616787076, -0.019314421340823174, 0.015470095910131931, 0.011440182104706764, -0.014224004000425339, -0.008384606800973415, 0.04035216197371483, 0.007827842608094215, 0.020640050992369652, 0.012129509821534157, 0.013262922875583172, 0.019672341644763947, -0.017034338787198067, -0.027758682146668434, -0.0020000434014946222, 0.02282734028995037, 0.010744227096438408, 0.018068330362439156, 0.00990908034145832, -0.017339233309030533, -0.007960405200719833, 0.015735222026705742, -0.001980158966034651, 0.050586022436618805, -0.01686200685799122, -0.005657123867422342, 0.0007647225284017622, -0.00799354538321495, -0.01657036878168583, -0.08123457431793213, -0.011420298367738724, 0.010704457759857178, 0.011751705780625343, -0.004420974291861057, 0.014847050420939922, 0.012315098196268082, 0.037435777485370636, -0.01360095851123333, 0.014025160111486912, 0.008709385991096497, -0.02596908248960972, 0.0029279841110110283, -0.003728332929313183, 0.01647757552564144, -0.007967033423483372, 0.01772366650402546, -0.02987968921661377, -0.005246178712695837, 0.017259696498513222, -0.020202593877911568, 0.010810508392751217, 0.005739975720643997, -0.03502313047647476, -0.01639803685247898, 0.0011118716793134809, -0.011115402914583683, 0.014886818826198578, 0.012315098196268082, 0.013614214956760406, -0.011373900808393955, -0.0035328026860952377, 0.007940520532429218, -0.017034338787198067, 0.0005020821699872613, -0.027387505397200584, -0.01714038848876953, -0.028660109266638756, 0.008404490537941456, -0.05016181990504265, -0.00538868410512805, -0.0055510737001895905, -0.0009577673044987023, -0.002846789313480258, -0.02950851246714592, -0.026817483827471733, -0.00886183325201273, 0.0262076947838068, 0.02720191702246666, -0.014555411413311958, -0.02159450389444828, 0.002498811576515436, -0.032371871173381805, -0.018121354281902313, 0.035288255661726, 0.009630698710680008, 0.0038774663116782904, -0.004994309041649103, -0.02253570221364498, -0.0033753840252757072, -0.024669965729117393, -0.010174206458032131, -0.033273302018642426, 0.014144466258585453, -0.005342287011444569, 0.0007382099283859134, -0.024829041212797165, -0.005862596444785595, 0.012315098196268082, 0.006220516283065081, -0.003897350747138262, 0.0049147712998092175, -0.0062172021716833115, 0.0312318317592144, -0.03968934714794159, -0.007602485362440348, -0.02253570221364498, -0.017299463972449303, 0.02151496708393097, -0.016185935586690903, -0.008417746983468533, -0.027016328647732735, -0.01386608462780714, -0.003372069913893938, 0.000855031015817076, 0.008271927945315838, 0.00997536163777113, 0.005498048383742571, -0.001930447993800044, -0.03828417882323265, 0.009133587591350079, 0.028209395706653595, -0.00721142441034317, -0.01409144140779972, 0.016795726493000984, 0.008643104694783688, 0.008769039064645767, -0.00994884967803955, -0.009630698710680008, 0.024842295795679092, 0.002821933710947633, -0.01805507391691208, -0.08404491096735, 0.02253570221364498, 0.003519546240568161, 0.005421824753284454, 0.00288987229578197, -0.0022535701282322407, 0.013547933660447598, -0.008132737129926682, 0.004082938656210899, 0.01386608462780714, -0.001870794571004808, 0.022800827398896217, 0.030489478260278702, -0.018651606515049934, -0.017087364569306374, -0.029084311798214912, -0.01166553981602192, -0.012513942085206509, -0.017869485542178154, 0.01222230400890112, -0.005577586125582457, 0.005087103229016066, -0.010591779835522175, 0.007589228916913271, -0.01281883753836155, 0.0052130380645394325, 0.017644127830863, 0.0011682109907269478, 0.005965332500636578, -0.00765551021322608, 0.0016180964885279536, -0.014939844608306885, -0.0083647221326828, 0.015589402988553047, -0.0006578436587005854, -0.016424549743533134, 0.0030439766123890877, 0.029031286016106606, -0.0005716776940971613, 0.024961603805422783, -0.02702958509325981, -0.02171381190419197, 0.012242188677191734, -0.016331754624843597, -0.010101296938955784, -0.006220516283065081, -0.006465757731348276, -0.0013239724794402719, 0.010167578235268593, 0.017644127830863, 0.023516668006777763, 0.022482676431536674, -0.0070656053721904755, 0.004785522352904081, 0.012136138044297695, -0.0057267192751169205, 0.03719716519117355, 0.005418510641902685, 0.015721965581178665, -0.003691878169775009, 0.029826663434505463, 0.01553637720644474, 0.014873563311994076, -0.0019635886419564486, 0.035500358790159225, 0.028527546674013138, -0.006409418769180775, -0.017962278798222542, 0.02018933743238449, -0.030940191820263863, -0.011745077557861805, -0.027917757630348206, 0.01772366650402546, 0.05238887667655945, 0.008165877312421799, 0.012242188677191734, 0.00793389230966568, -0.0007174969650804996, -0.019354190677404404, 0.009597557596862316, 0.021077508106827736, -0.0051931533962488174, -0.024736246094107628, 0.01073759887367487, -0.003047290723770857, 0.029641075059771538, 0.0006860132561996579, 0.016888519749045372, -0.0028484463691711426, 0.0034300663974136114, -0.006071383133530617, 0.007224680855870247, 0.004487256053835154, 0.043135982006788254, -0.011705308221280575, 0.006356393452733755, -0.01805507391691208, -0.02374202385544777, 0.007801329717040062, 0.01649083010852337, 0.019022783264517784, 0.0011292706476524472, 0.004566793795675039, -0.021130533888936043, 0.006372964009642601, 0.01364072784781456, -0.024988116696476936, -0.03234535828232765, -0.009325803257524967, -0.011095519177615643, 0.0015377302188426256, -0.01610639877617359, 0.009107074700295925, 0.02134263515472412, -0.009120331145823002, 0.0037681018002331257, -0.004354692995548248, -0.01825391873717308, -0.010320025496184826, 0.015801504254341125, 0.0013090590946376324, -0.013090590946376324, 0.005690264515578747, -0.015165201388299465, 0.04602585732936859, -0.023092465475201607, 0.01222230400890112, -0.009398712776601315, -0.009716863743960857, -0.01168542355298996, -0.004321552347391844, 0.0019901013001799583, -0.01825391873717308, -0.019871186465024948, -0.005329030565917492, 0.014330054633319378, 0.0009047421044670045, 0.02929641120135784, -0.041969429701566696, 0.07312172651290894, 0.01907580904662609, -0.014038416557013988, 0.012182534672319889, -0.029906202107667923, 0.01887696422636509, -0.009358944371342659, 0.006558551918715239, -0.008358093909919262, -0.00677065271884203, -0.0027440530247986317, 0.0012154364958405495, 0.0017150331987068057, -0.00787423923611641, -0.004387833643704653, 0.003814498893916607, -0.000541436776984483, 0.020918432623147964, -0.006452501751482487, -0.013998647220432758, 0.032530948519706726, -0.01643780618906021, 0.00225688423961401, -0.015523121692240238, 0.010247115977108479, 0.0035990839824080467, 0.004440858960151672, -0.014661462046205997, -0.007602485362440348, -0.03070157952606678, -0.008576822467148304, -0.012785696424543858, -0.027997294440865517, -0.022350113838911057, 0.028792671859264374, 0.012898375280201435, -0.0073439874686300755, 0.0011723536299541593, 0.025239985436201096, 0.025823261588811874, 0.0012585194781422615, 0.015032638795673847, -0.01850578747689724, -0.024126457050442696, 0.020984714850783348, 0.014170979149639606, 0.0021276352927088737, 0.019871186465024948, -0.03221279755234718]} +{"id": "test:10000011", "text": "\"Below you'll find some great videos that will encourage you, train you and build you up in hearing from GOD and being able to let HIM fulfill HIS plan in your life.\\nSOMETHING NEW THAT WILL HELP YOU HEAR GODS VOICE!\\nHow to Understand and Rightly Divide It & How It Applies to Life!\\nIn this important teaching Terry reveals the clear distinction between the Spirit & Soul and how critical it is that we learn how to operate in the Spirit and not be deceived. It's a serious matter that must be reviewed no matter what your level of faith and maturity.\\nMarriage & How it Applies to the Spirit & The Soul!\\nIn this enlightening class Terry reveals a key insight that helps us to better discern the Spirit vs the Soul as it's reflected in the Marriage relationship from the intuitive nature, the emotional component of the woman and the power aspect of a male in submission to the LORD.\\nIn this eye-opening teaching Terry shares how the Tabernacle as it represents the body soul and spirit of man and how it applies to our spiritual walk.\\nThis complete deliverance packet helps to walk you through what you can do to free yourself from the hidden forced buried deep in your soul that is hindering you from walking in the fullness that GOD paid for you to walk in.\\nIn today's video, Terry defines salvation better and the things that get in the way of our soul's salvation and ways we can increase our spiritual maturity.\"", "vector_field": [-0.012014966458082199, 0.0024973065592348576, 0.0035367438104003668, -0.021891282871365547, -0.008116246201097965, -0.0006865932373329997, -0.008016619831323624, -0.0351482592523098, -0.03188050910830498, -0.03299632668495178, -0.01498383842408657, 0.03273065388202667, -0.023153219372034073, 0.0028393578249961138, 0.011058551259338856, -0.002638444071635604, 0.03071155585348606, -0.005353268701583147, 0.0164184607565403, -0.015289360657334328, -0.010049001313745975, 0.009922808036208153, -0.004944799467921257, -0.01234041340649128, -0.013024515472352505, 0.0017484466079622507, 0.009863032028079033, -0.021120836958289146, -0.02922379970550537, -0.00866751279681921, 0.0024391908664256334, 0.009823181666433811, -0.018251590430736542, -0.019048603251576424, -0.004008309915661812, -0.011676236055791378, -0.0007480296189896762, 0.00893318373709917, 0.015315927565097809, -0.023392323404550552, 0.010726462118327618, 0.004383570048958063, -0.00312163308262825, 0.005266925785690546, -0.03846914693713188, 0.019938601180911064, -0.022555459290742874, -0.02792201191186905, -0.008435050956904888, 0.02271486259996891, 0.028586188331246376, 0.014678317122161388, -0.02096143364906311, -0.026620224118232727, 0.006661697756499052, -0.007332516834139824, -0.009916165843605995, 0.021479493007063866, 0.020642628893256187, -0.011868847534060478, 0.010693253949284554, -0.011018700897693634, -0.0060373712331056595, 0.006761324591934681, -0.00840184185653925, -0.004868419375270605, -0.009152362123131752, 0.01507682353258133, -0.006967219524085522, -0.004310510121285915, 0.03310259431600571, 0.021692028269171715, 0.004197600297629833, 0.02863932214677334, 0.042826149612665176, -0.031296033412218094, -0.03339483216404915, -0.012984665110707283, 0.014758018776774406, 0.005469499621540308, 0.0025305154267698526, 0.0023910379968583584, 0.0003549197281245142, 0.019287707284092903, -0.003364057745784521, -0.004350360948592424, -0.004579502157866955, 0.011291013099253178, -0.01159653440117836, 0.0018879238050431013, 0.014279810711741447, 0.010480716824531555, 0.01279869582504034, 0.032810356467962265, -0.021067703142762184, 0.01510339044034481, -0.03273065388202667, 0.03493572399020195, -0.014067274518311024, -0.02149277552962303, -0.016777116805315018, -0.0041178991086781025, -0.012506457976996899, -0.00917228776961565, 0.0030352899339050055, -0.011828997172415257, 0.002723126672208309, -0.007372367661446333, 0.014665033668279648, -0.009059377945959568, -0.016816968098282814, 9.891311236742695e-08, 0.0020805352833122015, -0.017853084951639175, 0.0312163308262825, 0.018278157338500023, -0.006322967354208231, -0.0020921584218740463, -0.029356634244322777, -0.014186825603246689, 0.019168155267834663, -0.0045628976076841354, 0.03169453889131546, -0.027868878096342087, 0.03209304437041283, -0.0014030744787305593, -0.004147787112742662, -0.01599338836967945, 0.007086771074682474, -0.00936489924788475, 0.040753915905952454, 0.01632547751069069, 0.01882278360426426, 0.026978880167007446, -0.005007896572351456, 0.022767996415495872, -0.009192213416099548, 0.013775035738945007, -0.014943988062441349, -0.025105899199843407, 0.020509792491793633, 0.0031448793597519398, 0.007312591653317213, -0.024640975520014763, 0.007392292842268944, 0.0027397312223911285, 0.022396055981516838, 0.012685785070061684, 0.014625183306634426, 0.01611294038593769, 0.024787094444036484, -0.020762180909514427, 0.004725621081888676, -0.00943795870989561, 0.026181867346167564, 0.01629890874028206, 0.006738078314810991, 0.0062797958962619305, 0.02432217076420784, 0.014638466760516167, 0.01559488195925951, 0.004911590833216906, 0.01620592549443245, 0.008156096562743187, 0.002527194330468774, 0.025252018123865128, -0.0025305154267698526, -0.010108777321875095, -0.015289360657334328, -0.00043877208372578025, 0.002034042729064822, 0.03347453474998474, -0.01265257690101862, 0.02016442082822323, -0.024760527536273003, 0.003918645903468132, 0.00874057225883007, -0.006313004530966282, -0.028825292363762856, -0.006960577331483364, -0.01535577792674303, 0.026142016053199768, 0.016750549897551537, 0.031003793701529503, -0.02408306673169136, -0.0003312584012746811, -0.006349534261971712, 0.0016737267142161727, 0.00022457493469119072, -0.03868168219923973, -0.008627662435173988, 0.0036197660956531763, -0.010812805965542793, -0.002626820933073759, -0.6452614665031433, 0.016723982989788055, 0.027191417291760445, -0.013031157664954662, 0.014239960350096226, -0.008561245165765285, 0.010899148881435394, -7.015372102614492e-05, -0.024521423503756523, 0.020762180909514427, -0.0034703260753303766, 0.019247855991125107, -0.0012063119793310761, -0.016909953206777573, -0.019792482256889343, -0.007624754682183266, -0.008986318483948708, -0.045775096863508224, -0.002288090530782938, -0.007372367661446333, -0.042773015797138214, 0.008361991494894028, 0.0007671247003600001, 0.013655484654009342, 0.007312591653317213, -0.010513925924897194, -0.0031830694060772657, 0.011204670183360577, -0.005253641866147518, 0.006396026816219091, -0.02604903094470501, 0.014040706679224968, 0.00585140148177743, 0.01906188763678074, 0.032332148402929306, 0.0015217961044982076, -0.021479493007063866, 0.023113368079066277, 0.019128303974866867, 0.03315572813153267, -0.00886012427508831, 0.015555031597614288, 0.02274142950773239, -0.019075170159339905, 0.005177261773496866, 0.006097147241234779, 0.011809071525931358, -0.014279810711741447, 0.009643853642046452, 0.01772024855017662, 0.01291824784129858, -0.008328782394528389, 0.008793707005679607, 0.008501469157636166, 0.008707364089787006, 0.00289913360029459, 0.05358581990003586, -0.016219208016991615, 0.0014562086435034871, 0.009125795215368271, -0.0029838162008672953, 0.009982584044337273, -0.01709592156112194, -0.009823181666433811, -0.03421841189265251, 0.003782489337027073, -0.0037393178790807724, 0.005645506549626589, 0.019287707284092903, 0.01575428433716297, 0.02098800055682659, 0.02616858296096325, -0.01983233168721199, -0.010188478976488113, -0.002875887556001544, 0.02057621069252491, 0.015661299228668213, -0.0002804073446895927, 0.012778770178556442, -0.0025770077481865883, 0.0057086036540567875, -0.01700293831527233, 0.00908594485372305, -0.01277212891727686, 0.016604430973529816, -0.009703629650175571, -0.01985890045762062, 0.02012457139790058, 0.004084690008312464, 0.01700293831527233, -0.0007704456220380962, 0.020709047093987465, 0.023126652464270592, -0.004111257381737232, 0.006226661615073681, 0.012692427262663841, -0.019008751958608627, 0.003349113743752241, -0.00872728880494833, -0.003945212811231613, 0.014625183306634426, 0.002429228276014328, -0.0055525219067931175, 0.007066845893859863, 0.031827375292778015, 0.009132437407970428, -0.016099656000733376, -0.009079302661120892, 0.022316355258226395, -0.013044441118836403, 0.009969300590455532, -0.01822502352297306, 0.0028210929594933987, -0.008926542475819588, -0.011855564080178738, -0.02715156599879265, 0.023764261975884438, 0.01127108745276928, 0.01236698031425476, -0.035015422850847244, 0.007392292842268944, -0.01611294038593769, -0.0005400591180659831, -0.015900403261184692, -0.004386890679597855, 0.029117530211806297, 0.004861777648329735, 0.011071834713220596, -0.013668768107891083, -0.0006077222060412169, -0.01947367750108242, 0.028054846450686455, 0.008295574225485325, 0.004280622582882643, 0.0037094298750162125, -0.00021118760923855007, 0.004894986283034086, -0.0077841575257480145, 0.005296813324093819, -0.02821424975991249, 0.004938157740980387, -0.017520995810627937, 0.016431745141744614, 0.003732676152139902, -0.017029505223035812, -0.005778342019766569, -0.029090963304042816, -0.019752630963921547, -0.03424498066306114, -0.030233347788453102, 0.00586468493565917, -0.009145720861852169, -0.019513526931405067, 0.007252815645188093, -0.002402661135420203, -0.0030120438896119595, -0.0021452924702316523, -0.018304724246263504, -0.013482797890901566, -0.03461691737174988, 0.008574528619647026, 0.01445249654352665, -0.022847697138786316, -0.009816539473831654, -0.022555459290742874, -0.02242262475192547, -0.01961979642510414, 0.0009954356355592608, -0.02122710458934307, -0.017494428902864456, 0.013449589721858501, 0.025345003232359886, -0.010560418479144573, 0.0253582876175642, -0.021625611931085587, 0.010646761395037174, 0.019101737067103386, -0.009969300590455532, 0.006894159596413374, 0.004609390161931515, 0.008069753646850586, -0.0015699489740654826, -0.021652178838849068, 0.005592372268438339, 0.019221289083361626, 0.015196375548839569, -0.01570115052163601, -0.006967219524085522, -0.0048152850940823555, 0.030631855130195618, 0.015780851244926453, 0.012599442154169083, -0.01858367957174778, 0.012181011028587818, -0.012034892104566097, 0.03775183483958244, -0.010859297588467598, -0.019699497148394585, 0.0023943588603287935, 0.03161483630537987, 0.012566233053803444, -0.027231266722083092, -0.00960400328040123, 0.005031142849475145, -0.0016604431439191103, -0.004074727185070515, -0.018185172230005264, -0.024521423503756523, 0.03833631053566933, -0.001643008436076343, 0.012898322194814682, -0.031110063195228577, -0.02004486881196499, -0.016166074201464653, -0.015807418152689934, 0.007611471228301525, -0.001770032336935401, -0.011291013099253178, -0.040063172578811646, -0.010161912068724632, 0.01235369686037302, -0.010241612792015076, -0.0031000473536551, -0.010892506688833237, -0.0425870455801487, 0.02917066402733326, 0.010899148881435394, -0.0027546752244234085, -0.02170531265437603, -0.0394255630671978, 0.005612297914922237, 0.012672501616179943, 0.04519062116742134, 0.027895445004105568, 0.0071332636289298534, -0.023538442328572273, -0.0018779612146317959, -0.03145543485879898, 0.03015364706516266, -0.007584904320538044, -0.0074719940312206745, 0.02607559785246849, 0.01006228569895029, -0.016458312049508095, 0.014386079274117947, 0.02482694573700428, 0.009125795215368271, -0.019633078947663307, -0.017906218767166138, -0.01608637347817421, 0.010600268840789795, 0.00620673643425107, -0.02453470788896084, -0.010646761395037174, -0.007525128312408924, -0.01473145093768835, -0.023644709959626198, -0.007046920713037252, 0.005771700292825699, 0.018743081018328667, 0.013761752285063267, 0.001997512998059392, 0.0304458849132061, -0.02482694573700428, 0.006010803859680891, 0.020908299833536148, -0.010885865427553654, -0.020496509969234467, -0.018357859924435616, 0.0016903311479836702, 0.015926970168948174, -0.005253641866147518, 0.0022631839383393526, -0.02060277760028839, 0.01189541444182396, 0.010885865427553654, 0.0045628976076841354, 0.012260711751878262, 0.014904137700796127, 0.011941906996071339, -0.021399790421128273, -0.021067703142762184, 0.02578336000442505, 0.0017683719052001834, -0.021572476252913475, -0.015183092094957829, -0.008215872570872307, -0.0020523075945675373, -0.0061536021530628204, 0.02098800055682659, -0.013482797890901566, 0.0002517646935302764, 0.011496908031404018, 0.022462474182248116, 0.003819019068032503, -0.0126791438087821, 0.008435050956904888, -0.007007069885730743, 0.02384396269917488, 0.0060473340563476086, 0.0009522641194052994, 0.0023561688140034676, -0.02066919580101967, 0.007704455871134996, 0.03015364706516266, 0.003845586208626628, -0.0011764239752665162, -0.003249487141147256, -0.017255324870347977, -0.03841601312160492, -0.004031555727124214, 0.0019726064056158066, -0.025623958557844162, -0.007956843823194504, 0.013436305336654186, -2.5279598503402667e-06, -0.009537585079669952, -0.020642628893256187, 0.009404749609529972, -0.009059377945959568, -0.01906188763678074, 0.00899295974522829, -0.008169380016624928, 0.010474075563251972, 0.11105043441057205, 0.006691585760563612, 0.014093841426074505, 0.025491122156381607, -0.00872728880494833, -0.015661299228668213, -0.017029505223035812, -0.021253671497106552, 0.03294319286942482, -0.002090497873723507, 0.0017384839011356235, -0.025172317400574684, 0.0066085634753108025, -0.019035320729017258, 0.020443376153707504, 0.01522294245660305, 0.01277212891727686, 0.015023689717054367, 0.0005242849001660943, -0.018689947202801704, 0.002264844486489892, -0.004891665652394295, 0.011948548257350922, 0.03161483630537987, 0.007279382552951574, -0.026872610673308372, 0.009683704003691673, 0.034059010446071625, 0.005904535762965679, 0.008806990459561348, 0.005506029352545738, 0.011736012063920498, 0.023445457220077515, 0.011124968528747559, -0.01170280296355486, -0.012705710716545582, 0.008428409695625305, 0.02539813704788685, 0.009404749609529972, 0.015528463758528233, -0.0012005004100501537, 0.035546764731407166, 0.01445249654352665, -0.026447538286447525, 0.018596962094306946, -0.013814887031912804, -0.007086771074682474, 0.004180995747447014, 0.02004486881196499, -0.026593657210469246, 0.035520199686288834, 0.004808643367141485, -0.007731023244559765, -0.008043186739087105, 0.014904137700796127, -0.013117500580847263, -0.010573701933026314, 0.019221289083361626, -0.0049315160140395164, -0.004453308414667845, -0.025677092373371124, -0.027789175510406494, 0.03150856867432594, -0.014532198198139668, 0.003789131296798587, 0.0055525219067931175, -0.013868020847439766, 0.012247428297996521, -0.02485351264476776, 0.015501896850764751, 0.0072196065448224545, -0.03243841603398323, -0.02684604376554489, -0.01522294245660305, 0.0115234749391675, 0.01140392292290926, 0.012327129952609539, -0.004948120564222336, -0.02012457139790058, -0.01178914587944746, 0.027390670031309128, -0.0232196357101202, 0.019380692392587662, -0.011257803998887539, -0.02747037075459957, -0.0037227135617285967, 0.002638444071635604, 0.00026401045033708215, 0.015501896850764751, 0.0053964401595294476, -0.019380692392587662, 0.01575428433716297, 0.01942054182291031, -0.026155300438404083, 0.015648016706109047, 0.025557540357112885, 0.023976799100637436, 0.00047405652003362775, 0.017268609255552292, -0.007631396409124136, -0.011543400585651398, -0.047501955181360245, -0.011736012063920498, -0.00942467525601387, 0.02345873974263668, 0.022024117410182953, -0.01216108538210392, 0.036317210644483566, -0.0022814488038420677, -0.014904137700796127, 0.017029505223035812, -0.014625183306634426, 0.008853483013808727, -0.013203843496739864, 0.010753029957413673, -0.00629640044644475, 0.001494398806244135, 0.04022257402539253, -0.005326701328158379, -0.011457057669758797, -0.007518486585468054, -0.05467507243156433, -0.0011564986780285835, 0.02530515380203724, -0.024269036948680878, 0.0027098432183265686, 0.00618349015712738, -0.010480716824531555, 0.008355350233614445, 0.014558765105903149, 0.02132008969783783, -0.020137853920459747, -0.01772024855017662, -0.012831903994083405, -0.021134119480848312, -0.019194722175598145, -0.017640547826886177, 0.0232196357101202, -0.018743081018328667, -0.01810547150671482, 0.004848493728786707, -0.008189305663108826, -0.027762608602643013, -0.022874264046549797, 0.025278586894273758, -0.02770947478711605, -0.025185601785779, -0.004778755363076925, -0.0011199688306078315, 0.04617360234260559, 0.009710270911455154, 0.011636385694146156, -0.018915768712759018, 0.027364103123545647, 0.0027779212687164545, -0.013369888067245483, -0.01973934844136238, -0.012579517439007759, 0.02182486467063427, 0.0173615925014019, 0.0244151558727026, 0.008601095527410507, -0.00010232480417471379, 0.0006650074501521885, 0.005589051637798548, 0.010420940816402435, -0.00966377928853035, -0.029090963304042816, -0.032039910554885864, 0.0253582876175642, 0.01547532994300127, -0.0024275677278637886, 0.013276902958750725, 0.004838531371206045, 0.004343719221651554, 0.026327986270189285, -0.023272771388292313, -0.02340560592710972, -0.009657137095928192, -0.04099301993846893, -0.009597361087799072, -0.002668332075700164, -0.026035748422145844, -0.01870323158800602, -0.02120053768157959, 0.002258202526718378, 0.025411421433091164, 0.0030319690704345703, -0.0025537614710628986, 0.00625987071543932, 0.029808275401592255, -0.018384426832199097, 0.004788717720657587, 0.01507682353258133, 0.01431966107338667, -0.030578721314668655, 0.0054263281635940075, -0.03225244581699371, -0.004672486800700426, 0.024375304579734802, -0.004692412447184324, -0.0014503970742225647, 0.001344958902336657, 0.0018796216463670135, -0.026181867346167564, 0.011623102240264416, -0.01459861546754837, -0.02247575856745243, -0.005788304843008518, -0.012997948564589024, -0.019513526931405067, -0.019699497148394585, -0.041763465851545334, -0.011576609686017036, 0.0024059819988906384, 0.007305949926376343, 0.01558159850537777, 0.015302644111216068, 0.0005134920356795192, -0.011569967493414879, 0.0008111264905892313, -0.009325048886239529, 0.017202191054821014, 0.020257405936717987, 0.016551297158002853, 0.028054846450686455, -0.0017152377404272556, -0.013761752285063267, 0.0189423356205225, 0.019579945132136345, -0.01509010698646307, 0.00891990028321743, -0.006894159596413374, 0.009398108348250389, 0.006927368696779013, 0.03796437010169029, -0.0012760505778715014, -0.02655380591750145, -0.028745591640472412, 0.0008775441674515605, 0.014412646181881428, -0.007053562439978123, -0.016697416082024574, -0.009344973601400852, -0.017029505223035812, 0.01485100295394659, -0.0011747635435312986, 0.014771302230656147, -0.025012915953993797, -0.0008850161684677005, -0.02871902473270893, 0.011496908031404018, 0.0052137915045022964, 0.00580490892753005, 0.012911605648696423, 0.001437943778000772, 0.01661771535873413, 0.0005483613349497318, -0.02340560592710972, -0.011470341123640537, 0.028400219976902008, 0.014811152592301369, -0.011031984351575375, -0.006791212130337954, 0.011244520545005798, 0.013489440083503723, 0.00013511856377590448, 0.025411421433091164, 0.0036430121399462223, 0.030366184189915657, -0.011198027990758419, -0.02482694573700428, -0.002671652939170599, -0.025185601785779, 0.01954009383916855, -0.0021934453397989273, -0.006651734933257103, 0.0033989271614700556, -0.010972208343446255, -0.024122918024659157, 0.004778755363076925, 0.018025770783424377, 0.003447079798206687, -0.031269464641809464, 0.008408484049141407, -0.023179786279797554, -0.007750948425382376, 0.0039219665341079235, 0.017667114734649658, -0.009358257055282593, -0.02646082080900669, -0.02268829569220543, -0.019008751958608627, 0.019274422898888588, -0.01837114244699478, 0.00886012427508831, 0.0019659646786749363, 0.04325122386217117, -0.02048322558403015, -0.012891680002212524, -0.014625183306634426, 0.007086771074682474, -0.024694109335541725, 0.002938984427601099, 0.002027401002123952, -0.026301419362425804, 0.026009181514382362, -0.01495727151632309, -0.013801603578031063, -0.020416809245944023, 0.014532198198139668, 0.010042360052466393, -0.023352472111582756, -0.003935249987989664, -0.009325048886239529, 0.004958082921802998, 0.0036529749631881714, 0.00011882546095876023, -0.04086018353700638, -3.8559901440748945e-06, 0.007193039637058973, -0.021067703142762184, 0.018397709354758263, -0.02652723900973797, 0.03987720236182213, 0.020390242338180542, -0.005775021389126778, 0.01291824784129858, -0.014784585684537888, -0.005147373769432306, 0.017773382365703583, -0.03676885366439819, 0.013708618469536304, 0.007119980175048113, 0.0015990067040547729, -0.005937744397670031, -0.01861024647951126, -0.005379835609346628, -0.010726462118327618, -0.006127035245299339, -0.002299713669344783, 0.006508937105536461, -0.0036529749631881714, 0.007325875107198954, -0.007478635758161545, -0.0012361998669803143, 0.0014512272318825126, -0.021572476252913475, -0.02672649174928665, -0.030233347788453102, -0.004250734578818083, 0.04734255373477936, -0.006180169060826302, -0.01458533201366663, 0.008003335446119308, 0.025437988340854645, -0.054249998182058334, -0.006143639329820871, -0.009139078669250011, 0.03570616990327835, 0.01843756064772606, 0.006538825109601021, 0.017414728179574013, 0.028612755239009857, -0.0046890913508832455, 0.004762150812894106, -0.020297257229685783, -0.023511875420808792, 0.007584904320538044, 0.006150281522423029, 0.01146369893103838, -0.0016629337333142757, -0.03498885780572891, -0.023272771388292313, 0.008421767503023148, -0.009816539473831654, 0.008315498940646648, 0.0043304357677698135, -0.028293950483202934, -0.004994613118469715, 0.02179829776287079, 0.024308886379003525, 0.016166074201464653, 0.0022332959342747927, 0.009105870500206947, -0.041497793048620224, 0.012307204306125641, -0.009325048886239529, 0.00897967629134655, -0.014133691787719727, 0.004383570048958063, 0.02770947478711605, -0.012141159735620022, -0.01498383842408657, -0.0057086036540567875, -0.013582424260675907, -0.002894152421504259, 0.011144894175231457, 0.026859328150749207, 0.007452068850398064, -0.00853467732667923, 0.004131182562559843, 0.02964887209236622, -0.008049828000366688, -0.0011672915425151587, 0.011569967493414879, -0.0024358700029551983, 0.008215872570872307, -0.0008277308661490679, -0.02578336000442505, 0.002379415091127157, -0.033315129578113556, 0.013409738428890705, -0.0034703260753303766, 0.02149277552962303, -0.023644709959626198, 0.004841852001845837, -0.012360338121652603, 0.023272771388292313, 0.011742653325200081, -0.011815712787210941, -0.0008422597893513739, -0.0024906645994633436, -0.03294319286942482, -0.027284400537610054, -0.00862102024257183, 0.017919501289725304, -0.01995188370347023, -0.0036131241358816624, 0.009683704003691673, 0.005844759754836559, -0.0077841575257480145, -0.029808275401592255, 0.006927368696779013, -0.00975012220442295, 0.005937744397670031, 0.23357784748077393, -0.007551695220172405, 0.004250734578818083, 0.0312163308262825, -0.00592446094378829, 0.022914115339517593, 0.015674583613872528, 0.006787891499698162, -0.023060234263539314, -0.028347084298729897, -0.03565303608775139, -0.0013017874443903565, -0.013496081344783306, -0.0037227135617285967, 0.006150281522423029, -0.0366891510784626, -0.027045298367738724, -0.02242262475192547, -0.008866766467690468, -0.024069784209132195, 0.010560418479144573, -0.015515180304646492, 0.014558765105903149, -0.013854737393558025, 0.00582151347771287, -0.021652178838849068, 0.0017866367707028985, 0.00024200958432629704, 0.00902616884559393, 0.003550027497112751, -0.027868878096342087, -0.005293492693454027, 0.004732262808829546, 0.0032096365466713905, -0.015262792818248272, -0.007119980175048113, 0.02693902887403965, -0.0017949390457943082, 0.010009150952100754, 0.0072594573721289635, 0.005549200810492039, 0.003978421911597252, -0.0002268580428790301, -0.017560847103595734, 0.016166074201464653, 0.03411214426159859, -0.01632547751069069, -0.008288932032883167, 0.005423007067292929, -0.01632547751069069, -0.026952313259243965, 0.01510339044034481, 0.024255752563476562, 0.026248285546898842, -0.01311085931956768, 0.010513925924897194, 0.01318391878157854, 0.0019908712711185217, 0.0016064787050709128, 0.035520199686288834, -0.03172110393643379, 0.015143240801990032, -0.017268609255552292, 0.03583900257945061, -0.00573184946551919, 0.04120555520057678, -0.014000856317579746, -0.0018713193712756038, 0.0005442102556116879, -0.04048824682831764, 0.024122918024659157, -0.0071332636289298534, -0.024401871487498283, 0.0027812421321868896, -0.013516006991267204, -0.0031083496287465096, 0.029542604461312294, -0.002009136136621237, 0.01251974143087864, 0.034776318818330765, -0.0181718897074461, -0.027496937662363052, -0.015315927565097809, -0.010009150952100754, 0.005711924284696579, -0.03490915521979332, 0.007797440979629755, -0.03387304022908211, 0.025105899199843407, 0.0015193053986877203, -0.013170634396374226, -0.014518914744257927, -0.002347866538912058, -0.01235369686037302, -0.0017932786140590906, -0.00029638910200446844, 0.0030535547994077206, -0.0037094298750162125, -0.02230307273566723, 0.02578336000442505, -0.004068085458129644, 0.021811580285429955, 0.016763834282755852, 0.03286349028348923, -0.013296828605234623, 0.012599442154169083, 0.015395628288388252, 0.010420940816402435, 0.020589495077729225, -0.024215903133153915, -0.004194279201328754, -0.018118755891919136, 0.009358257055282593, -0.007232889998704195, 0.012951456010341644, 0.020390242338180542, 0.0197127815335989, -0.015833985060453415, 0.02129352279007435, -0.03687512129545212, -0.015249509364366531, -0.004782075993716717, 0.00960400328040123, 0.010467433370649815, 0.003453721757978201, 0.00599087867885828, 0.0040780482813715935, 0.013655484654009342, -0.01483771950006485, -0.029303500428795815, 0.006754682399332523, -0.006684944033622742, 0.005492745898663998, 0.012958098202943802, 0.010228329338133335, -0.018012486398220062, 0.01299130730330944, 0.0032926585990935564, -0.018211741000413895, 0.01266586035490036, -0.019048603251576424, 0.007824008353054523, -0.009922808036208153, 0.025889629498124123, -0.002321299398317933, -0.04157749563455582, 0.010068926960229874, -0.002761316951364279, -0.004254055209457874, 0.0017949390457943082, 0.00020693271653726697, 0.004247413482517004, -0.002731428947299719, 0.004018272273242474, 0.021904565393924713, 0.013668768107891083, -0.02539813704788685, -0.004599427338689566, -0.002842678688466549, -0.008820273913443089, -0.02943633496761322, 0.0051440526731312275, 0.010938999243080616, -0.004782075993716717, -0.011237879283726215, 0.004410136956721544, -0.16896669566631317, 0.024999631568789482, 0.019128303974866867, -0.03188050910830498, 0.05284194275736809, -0.014213393442332745, 0.008302215486764908, 0.02797514572739601, -0.030499018728733063, 0.0056289019994437695, -0.012247428297996521, 0.01973934844136238, -0.015063540078699589, -0.014160258695483208, -0.008640945889055729, 0.0038389444816857576, -0.022462474182248116, 0.01959322951734066, 0.019022036343812943, 0.004868419375270605, 0.023139934986829758, -0.016192641109228134, 0.015422196127474308, -0.03296975791454315, 0.00814945437014103, 0.016219208016991615, -0.014279810711741447, 0.0035267812199890614, -0.0012486532796174288, 6.58469507470727e-05, -0.004303868394345045, 0.008886691182851791, 0.006738078314810991, -0.016272341832518578, -0.012021607719361782, 0.0033059422858059406, 0.0016919915797188878, -0.03071155585348606, -0.014266527257859707, 0.027842309325933456, 0.02485351264476776, -0.014439213089644909, -0.0006969710229896009, 0.01582070253789425, 0.0012104630004614592, 0.039584964513778687, -0.021931132301688194, 0.0019045282388105989, 0.004449987784028053, -0.03541393205523491, 0.006671660579741001, -0.005711924284696579, -0.009776689112186432, 0.01510339044034481, 0.01605980657041073, -0.028798725455999374, -0.01870323158800602, 0.004161070566624403, 0.0002025740541284904, -0.038814518600702286, -0.011682877317070961, 0.01195519044995308, -0.01620592549443245, 0.0015815721126273274, 0.016551297158002853, -0.0037193926982581615, 0.008109604008495808, -0.012552949599921703, -0.02324620448052883, 0.01584726944565773, -0.01184892188757658, -0.025225451216101646, -0.013947722502052784, -0.010653402656316757, -0.008561245165765285, 0.0031598233617842197, -0.01920800656080246, 0.015289360657334328, 0.016511445865035057, 0.00027106734341941774, 0.003965137992054224, 0.017507711425423622, 0.0011506871087476611, -0.008747214451432228, 0.0007156509673222899, -3.3442363928770646e-05, 0.006352855358272791, -0.018543828278779984, 0.015926970168948174, 0.0045728604309260845, -0.007731023244559765, -0.011915340088307858, -0.007764231879264116, -0.01849069446325302, 0.020908299833536148, 0.008109604008495808, -0.004738904535770416, 0.004260696936398745, -0.001304278033785522, -0.029250366613268852, 0.014930704608559608, 0.001027814345434308, -0.010693253949284554, 0.027284400537610054, 0.04364972934126854, -0.018530545756220818, 0.0007779176230542362, 0.009936091490089893, 0.027191417291760445, -0.00905273575335741, -0.02893155999481678, -0.004798680543899536, 0.014943988062441349, 0.01798591949045658, 0.00034101351047866046, 0.011211312375962734, 0.00930512323975563, -0.02191784977912903, 0.015541747212409973, -0.00573849119246006, 0.057012975215911865, -0.014027423225343227, -0.007000428158789873, 0.01605980657041073, -0.030525585636496544, -0.03684855252504349, -0.10292090475559235, -0.01692323572933674, 0.016272341832518578, 0.05419686436653137, 0.026142016053199768, 0.014372795820236206, -8.006449206732213e-05, 0.010314672254025936, -0.002880868734791875, 0.014386079274117947, -0.0011838959762826562, -0.02893155999481678, -0.008215872570872307, -0.007379009388387203, -0.020084720104932785, -0.006053975783288479, 0.0033441323321312666, 0.004104615189135075, -0.02316650189459324, -0.00298049533739686, 0.0021071024239063263, 0.009936091490089893, -0.01544876303523779, -0.027018729597330093, 0.019885467365384102, -0.018291441723704338, -0.02566380798816681, 0.010334597900509834, 0.006572033744305372, 0.011629743501543999, -0.014943988062441349, 0.014558765105903149, 0.00890661682933569, 0.005502708721905947, 0.013642200268805027, 0.010885865427553654, 0.006598601117730141, -0.00933833234012127, 0.02845335379242897, -0.040780484676361084, 0.008554602973163128, 0.02684604376554489, 0.008660871535539627, -0.0028210929594933987, 0.00908594485372305, -0.012061459012329578, -0.04266674816608429, 0.023870529606938362, 0.018570395186543465, -0.020629344508051872, -0.027603207156062126, 0.0025022877380251884, -0.024029932916164398, -0.0232196357101202, 0.010932357050478458, -0.0013358264695852995, 0.020908299833536148, -0.011483624577522278, -0.010088852606713772, 0.00018171474221162498, 0.007863858714699745, 0.014492347836494446, 0.004317152313888073, 0.016338760033249855, 0.00611043069511652, -0.009152362123131752, -0.004888344556093216, -0.01918143965303898, 0.026633508503437042, -0.014186825603246689, 0.005844759754836559, -0.0021436321549117565, -0.007186397910118103, 0.048989713191986084, -0.026885895058512688, 0.012705710716545582, -0.002512250328436494, -0.013170634396374226, 0.023485306650400162, 7.212549826363102e-05, -0.012778770178556442, -0.008694079704582691, 0.0032876774203032255, -0.010427583009004593, 0.0371142253279686, -0.011483624577522278, 0.0028974732849746943, -0.004968045745044947, -0.007179756183177233, -0.04213540628552437, -0.0035367438104003668, 0.020868448540568352, -0.010686611756682396, -0.0119086978957057, -0.006708190310746431, 0.013409738428890705, -0.005565805360674858, -0.011012058705091476, -0.013436305336654186, 0.02211710251867771, 0.00081942870747298, -0.008421767503023148, -0.05791625753045082, -0.0021486133337020874, -0.004310510121285915, 0.008454976603388786, -0.0038621907588094473, 0.00924534723162651, -0.002312997356057167, -0.012699068523943424, -0.024282319471240044, 0.011755937710404396, -0.014744735322892666, 0.02016442082822323, 0.012227503582835197, -0.013356604613363743, -0.02167874574661255, -0.012632651254534721, 0.01311085931956768, -0.007584904320538044, -0.012001683004200459, -0.008614378981292248, -0.004031555727124214, 0.020616061985492706, -0.009657137095928192, 0.04287928342819214, 0.014771302230656147, 0.016391893848776817, 0.0018215060699731112, -0.02209053561091423, 0.0024275677278637886, -0.02410963363945484, 0.007272740826010704, -0.014253243803977966, 0.0069871447049081326, 0.043038684874773026, -0.004788717720657587, -0.006575354840606451, 0.008468260057270527, 0.005711924284696579, 0.02578336000442505, 0.013642200268805027, 0.0047754342667758465, -0.03270408883690834, -0.010812805965542793, 0.012406830675899982, -0.004652561619877815, 0.006814458407461643, 0.0030170250684022903, 0.011569967493414879, 0.010128702968358994, 0.010885865427553654, 0.03860198333859444, 0.01637861132621765, -0.026009181514382362, -0.017401443794369698, -0.03708765655755997, -0.009457884356379509, 0.009703629650175571, -0.005625581368803978, -0.0006180999334901571, -0.02432217076420784, 0.04962732270359993, 0.023591576144099236, 0.0064059896394610405, 0.003941892180591822, 0.004576181061565876, 0.018264874815940857, -0.002879208419471979, -0.004828568547964096, 0.002004154957830906, 0.004147787112742662, -0.013775035738945007, 0.00941139180213213, 0.007119980175048113, 0.019646363332867622, 0.00636281818151474, 0.0046492405235767365, -0.017414728179574013, 0.009922808036208153, -0.019898749887943268, 0.010241612792015076, 0.014027423225343227, 0.002872566692531109, -0.014425929635763168, 0.020536361262202263, -0.000521794252563268, 0.013801603578031063, 0.0036895046941936016, 0.013210485689342022, 0.011496908031404018, -0.00629640044644475, 0.0003584481601137668, -0.0031432188116014004, -0.006518899463117123, -0.01236698031425476, 0.014093841426074505, -0.0014213393442332745, 0.011344147846102715, -0.005838118027895689, 0.016763834282755852, -0.0050975605845451355, 0.010978849604725838, -0.0037791684735566378, 0.0001745540794217959, 0.00649565365165472, -0.010706537403166294, -0.00942467525601387, -0.0232196357101202, -0.01470488402992487, -0.009198854677379131, 0.008043186739087105, 0.015116673894226551, -0.0020506472792476416, 0.0019344162428751588, -0.0013009571703150868, -0.00902616884559393, 0.002201747614890337, 0.031030360609292984, -0.009909524582326412, -0.0193275585770607, 0.031349167227745056, 0.002641764935106039, 0.029994243755936623, 0.03714079037308693, -0.012938172556459904, 0.01153675839304924, 0.03490915521979332, 0.00840184185653925, 0.0026467463467270136, 0.0024640976916998625, 0.007611471228301525, 0.005788304843008518, -0.014266527257859707, -0.00580490892753005, -0.010806163772940636, -0.001617271569557488, 0.006711510941386223, 0.02242262475192547, -0.007578262593597174, -0.02598261460661888, 0.07757589966058731, 0.0030469130724668503, 0.006887517869472504, -0.009212138131260872, 0.006907443515956402, 0.03344796597957611, 0.015528463758528233, 0.0331822969019413, -0.012811979278922081, -0.03424498066306114, 0.018132038414478302, -0.03323543071746826, 0.005476141348481178, -0.003118312219157815, -0.010872581973671913, 0.0023345830850303173, -0.022037401795387268, 0.03169453889131546, -0.01234041340649128, -0.002346206223592162, 0.018118755891919136, -0.03488259017467499, 0.01620592549443245, -0.002175180474296212, 0.0019161513773724437, 0.00865422934293747, 0.02057621069252491, 0.009962658397853374, -0.012506457976996899, -0.040807049721479416, 0.007525128312408924, 0.010859297588467598, -0.0371142253279686, -0.017056072130799294, -0.004997933749109507, -0.00652554165571928, -0.014080557972192764, -0.015780851244926453, 0.0010850995313376188, 0.009909524582326412, 0.018264874815940857, 0.031269464641809464, -0.030020810663700104, -0.01594025455415249, 0.004675807897001505, 0.017255324870347977, -0.024149484932422638, -0.0014686619397252798, -0.02182486467063427]} +{"id": "test:10000012", "text": "\"\\\"Unemployment, Relative Price Dispersion and the Implicit Contract Model.\\\"\\n\\\"The Federal Reserve's Preferences for Inflation and Unemployment: An Analysis of Fed Chairmen.\\\"\\n\\\"Do Presidential Administrations Have Preferred Rates of Unemployment?\\\"\\n\\\"The Baseball Writer and The National Baseball Hall of Fame: Are Election Outcomes Affected by Race or Ethnicity?,\\\" (with Cliff Reid).\\n\\\"Race, Ethnicity and The National Baseball Hall of Fame: Is There Discrimination in the Nomination Process?,\\\" (with Cliff Reid and John Santos).\\n\\\"Discrimination, Voting Behavior and the National Baseball Hall of Fame: The Case of Pitchers,\\\" (with Cliff Reid).\\n\\\"A Comparison of Two Models to Forecast Voting for Membership into the National Baseball Hall of Fame,\\\" (with Cliff Reid).\\n\\\"Jackie Robinson and The National Baseball Hall of Fame,\\\" (with Cliff Reid).\\n\\\"Race, Ethnicity and the Market(s) for Baseball Cards,\\\" (with Jim Meehan and Cliff Reid).\\n\\\"Further Analysis of the Determinants of Sovereign Credit Ratings,\\\" (with Shannon Landauer, '99).\\n\\\"An Empirical Analysis of the Relationship between the Yield Spread and the Probability of Recessions,\\\" (with Sunil Thakor, '99).\"", "vector_field": [-0.015737121924757957, -0.010903381742537022, 0.0069450694136321545, -0.032380569726228714, -0.028974981978535652, -6.82856043567881e-05, -0.01973319798707962, -0.00890534371137619, -0.01965080387890339, -0.026091214269399643, -0.016162822023034096, 0.004569395910948515, 0.012434523552656174, 0.003316330723464489, -0.0240313820540905, -0.01151446532458067, 0.030650312080979347, -0.014940653927624226, 0.010683665983378887, -0.03902696445584297, -0.01590190827846527, 0.028727801516652107, 0.005506619811058044, -0.026352128013968468, -0.0167533066123724, 8.394891483476385e-05, 0.03268267959356308, -0.001943108974955976, 0.003566943807527423, 0.013011276721954346, -0.012489452958106995, 0.002789356978610158, -0.0024838149547576904, -0.004984795115888119, -0.028027458116412163, -0.019019123166799545, -0.0009449482895433903, -0.01684943214058876, 0.021504653617739677, -0.004054437391459942, -0.002718979259952903, -0.008836682885885239, -0.010869051329791546, 0.00011082329729106277, 0.02062579244375229, 0.015462477691471577, 0.016094159334897995, -0.03076016902923584, -0.00010808758088387549, 0.020996561273932457, 0.038642462342977524, 0.021861691027879715, 0.008946540765464306, 0.0038999500684440136, 0.004665521439164877, -0.01917017623782158, 0.0023121621925383806, 0.02791760116815567, -0.008047079667448997, -0.013141732662916183, 0.016725841909646988, 0.004789111204445362, -0.015531139448285103, 0.010663067921996117, -0.0192251056432724, -0.005819027777761221, -0.012434523552656174, 0.0026468851137906313, -0.011802841909229755, 0.006948502734303474, 0.023331038653850555, 0.013155465014278889, -0.0011775377206504345, -0.0002564062597230077, 0.025322210043668747, -0.006938203237950802, -0.033369291573762894, -0.0018881800351664424, -0.015517407096922398, 0.00842471607029438, -0.002509562997147441, 0.012434523552656174, -0.022713089361786842, -0.016121624037623405, 0.00038664776366204023, 0.009859733283519745, -0.0024838149547576904, 0.02686021849513054, -0.018030403181910515, -0.021834228187799454, 0.007415398024022579, 0.0026571843773126602, 0.030677776783704758, 0.00012595018779393286, -0.028782730922102928, 0.010704264044761658, -0.00012927596981171519, 0.02783520705997944, 0.010532611981034279, -0.025157423689961433, 0.007387933786958456, 0.03759881481528282, -0.03759881481528282, -0.015778319910168648, -0.0031189301516860723, 0.005657673813402653, -0.007566452492028475, -0.016080427914857864, -0.010409021750092506, -0.04559096693992615, -0.00684551103040576, 0.004081902094185352, 0.018359975889325142, -0.05578027293086052, 0.0006548551609739661, -0.04097694158554077, -0.008136339485645294, -0.02982637844979763, -0.010271699167788029, -0.023221181705594063, 0.023234913125634193, 0.029963700100779533, 0.021724369376897812, -0.015187833458185196, 0.004713584203273058, 0.00590485380962491, -0.04531631991267204, -0.02732711471617222, 0.017590971663594246, 0.01133594661951065, -0.0007149336161091924, 0.01634133979678154, -0.0013620643876492977, 0.01348503865301609, -0.03608826920390129, 0.02068072184920311, -0.01965080387890339, -0.009743008762598038, -0.0019825890194624662, -0.023207448422908783, 0.005091220140457153, 0.011967628262937069, -0.01181657426059246, -0.018085332587361336, 0.022987734526395798, 0.018469834700226784, 0.02397645264863968, 0.019005389884114265, 0.01882687211036682, 0.002125060884281993, -0.010209904052317142, -0.002906080801039934, 0.0004887811373919249, 0.031089743599295616, -0.01037469133734703, 0.005008826497942209, -0.006639527622610331, 0.0014633395476266742, -0.004950464703142643, -0.031089743599295616, 0.0062721907161176205, -0.0008518266840837896, 0.007401665672659874, 0.014102988876402378, -0.011905833147466183, 0.027052471414208412, 0.01966453716158867, -0.01592937298119068, -0.0035034322645515203, 0.018071599304676056, -0.005616477224975824, 0.007113289088010788, -0.045920539647340775, 0.011184891685843468, 0.006083372980356216, 0.012413925491273403, 0.012647373601794243, 0.00673222029581666, -0.035401660948991776, -0.019376160576939583, -0.0008917359518818557, 0.004844040144234896, 0.028700336813926697, 0.03617066517472267, 0.02735457941889763, 0.01587444543838501, 0.026228537783026695, -0.020515933632850647, -0.008747423067688942, -0.014363900758326054, 0.012894553132355213, 0.024182436987757683, 0.008527707308530807, 0.00017723144264891744, -0.6424481272697449, -0.02205394208431244, 0.0027035304810851812, -0.0045796949416399, 8.845479896990582e-05, 0.003759194863960147, 0.020941633731126785, 0.01631387509405613, 0.014075524173676968, 0.023276111111044884, -0.002425453159958124, -0.007882293313741684, -0.005753799341619015, -0.01917017623782158, -0.029167233034968376, -0.006539969239383936, 0.017536042258143425, -0.010985774919390678, 0.014020594768226147, -0.0028477187734097242, -0.02745070494711399, 0.03927414491772652, -0.01919764094054699, -0.003570376895368099, 0.017206469550728798, 0.010024519637227058, -0.0005153873353265226, 0.009509561583399773, 0.00540362810716033, -0.008582636713981628, -0.01538008451461792, 0.03227071464061737, -0.020309951156377792, 0.010038251988589764, 0.0491064116358757, 0.01686316356062889, 0.0007342445314861834, 0.006450709421187639, 0.017673365771770477, 0.02982637844979763, -0.017192738130688667, -0.030622847378253937, 0.011274151504039764, -0.01106130238622427, 0.009317310526967049, 0.022864144295454025, -0.0018143693450838327, 0.02977144904434681, 0.00271211308427155, -0.005499753635376692, -0.012812159955501556, 0.013759682886302471, -0.00271382974460721, 0.023948989808559418, 0.023674344643950462, -0.026283465325832367, -0.0015414415393024683, -0.021930353716015816, 2.2864680431666784e-05, -0.007559586316347122, 0.012098084203898907, 0.01300441101193428, -0.01779695600271225, -0.013986264355480671, -0.02839822880923748, 0.021738102659583092, -0.0006758826202712953, 0.00865129753947258, 0.017700830474495888, -0.008307992480695248, -0.018442369997501373, 0.028672872111201286, -0.004198625683784485, 0.004960763733834028, -0.006783715914934874, -0.0129563482478261, 0.010704264044761658, -0.011754779145121574, -0.016519859433174133, 0.028700336813926697, -0.009852866642177105, -0.034440405666828156, 0.018030403181910515, -0.0028202543035149574, 0.004263853654265404, -0.020969098433852196, -0.009008334949612617, -0.0035909751895815134, 0.016945557668805122, -0.01294948160648346, 0.003759194863960147, 0.021793030202388763, 0.005956349894404411, -0.016629716381430626, -0.032435499131679535, 0.02889258787035942, -0.0065125045366585255, -6.73200556775555e-05, 0.018936729058623314, -0.03798331692814827, 0.0017457082867622375, -0.012413925491273403, 0.01176164485514164, 0.013382046483457088, -0.02113388478755951, 0.011782243847846985, 0.027340847998857498, 0.011390875093638897, 0.039191752672195435, 0.0011088766623288393, -0.010031385347247124, 0.0001405406801495701, 0.0020632657688111067, -0.0005947767058387399, 0.02313878759741783, -0.03985089808702469, 0.022768018767237663, -0.025363408029079437, -0.0030142220202833414, 0.020337415859103203, 0.009804803878068924, 0.02791760116815567, 0.007387933786958456, -0.03081509843468666, 0.0023928391747176647, 0.0019362427992746234, 0.002394555602222681, -0.006011278834193945, -0.0062721907161176205, -0.0017800388159230351, 0.0005346982507035136, 0.009626285172998905, 0.053665511310100555, -0.005757232662290335, 0.011981360614299774, -0.0001231608330272138, 0.030403131619095802, -0.01738498918712139, 0.008719958364963531, -0.03369886428117752, -0.006430111359804869, 0.014295239932835102, 0.00865129753947258, -0.01590190827846527, -0.015091707929968834, -0.04798037186264992, -0.00017422751989215612, 0.005774397868663073, 0.0015817799139767885, -0.005334966816008091, -0.01199509296566248, -0.018593423068523407, -0.011823439970612526, 0.008774887770414352, -0.014048059470951557, -0.01110936515033245, 0.006612063385546207, -0.023770470172166824, -0.028700336813926697, 0.011528197675943375, -0.005197644699364901, 0.01823638565838337, -0.04122411832213402, 0.0055375169031322, -0.0096125528216362, -0.013773415237665176, 0.002526728203520179, 0.0023619416169822216, -0.0013732218649238348, -0.02315252088010311, 0.010058850049972534, -0.008562038652598858, -0.005966648925095797, 0.006203529890626669, -0.0035154479555785656, 0.02114761620759964, -0.00910446047782898, 0.006831778679043055, 0.008685627952218056, 0.004898969084024429, 0.01492692157626152, 0.0221500676125288, -0.01010004710406065, -0.0024443347938358784, 0.03564883768558502, -0.003721431130543351, 0.0014736386947333813, 0.026667967438697815, 0.017110344022512436, 0.019005389884114265, -0.02640705555677414, 0.035291802138090134, -0.03163903206586838, 0.01581951603293419, 0.003999508451670408, 0.020035305991768837, -0.0022057376336306334, 0.0027275618631392717, -0.01012064516544342, 0.010539477691054344, 0.01462481264024973, 0.01154193002730608, -0.0025232951156795025, -0.013478172942996025, 0.0010513729648664594, -0.005355565343052149, 0.015709657222032547, -0.006495339330285788, 0.00935850664973259, 0.023317307233810425, 0.026324663311243057, -0.019444821402430534, -0.013450708240270615, -0.01222167443484068, -0.005843059159815311, 0.007532122079282999, -0.04147129878401756, 0.028027458116412163, -0.01868954859673977, 0.008871013298630714, -0.004418341442942619, 0.002334477147087455, 0.025308478623628616, 0.011198624037206173, -0.021367331966757774, -0.0004299900901969522, -0.036774881184101105, -0.007010297384113073, 0.03216085582971573, -0.005661107134073973, 0.004844040144234896, 0.014013729058206081, 0.00867876224219799, 0.02694261260330677, -0.00042870271136052907, 0.003259685356169939, 0.00989406369626522, -0.0066566928289830685, 0.03801077976822853, -0.01735752448439598, -0.000548430485650897, 0.015558603219687939, 0.0041883266530931, 0.00862383283674717, 0.018620887771248817, -0.009269247762858868, 0.02842569164931774, 0.03185874596238136, -0.016739575192332268, 0.02503383345901966, -0.010896515101194382, 0.032435499131679535, -0.01772829331457615, 0.014830796048045158, 0.034879833459854126, 0.013972532004117966, -0.002202304545789957, 0.0013517652405425906, 0.01199509296566248, 0.023921525105834007, 0.014377633109688759, 0.005053456407040358, 0.0022572334855794907, -0.01592937298119068, 0.012077486142516136, -0.013526235707104206, -0.023825399577617645, -0.010862184688448906, -0.015503674745559692, 0.005448257550597191, -0.02212260477244854, -0.015627264976501465, -0.006986266002058983, 0.005379596725106239, 0.005249140318483114, 0.012908285483717918, 0.007250611204653978, -0.008774887770414352, 0.04556350037455559, 0.005135849583894014, -9.263883111998439e-05, -0.022919071838259697, 0.02554192580282688, 0.017398720607161522, -0.010134377516806126, -0.021037759259343147, 0.0024391852784901857, 0.005039724055677652, -0.005589012987911701, 0.022987734526395798, -0.01155566144734621, 0.01011377852410078, 0.0014152767835184932, -0.003372976090759039, -0.008383519016206264, 0.033836185932159424, 0.032023534178733826, -0.01246885396540165, -0.009495829232037067, 0.007635113317519426, 0.013629226945340633, 0.015998033806681633, -0.007909758016467094, -0.021065223962068558, 0.03180381655693054, -0.007573318667709827, -0.0024443347938358784, -0.008115741424262524, 0.00563020957633853, 0.0024632166605442762, 0.017426185309886932, -0.019897984340786934, -0.009303578175604343, 0.007648845668882132, 0.01873074658215046, -0.014336436055600643, 0.002626286819577217, 0.0016101025976240635, 0.03408336639404297, -0.0049298666417598724, 0.0011878368677571416, -0.004717017058283091, -0.019019123166799545, 0.018977925181388855, 0.10727608948945999, 0.03523687273263931, -0.01103383768349886, 0.027574295178055763, -0.006941636558622122, -0.014762135222554207, 0.005249140318483114, -0.018977925181388855, -0.014501222409307957, -0.0023928391747176647, -0.005870523396879435, -0.024910245090723038, 0.02738204412162304, 0.0018435503588989377, 0.02982637844979763, -0.010985774919390678, 0.009324176236987114, -0.03023834526538849, 0.009681213647127151, -0.01034722663462162, -0.023715540766716003, -0.022314855828881264, -0.006553701125085354, 0.026118678972125053, -0.011754779145121574, 0.001143207191489637, 0.05045216903090477, 0.016519859433174133, -0.0033678265754133463, -0.03331436216831207, -0.0069794002920389175, 0.002336193574592471, 0.00636488338932395, 0.0335066132247448, -0.004964197054505348, 0.00034545111702755094, 0.014583616517484188, -0.00384502112865448, 0.010910247452557087, -0.005115251522511244, 0.009605687111616135, 0.0055031864903867245, -0.009598820470273495, -0.022768018767237663, -0.009873464703559875, -0.006509071681648493, 0.0027172628324478865, 0.006327119655907154, -0.001749999588355422, -0.006193230394273996, 0.022740554064512253, -0.011246686801314354, -0.005836192984133959, 0.0015826381277292967, -0.00686610909178853, -0.014185382053256035, 0.0043531134724617004, -0.012111816555261612, 0.019403625279664993, -0.006780283059924841, -0.013704754412174225, -0.022754285484552383, 0.011171159334480762, -0.007394799496978521, -0.007648845668882132, -0.01467974204570055, -0.011905833147466183, 0.0017285429639741778, -0.036802344024181366, 0.007847962900996208, 0.016135357320308685, -0.02253456972539425, -0.03262775018811226, 0.0007672876818105578, 0.025226084515452385, 0.0003046836063731462, 0.019623341038823128, 0.015998033806681633, 0.0039686113595962524, -0.012235406786203384, -0.02018636092543602, -0.008774887770414352, 0.005973515100777149, -0.0394938588142395, 0.0005076629458926618, -0.01394506823271513, -0.013093669898808002, 0.01970573328435421, -0.0013311669463291764, 0.013588029891252518, 0.011487000621855259, -0.020309951156377792, 0.00036776598426513374, -0.01875821128487587, 0.015654729679226875, 0.022507106885313988, 0.027519365772604942, 0.009028933942317963, 0.01303187571465969, 5.557257100008428e-05, -0.008287393487989902, -0.01130848191678524, -0.014027461409568787, 0.006660126149654388, 0.006735653150826693, -0.011857770383358002, -0.020927900448441505, 0.001672755810432136, 0.0012256004847586155, -0.012393327429890633, -0.012839624658226967, -0.02161451242864132, 0.008074544370174408, 0.001211868249811232, 0.020845508202910423, 0.012201076373457909, -0.0067390864714980125, -0.016931826248764992, 0.002660617232322693, -0.03693966940045357, -0.0009543891646899283, -0.013931335881352425, 0.017659632489085197, 0.02746443636715412, -0.018071599304676056, 0.021381063386797905, -0.004126531537622213, -0.04133397713303566, -0.0031961738131940365, 0.005963216070085764, -0.004239822272211313, 0.042102981358766556, -0.020337415859103203, -0.008802352473139763, -0.02160077914595604, 0.0010556642664596438, -0.012111816555261612, -0.00165387406013906, -0.00398234324529767, -0.0011844038963317871, 0.01919764094054699, -0.026709165424108505, 0.00720941461622715, -0.0007153627811931074, 0.04707404598593712, -0.029936237260699272, 0.007374201435595751, 0.008692494593560696, -0.015270226635038853, 0.01819518953561783, -0.0036562031600624323, 0.008644431829452515, -0.002507846336811781, 0.012647373601794243, -0.010038251988589764, -0.003645904129371047, 0.00379009242169559, -0.006852377206087112, 0.02552819438278675, 0.025775374844670296, 0.02699754200875759, -0.004741048440337181, 0.009434034116566181, 0.01592937298119068, 0.017014218494296074, 0.01538008451461792, 0.026571841910481453, 0.009914661757647991, -0.011267284862697124, 0.01873074658215046, 0.00988719705492258, 0.005365864373743534, 0.01323099248111248, -0.0037351634819060564, 0.010340360924601555, 0.004950464703142643, 0.0012728050351142883, -0.017989207059144974, -0.03518194332718849, -0.02401765063405037, -0.008802352473139763, 0.013862675055861473, -0.025308478623628616, -0.006618929095566273, -0.02930455468595028, -0.016684645786881447, 0.04427267238497734, -0.02982637844979763, -0.014597347937524319, -0.011713582091033459, 0.03218831866979599, -0.017645901069045067, 0.04317409545183182, -0.0037729269824922085, -0.013134866952896118, -0.006687590386718512, -0.013478172942996025, -0.03666502237319946, -0.017920544371008873, 0.024717994034290314, 0.009248648770153522, 0.03218831866979599, -0.024690529331564903, -0.004644922912120819, 0.009852866642177105, 0.0008385236142203212, -0.012846490368247032, 0.0022949969861656427, -0.013265322893857956, -0.01586071215569973, 0.013697887770831585, -0.034440405666828156, -0.010862184688448906, -0.018648352473974228, 0.018868068233132362, 0.022314855828881264, 0.004823441617190838, 0.01132907997816801, 0.0010496564209461212, -0.027052471414208412, -0.00540362810716033, 0.009770473465323448, 0.02264442853629589, 0.004682686645537615, 0.018085332587361336, 0.012056888081133366, -0.005482588429003954, -0.02684648707509041, 0.00601814454421401, 0.017124077305197716, 0.014006863348186016, 0.018964193761348724, 0.006618929095566273, -0.022919071838259697, -0.020859239622950554, 0.03232564404606819, 0.012153013609349728, -0.021971549838781357, -0.045014213770627975, 0.03284746780991554, 0.01881313882768154, -0.013862675055861473, -0.008108874782919884, 0.008500243537127972, -0.018524762243032455, 0.004078468773514032, 0.003693966893479228, -0.014569884166121483, -0.0050259921699762344, -0.0293594840914011, -0.004747914616018534, 0.02168317325413227, 0.0019242271082475781, 0.031419314444065094, 0.005952916573733091, -0.0038621865678578615, -0.005084353964775801, -0.007930356077849865, -0.006872975267469883, 0.016245214268565178, -0.0026983809657394886, 0.015984302386641502, -0.01462481264024973, 0.010463950224220753, 0.0037385965697467327, -0.0029301121830940247, 0.006989699322730303, -0.008706226944923401, -0.002228052355349064, 0.02927708998322487, -0.018895532935857773, -0.005396761931478977, -0.006306521128863096, -0.007092691026628017, -0.019101515412330627, 0.003230504458770156, -0.015325156040489674, -0.01587444543838501, 0.0006200954667292535, 0.010656201280653477, 0.01247572060674429, -0.003807257628068328, -0.02938694693148136, 0.0018023536540567875, -0.005637075752019882, 0.0023396266624331474, 0.003518881043419242, -0.006711621768772602, -0.005609611049294472, -0.02414124086499214, 0.0003480259038042277, 0.0144188292324543, 0.011754779145121574, 0.006261891685426235, 0.0005359856295399368, 0.0027327113784849644, 1.610585422895383e-05, -0.0014281506882980466, -0.017948009073734283, 0.03746149316430092, 0.00251814560033381, 0.001953408122062683, -0.027546830475330353, -0.020996561273932457, -0.024347223341464996, -0.007401665672659874, 0.02697007730603218, -0.010079448111355305, -0.029579197987914085, -0.022809214890003204, -0.0017422751989215612, 0.02015889622271061, 0.0007625672733411193, 0.017165273427963257, 0.0027687584515661, -0.010415887460112572, 0.008747423067688942, -0.008259929716587067, -0.006347718182951212, 0.029222160577774048, -0.005211377050727606, -0.013752817176282406, 0.01540754921734333, -0.009694945998489857, 0.01727513037621975, -0.012139281257987022, 0.00964001752436161, -0.007614515256136656, -0.030375666916370392, -0.014885724522173405, -0.013842076063156128, 0.02784893848001957, -0.00044458056800067425, -0.009763607755303383, 0.008019615896046162, -0.017948009073734283, 0.0040132408030331135, 0.036362916231155396, -0.04185580089688301, -0.016918092966079712, 0.010038251988589764, 0.01823638565838337, -0.0010410738177597523, 0.014528687112033367, 0.0005685996729880571, -0.020955365151166916, 0.007106422912329435, 0.0327376089990139, -0.010745461098849773, 0.013148599304258823, -0.008060812018811703, 0.04699165001511574, 0.021820494905114174, -0.0215595830231905, -0.013533101417124271, 0.002442618366330862, -0.02977144904434681, -0.0012230257270857692, -0.017124077305197716, -0.0028236873913556337, 0.02253456972539425, -0.023454628884792328, 0.007044628262519836, -0.003577242838218808, -0.011576260440051556, 0.004559096414595842, -0.009674347937107086, -0.01207062043249607, 0.005334966816008091, -0.01489945687353611, 0.016931826248764992, 0.037681207060813904, -0.016217749565839767, 0.003433054545894265, 0.0007840238395147026, 0.014350168406963348, 0.01970573328435421, 0.02894751727581024, 0.008568904362618923, 0.009282980114221573, 0.01080038957297802, 0.01870328187942505, 0.0008616967243142426, 0.0183187797665596, 0.004226090386509895, -0.03652770072221756, -0.022740554064512253, 0.023427164182066917, -0.012146146968007088, -0.03229817748069763, -0.004229523241519928, 0.01779695600271225, -0.02254830300807953, 0.016588520258665085, -0.023866595700383186, -0.029743986204266548, -0.0004565962590277195, -0.02107895538210869, 0.00023881185916252434, 0.00590828713029623, 0.0037385965697467327, 0.03553898259997368, -0.008115741424262524, -0.03603333979845047, 0.002221186412498355, -0.012056888081133366, 0.009921527467668056, -0.00026112672640010715, 0.0035806759260594845, 0.008362920954823494, 0.007799900136888027, -0.023688076063990593, 0.015682194381952286, 8.984947612589167e-07, 0.019348695874214172, -0.036253057420253754, -0.019939180463552475, -0.03400097414851189, 0.02265815995633602, 0.00686954241245985, -0.022836679592728615, 0.015023047104477882, -0.0026005387771874666, -0.007120155263692141, -0.02069445326924324, -0.01870328187942505, 0.00887787900865078, -0.023770470172166824, -0.003288866253569722, -0.007401665672659874, -0.02120254561305046, 0.015146637335419655, 0.013375180773437023, -0.023825399577617645, -0.006141734775155783, 0.04435506463050842, 0.20356640219688416, 0.0027653255965560675, 0.02894751727581024, 0.0178656168282032, -0.009543891996145248, -0.017522310838103294, 0.021271206438541412, -0.0018023536540567875, -0.026571841910481453, -0.0010333494283258915, -0.011548795737326145, 0.018868068233132362, 0.00887787900865078, 0.006749385502189398, -0.005554682575166225, -0.01581951603293419, -0.042542412877082825, -0.03380872309207916, -0.018565960228443146, -0.004044138360768557, 0.010766059160232544, -0.0031687093432992697, -0.006220695096999407, -0.01738498918712139, 0.033479150384664536, 0.0016332757659256458, -0.003934280481189489, -0.0009312160545960069, -0.006831778679043055, 0.019623341038823128, -0.018002938479185104, -0.006333985831588507, -0.012146146968007088, 0.006818046327680349, -0.019142713397741318, -0.005142715759575367, -0.009955858811736107, -0.00192251056432724, 0.005544383078813553, 0.005461989901959896, 0.027711616829037666, -0.018950462341308594, -0.0005900562391616404, -0.017192738130688667, 0.020090235397219658, 0.04081215336918831, 0.001874447800219059, -0.010944577865302563, -0.009681213647127151, -0.016670912504196167, -0.03762627765536308, -0.015640996396541595, 0.013388913124799728, 0.019128980115056038, 0.0027241287752985954, -0.004634623881429434, 0.013443841598927975, -0.010608138516545296, -0.004967629909515381, 0.01054634340107441, 0.006999998353421688, 0.024896511808037758, -0.0020255022682249546, 0.00505688926205039, -0.019623341038823128, 0.02204021066427231, -0.011583126150071621, -0.013052473776042461, 0.014295239932835102, -0.01970573328435421, 0.019005389884114265, -0.004655221942812204, -0.009646883234381676, -0.004456104710698128, -0.018662085756659508, -0.021065223962068558, 0.02930455468595028, 0.010930845513939857, 0.016670912504196167, 0.011720448732376099, -0.014377633109688759, 0.02842569164931774, -0.011898967437446117, -0.0183187797665596, 0.0017517161322757602, -0.025871500372886658, 0.03419322520494461, -0.00940656941384077, -0.009495829232037067, 0.014638544991612434, -0.0013577730860561132, -0.009983322583138943, -0.021394796669483185, -0.011754779145121574, 0.003220205195248127, -0.018662085756659508, 0.027079934254288673, -0.01558606792241335, -0.02551446110010147, 0.018565960228443146, -0.013972532004117966, 0.096125528216362, 0.03081509843468666, -0.015229030512273312, -0.009022067300975323, -0.008706226944923401, 0.0029352616984397173, 0.013443841598927975, 0.027588026598095894, -0.012407059781253338, -0.0024769490119069815, -0.0008140630670823157, -0.004504167474806309, -0.018030403181910515, -0.012791561894118786, -0.019952913746237755, -0.00672878697514534, -0.017577240243554115, 0.03998821973800659, 0.000528690405189991, -0.014542419463396072, -0.02256203442811966, 0.021751834079623222, 0.023207448422908783, -0.00819126795977354, 0.0008024765411391854, -0.008967138826847076, -8.593365055276081e-05, -0.0015131188556551933, -0.009674347937107086, -0.0024975473061203957, -0.016300143674016, 0.0006664417451247573, 0.0002570499782450497, 0.01469347346574068, -0.001019617193378508, 0.020502202212810516, -0.013890138827264309, -0.008328590542078018, 0.00096125528216362, -0.02404511533677578, -0.00494703184813261, 0.014041193760931492, -0.003058851696550846, -0.012077486142516136, -0.019540946930646896, 0.02839822880923748, -0.001433300320059061, -0.016588520258665085, -0.016670912504196167, 0.00013410369865596294, -0.007696908432990313, 0.005564981605857611, -0.04331141710281372, 0.012832758016884327, -0.01926630176603794, -0.027230989187955856, -0.014583616517484188, -0.014803331345319748, 0.028535550460219383, -0.05135849863290787, 0.022397248074412346, 0.014391365461051464, -0.019046587869524956, -0.0069931321777403355, 0.0009380821720696986, -0.17676112055778503, 0.020515933632850647, 0.013265322893857956, -0.02886512316763401, 0.02066698856651783, -0.008431581780314445, 0.029963700100779533, -0.002046100562438369, -0.031529173254966736, 0.01513290498405695, 0.011129963211715221, 0.010154975578188896, -0.016986753791570663, -0.013642959296703339, 0.0024855316150933504, -0.010669933632016182, -0.012860222719609737, 0.004741048440337181, 0.01247572060674429, -0.002959293080493808, 0.03284746780991554, -0.028205977752804756, 0.014377633109688759, -0.010141243226826191, 0.004322215914726257, -0.005125550553202629, -0.0058876886032521725, 0.03842274844646454, -0.01156252808868885, -0.010257966816425323, -0.0034759677946567535, -0.0031086308881640434, 0.03356154263019562, -0.010566942393779755, -0.0022709656041115522, 0.0027910734061151743, -0.001192128169350326, 0.01635507307946682, 0.0035463455133140087, 0.0235782191157341, 0.030567917972803116, 0.014185382053256035, -0.003343795193359256, 0.009557624347507954, -0.016258947551250458, 0.02924962528049946, -0.0215595830231905, 0.001527709304355085, 0.017137808725237846, -0.03405590355396271, -0.006457575596868992, -0.02307012677192688, 0.007051494438201189, 0.0032751341350376606, 0.035264335572719574, 0.000634685973636806, -0.001336316461674869, 0.001018758979626, 0.002365374704822898, -0.019005389884114265, -0.023921525105834007, -0.007573318667709827, 0.02599508874118328, -0.013285920955240726, -0.007133887615054846, -0.013038741424679756, -0.011933297850191593, -0.0015792051563039422, -0.036719951778650284, -0.004483569413423538, 0.006152033805847168, -0.009646883234381676, -0.002547326497733593, -0.014830796048045158, 0.009468364529311657, 0.038257960230112076, -0.006715054623782635, 0.009070130065083504, 0.0007930356077849865, -0.006038743071258068, -0.02062579244375229, 0.041526228189468384, -0.004655221942812204, -0.006824912503361702, -0.0053143687546253204, 0.00505688926205039, -0.004620891530066729, 0.017632167786359787, 0.015682194381952286, -0.017220202833414078, 0.0007896025781519711, -0.008314858190715313, -0.001529425848275423, -0.003142961533740163, -0.01540754921734333, 0.010443352162837982, 0.013368315063416958, -0.008761155419051647, 0.017604704946279526, -0.007188816554844379, 0.007456594612449408, -0.025651784613728523, -0.011425205506384373, 0.03603333979845047, 0.02787640318274498, -0.007243745494633913, -0.027670420706272125, 0.018112795427441597, 0.03218831866979599, -0.021751834079623222, -0.0055031864903867245, 0.0011474984930828214, 0.005331533960998058, 0.031062278896570206, -0.00353604624979198, 0.023907791823148727, -0.01738498918712139, -0.009234917350113392, 0.01105443574488163, 0.004198625683784485, 0.06492593139410019, 0.0023894060868769884, -0.01737125590443611, 0.005177046172320843, 0.01635507307946682, -0.014762135222554207, -0.09563116729259491, -0.011191758327186108, -0.011088766157627106, 0.01540754921734333, 0.016135357320308685, 0.03076016902923584, -0.008307992480695248, 0.017453650012612343, -0.010930845513939857, 0.03987836092710495, -0.013869540765881538, -0.0027550263330340385, -0.004191759508103132, 0.00325110275298357, 0.007044628262519836, 0.01414418499916792, 0.013382046483457088, -0.028755266219377518, -0.03666502237319946, 0.03883471339941025, -0.020049039274454117, -0.020007843151688576, -0.005788130220025778, -0.021999014541506767, 0.0012376161757856607, -0.0027344280388206244, -0.016052963212132454, 0.025294747203588486, 0.007020596880465746, -0.002815104788169265, -0.0004969346337020397, -0.008582636713981628, 0.008314858190715313, -0.014281507581472397, -0.006581165827810764, -0.0018967626383528113, -0.038257960230112076, -0.003539479337632656, 0.013148599304258823, -0.03669248893857002, -0.009509561583399773, 0.009708678349852562, 0.023811666294932365, 0.013876406475901604, -0.005595879163593054, -0.006999998353421688, -0.029057374224066734, 0.02112015150487423, 0.019980378448963165, -0.008761155419051647, 0.00579499639570713, -0.008053946308791637, -0.013107402250170708, 0.0002643452025949955, 0.01462481264024973, 0.027176059782505035, -0.013155465014278889, 0.019486017525196075, -0.015009314753115177, 0.0005265447543933988, 0.0030056394170969725, 0.027107398957014084, -0.00910446047782898, 0.003189307637512684, 0.032957322895526886, -0.024869047105312347, -0.00506032258272171, -0.009743008762598038, -0.02453947439789772, -0.028041189536452293, -0.029579197987914085, 0.04270720109343529, -0.01399313099682331, 0.009935259819030762, 0.015517407096922398, 0.009859733283519745, -0.0316939614713192, -0.01592937298119068, 0.019911717623472214, -0.015558603219687939, -0.018620887771248817, -0.009427168406546116, 0.009729276411235332, -0.005819027777761221, 0.025624319911003113, 0.026791557669639587, -0.0053589981980621815, -0.0006767408922314644, -0.002500980393961072, -0.006289355922490358, 0.008115741424262524, 0.035758696496486664, 0.025844035670161247, -0.031446781009435654, -0.006986266002058983, 0.011363410390913486, -0.019829323515295982, -0.017659632489085197, 0.012304067611694336, -0.0052182432264089584, -0.03282000124454498, -0.0023224614560604095, -0.056466881185770035, 0.031529173254966736, -0.022410981357097626, -0.027821475639939308, -0.020914169028401375, 4.0365212043980137e-05, 0.00867189560085535, -0.006351151037961245, -0.0374065637588501, 0.012530649080872536, -0.04949091374874115, 0.031089743599295616, -0.013011276721954346, -0.021408528089523315, -0.016245214268565178, -0.03515448048710823, 0.019939180463552475, -0.012805293314158916, 0.020461006090044975, 0.018401172012090683, 0.002192005282267928, 0.012098084203898907, 0.002912946743890643, 0.010457084514200687, -0.019431089982390404, -0.01976066268980503, -0.035346731543540955, 0.000634685973636806, -0.00963315088301897, -0.015572335571050644, -0.0016761888982728124, -0.026310930028557777, -0.011322214268147945, -0.007188816554844379, -0.01868954859673977, -0.014514954760670662, 0.021820494905114174, 0.01636880449950695, 0.034797441214323044, 0.04317409545183182, -0.025692980736494064, -0.008562038652598858, -0.005637075752019882, -0.013292787596583366, -0.023303573951125145, -0.006090238690376282, -0.02790386788547039, 0.01561353262513876, -0.0014762134524062276, -0.008239330723881721, 0.02886512316763401, 0.001422142842784524, -0.01917017623782158, -0.02161451242864132, -0.00961941946297884, 0.008019615896046162, 0.007387933786958456, 0.010525745339691639, 0.0035806759260594845, -0.014913189224898815, 0.03886217996478081, 0.0007170792669057846, 0.0016332757659256458, -0.020914169028401375, 0.00814320519566536, 0.019897984340786934, -0.0032322208862751722, 0.017137808725237846, 0.0048818038776516914, 0.004267286974936724, -0.03081509843468666, 0.01320352777838707, 0.00675625167787075, 0.011157427914440632, 0.01586071215569973, 0.005101519171148539, 0.01007258240133524, -0.0024014217779040337, 0.0018212355207651854, 0.015517407096922398, -0.004339381121098995, 0.0019843056797981262, -0.004178027622401714, 0.004607159178704023, 0.004250121768563986, 0.011164293624460697, -0.019019123166799545, -0.00986659899353981, -0.01469347346574068, 0.009942126460373402, 0.007573318667709827, 0.01682196743786335, -0.03471504896879196, 0.007085824850946665, 0.011837172321975231, 0.008417850360274315, -0.008589502424001694, -0.006467875093221664, 0.008417850360274315, 0.024731725454330444, 0.006333985831588507, -0.00379009242169559, -0.0007797325379215181, -0.0168906282633543, -0.003239087061956525, 0.009749875403940678, -0.04229523241519928, -0.017124077305197716, 0.0144188292324543, 0.004263853654265404, 0.014913189224898815, 0.015325156040489674, -0.009571356698870659, -0.005383029580116272, -0.02414124086499214, 0.008088276721537113, -0.03696713224053383, -0.021834228187799454, -0.009440899826586246, 0.01781068742275238, -0.002076998120173812, 0.01224913913756609, 0.0017474248306825757, -0.006965667940676212, 0.029112303629517555, 0.02256203442811966, 0.020090235397219658, -0.012310934253036976, 0.015366352163255215, -0.007120155263692141, 0.020831774920225143, -0.004902401939034462, -0.00789602566510439, -0.016437465324997902, -0.017508579418063164, -0.003666502423584461, -0.0196782685816288, -0.0010462234495207667, 0.017082879319787025, 0.06311327964067459, 0.02550072968006134, 0.0019276601960882545, 0.019348695874214172, 0.007223146967589855, 0.015572335571050644, 0.003817556658759713, -0.009070130065083504, 0.008232465013861656, -0.014844528399407864, 0.0183187797665596, -0.023839130997657776, -0.0033884248696267605, -0.02990877255797386, 0.010264833457767963, -0.002779057715088129, -0.009626285172998905, 0.004459538031369448, -0.03413829579949379, -0.02933201938867569, -0.0005132416845299304, 0.006605197209864855, 0.03474251180887222, -0.001671897596679628, -0.012510051019489765, 0.004538498353213072, 0.008472778834402561, -0.01926630176603794, -0.012166745960712433, -0.02205394208431244, -0.0004535923362709582, 0.015517407096922398, -0.01782441884279251, -0.007765569724142551, 0.008562038652598858, -0.013121134601533413, -0.013574298471212387, 0.003199606901034713, 0.0007346736965700984, -0.013265322893857956, -0.0032081895042210817, -0.008410983718931675, -0.008692494593560696, -0.02930455468595028, -0.0019843056797981262, 0.011583126150071621, -0.003154977224767208, 0.0020220691803842783, -0.011157427914440632]} +{"id": "test:10000013", "text": "\"I have existing web site brittlloyd.org. I opened a new domain sayrebuildersrockport.com however I don't have a way to start building this new web site online like I do the other one. Please list the steps to take.\\nI see you deleted the domain. Did you no longer want to use it? You can see how to create a new site here, https://community.homestead.com/homestead/topics/how-do-i-create-a-new-site.\"", "vector_field": [-0.006357844453305006, 0.0038341970648616552, 0.006018445827066898, -0.010558323003351688, -0.005954598542302847, 0.01287698745727539, -0.026815855875611305, -0.0243829395622015, 0.0032679724972695112, -0.009839201346039772, 0.007507095579057932, -0.001174453878775239, 0.014355556108057499, -0.012756014242768288, 0.004328173119574785, 0.01285682525485754, 0.010235726833343506, -0.003642655210569501, 0.0053194863721728325, -0.014664711430668831, -0.007849854417145252, -0.007809530012309551, 0.02131826989352703, -0.008629463613033295, -0.01178822387009859, -0.0266142338514328, 0.012218352407217026, 0.009368747472763062, -0.029974617063999176, 0.012124261818826199, -0.01904665119946003, 0.019799375906586647, -0.0009745110874064267, -0.012325884774327278, -0.030350979417562485, -0.0004305490874685347, -0.005836985073983669, -0.0027991989627480507, 0.0023186642210930586, -0.022339826449751854, 0.010040824301540852, 0.01486633438616991, -0.006425052415579557, -0.023563005030155182, -0.027340076863765717, -0.008246379904448986, -0.004912879783660173, -0.01045079156756401, -0.026708323508501053, 0.005198512692004442, -0.007964108139276505, 0.011606763117015362, -0.02043112926185131, -0.02087469957768917, 0.0019322201842442155, 0.004129910841584206, -0.005971400532871485, 0.032098378986120224, 0.011109426617622375, -0.0023203445598483086, 0.00010327927157049999, -0.006196546368300915, -0.009207449853420258, -0.005326207261532545, -0.023697420954704285, -0.0013643155107274652, -0.003335180226713419, -0.013871661387383938, -0.022057553753256798, 0.009160404093563557, 0.04325484856963158, -0.017460549250245094, 0.004926321562379599, -0.020216064527630806, 0.020202621817588806, 0.014933542348444462, -0.020054765045642853, -0.010927965864539146, 0.00744660897180438, 0.0023942729458212852, 0.0050540161319077015, -0.011680691502988338, -0.021372035145759583, 0.039625637233257294, 0.033953309059143066, -0.013670038431882858, 0.015256138518452644, 0.030377862975001335, 0.0027991989627480507, 0.004489471670240164, -0.00643177330493927, 0.05228755995631218, 0.0010249167680740356, 0.02212476171553135, -0.01526958029717207, 0.02212476171553135, -0.011600041761994362, 0.014342114329338074, -0.008743716403841972, -0.01375068724155426, -0.0011929359752684832, 0.015000749379396439, 0.0013315518153831363, -0.009321702644228935, -0.004264325834810734, -0.016694383695721626, 0.002552210818976164, -0.019221391528844833, -0.006052049808204174, -0.01949022151529789, -0.03760940581560135, 0.030700460076332092, -0.0056555247865617275, -0.017971327528357506, -0.01131776999682188, 0.0017692416440695524, -0.003575447481125593, -0.007896900177001953, -0.009926571510732174, -0.002767275320366025, 0.023657096549868584, 0.0013559145154431462, 0.0033385404385626316, -0.007171057164669037, 0.018374573439359665, 0.004271046724170446, -0.031748898327350616, -0.003422550158575177, 0.045889388769865036, -0.004472669679671526, 0.023347940295934677, 0.02082093246281147, -0.0014021197566762567, 0.003350301878526807, 0.005705930292606354, 0.02532384730875492, 0.012245235964655876, 0.017447108402848244, -0.021882813423871994, -0.029410071671009064, 0.022044112905859947, 0.02041768655180931, -0.00411646906286478, -0.019826259464025497, -0.0034578340128064156, 0.016197046265006065, 0.0026042968966066837, -0.027635790407657623, 0.0033217386808246374, -0.020269829779863358, 0.013723804615437984, 0.008320308290421963, -0.004139991942793131, -0.022420475259423256, 0.031614482402801514, 0.03142630308866501, -0.008582417853176594, 0.01399263460189104, 0.00908647570759058, -0.002748793223872781, 0.010806991718709469, 0.04381939396262169, 0.0354856438934803, -0.007009759079664946, 0.001876773894764483, 0.018885351717472076, 0.029813317582011223, -0.015484645031392574, -0.021990345790982246, -0.01684224046766758, -0.008058198727667332, 0.0003414989332668483, -0.03812018409371376, 0.011183355003595352, -0.006041968706995249, 0.002478282433003187, -0.0132667925208807, 0.021089764311909676, 0.0036997816059738398, 0.004163514357060194, -0.014489972032606602, -0.0026631036307662725, 0.019154183566570282, 0.07785335183143616, -0.012574553489685059, -0.004842312075197697, 0.012325884774327278, -0.0121713075786829, 0.01998755894601345, -0.008185893297195435, 0.014651269651949406, 0.007473491597920656, 0.0075138164684176445, -0.00243291724473238, -0.6322896480560303, 0.00332509889267385, 0.01998755894601345, -0.013871661387383938, 0.0037669893354177475, 0.01729925163090229, -0.005174989812076092, -0.012803059071302414, -0.04172251373529434, 0.01482600998133421, -0.007930504158139229, 0.0020800770726054907, -0.005665605887770653, -0.004358416888862848, -0.013495298102498055, 0.005232116207480431, 0.01647931896150112, 0.0014903298579156399, 0.010464232414960861, 0.01016179844737053, 0.005232116207480431, -0.004089585971087217, -0.015592177398502827, 0.016277695074677467, 0.02662767469882965, 0.010860757902264595, 0.0030209843534976244, -0.004207199439406395, -0.008952060714364052, 0.01419425755739212, -0.016223929822444916, 0.010585206560790539, 0.0005695848958566785, -0.007150894962251186, 0.05067457631230354, 0.005185070913285017, -0.0022464159410446882, 0.022003788501024246, 0.021882813423871994, 0.032555390149354935, 0.01348857767879963, -0.014463088475167751, 0.03930303826928139, 0.001979265594854951, 0.00377371022477746, -0.028657346963882446, 0.004328173119574785, -0.010376863181591034, 0.017581524327397346, -0.007688556332141161, 0.025619560852646828, -0.0007048403494991362, -0.010551602579653263, 0.0012819861294701695, 0.0013693560613319278, -0.007103849668055773, 0.05180366337299347, 0.006011725403368473, 0.009630857966840267, 0.03449097275733948, -0.0001434463483747095, 0.008253100328147411, -0.019342364743351936, -0.03467915207147598, -0.013394487090408802, -0.024261964485049248, -0.023267291486263275, -0.01591477356851101, 0.0010249167680740356, 0.000882940657902509, -0.011915918439626694, 0.0072920313104987144, -0.03645343333482742, -0.0006737568182870746, -0.02475930191576481, 0.05247574299573898, 0.00642841262742877, 0.0025824543554335833, 0.005870589055120945, -0.009516604244709015, 0.01377757079899311, 0.010638972744345665, 0.021976904943585396, -0.027662672102451324, 0.04570120945572853, -0.012218352407217026, -0.020995672792196274, -0.021076321601867676, 0.0005242197657935321, -0.0017272369004786015, 0.010712901130318642, 0.02266242355108261, -0.010349979624152184, -0.02610345557332039, -0.020095091313123703, 0.03199084475636482, 0.01348185632377863, 0.017541199922561646, 0.004398741293698549, -0.036964211612939835, -0.011089264415204525, -0.025001250207424164, 0.03018968179821968, 0.0034779964480549097, 0.04072784259915352, -0.004069423768669367, -0.01725892722606659, 0.005752975586801767, 0.02255489118397236, -0.0221516452729702, 0.010114752687513828, -0.0036997816059738398, -0.010363421402871609, 0.010813712142407894, -0.004808708094060421, -0.03056604415178299, 0.006475457921624184, -0.015471203252673149, -0.010820433497428894, -0.018777821213006973, 0.007157615851610899, 0.03290487080812454, -0.005060737021267414, -0.006522503215819597, 0.01086747832596302, 0.04524419829249382, 0.020014440640807152, 0.011398418806493282, 0.01066585537046194, 0.0010064346715807915, -0.010141636244952679, -0.007312193512916565, 0.023226967081427574, -0.011633645743131638, 0.018845027312636375, 0.02611689642071724, 0.011122867465019226, -0.01595509797334671, -0.0009577091550454497, -0.033469416201114655, -0.006257032975554466, 0.006394809111952782, 0.00521195400506258, -0.00953004602342844, -0.009146962314844131, -0.02700403705239296, -0.005910913925617933, 0.0033788650762289762, -0.011512672528624535, 0.010370141826570034, -0.004314731806516647, -0.0088848527520895, -0.0176756139844656, 0.011485788971185684, -0.032582275569438934, -0.00950988382101059, -0.006277195643633604, -0.03889979422092438, 0.003504879539832473, 0.012567832134664059, 0.014785685576498508, 0.01286354660987854, -0.03244785964488983, -0.028684228658676147, 0.0004742340533994138, 0.007520537357777357, -0.0021237621549516916, 0.018280483782291412, 0.005742894485592842, -0.02482650987803936, -0.004808708094060421, 0.012574553489685059, -0.02575397491455078, 0.0008879812085069716, 0.029329422861337662, -0.011922638863325119, -0.013535622507333755, 0.0026463016401976347, 0.0049397628754377365, -0.004731419496238232, -0.003508239984512329, -0.0021724875550717115, 0.006159582175314426, 0.0013248310424387455, -0.004906159359961748, 0.012829942628741264, -0.011875594034790993, 0.026305077597498894, -0.01911385916173458, 0.007500374689698219, 0.016183605417609215, 0.0220978781580925, 0.0024850033223628998, 0.028254101052880287, 0.004281127825379372, -0.0008480766555294394, 0.016264254227280617, 0.006720765959471464, -0.023200083523988724, 0.012803059071302414, 0.017998211085796356, 0.02044457010924816, 0.018361132591962814, 0.0025656523648649454, 0.002436277689412236, -0.009173845872282982, -0.0007754084072075784, -0.016519643366336823, 0.0069828759878873825, -0.0064956205897033215, -0.016976654529571533, -0.030055265873670578, 0.006720765959471464, -0.009073033928871155, -0.00077582843368873, 0.0034779964480549097, 0.008568977005779743, 0.02443670481443405, -0.0007691076607443392, 0.011109426617622375, -0.01733957603573799, 0.016102954745292664, 0.008898294530808926, -0.006351124029606581, -0.01599542237818241, 0.0007266828324645758, 0.0021540054585784674, -0.02125106193125248, -0.0008249740349128842, -0.02297157794237137, 0.0011752939317375422, 0.0025505307130515575, 0.020955348387360573, 0.016586851328611374, 0.023280734196305275, 0.02346891537308693, 0.016627175733447075, -0.0036560967564582825, -6.73126705805771e-05, -0.008347191847860813, 0.008454723283648491, -0.0028932897839695215, -0.0029470559675246477, -0.009637578390538692, 0.04578185826539993, 0.03693732991814613, 0.027326634153723717, 0.0014550458872690797, -0.04121173545718193, 0.027178777381777763, -0.013616272248327732, 0.0033990275114774704, -0.009099917486310005, -0.000985432299785316, 0.02040424570441246, -0.009133521467447281, -0.0015961818862706423, -0.004328173119574785, 0.013676758855581284, -6.267901881074067e-06, 0.013898544013500214, 0.02263553999364376, 0.017541199922561646, -0.005178350023925304, -0.003299896139651537, 0.017111070454120636, -0.02388560213148594, -0.013670038431882858, 0.0034443926997482777, -0.006683801766484976, -0.01947678066790104, -0.010464232414960861, 0.03322746604681015, 0.004593643359839916, 0.009422513656318188, -0.009321702644228935, -0.003840917721390724, -0.005756336264312267, 0.016076073050498962, -0.00801787432283163, -0.030350979417562485, -0.023159759119153023, 0.04260965809226036, -0.006401529535651207, -0.014032959938049316, -0.01643899455666542, -0.0008358952472917736, -0.0009089835803024471, -0.005030493251979351, 0.03763629123568535, 0.02127794548869133, -0.008683229796588421, -0.002742072567343712, 0.017527757212519646, -0.001955742947757244, -0.02844228222966194, 0.022339826449751854, -0.024167874827980995, 0.021358594298362732, -0.002256497275084257, 0.009039429947733879, 0.006962713785469532, -0.003316698130220175, -0.030781108886003494, 0.04642705246806145, 0.0009484681067988276, -0.0051245843060314655, -0.0013903584331274033, -0.013394487090408802, -0.03164136782288551, -0.012762734666466713, 0.013629713095724583, 0.008602580986917019, -0.003719944041222334, -0.016748148947954178, 0.008710112422704697, -0.022904369980096817, -0.018468664959073067, -0.011909197084605694, -0.0008262341725639999, 0.0012811460765078664, -0.019718727096915245, -0.02486683428287506, 0.015081399120390415, 0.08274607360363007, -0.004361777100712061, -0.00320412521250546, 0.02571365050971508, -0.021842489019036293, 0.014906658791005611, -0.012648481875658035, -0.050755225121974945, 0.0221516452729702, -0.007016479969024658, 0.00519179180264473, 0.008898294530808926, 0.013333999551832676, 0.012776176445186138, 0.012991241179406643, -0.013468415476381779, 0.02349579893052578, -0.012917311862111092, -0.024745861068367958, -0.021573659032583237, -0.01904665119946003, 0.006220068782567978, -0.0027471131179481745, 0.04123862087726593, 0.007493654265999794, 0.012870267033576965, 0.042233292013406754, -0.010524719953536987, 0.005393414758145809, -0.006122617982327938, -0.010746505111455917, 0.010585206560790539, 0.019342364743351936, 0.020148856565356255, 0.012204911559820175, 0.010054266080260277, -0.002461480675265193, 0.002913451986387372, 0.009677902795374393, -0.006838379427790642, -0.0066501977853477, 0.020135415717959404, 0.004348335787653923, -0.00519179180264473, 0.012540949508547783, -0.01438243966549635, -0.010470953769981861, 0.010054266080260277, -0.007856575772166252, -0.027366958558559418, 0.0265604667365551, 0.010773387737572193, -0.04161498323082924, 0.00013158000365365297, 0.014516854658722878, 0.0034779964480549097, 0.0056252810172736645, 0.004482750780880451, -0.013925427570939064, 0.0025152466259896755, -0.03771694004535675, -0.013676758855581284, 0.01489321794360876, -0.0007888498948886991, -0.012083937413990498, -0.03406084328889847, -0.008972222916781902, 0.013791011646389961, -0.03013591468334198, 0.0001343103067483753, 0.018750937655568123, -0.033039286732673645, -0.019167624413967133, 0.021156972274184227, 0.011707574129104614, 0.012319164350628853, -0.006737567950040102, 0.023778069764375687, 0.0016323060262948275, 0.016277695074677467, -0.017097629606723785, -0.02297157794237137, 0.01818639226257801, 2.8090702471672557e-05, -0.007910341955721378, 0.004193758126348257, -0.013381045311689377, -0.013340720906853676, 0.006969434209167957, 0.0531209334731102, 0.019638078287243843, 0.015793800354003906, 0.014342114329338074, -0.027635790407657623, 0.009946733713150024, 0.025848066434264183, 0.01645243540406227, 0.024638328701257706, 0.003998855594545603, 0.0013592749601230025, -0.004264325834810734, -0.008367354050278664, 0.007681835442781448, -0.021076321601867676, 0.0001453365694032982, 0.0018801343394443393, 0.012708968482911587, 0.009617416188120842, -0.02793150395154953, -0.00973839033395052, 0.025592677295207977, 0.012977799400687218, 0.022931253537535667, 0.007789367809891701, 0.004660851322114468, 0.03846966475248337, 0.02087469957768917, 0.004459228366613388, 0.010349979624152184, 0.011882314458489418, 0.024611445143818855, -0.041480567306280136, 0.01598198153078556, 0.01265520229935646, -0.022420475259423256, 0.03846966475248337, 0.0014382438967004418, -0.04285160452127457, -0.01309877261519432, 0.005699209403246641, 0.002579093910753727, 0.007527257781475782, -0.01600886508822441, -0.03056604415178299, -0.01958431303501129, -0.031748898327350616, 0.0019154183100908995, 0.004781825002282858, -0.0014458048390224576, -0.022084437310695648, -0.024745861068367958, -0.001809566281735897, 0.008743716403841972, 0.014503412880003452, -0.022339826449751854, -0.025592677295207977, -0.010746505111455917, -0.003908125218003988, -0.004825510084629059, 0.014463088475167751, 0.009442675858736038, 0.016654057428240776, -0.023764628916978836, 0.010699459351599216, 0.00707696657627821, -0.046722766011953354, -0.005353090353310108, 0.007473491597920656, 0.0010324777103960514, 0.03505551442503929, 0.02002788335084915, 0.013165980577468872, 0.033012401312589645, -0.012581273913383484, 0.012628319673240185, -0.01682879775762558, 0.013522181659936905, -0.021923139691352844, -0.0051649087108671665, -0.014584062620997429, -0.00039337482303380966, 0.011794944293797016, 0.018764378502964973, 0.01020212285220623, 0.0310768224298954, 0.04040524363517761, -0.0011803344823420048, -0.019664961844682693, -0.01735301874577999, -0.048497047275304794, 0.0031553995795547962, 0.0023707502987235785, -0.007258427329361439, 0.008972222916781902, -0.03064669296145439, -0.0008938618702813983, 0.022070996463298798, -0.006371286232024431, 0.022890929132699966, -0.010121474042534828, 0.022796837612986565, -0.007063525263220072, 0.034356556832790375, 0.013347441330552101, 0.0013491937424987555, -0.024127550423145294, -0.019234832376241684, -0.029517604038119316, 0.0038442781660705805, -0.0014500052202492952, 0.006435133516788483, 0.00024152752303052694, -0.00519179180264473, 0.006357844453305006, -0.012581273913383484, 0.031722016632556915, -0.008696671575307846, -0.024087226018309593, -0.02575397491455078, -0.00498680816963315, 0.013596110045909882, -0.030781108886003494, 0.005699209403246641, 0.016492759808897972, -0.012130982242524624, -0.006841739639639854, -0.0014113609213382006, 0.017111070454120636, 0.000857737788464874, -0.0310768224298954, 0.00030138433794490993, -0.039598751813173294, 0.021587099879980087, 0.03634590283036232, 0.02570020966231823, 0.03056604415178299, 0.015444320626556873, -0.01639866828918457, 0.013965751975774765, 0.01861652173101902, -0.009879525750875473, 0.033926427364349365, 0.03513616323471069, -0.014315231703221798, 0.003730025142431259, -0.020458010956645012, 0.015430878847837448, -0.02301190234720707, -0.016183605417609215, 0.0019406211795285344, -0.021829048171639442, 0.004822149407118559, -0.016640616580843925, 0.01728581078350544, -0.0020951987244188786, 0.023186642676591873, -0.023200083523988724, -0.017447108402848244, -0.016102954745292664, 0.008777320384979248, -0.03403395786881447, -0.018737494945526123, -0.02310599386692047, 0.007352517917752266, 0.018697170540690422, 0.01016179844737053, 0.012561111710965633, -0.007211382035166025, -0.001268544583581388, 0.024477029219269753, -0.009993779473006725, 0.00841439887881279, -0.014207699336111546, -0.009462838992476463, 0.017984770238399506, 0.021842489019036293, -0.005423658061772585, -0.02830786630511284, -0.013582668267190456, 0.03368448093533516, 0.0033805454149842262, -0.0045667607337236404, -0.036184605211019516, 0.002668144181370735, 0.00233210576698184, 0.02830786630511284, -0.000742224627174437, 0.0015272940509021282, -0.0288455281406641, -0.00794394500553608, 0.025498585775494576, 0.013542343862354755, -0.020995672792196274, -0.01997411623597145, 0.0003517901059240103, 0.003575447481125593, 0.00953004602342844, 0.0008779000490903854, 0.03150695189833641, -0.018482105806469917, -0.042797837406396866, -0.02302534505724907, -0.03131876885890961, -0.005440460052341223, 0.00454323785379529, 0.0024312371388077736, 0.01020212285220623, 0.039760053157806396, -0.030888641253113747, 0.011505951173603535, 0.011754619888961315, 0.0045667607337236404, -0.020699959248304367, 0.02162742428481579, 0.01172101590782404, -0.0050103310495615005, 0.002479962771758437, -0.020014440640807152, -0.024127550423145294, -0.03814706951379776, 0.03131876885890961, -0.005581596400588751, 0.004855753388255835, 0.0007989310543052852, 0.019396129995584488, 0.007325634825974703, 0.011505951173603535, 0.001851571025326848, -0.028065919876098633, 0.007654952351003885, -0.011391698382794857, -0.0038509988225996494, -0.0005817663040943444, 0.007614627946168184, 0.009388910606503487, -0.00866306759417057, 0.011203517206013203, 0.02930253930389881, -0.0036661778576672077, -0.02564644254744053, 0.021035997197031975, -0.008031315170228481, -0.013374323956668377, -0.014973866753280163, -0.021452685818076134, -0.038819145411252975, -0.026426052674651146, -0.009839201346039772, -0.002821041503921151, -0.022447358816862106, -0.0022464159410446882, 0.013118935748934746, 0.010780109092593193, 0.014530296437442303, 0.0005292603163979948, -0.005900832358747721, -0.012110820040106773, -0.0034410322550684214, -0.015632500872015953, 0.004539877641946077, -0.004980087745934725, 0.0800039991736412, -0.018401456996798515, -0.017890678718686104, -0.01779658906161785, -0.01354906428605318, -0.03803953528404236, 0.018777821213006973, 0.012036891654133797, 0.015256138518452644, 0.0354049950838089, -0.0060755726881325245, 0.01442276407033205, 0.024692093953490257, -0.022447358816862106, 0.002340506762266159, 0.0016297857509925961, -0.012480462901294231, -0.010276051238179207, 0.02701747976243496, -0.009395631030201912, -0.007278589531779289, -0.02610345557332039, -0.0036292136646807194, 0.008333750069141388, -0.006811496336013079, 0.002752153668552637, 0.012930753640830517, 0.0005343009252101183, -0.017030421644449234, 0.0071777780540287495, -0.004230722319334745, 0.011667249724268913, 0.022958137094974518, 0.012379650957882404, -0.038819145411252975, -0.02346891537308693, 0.01241325493901968, -0.009120079688727856, -0.003061308991163969, -0.007244986016303301, 0.002698387484997511, -0.011432022787630558, -0.010766667313873768, -0.00037006218917667866, -0.004526435863226652, -0.0038476386107504368, 0.010733063332736492, 0.03516304865479469, 0.0017003538087010384, -0.0028781681321561337, 0.016291137784719467, 0.0036997816059738398, -0.005376612767577171, 0.0041467128321528435, 0.004428984597325325, -0.00928809866309166, 0.010968290269374847, -0.010108032263815403, -0.059142742305994034, 0.0022413753904402256, -0.003514960641041398, 0.0034477529115974903, -0.011290887370705605, -0.009724948555231094, -0.009220890700817108, 0.0038711612578481436, -0.03629213571548462, -0.0003503199259284884, 0.024087226018309593, 0.011391698382794857, -0.029813317582011223, -0.010309655219316483, -0.024463588371872902, 0.016237370669841766, -0.031265001744031906, 0.00928809866309166, -0.038308367133140564, 0.0036560967564582825, 0.009449397213757038, -0.004365137312561274, -0.021398918703198433, 0.03204461187124252, 0.0032444498501718044, -0.0050943405367434025, 0.010934686288237572, 0.22022606432437897, 0.001311389496549964, 0.007520537357777357, 0.018845027312636375, 0.004694455303251743, -0.01689600571990013, 0.017568081617355347, 0.001730597228743136, -0.01328695472329855, 0.008777320384979248, -0.032179027795791626, -0.005971400532871485, -0.03774382174015045, 0.00160206260625273, -0.002863046247512102, -0.021519891917705536, -0.03631902113556862, -0.014301789924502373, -0.012675364501774311, -0.0037602686788886786, 0.023697420954704285, 0.004271046724170446, -0.036130838096141815, -0.014167374931275845, 0.04691094532608986, 0.02033703774213791, -0.025108782574534416, 0.012984519824385643, -0.008145568892359734, 0.0030982731841504574, -0.014879776164889336, 0.013172701001167297, -0.001071122125722468, 0.006929109804332256, 0.001037518261000514, -0.004912879783660173, 0.0398675836622715, -0.0011181674199178815, 0.01681535691022873, 0.03852343186736107, -0.0044525074772536755, -0.01287698745727539, -0.012010009028017521, -0.018777821213006973, -0.01724548637866974, -0.012319164350628853, -0.0014080004766583443, -0.02002788335084915, 0.018280483782291412, 0.0051649087108671665, -0.02133171074092388, -0.01810574345290661, 0.004791906103491783, 0.010477674193680286, 0.017877237871289253, -0.008279983885586262, 0.002587494906038046, -0.00014418143837247044, -0.006818217225372791, 0.020968789234757423, -0.02031015418469906, 0.03164136782288551, -0.006623314693570137, 0.037071745842695236, -0.0068719834089279175, 0.01062553096562624, 0.013898544013500214, -0.019382689148187637, 0.011022056452929974, -0.004472669679671526, 0.0020683156326413155, 0.00030516478000208735, 0.03322746604681015, 0.002315304009243846, 0.004798626992851496, -0.026493258774280548, 0.0121645862236619, 0.01733957603573799, 0.042313944548368454, 0.04293225333094597, -0.042744070291519165, 0.003481356892734766, -0.013938868418335915, 0.0009022628655657172, 0.021398918703198433, -0.024020018056035042, 0.01590133272111416, -0.013696921057999134, 0.0008564775926060975, -0.020014440640807152, 0.007271868642419577, 0.012440137565135956, -0.01818639226257801, -0.029114358127117157, 0.006969434209167957, 0.009442675858736038, 0.017070746049284935, 0.019436456263065338, -0.005477424245327711, 0.024248523637652397, -0.017151394858956337, 0.028200333938002586, 0.009409072808921337, 0.03274357318878174, -0.010746505111455917, -0.005554713308811188, 0.013885102234780788, 0.03400707617402077, 0.0031587600242346525, -0.008004432544112206, 0.0041030277498066425, -0.005443820264190435, -0.00821277592331171, 0.000698539603035897, 0.00541693763807416, 0.008568977005779743, 0.011358094401657581, -0.016976654529571533, -0.018710613250732422, -0.007063525263220072, -0.020283272489905357, -0.003746827132999897, -0.0029941012617200613, -0.007567582651972771, -0.004647409543395042, 0.009886247105896473, -0.029544487595558167, -0.0399482324719429, -0.03150695189833641, -0.05115847289562225, 0.011559717357158661, 0.02973266877233982, -0.00023627693008165807, 0.005591677501797676, -0.013333999551832676, -0.004234082531183958, -0.00354856438934803, -0.03196396306157112, -0.006700603757053614, 0.0043416148982942104, 0.013858219608664513, 0.03406084328889847, -0.013683479279279709, -0.008347191847860813, -0.003928287886083126, -0.05086275562644005, -0.011835268698632717, -0.0016827117651700974, -0.026748647913336754, 0.0025253277271986008, -0.021022556349635124, 0.014207699336111546, 0.007614627946168184, -0.012675364501774311, 0.0011450505116954446, 0.004502913448959589, -0.021009115502238274, -0.029410071671009064, 0.011889034882187843, 0.002553891157731414, -0.022474242374300957, -0.00885796919465065, 0.020162297412753105, 0.008851248770952225, -0.018388016149401665, -0.01602230593562126, -0.17097628116607666, 0.016291137784719467, 0.029813317582011223, -0.0071374536491930485, 0.030028382316231728, 0.003951810300350189, 0.037474989891052246, -0.0036594572011381388, -0.009220890700817108, 0.002441318240016699, 0.02563300170004368, 0.0007766685448586941, -0.0037098629400134087, -0.01631801947951317, -0.024611445143818855, -0.0199606753885746, -0.013898544013500214, 0.020552102476358414, 0.03621148690581322, 0.012097379192709923, 0.028926176950335503, 0.003214206313714385, -0.005427018739283085, -0.008710112422704697, 0.01219146978110075, 0.010397025384008884, 0.0034511133562773466, 0.012561111710965633, 0.005376612767577171, -0.012890429235994816, -0.02975955232977867, 0.02178872376680374, 0.008071639575064182, -0.015807241201400757, 0.003514960641041398, -0.016707824543118477, 0.015807241201400757, -0.00764823192730546, -0.01459750346839428, 0.0023085831198841333, 0.01685568131506443, 0.007285310421139002, 0.02252800762653351, -0.01864340528845787, 0.007507095579057932, 0.018912235274910927, 0.023683980107307434, -0.0007254226948134601, 0.0011299287434667349, -0.001372716506011784, -0.008313587866723537, 0.012110820040106773, -0.01864340528845787, -0.01489321794360876, 0.0037434666883200407, 0.0020111892372369766, 0.0023421868681907654, 0.027272868901491165, -0.01868372969329357, 0.009200728498399258, -0.008387516252696514, -0.007917062379419804, -0.014274907298386097, -0.011875594034790993, 5.12983460794203e-05, -0.019691845402121544, -0.028711112216114998, -0.016707824543118477, -0.019302040338516235, 0.011694133281707764, 0.00034296908415853977, -0.04129238799214363, -0.018764378502964973, 0.003696421394124627, 0.012117541395127773, -0.009234332479536533, -0.04123862087726593, 0.019409572705626488, 0.0007678475230932236, -0.004062702879309654, 0.0155384112149477, 0.030431628227233887, -0.0019339004065841436, 0.008118685334920883, -0.022017229348421097, 0.011647087521851063, -0.024087226018309593, 0.004425624385476112, -0.005524469539523125, 0.013636434450745583, 0.007244986016303301, -0.015699708834290504, -0.01721860282123089, -0.03110370598733425, 0.0023035425692796707, 0.02574053406715393, 0.023791512474417686, 0.012682085856795311, -0.007675115019083023, 0.015457761473953724, 0.02395281009376049, 0.003231008304283023, -0.011250562034547329, 0.018307367339730263, 0.037878237664699554, 0.0005364011158235371, -0.003160440130159259, 0.01993379183113575, 0.028469165787100792, -0.010746505111455917, -0.019853143021464348, 0.01258799433708191, 0.01018196064978838, 0.016210487112402916, -0.004835591185837984, -0.011882314458489418, 0.02614377997815609, -0.009993779473006725, -0.008347191847860813, 0.008716833777725697, 0.017527757212519646, 0.0007749883225187659, 0.00036523162270896137, 0.008132127113640308, -0.02387216128408909, 0.001885174890048802, -0.08376763015985489, -0.02930253930389881, 0.020068207755684853, -0.01038358360528946, -0.03234032541513443, 0.016546525061130524, 0.015807241201400757, -0.0012467020424082875, 0.008548814803361893, 0.003770349780097604, -0.030243447050452232, -0.03228655830025673, -0.010571764782071114, -0.006619954481720924, 0.005141385830938816, -0.009987058117985725, 0.01910041645169258, -0.006357844453305006, -0.02166775055229664, 0.0016440673498436809, -0.03164136782288551, -0.003558645723387599, -0.012729130685329437, -0.01015507709234953, 0.00411646906286478, -0.013219746761023998, -0.017769705504179, -0.003881242359057069, -0.010860757902264595, 0.009341864846646786, -0.01307861041277647, -0.014745360240340233, 0.014100166969001293, -0.0031066741794347763, 0.007365959696471691, -0.026278195902705193, -0.02609001286327839, 0.003120115725323558, 0.011922638863325119, -0.02173495665192604, -0.012829942628741264, -0.001997747691348195, -0.01370364148169756, -0.03946433961391449, -0.03946433961391449, -0.0004981767851859331, -0.022286059334874153, 0.021976904943585396, -0.008871410973370075, 0.0008237138972617686, 0.004123189952224493, -0.0010686018504202366, -0.006475457921624184, -0.04226017743349075, 0.018710613250732422, -0.026049688458442688, 0.005978121422231197, 0.034732919186353683, -0.035297464579343796, -0.003130196826532483, 0.001414721249602735, -0.009469559416174889, -0.012924033217132092, 0.007594465743750334, 0.024651769548654556, 0.014570620842278004, 0.007863296195864677, 0.004230722319334745, 0.0009451077203266323, -0.014718477614223957, -0.010013941675424576, 0.017890678718686104, -0.016734708100557327, 0.023173201829195023, -0.018388016149401665, -0.002185929100960493, -0.03363071382045746, -0.016331462189555168, 0.029463838785886765, -0.021815607324242592, -0.004936402663588524, -0.007890179753303528, 0.011479068547487259, -0.025995923206210136, 0.024127550423145294, -0.0032192468643188477, 0.008710112422704697, 0.025538910180330276, -0.00042130801011808217, -0.053819894790649414, -0.00687534362077713, 0.020982231944799423, 0.004139991942793131, -0.006488899700343609, 0.002137203700840473, -0.018078859895467758, -0.012668644078075886, 0.004711256828159094, -0.025955598801374435, 0.0016222249250859022, -0.04295913502573967, 0.0012156185694038868, -0.052018728107213974, 0.016304578632116318, -0.0052085937932133675, -0.04919600859284401, 0.007379401009529829, -0.005104421637952328, -0.006240231450647116, -0.012850104831159115, -0.025391053408384323, 0.0243694968521595, -0.02130482904613018, 0.028979944065213203, 0.015766916796565056, 0.011700853705406189, -0.011190075427293777, -0.008918456733226776, 0.02564644254744053, -0.006683801766484976, 0.003931648097932339, -0.0200009997934103, -0.004647409543395042, -0.022864045575261116, 0.026331961154937744, 0.021923139691352844, -0.009724948555231094, 0.02576741762459278, -0.028549814596772194, 0.026909947395324707, 0.020673075690865517, -0.02662767469882965, 0.0019943872466683388, -0.03580824285745621, -0.013831336051225662, 0.02310599386692047, -0.0071979402564466, -0.016694383695721626, -0.026036247611045837, 0.026936830952763557, 0.003693060949444771, 0.03599642217159271, -0.015162047930061817, -0.014758802019059658, -0.0036392947658896446, -0.006690522655844688, 0.0056555247865617275, -0.005460622254759073, -0.001114807091653347, -0.0020733564160764217, 0.019221391528844833, 0.009684624150395393, 0.033012401312589645, 0.0027907982002943754, -0.010773387737572193, -0.027353517711162567, -0.004570120945572853, -0.030861757695674896, 0.010800271295011044, -0.008609301410615444, 0.017111070454120636, 0.002698387484997511, 0.027797088027000427, -0.006740928161889315, 0.007507095579057932, -0.005776498466730118, -0.000291093165287748, -0.012581273913383484, 0.011411860585212708, 0.019369248300790787, -0.0026194185484200716, -0.03333500027656555, -0.010578486137092113, 0.014355556108057499, -0.021560218185186386, 0.020054765045642853, 0.02121073752641678, 0.00642841262742877, -0.01729925163090229, -0.01482600998133421, 0.002400993602350354, 0.016183605417609215, 0.015834124758839607, -0.014059842564165592, -0.020928464829921722, 0.020565543323755264, -0.014449646696448326, 0.011922638863325119, -0.005994923412799835, 0.010706180706620216, 0.002187609439715743, -0.00035704069887287915, -0.0009299860103055835, -0.0006409930647350848, -0.01303828600794077, 0.0057798586785793304, 0.011015335097908974, -0.004328173119574785, -0.002913451986387372, 0.006878703832626343, 0.008474886417388916, 0.01907353475689888, 0.022353267297148705, 0.02973266877233982, -0.006357844453305006, -0.0038980443496257067, -0.02090158313512802, 0.020955348387360573, 0.0004645729495678097, -0.0177293810993433, -0.01238637138158083, 0.005027133040130138, -0.038711611181497574, 0.023630212992429733, 0.002453079679980874, 0.0075541408732533455, -0.015054515562951565, 0.011640367098152637, -0.028603579849004745, -0.017541199922561646, -0.026049688458442688, 0.013199584558606148, 0.008347191847860813, -0.0007388642407022417, 0.014906658791005611, -0.006798055022954941, 0.03637278452515602, -0.008078360930085182, 0.0034477529115974903, -0.008931898511946201, -0.024114107713103294, -0.03193708136677742, 0.005668966099619865, 0.008911735378205776, -0.013213026337325573, 0.0013869981048628688, 0.002963857725262642, -0.015484645031392574, -0.002140563912689686, 0.028496047481894493, -0.018522432073950768, 0.057368457317352295, 0.02121073752641678, -0.015713151544332504, 0.01684224046766758, 0.010733063332736492, 0.041426800191402435, 0.01681535691022873, 0.003049547551199794, -0.010101310908794403, -0.028012152761220932, 0.03857719525694847, 0.011129588820040226, 0.002743752673268318, -0.00797082856297493, -0.037958886474370956, 0.008649625815451145, -0.0049700066447257996, 0.028200333938002586, -0.01422114111483097, 0.021143529564142227, 0.010007220320403576, 0.0026799053885042667, 0.01350201852619648, -0.003157079918310046, -0.022245734930038452, -0.0011509312316775322, 0.030888641253113747, -0.023321058601140976, -0.02305222675204277, -0.030700460076332092, 0.003696421394124627, 0.011714295484125614, 0.0031822826713323593, -0.011633645743131638, 0.011196795850992203, 0.015793800354003906, -0.021546775475144386, 0.011122867465019226, 0.009099917486310005, 0.00488599669188261, 0.002752153668552637, 0.009644298814237118, -0.01198984682559967, -0.021116646006703377, -0.019880026578903198, -0.0036124116741120815, -0.005574875511229038, -0.004593643359839916, -0.02525663934648037]} +{"id": "test:10000014", "text": "\"Embrace world class facilities at East Bourne Resort & Spa Shimla. Facilities at East Bourne Resort & Spa Shimla comprise multi cuisine restaurant, tours and travel desk. Avail facilities of East Bourne Resort & Spa Shimla.\"", "vector_field": [-0.0064783766865730286, 0.01880580186843872, -0.019290417432785034, 0.004630777053534985, -0.019707726314663887, 0.032065439969301224, -0.00830578338354826, 0.004092314746230841, -0.008474052883684635, -0.014336563646793365, -0.012842330150306225, 0.023867348209023476, -0.018294261768460274, -0.005175970494747162, -0.016961567103862762, 0.01688079908490181, 0.014269255101680756, -0.04280776530504227, -0.009611555375158787, 0.017850030213594437, -0.016921183094382286, 0.01109232660382986, 0.02428465709090233, -0.011819250881671906, -0.009988478384912014, -0.008750014938414097, -0.002943033818155527, -0.0042033726349473, -0.030880821868777275, 0.020609650760889053, 0.0016675509978085756, -0.013636562041938305, -0.018657725304365158, -0.03112313151359558, -0.012849060818552971, 0.015117334201931953, -0.007154820021241903, -0.024405810981988907, 0.012694252654910088, 0.028188509866595268, 0.019546188414096832, -0.010035593993961811, -0.012505791150033474, 0.030369283631443977, -0.016800029203295708, 0.006636549718677998, 0.012600021436810493, -0.03131159394979477, -0.04130006954073906, 0.01370386965572834, -0.0028992837760597467, 0.014646179042756557, -0.009221170097589493, -0.008103860542178154, -0.020138496533036232, 0.007706744130700827, -0.026855815201997757, -0.004364911466836929, 0.013973101042211056, 0.0024701966904103756, 0.007356743328273296, -0.007922128774225712, -0.020367342978715897, 0.0019048110116273165, -0.014026947319507599, 0.0018290897132828832, -0.011994251050055027, 0.012081751599907875, -0.0008039076346904039, 0.011334635317325592, 0.028053894639015198, 0.008581745438277721, 0.0012796897208318114, 0.01674618199467659, 0.01938464865088463, -0.016921183094382286, -0.007228858768939972, 0.016032719984650612, 0.022278884425759315, 0.0032139478716999292, 0.027286585420370102, -0.04076160863041878, -0.003621160052716732, -0.00655578076839447, 0.024325041100382805, -0.0012292088940739632, 0.005229816772043705, 0.018375031650066376, 0.008494245819747448, -0.011805789545178413, 0.008447130210697651, 0.00695289671421051, -0.0036649100948125124, 0.013030791655182838, -0.032738517969846725, 0.007558667100965977, -0.01919618621468544, 0.03669621795415878, -0.01907503232359886, -0.017688492313027382, -0.0066331843845546246, -0.0009078140719793737, -0.007915398105978966, -0.00475193141028285, -0.04130006954073906, 0.018253877758979797, -0.018092339858412743, -0.016073103994131088, 0.01290963776409626, 0.024540426209568977, -0.03012697584927082, 0.012034635990858078, 0.01568271964788437, -0.01376444660127163, -0.014053869992494583, 0.0009709151345305145, 0.011859635822474957, -0.023275040090084076, -0.003964429721236229, 0.0058019328862428665, -0.0002526146126911044, 0.019115418195724487, 0.010789441876113415, -0.011341365985572338, 0.02164619043469429, -0.00034474217682145536, -0.023315424099564552, -0.009981747716665268, 0.020946189761161804, 0.008231745101511478, -0.005229816772043705, -0.009086553938686848, 0.009988478384912014, 0.023853886872529984, -0.0031802940648049116, 0.003560583107173443, -0.01422887109220028, -0.008837515488266945, -0.004176449496299028, -0.004344718996435404, 0.022871192544698715, 0.015373103320598602, -0.02065003477036953, -0.02575196698307991, -0.011112519539892673, 0.008649053052067757, 0.008157706819474697, -0.016786567866802216, 0.0223058070987463, 0.016934644430875778, 0.017661569640040398, -0.04205391928553581, 0.02303273230791092, 0.008453860878944397, -0.00484952749684453, 0.005734625272452831, 0.02032695896923542, 0.024405810981988907, -0.029292358085513115, -0.02151157520711422, 0.009322131052613258, -0.009591362439095974, 0.004371642135083675, -0.006407703272998333, 0.02536158263683319, 0.005539432633668184, 0.013003868982195854, -0.015426949597895145, -0.01754041574895382, -0.004987508524209261, -0.002264907816424966, 0.0293192807585001, -0.030557744204998016, 0.00800962932407856, 0.01367021631449461, 0.013347138650715351, -0.0026973604690283537, 0.011206749826669693, 0.005024528130888939, -0.003210582537576556, -0.044261615723371506, 0.0002686002117116004, 0.03675006330013275, 0.01747310720384121, 0.0003045678313355893, 0.008554822765290737, 0.014740410260856152, 0.004711546469479799, 0.02686927653849125, -0.025267351418733597, 0.008231745101511478, 0.022534653544425964, -0.001029809471219778, 0.02198272943496704, -0.6496011018753052, -0.035646215081214905, -0.035376984626054764, -0.014780795201659203, 0.016463490203022957, 0.0031213995534926653, 0.014053869992494583, 0.04019622504711151, 0.000238732376601547, -0.004556738771498203, -0.0029632262885570526, 0.019048109650611877, -0.00410241074860096, -0.005620202049612999, -0.0023911099415272474, -0.015332718379795551, 0.009524054825305939, -0.00010627271694829687, -0.0016338970744982362, -0.026263507083058357, -0.023651963099837303, 0.03238851577043533, -0.005428374744951725, 0.018819263204932213, 0.01177886687219143, 0.005556259769946337, -0.010143286548554897, 0.010345210321247578, -0.027703894302248955, 0.015426949597895145, 0.012633675709366798, 0.01933080330491066, 0.031230824068188667, 4.506468030740507e-05, 0.04918854683637619, -0.020744265988469124, -0.03642698749899864, 0.022076960653066635, 0.024513503536581993, 0.05357701703906059, 0.008750014938414097, -0.01314521487802267, 0.025011582300066948, -0.007504820823669434, 0.007915398105978966, 0.012586560100317001, 0.03341159597039223, 0.022467346861958504, -0.01853657141327858, 0.003977891523391008, 0.006118279881775379, -0.005711067467927933, -0.004893277771770954, 0.0403846837580204, -0.0005422485410235822, -0.027313508093357086, 0.026048121973872185, -0.021161574870347977, -0.002204330638051033, 0.012983676046133041, 0.001839185832068324, 0.024836581200361252, -0.009335593320429325, -0.0508577786386013, 0.004893277771770954, 0.021821191534399986, -0.01155001949518919, 0.016436567530035973, 0.018644262105226517, -0.005902894772589207, -0.004984143190085888, 0.01973464898765087, -0.02039426565170288, -0.004085584077984095, 0.00380625668913126, 0.019236572086811066, 0.020286573097109795, 0.0008985592285171151, -0.025375043973326683, -0.0025863025803118944, 0.008978861384093761, -0.02680196985602379, -0.020219266414642334, -0.01966734230518341, 0.027219276875257492, -0.032469287514686584, -0.006024048663675785, 0.004496161825954914, 0.0415693037211895, -0.0058456831611692905, 0.01662502810359001, 0.021282728761434555, 0.004903373774141073, -0.019034648314118385, -0.02157888375222683, 0.013582715764641762, -0.009948094375431538, -0.008972130715847015, 0.0147000253200531, -0.028107739984989166, -0.025253890082240105, 0.013932716101408005, -0.0026435141917318106, 0.017984647303819656, 0.013414446264505386, -0.010345210321247578, -0.014619256369769573, 0.009025976993143559, 0.02079811319708824, -0.016584644094109535, 0.0034024096094071865, -0.018832724541425705, 0.002118513220921159, 0.011980789713561535, -0.009288477711379528, -0.03276544064283371, -0.010823095217347145, -0.003977891523391008, 0.011341365985572338, 0.005384624470025301, -0.003809622023254633, -0.006946166045963764, 0.00966540165245533, -0.026061583310365677, -0.007188473828136921, 0.03427313640713692, 0.017890416085720062, -0.007558667100965977, -0.016382720321416855, 0.001877887872979045, -0.003479813691228628, 0.017109645530581474, 0.0361577533185482, -0.011792328208684921, 0.013037522323429585, 0.01939810998737812, 0.02303273230791092, -0.016436567530035973, 0.025132736191153526, -0.02270965464413166, -0.012929829768836498, -0.014888486824929714, 0.016571182757616043, -0.014080793596804142, -0.0180788766592741, -0.011684635654091835, -0.022359654307365417, 0.0033603422343730927, -0.022669270634651184, 0.001551444991491735, -0.02615581452846527, 0.024863503873348236, -0.030288513749837875, 0.0232346560806036, 0.020448112860322, -0.0031920727342367172, -0.009645208716392517, -0.0378539115190506, -0.026384660974144936, -0.021148113533854485, 0.0034528905525803566, 0.026451969519257545, -0.029076972976326942, 0.01767503097653389, 0.002414667746052146, 0.02549619786441326, -0.009840401820838451, 0.010977903380990028, 0.006596165243536234, -0.0398731455206871, 0.020286573097109795, -0.0019754841923713684, 0.008945208042860031, 0.018240416422486305, -0.029911590740084648, 0.0046644313260912895, -0.020407726988196373, -0.026371199637651443, 0.0058389524929225445, -0.002293513622134924, 0.003489909926429391, 0.01195386704057455, -0.016988491639494896, -0.0072019356302917, 0.02629042975604534, -0.0006671886658295989, 0.012458675540983677, -0.022413499653339386, -0.0014244015328586102, -0.02654619887471199, -0.0006234385655261576, -0.002978370524942875, -0.0036110638175159693, 0.03026159107685089, -0.00445241155102849, -0.006727415602654219, 0.007316358853131533, 0.009732709266245365, 0.015130795538425446, 0.012418290600180626, 0.020501958206295967, 0.018509646877646446, 0.011011557653546333, -0.006808185018599033, 0.020151957869529724, -0.004788950551301241, 0.011422134935855865, -0.0133067537099123, 0.03136543929576874, 0.007807706017047167, 0.0042302957735955715, -0.033196210861206055, -0.01867118664085865, -0.006273087579756975, -0.0036278909537941217, 0.006700492464005947, 0.0011947136372327805, 0.04243084415793419, 0.0008451336761936545, -0.011287519708275795, 0.015480795875191689, 0.008036551997065544, -0.016301950439810753, 0.013448100537061691, -0.018644262105226517, 0.011765404604375362, -0.006639915518462658, 0.03965776041150093, 0.004213468637317419, 0.017984647303819656, -0.0028033701237291098, 0.02203657664358616, 0.0012494012480601668, 0.022480808198451996, 0.024796197190880775, 0.018684647977352142, 0.007558667100965977, -0.02688273787498474, 0.0447731539607048, -0.01035867165774107, 0.017190413549542427, 0.015144256874918938, 0.014444255270063877, -0.011442327871918678, 0.009631747379899025, 0.013131753541529179, 0.027246201410889626, 0.002047840040177107, 0.0006823329022154212, -0.017526952549815178, -0.02257503941655159, 0.01720387674868107, -0.003259380580857396, 0.02217119187116623, 0.013946178369224072, 0.009544246830046177, -0.01122021209448576, 0.00536106713116169, 0.0032492843456566334, 0.01601925864815712, -0.010190402157604694, 0.01297021470963955, 0.031203899532556534, 0.004563469439744949, 0.012768290936946869, 0.014969256706535816, 0.012095212936401367, -0.021700037643313408, 0.0014445938868448138, -0.03144620731472969, -0.017459645867347717, -0.004839431494474411, 0.014107716269791126, 0.0009044486796483397, 0.008723092265427113, -0.014848102815449238, -0.01674618199467659, 0.004385103937238455, -0.0020192342344671488, 0.03416544198989868, -0.024809658527374268, -0.020340420305728912, 0.023544270545244217, -0.003429332748055458, -0.0013082955265417695, -0.006145203020423651, -0.008130783215165138, 0.012896176427602768, -0.020488496869802475, 0.04964623972773552, 0.014000024646520615, 0.00996155571192503, -0.007975975051522255, -0.009436555206775665, -0.009059631265699863, -0.001736541511490941, 0.0066264537163078785, -0.004906739108264446, 0.025455813854932785, -0.009133669547736645, 0.0036480831913650036, -0.00495048938319087, -0.0122096361592412, -0.009349054656922817, -0.002552648540586233, -0.023786580190062523, -0.022601962089538574, -0.00856828410178423, -0.001530411303974688, -0.02734043262898922, 0.002495436929166317, -0.027313508093357086, -0.00014607897901441902, -0.012088482268154621, 0.0022245231084525585, 0.004873085301369429, -0.011644250713288784, 0.025671198964118958, 0.022682731971144676, -0.021821191534399986, -0.014269255101680756, 0.01635579764842987, -0.008561553433537483, 0.004533180966973305, 0.11296942830085754, -0.015925027430057526, -0.011496174149215221, -0.004018275998532772, 0.011846174485981464, -0.029884666204452515, -6.399500125553459e-05, -0.04523084685206413, -0.0162211824208498, 0.0023490425664931536, 0.006266356911510229, -0.003218995872884989, 0.005186066497117281, 0.022076960653066635, -0.006878857966512442, -0.017796184867620468, -0.026761583983898163, -0.026061583310365677, -0.004449046216905117, -0.005189431831240654, 0.005704336799681187, 0.0004234081716276705, 0.005687509663403034, 0.030234668403863907, 0.0017247626092284918, 0.024405810981988907, 0.008992323651909828, 0.011186557821929455, 0.010338479653000832, -0.028942357748746872, -0.006986550521105528, -0.001484137144871056, 0.0015960363671183586, 0.01814618520438671, -0.02859235741198063, -0.0048158736899495125, -0.0014631034573540092, -0.019344264641404152, 0.010775980539619923, -0.02039426565170288, 0.02389427274465561, 0.007881744764745235, 0.006710588466376066, 0.005229816772043705, 0.0029598609544336796, 0.01354233082383871, -0.006761069409549236, 0.005882702302187681, -0.02384042553603649, 0.0015951950335875154, 0.014619256369769573, 0.006532222963869572, -0.00027617235900834203, 0.002645196858793497, 0.01390579342842102, 0.02210388518869877, 0.0034495252184569836, -0.006791357882320881, -0.026909662410616875, -0.0288077425211668, -0.025900045409798622, -0.02396157942712307, 0.01274809893220663, 0.0009751218603923917, -0.024540426209568977, 0.013825023546814919, 0.012324059382081032, 0.008238475769758224, -0.00765962852165103, 0.013878869824111462, 0.0032391883432865143, -0.029911590740084648, -0.010143286548554897, 0.0012098578736186028, 0.02303273230791092, -0.009409631602466106, -0.0006360588013194501, 0.0013040887424722314, 0.004674527328461409, 0.0010491604916751385, 0.026236584410071373, -0.0017415895126760006, -0.004257218912243843, -0.027205815538764, 0.002328850096091628, 0.017109645530581474, -0.007565397769212723, 0.01025770977139473, -0.006441357079893351, 0.009779824875295162, 0.027044277638196945, -0.005815394688397646, 0.0232346560806036, -0.008480783551931381, -0.012532713823020458, 0.013784639537334442, 0.018119262531399727, -0.013892332091927528, 0.010028863325715065, 0.006225972436368465, 0.015844257548451424, -0.015548103488981724, 0.007390397600829601, 0.007329820189625025, 0.028511587530374527, -0.0008716361480765045, -0.01601925864815712, 0.009234631434082985, 0.006249530240893364, -0.00880386121571064, 0.016705797985196114, -0.008810591883957386, 0.006713953800499439, -0.009227900765836239, -0.028107739984989166, -0.0010685115121304989, -0.0030086589977145195, 0.013986562378704548, -0.00736347446218133, 0.021767346188426018, 0.0011955549707636237, -0.014417332597076893, 0.026694277301430702, 0.01728464476764202, -0.016328874975442886, 0.005912990774959326, -0.0052129896357655525, -0.004075487609952688, 0.002596398815512657, -0.008130783215165138, -0.004213468637317419, 0.0040283724665641785, -0.015426949597895145, -0.014417332597076893, -0.03771929442882538, -0.0260211993008852, -0.03467698395252228, 0.0013301705475896597, -0.01649041287600994, -0.012478867545723915, 0.018900033086538315, -0.0074509745463728905, 0.0007471166900359094, 0.026667354628443718, 0.014538486488163471, -0.01642310619354248, 0.00014366010145749897, -0.007343281991779804, 0.013946178369224072, 0.005478855688124895, -0.0026973604690283537, -0.01506348792463541, -0.034919291734695435, 0.0183346476405859, -0.00910674687474966, -0.038096219301223755, -0.034515444189310074, -0.009591362439095974, -0.000678126176353544, 0.03090774640440941, -0.00160108448471874, -0.022426960989832878, 0.015332718379795551, 0.01105867326259613, -0.01502310298383236, -0.012862522155046463, -0.018846185877919197, -0.01015674788504839, -0.010977903380990028, 0.005812029354274273, 0.018186569213867188, -0.00037881676689721644, 0.005933183245360851, 0.0027175527065992355, -0.01568271964788437, 0.04062699154019356, 0.011832713149487972, -0.013919254764914513, -0.01270771399140358, -0.015171180479228497, 0.001466468907892704, -3.5257722629467025e-05, -0.030826976522803307, 0.013777908869087696, -0.024917351081967354, 0.018886571750044823, 0.025792352855205536, 0.008696168661117554, -0.000731131061911583, 0.0035942369140684605, 0.002147119026631117, -0.004889912437647581, 0.04649623483419418, -0.005475490354001522, -0.006525491829961538, -0.0054081822745501995, -0.0025543314404785633, -0.024015426635742188, -0.006730780936777592, 0.020892344415187836, -0.0014513246715068817, -0.018953878432512283, 0.01386540848761797, -0.01887311041355133, -0.01701541431248188, 0.03833852708339691, -0.01642310619354248, -0.010392325930297375, -0.004368276800960302, -0.006939434912055731, 0.0019721188582479954, -0.03755775839090347, 0.01912887953221798, -0.031338516622781754, -0.014646179042756557, -0.00042445986764505506, -0.038823142647743225, 0.032926980406045914, -0.041084688156843185, 0.01772887632250786, -0.001161059713922441, -0.019357725977897644, 0.01035867165774107, 0.009174054488539696, -0.0027276489417999983, 0.013320215046405792, -0.0009305304265581071, -0.00630674185231328, 0.0012838965049013495, -0.018092339858412743, -0.0052163549698889256, 0.01227694470435381, -0.004311065189540386, 0.011401942931115627, -0.021861575543880463, -0.012700983323156834, 0.01015674788504839, -0.018576955422759056, -0.01350867748260498, 0.01640964299440384, -0.019761573523283005, -0.013091368600726128, -0.011570212431252003, 0.0183346476405859, -0.012997137382626534, 0.018496185541152954, -0.013151945546269417, 0.0168134905397892, -0.032200057059526443, -0.03858083486557007, -0.023073116317391396, 0.026828892529010773, -0.032738517969846725, 0.009847132489085197, 0.019451957195997238, -0.00946347787976265, -0.0015396660892292857, 0.004041833803057671, 0.007982705719769001, 0.0007954941829666495, 0.0029093800112605095, 0.033734675496816635, -0.03957699239253998, 0.0021959173027426004, 0.014053869992494583, 0.011819250881671906, -0.017971184104681015, 0.0019788495264947414, 0.036184679716825485, 0.023732732981443405, -0.029507743194699287, -0.019101956859230995, 0.013279831036925316, -0.004539911635220051, 0.0293192807585001, -0.0098000168800354, -0.017500029876828194, -0.008285591378808022, 0.01095771137624979, 0.013434638269245625, 0.013320215046405792, 0.008992323651909828, -0.0068216463550925255, -0.034003905951976776, -0.009995209984481335, -0.017109645530581474, -0.0016566134290769696, -0.00800962932407856, -0.011139442212879658, -0.01721733808517456, -0.0042336611077189445, -0.0022665904834866524, -0.0072153969667851925, 0.028969280421733856, 0.0299923587590456, 0.010823095217347145, -0.01872503198683262, 0.021134652197360992, -0.03136543929576874, 0.0008893044432625175, 0.01642310619354248, 0.028646202757954597, -0.025321196764707565, 0.011334635317325592, -0.004731738939881325, -0.004267314914613962, -0.0024988024961203337, -0.005700971465557814, 0.001455531339161098, -0.027892356738448143, 0.004277411382645369, -0.008931745775043964, -0.0024230810813605785, -0.014511563815176487, 0.001580050797201693, -0.025307735428214073, 0.004974047187715769, -0.027892356738448143, -0.05360393971204758, 0.00561010604724288, -0.006811550352722406, -0.00395433371886611, 0.016140412539243698, -0.03203851729631424, 0.024082733318209648, 0.006986550521105528, 0.00040868460200726986, -0.010264440439641476, 0.00036030710907652974, -0.0050615472719073296, 0.008117321878671646, 0.0014286082005128264, 0.014551947824656963, -0.011287519708275795, -0.007693282328546047, -0.0031466400250792503, 0.004691354464739561, 0.023463502526283264, -0.013959639705717564, 0.005939913913607597, 0.012256751768290997, 0.0024483215529471636, 0.01000194065272808, 0.0035740446764975786, 0.00956443976610899, -0.0039846221916377544, 0.00906636193394661, -0.016719259321689606, -0.01860387809574604, -0.006451453547924757, 0.003131495788693428, 0.008635591715574265, 0.017661569640040398, -0.009975017048418522, 0.00039080597343854606, -0.023651963099837303, -0.06386164575815201, 0.017823107540607452, 0.013286561705172062, 0.041084688156843185, 0.01556156575679779, 0.03004620596766472, 0.0010500018252059817, -0.024594273418188095, 0.0004762027529068291, 0.021161574870347977, 0.018294261768460274, 0.00070126325590536, 0.019438495859503746, -0.017055798321962357, -0.006612992379814386, 0.014323101378977299, -0.031203899532556534, -0.010688479989767075, 0.024998120963573456, 0.014780795201659203, -0.0022093788720667362, 0.005717798136174679, -0.010102901607751846, -0.03421929106116295, 0.013125022873282433, 0.0028925531078130007, 0.038688529282808304, 0.02622312307357788, 0.007134627550840378, 0.0007618402596563101, -0.028350049629807472, 0.013629831373691559, -0.005690874997526407, -0.007471166551113129, 0.010924057103693485, 0.004893277771770954, 0.014538486488163471, 0.022023115307092667, -0.03117697685956955, -0.01595195010304451, 0.005179335828870535, -0.01257309876382351, 0.022265423089265823, -0.005711067467927933, -0.0016532480949535966, 0.012478867545723915, -0.008070206269621849, -0.012021174654364586, -0.0034528905525803566, -0.013629831373691559, -0.017850030213594437, -0.021619267761707306, -0.005303855519741774, -0.018186569213867188, -0.006690395995974541, -0.022871192544698715, -0.020905805751681328, -0.003920679911971092, -0.017823107540607452, -0.012875983491539955, 0.0002911062620114535, -0.0017533684149384499, 0.016503874212503433, 0.03195774927735329, -0.01171828992664814, -0.0260211993008852, -0.038688529282808304, -0.006592799909412861, 0.00655578076839447, 0.006374049466103315, 0.02443273365497589, -0.023651963099837303, -0.005381259135901928, -0.006061068270355463, -0.0028706779703497887, -0.014457717537879944, 0.013528869487345219, 0.003796160453930497, -0.015453873202204704, 0.023261578753590584, 0.22400039434432983, -0.0015186324017122388, -0.021821191534399986, 0.04375007376074791, 0.011576943099498749, -0.0011299297912046313, 0.01079617254436016, 0.00670385779812932, -0.007726936601102352, 0.0030221205670386553, -0.0452846921980381, 0.0014656275743618608, 0.006508665159344673, 0.0006600372144021094, 0.013030791655182838, -0.021134652197360992, -0.05831548571586609, -0.025105813518166542, -0.004240391775965691, -0.015211564488708973, 0.02073080465197563, -0.026115430518984795, 0.008736553601920605, -0.016705797985196114, 0.015117334201931953, 0.0018240415956825018, -0.023328887298703194, 0.001807214692234993, 0.02899620309472084, 0.029642358422279358, 0.01270771399140358, -0.006222607102245092, -0.0180788766592741, 0.008090398274362087, -0.001071876846253872, 0.013959639705717564, 0.03150005266070366, -0.0147000253200531, 0.00930193904787302, 0.032011594623327255, -0.025671198964118958, 0.010405787266790867, 0.02713850885629654, -0.01772887632250786, 0.002460100455209613, -0.0031382266897708178, -0.012559637427330017, 0.0020579362753778696, 0.026451969519257545, 0.010897134430706501, -0.04183853417634964, 0.0014883439289405942, 0.0018913494423031807, 0.01344136893749237, 0.0025122640654444695, 0.001931734150275588, 0.018900033086538315, 0.0013638244708999991, -0.004533180966973305, -0.009429823607206345, 0.0033250057604163885, 0.033653903752565384, -0.02495773509144783, 0.008063475601375103, -0.011772135272622108, 0.009416362270712852, -0.0006545684300363064, -0.006565876770764589, 0.01806541532278061, 0.006858665496110916, 0.013454831205308437, -0.025132736191153526, 0.005583182908594608, 0.011893290095031261, -0.02767697162926197, -0.025644274428486824, 0.0017735607689246535, -0.036103907972574234, 0.03653467819094658, 0.043803922832012177, -0.007471166551113129, -0.025671198964118958, -0.016073103994131088, 0.013959639705717564, 0.01747310720384121, -0.037369295954704285, 0.0281346645206213, 0.014376947656273842, 0.011072134599089622, 0.006532222963869572, 0.012976945377886295, 0.004051930271089077, 0.011684635654091835, -0.00711443554610014, 0.006777896545827389, 0.0359962172806263, -0.001295675290748477, 0.0023372636642307043, -0.014269255101680756, 0.01793080009520054, 0.005425009410828352, 0.021201958879828453, -0.001739906845614314, -0.011159634217619896, -0.005912990774959326, -0.009409631602466106, -2.5726832973305136e-05, 0.0033502462320029736, 0.017836568877100945, 0.006094722077250481, -0.008662515319883823, -0.005549528636038303, -0.008716361597180367, -0.002990149427205324, -0.01667887531220913, 0.004903373774141073, 0.005892798770219088, -0.04706161841750145, 0.019411571323871613, -0.0174865685403347, 0.009584631770849228, -0.01079617254436016, 0.005737990606576204, 0.018428878858685493, -0.004846162162721157, -0.010351940989494324, -0.03031543642282486, -0.01772887632250786, 0.014888486824929714, -0.026425044983625412, -0.007006742991507053, 0.012021174654364586, -0.0005292076384648681, -0.01814618520438671, -0.018886571750044823, -0.004556738771498203, 0.013246176764369011, -0.035376984626054764, 0.0026889469008892775, 0.029938513413071632, 0.035323139280080795, -0.0050581819377839565, 0.0006671886658295989, -0.010284633375704288, -0.003272842150181532, -0.020084649324417114, 0.004220199771225452, 0.00021464881137944758, -0.014026947319507599, 0.0011013239854946733, -0.008480783551931381, -0.014605794101953506, -0.005812029354274273, -0.042242381721735, 0.02038080431520939, -0.021215422078967094, -0.0380154512822628, -0.015426949597895145, -0.0012367810122668743, -0.005579817108809948, -0.03411159664392471, 0.01260675210505724, 0.021767346188426018, 0.0008177898707799613, -0.016988491639494896, 0.010506749153137207, -0.17198491096496582, 0.014551947824656963, 0.01141540426760912, -0.018819263204932213, 0.025994274765253067, 0.02515965886414051, 0.034919291734695435, 0.0008741601486690342, -0.025536581873893738, 0.003082697745412588, 0.013878869824111462, 0.0052062589675188065, -0.053065475076436996, -0.02507888898253441, -0.0034562558867037296, 0.024648118764162064, -0.0018980802269652486, -0.004661065526306629, 0.026532737538218498, -0.007127896882593632, 0.0380154512822628, -0.023988502100110054, 0.011994251050055027, 0.004486065357923508, 0.010392325930297375, 0.01045963354408741, -0.00759905157610774, 0.0048226043581962585, 0.002232936443760991, -0.001178727950900793, -0.014848102815449238, 0.01344136893749237, 0.031203899532556534, -0.010304825380444527, -0.005320682190358639, 0.007154820021241903, -0.00031108828261494637, -0.03166159242391586, -0.0095038628205657, 0.005552894435822964, 0.0019939937628805637, 0.0016027671517804265, 0.011570212431252003, -0.002441590651869774, 0.018550032749772072, 0.014026947319507599, 0.007834629155695438, -0.004358180798590183, 0.005303855519741774, -0.0028420721646398306, 0.009786555543541908, -0.020165419206023216, 0.006222607102245092, 0.007309628184884787, 0.014834641478955746, -0.006727415602654219, 0.014457717537879944, 0.029642358422279358, 0.004237026441842318, -0.0026788508985191584, -0.021753882989287376, -0.01141540426760912, 0.003035582136362791, -0.012593290768563747, 0.005364432465285063, -0.029103895649313927, -0.009705785661935806, -0.012761560268700123, -0.02058272808790207, 0.006175491493195295, 0.0162211824208498, 0.029050050303339958, 0.010042324662208557, -0.010432709939777851, 0.017190413549542427, -0.0025863025803118944, -0.005889433436095715, 0.00814424455165863, -0.019303878769278526, -0.03728852421045303, -0.013643292710185051, 0.05196162685751915, -0.019963495433330536, -0.013488484546542168, 0.021363498643040657, 0.02529427409172058, 0.002175724832341075, -0.02541542798280716, 0.01939810998737812, -0.0036514485254883766, 0.04294238239526749, -0.03139236196875572, -0.015103871934115887, 0.006875492632389069, 0.014901949092745781, 0.016638489440083504, 0.02985774353146553, 0.003688467899337411, -0.0017668299842625856, -0.006528857629746199, 0.020286573097109795, 0.011597135104238987, 0.004566834773868322, 0.02958851307630539, 0.029211588203907013, 0.0006760227843187749, -0.0016582960961386561, 0.006360588129609823, 0.015669258311390877, -0.012963484041392803, 0.01519810315221548, 0.005714432802051306, 0.02098657563328743, 0.00940290093421936, 0.0037793335504829884, 0.029076972976326942, 0.00012946236529387534, -0.015036564320325851, 0.006545684300363064, -0.005128854885697365, 0.011671174317598343, 0.009375977329909801, -0.03209236264228821, 0.006892319768667221, -0.014969256706535816, -0.01608656533062458, -0.0893847718834877, 0.007477897685021162, 0.010419248603284359, -0.004438949748873711, -0.004459142219275236, 0.01873849332332611, 0.001221636775881052, 0.01416156254708767, 0.00959809310734272, 0.004125968553125858, -0.01356925442814827, -0.0427539199590683, -0.0009254823671653867, 0.00360096781514585, 0.03742314130067825, -0.0046677966602146626, 0.012579829432070255, -0.022023115307092667, -0.010950980708003044, 0.013737523928284645, 0.011792328208684921, 0.0008867803844623268, 0.007518282160162926, -0.019425032660365105, 0.03588852286338806, 0.013697138987481594, -0.011765404604375362, 0.0011030067689716816, 0.03613083064556122, 0.018819263204932213, -0.019317341968417168, -0.011449058540165424, -0.01406733226031065, 0.006939434912055731, 0.01370386965572834, -0.017769262194633484, -0.04148853197693825, -0.0011375019093975425, 0.011792328208684921, -0.009335593320429325, 0.0011938723037019372, 0.0032425536774098873, -0.0015211565187200904, -0.023140424862504005, -0.0060846260748803616, -0.021363498643040657, -0.008447130210697651, 0.02191542275249958, -0.014592332765460014, -0.01175194326788187, -0.016732720658183098, 0.005943279713392258, -0.023396193981170654, -0.01786349155008793, -0.006094722077250481, -0.007370205130428076, 0.010210594162344933, 0.013596177101135254, 0.008850976824760437, -0.007329820189625025, -0.014013485983014107, 0.008723092265427113, -0.010897134430706501, 0.013522138819098473, 0.019815418869256973, 0.01662502810359001, -0.010526941157877445, 0.0012510839151218534, -0.0019300513667985797, -0.004963950719684362, -0.0006810709019191563, -0.0033367846626788378, -0.0076528978534042835, 0.02017888054251671, -0.028242357075214386, -0.01576348952949047, -0.004546642303466797, -0.006538953632116318, -0.00040595020982436836, -0.00880386121571064, 0.007632705383002758, -0.017432723194360733, 0.01185290515422821, -0.004314430523663759, 0.003298082621768117, 0.01506348792463541, 0.0062293377704918385, -0.018105801194906235, -0.008050014264881611, -0.024607734754681587, 0.008339437656104565, 0.009490400552749634, -0.004263949580490589, -0.0223058070987463, 0.005875971633940935, 0.006050971802324057, -0.009524054825305939, -0.023019270971417427, -0.019896188750863075, 0.0021959173027426004, -0.01780964620411396, -0.01654426008462906, -0.08453860878944397, 0.0190077256411314, -0.0010373815894126892, -0.027757739648222923, 0.007504820823669434, -0.012754829600453377, 0.030019283294677734, -0.0005405658157542348, 0.007275973912328482, -0.009019246324896812, -0.01589810475707054, -0.008790399879217148, 0.013232715427875519, 0.008184629492461681, -0.023409655317664146, -0.016638489440083504, -0.0013680312549695373, -0.023153886198997498, 0.01879234053194523, 0.01867118664085865, -0.021551961079239845, -0.013333677314221859, 0.0378539115190506, -0.0017651473172008991, -0.05056162551045418, -0.005095201078802347, -0.016907721757888794, 0.013400984928011894, -0.002404571510851383, -0.01846926286816597, 0.018765417858958244, -0.028350049629807472, -0.0008951938361860812, 0.02958851307630539, -0.031150054186582565, -0.015857718884944916, -0.01370386965572834, 0.01131444238126278, 0.010728864930570126, 0.022871192544698715, -0.038823142647743225, 0.0015497623244300485, 0.02046157419681549, -0.006690395995974541, 0.002936303149908781, -0.007289435714483261, 0.006421165075153112, -0.0014050505124032497, 0.015480795875191689, -0.005004335660487413, 0.015103871934115887, 0.022346192970871925, 0.008056744933128357, -0.0021538499277085066, 0.0036278909537941217, -0.013959639705717564, -0.008871168829500675, 0.01465964037925005, 0.026694277301430702, -0.010863480158150196, 0.011630789376795292, 0.017755798995494843, 0.024998120963573456, 0.007881744764745235, 0.008709629997611046, 0.012061559595167637, -0.0003518936282489449, -0.00536106713116169, -0.007592320907860994, -0.017836568877100945, -0.011381749995052814, 0.01177886687219143, 0.0015421902062371373, 0.023665426298975945, 0.010775980539619923, -0.011765404604375362, -0.0010600979439914227, -0.01122021209448576, -0.026909662410616875, 0.009490400552749634, -0.0006793881766498089, 0.0050110663287341595, -0.030961591750383377, 0.00854136049747467, 0.0060442411340773106, -0.013475023210048676, -0.009611555375158787, -0.010823095217347145, -0.007666359189897776, 0.008433668874204159, -0.019694264978170395, 0.006259626243263483, 0.005670682992786169, 0.009894248098134995, 0.012027905322611332, -0.001909859012812376, 0.02316734753549099, -0.025334659963846207, 0.02245388552546501, 0.026761583983898163, 0.011072134599089622, 0.021861575543880463, -0.0062394337728619576, -0.021067343652248383, -0.014188486151397228, -0.010452902875840664, -0.010971172712743282, -0.01195386704057455, 0.0144038712605834, 0.0017971184570342302, 0.015480795875191689, 0.01894041709601879, 0.0017331760609522462, -0.0011088961036875844, -0.023584656417369843, -0.019034648314118385, 0.016961567103862762, -0.016598105430603027, 0.008931745775043964, -0.005243278108537197, 0.01277502253651619, 0.03494621440768242, 0.03828468173742294, 0.010385595262050629, 0.001476565026678145, -0.017459645867347717, -0.008527899160981178, -0.0002850065066013485, -0.012270213104784489, -0.0030153898987919092, -0.006542318966239691, 0.0002433597983326763, -0.03012697584927082, -0.014390409924089909, -0.0038028911221772432, -0.006693761795759201, -0.003330053761601448, 0.026196198537945747, -0.010843288153409958, 0.054411631077528, 0.022669270634651184, -0.01642310619354248, 0.01390579342842102, -0.01231059804558754, 0.02867312729358673, 0.0001604870631126687, -0.008958669379353523, -0.01826733909547329, -0.025765428319573402, 0.019963495433330536, -0.024446196854114532, -0.009160593152046204, -0.012189444154500961, -0.013151945546269417, 0.003473083022981882, 0.01091732643544674, 0.01581733487546444, 0.0003931196697521955, -0.007861551828682423, -0.0024298119824379683, -0.021605806425213814, 0.03330390527844429, -0.005159143358469009, -0.03553852438926697, 0.020475035533308983, 0.01472694892436266, 0.002069715177640319, -0.0038533720653504133, -0.024607734754681587, 0.009046169929206371, -0.0002187503851018846, -0.024244273081421852, -0.00916732382029295, 0.022938501089811325, -0.00137392058968544, -0.011570212431252003, -0.004587027244269848, 0.012068290263414383, 0.03411159664392471, -0.01254617515951395, 0.00549568235874176, -0.012027905322611332, -0.042726997286081314, 0.008763476274907589, 0.010513479821383953, -0.0035101021640002728, -0.008150975219905376, -0.019357725977897644]} +{"id": "test:10000015", "text": "\"\\\"Bake me a pie or go away,\\\" I've literally had my son say that to me a few times. So, what was I to do except fulfilling his demand? Baking (not only pies), is a cooking method where tools DO actually make the man, which means that it is very hard or nearly impossible to do it successfully without proper bakeware.\\nWith years of experience under my belt, I\u2019ve learned about and discovered what truly matters when looking for bakeware to buy, and I am going to teach you later in the article.\\nAlso, I have reviewed what I believe to be the best five non-stick bakeware sets out there, so make sure to check them out if you are in search of one.\\nThe first and most obvious reason is \u2013 because food doesn\u2019t stick to it! I do not want to sound overly simplistic, but any chef (or mom, as a matter of fact) knows that pans that stick will absolutely mess up the outcome.\\nPeople normally solve this problem by oiling up, but I don\u2019t like it since it makes the food greasy. Using non-stick bakeware is a healthier, more convenient, and more practical way of baking.\\nThe next thing that you might not have known is that non-stick bakeware is so much easier to clean than more conventional types. It is a no-brainer really \u2013 if nothing sticks to the surface then there won\u2019t be anything to scrub, scrape or wash off.\\nThis trait right here will save you tons of time as well as frustration, and it might motivate you to bake more often because of the hassle-free cleaning process. Just make sure to maintain the bakeware.\\n\u200bLastly, even heat distribution is another thing that you should look for, and this type of bakeware enables it.\\nHere's the deal: heat distribution most likely won't affect the quality of the result, but it will make the bakeware heat up more quickly which will ultimately decrease the cooking time.\\nInvesting a little bit of time into research can save you a lot of money and trouble in the long run, and that is why you need to read this section carefully.\\nBakeware sets differ regarding materials, abilities, cost, and maintenance, so learning about some general features of these products will help you decide which type will fulfill your demands.\\nBakeware can be made from different kinds of materials including iron, glass, copper, steel, etc.\\nToday I will mention only the types that are related to products I reviewed in this article.\\nIt is important to know that each material yields different cooking properties so study these paragraphs to find out which one is right for you.\\nCarbon steel \u2013 this material is light, durable, and reacts to temperature changes very quickly, meaning that it can heat up fast. Even though is lightweight, it is a bit heavier than copper and doesn't retain heat as well as iron. It's a good choice for baking meats, veggies, bread, and cakes.\\nCopper \u2013 it is an excellent heat conductor and doesn\u2019t need pre-heating since it gets hot very quickly. Very light, but also quite expensive compared to other bakeware materials. Also, it isn\u2019t the best choice for recipes that require baking under very high heat.\\nSilicon \u2013 good for foods where sticking is an issue, such as breads, cookies, and cakes. It is light, affordable and dishes will slide out of it easily. The one downside is that it can be frustrating to wash because it is so elastic, but luckily, most of the silicon bakeware is dishwasher safe.\\nSince you\u2019ll be buying an entire set of products, it would be a good idea to find one that is suitable for all kinds of baking jobs. Make sure that all the basic items such as pie pan, muffin pan, loaf pan, cookie sheet, cake pan, or whatever you need, is included in the package.\\nThis is a feature that most folks overlook, and it makes a world of difference when it comes to handling the pan. It's easy to move the dish around when it's cold but taking it out of a scorching hot oven is a different story (trust me, I know). You\u2019ll try to be cautious not to burn your fingers so it is easy not to pay attention to the contents of the pan which can lead to spillage and a messy kitchen.\\nLook for bakeware that has nice, ergonomic handles or handling areas. Some models have silicon added to these parts for extra protection and ease of use.\\nNot all types of bakeware can withstand the same temperatures; as a matter of fact, this trait greatly differs from product to product. Certain pans are oven safe to up to 500 degrees Fahrenheit, while others cannot take more than 350.\\nThis is a significant difference so pay close attention to this detail when buying a set.\\nNow, let\u2019s get into the meaty part of this article \u2013 the reviews.\\nAfter trying out many sets, I\u2019ve reached some conclusions and decided to put together a list of my favorite ones.\\nKeep in mind that these are my personal choices, but they are also top rated products.\\nRachel Ray is a world-renowned chef and TV personality, and her famous Cucina line of products never fails to deliver. Here, we are dealing with an excellent, 10-piece set that I personally think very highly of.\\nThe material that this bakeware is made of is carbon steel, and the non-stick, latte-colored coating looks very pretty and does the job well.\\nA thing that I didn\u2019t like about the construction of this set is that everything is so bulky. It is not a huge issue, but folks who lack storage space will not be thrilled with these products.\\nAs far as items are concerned, the set includes two loaf pans, two cake pans, two cookie sheets, two pie plates, a muffin tin, and a plastic cake pan topper.\\nIn my opinion, this is more than enough, but I\u2019m sure some folks are going to be disappointed because there\u2019s no pizza pan included.\\nThe handles have silicon grips attached to them, and this is a big bonus when it comes to practicality and handling. Also, the rolled edges allow you to grab the pan more easily and move it around securely.\\nThe pan can withstand heat of up to 450 F, and I think that this should be sufficient for most baking jobs. Make sure not to go over this temperature because you could burn the pans.\\nCopper has gained a lot of popularity in the cooking world over the last few years, and honestly, I can only say good things about it. Copper Chef is known for a plethora of cookware, and now we are taking a look at their baking set.\\nAs I've mentioned before, copper is a fantastic heat conductor, and the heat distribution in this material is as even as it gets.\\nPersonally, I find the latter feature very important because I like to have consistency in my cooking.\\nThis is a 12-piece set that consists of a 12-cup muffin pan, a cookie sheet, a loaf pan, a square pan, and eight pieces of silicon ramekin cups with lids.\\nOk, so I think it is a bit unfair for a company to advertise a set of twelve copper products while only four of them are actually made of this material. While the ramekin cups are lovely, they seem to be a cheap filler in this set.\\nAs far as handling is concerned, most of the products in this pack do not have handles, so you have to rely on the rolled edges for gripping. This is a big disadvantage in my opinion, and greatly handicaps the ease of use.\\nThese black, sturdy-looking baking dishes remind me of old-school bakeware sets that people used to bake with a few decades ago. However, this is a modern and versatile set that should fulfill most of your demands.\\nThe ChefLand set is made of carbon steel; it looks pretty plain but has a non-stick coating that's quite effective. One thing that this material is good at is even heat distribution, so you won\u2019t need to worry about hotspots.\\nThe products are dishwasher safe, however, after several months of kind of cleaning, some items started to form rust on the outer edges, which is a big minus. I would recommend washing them by hand.\\nWhen it comes to the items, ChefLand\u2019s set includes a roasting pan, a round pizza pan, a large and medium cookie sheet, two round cake pans, a square cake pan, a loaf pan, an oven crisper pan, and a 12-cup muffin pan. As far as I\u2019m concerned this set has everything covered!\\nMost of the dishes have handles on them, and even those which don't, have a nice grip area that allows for good handling.\\nUnfortunately, there is no silicon padding or protective surface so you will need to use gloves or cloth.\\nThis bakeware set comes to us from Sunbeam; it is a bit smaller set in terms of the number of items in it, but a worthy addition to today's list. Oh, I forgot to mention it\u2019s cheap as chips!\\nThe material these pans are made of is carbon steel, and its non-stick features are achieved with a xylan coating interior and exterior. The black color gives it a pretty plain, non-exciting look, but I personally do not mind this.\\nThe Sunbeam set has only 5-pieces, which is significantly less than other sets we reviewed today. The dishes included are a loaf pan, a cookie sheet, a 6-cup muffin pan, and two round cake pans.\\nOne disadvantage of this set is that it cannot handle temperatures higher than 400 F, which is quite limiting because some recipes require more heat.\\nAll of the pans have handles or enough room to grab them properly and handle safely.\\nNow as I was reading some online commentary, many people complained that pans arrived dented or bent when they ordered them, so this might be an issue.\\nI haven\u2019t experienced this myself, so I just want to put it out as a warning.\\nThis set differs from others I reviewed today because it is the only one that\u2019s not made of metal.\\nBoxiki Kitchen brings us this 3-piece silicone set that bread and cake lovers might find interesting.\\nAs I\u2019ve said, these products are made of silicone which has its pros and cons.\\nThe good thing is that food will just slide out of it, but it might be irritating when trying to wash it by hand because it\u2019s so bendy. A lot of people raise concerns about chemicals in this material, so you\u2019ll be glad to know that this set is FDA approved and non-toxic.\\nThe set comes with three items \u2013 a round cake pan, a square brownie pan, and a banana bread/meatloaf pan.\\nAll of the dishes are equipped with metal handles which allows for a nice and solid grip, so you won\u2019t have to worry about the pans slipping out of your hands.\\nSilicone is generally oven safe up to 500 degrees F, which is fantastic and much better than some metal products, so you won\u2019t need to worry about burning your bakeware.\\nHowever, the metal handles don't seem to take high temperatures very well, and I've heard some reports of them flaking and bubbling at 400 F.\\nI hope that the article was helpful and that it got you acquainted with the basics of bakeware.\\nNow, the time has come for me to declare my top pick of the day, and the product that got most of my sympathies is Rachel Ray\u2019s 10-piece Cucina Set.\\nIts beautiful design is what attracted me to this bakeware set, but the excellent performance made an even stronger impression.\\nNon-stick surface does its job flawlessly, and the carbon steel construction is sturdy and strong.\\nThis is a very good list, i have tested myself the Rachel Ray\u2019s 10-Piece Cucina Bakeware Set and i\u2019m satisfied with it, it fulfill most of my daily cooking.\"", "vector_field": [0.00780827458947897, -0.009468036703765392, 0.009642749093472958, -0.021180449053645134, -0.020710071548819542, -0.006850719451904297, 0.003904137294739485, -0.024661246687173843, -0.030507372692227364, -0.05531645193696022, 0.011053882539272308, 0.011947601102292538, -0.01708144135773182, -0.015199929475784302, -0.0027785899583250284, 0.018075954169034958, 0.03203945979475975, 0.03023858554661274, 0.006840640213340521, -0.04969879612326622, -0.04416177421808243, 0.0016614423366263509, 0.017175516113638878, 0.007774676196277142, -0.020212814211845398, 0.0061955503188073635, 0.011772889643907547, -0.005684853997081518, -0.000865999492816627, 0.013278098776936531, 0.0023350908886641264, 0.01868072710931301, -0.025145063176751137, -0.006975033786147833, -0.02654275856912136, -0.0026979537215083838, -0.0062224287539720535, -0.0019184702541679144, 0.008251774124801159, -0.016019729897379875, 0.014944581314921379, -0.005157358944416046, 0.00289618456736207, 0.0015992851695045829, -0.02665027417242527, 0.0052951122634112835, -0.00809722114354372, 0.0003700027009472251, -0.023008203133940697, -0.011786328628659248, 0.04730658605694771, 0.013533446937799454, -0.022510947659611702, 0.001473291078582406, 0.005873005371540785, -0.011262193322181702, -0.0054026274010539055, 0.010576785542070866, 0.031878188252449036, -0.02181209996342659, 0.001439692685380578, 0.027496954426169395, -0.0024694844614714384, 0.004999446216970682, 0.02238999307155609, -0.021529873833060265, -0.01056334562599659, -0.018250666558742523, -0.015267126262187958, 0.005197676829993725, 0.013479689136147499, 0.024137111380696297, 0.027281923219561577, -0.007552926894277334, 0.025386972352862358, 0.0036958272103220224, -0.0052783130668103695, 0.01865384727716446, 0.006299705244600773, 0.003022178541868925, 0.020400965586304665, -0.02765822596848011, 0.0003334644134156406, 0.04652710258960724, 0.011168117634952068, 0.012149191461503506, -0.03058800846338272, 0.0219330545514822, -0.008701993152499199, -0.0025366812478750944, -0.015347762033343315, -0.0021335002966225147, 0.013459530659019947, -0.008614636957645416, -0.013674560002982616, 0.05238667130470276, -0.018143150955438614, 0.03679699823260307, -0.01640947163105011, -0.03362530842423439, 0.009709945879876614, 0.008869985118508339, -0.011242033913731575, -0.02182553894817829, -0.03878602758049965, 0.0072975787334144115, -0.017054561525583267, -0.028760254383087158, 0.02519882097840309, -0.03069552406668663, -0.010449111461639404, 0.03128685802221298, -0.0002883165143430233, -0.03115246258676052, 0.008030024357140064, -0.017578696832060814, 0.0009819141123443842, -0.03104494698345661, -0.026394926011562347, -0.04104384034872055, 0.031206220388412476, 0.004814654588699341, 0.013513287529349327, -0.015253686346113682, 0.023236673325300217, -0.0059200432151556015, -0.04171580821275711, -0.0028189080767333508, 0.024782201275229454, -0.016705138608813286, 0.007915790192782879, 0.0013422572519630194, 0.02089822292327881, 0.012055115774273872, 0.0036118309944868088, -0.009017818607389927, -0.027026575058698654, -0.008628076873719692, -0.010932928882539272, -0.02112669125199318, 0.010140005499124527, 0.008970780298113823, -0.003994853235781193, -0.0003912117099389434, 0.0028659459203481674, 0.019500527530908585, 0.0021519793663173914, 0.022457189857959747, -0.009972013533115387, -0.0070220716297626495, 0.03298021852970123, -0.0018663926748558879, 0.010019050911068916, 0.003564793150871992, 0.026717470958828926, 0.012176070362329483, -0.019298937171697617, 0.013869430869817734, -0.002708033425733447, -0.020038101822137833, 0.02315603755414486, 0.011940881609916687, 0.0023266910575330257, 0.036689482629299164, -0.019204862415790558, 0.019702117890119553, 0.024432776495814323, 0.0032960057724267244, -0.002954981755465269, -0.006968313828110695, 0.006528174504637718, 0.008628076873719692, -0.0415814146399498, 0.04214586690068245, 0.010744777508080006, 0.009756983257830143, 0.011221875436604023, 0.01310338731855154, -0.0009491555974818766, 0.00842648558318615, 0.007767956703901291, 0.00045945850433781743, 0.01511929277330637, 0.0326307937502861, -0.015791261568665504, -0.01866728626191616, 0.04225338250398636, 0.03529179096221924, -0.001358216512016952, -0.01935269497334957, 0.014998338185250759, 0.034243516623973846, 0.0073244571685791016, -0.006665928289294243, -0.6326180696487427, -0.016906728968024254, 0.0018747922731563449, -0.00503304461017251, 0.025239139795303345, 0.030561130493879318, 0.009011098183691502, -0.0019218301167711616, -0.02057567797601223, -0.01282116025686264, -0.017578696832060814, 0.012458297424018383, 0.03338339924812317, 0.001982307294383645, -0.016799213364720345, -0.015535913407802582, 0.0032792065758258104, -0.011262193322181702, -0.022107765078544617, 0.011638496071100235, -0.025212259963154793, 0.020831026136875153, -0.037710875272750854, -0.0074655706994235516, 0.0060544367879629135, 0.011322670616209507, 0.008150978945195675, -0.0003956214932259172, 0.018828559666872025, 0.0012523813638836145, -0.029862282797694206, 0.04217274859547615, -0.007304298225790262, -0.0016219641547650099, 0.054241303354501724, -0.010321437381207943, -0.017256153747439384, 0.043839231133461, 0.019191421568393707, 0.038947299122810364, -0.018156589940190315, -0.01865384727716446, 0.006168671417981386, -0.008742311038076878, 0.0043005989864468575, -0.005221195984631777, 0.008614636957645416, 0.0061283535324037075, 0.014272612519562244, 0.006595371291041374, -0.016866410151124, -0.0067129661329090595, 0.0008235814748331904, -0.008749030530452728, 0.01572406478226185, -0.01596597395837307, 0.020992297679185867, -0.012801000848412514, 0.0046433028765022755, 0.012552373111248016, 0.005234634969383478, 0.003247288055717945, -0.019419891759753227, -0.014111340045928955, -0.048838675022125244, 0.02419086918234825, -0.02506442740559578, -0.011067322455346584, 0.030184827744960785, -0.012780842371284962, -0.01270020566880703, -0.0025853989645838737, -0.02686530351638794, -0.0045189885422587395, 0.011524260975420475, 0.019809633493423462, 0.03397472947835922, -0.015912216156721115, 0.017175516113638878, -0.004149405751377344, 0.024889715015888214, -0.0053455098532140255, -0.024029595777392387, -0.027470074594020844, 0.029432224109768867, -0.013116826303303242, -0.0004661782004404813, -0.015535913407802582, 0.006057796534150839, -0.004549227189272642, -0.0048348139971494675, 0.013788795098662376, -0.0207907073199749, -0.03752272576093674, 0.00915893167257309, 0.006934715434908867, -0.009797302074730396, -0.004099008161574602, -0.00686079915612936, -0.02428494393825531, -0.009676347486674786, -0.004095648415386677, -0.0024442856665700674, -0.0028911447152495384, -0.0040855687111616135, 0.0028491467237472534, -0.012001358903944492, 0.015052095986902714, 0.027375999838113785, -0.016342274844646454, 0.008520561270415783, -0.02970101125538349, 0.01562998816370964, -0.011685533449053764, 0.022174961864948273, -0.030319221317768097, -0.011981199495494366, 0.007835153490304947, 0.0003315744979772717, -0.009313484653830528, 0.02091166190803051, -0.022336235269904137, -0.016207881271839142, -0.005741971544921398, -0.019070468842983246, 0.010079528205096722, 0.030104191973805428, 0.0034320794511586428, -0.029647253453731537, -0.007082548923790455, 0.01597941294312477, 0.011188277043402195, 0.0019772674422711134, -0.014662354253232479, 0.0010877491440623999, 0.02730880305171013, -0.005923402961343527, -0.01889575645327568, 0.03690451383590698, -0.04069441556930542, 0.00045105890603736043, 0.002570279873907566, -0.0036958272103220224, -0.012512054294347763, 0.00624258816242218, -0.023303870111703873, -0.023881763219833374, -0.009669627994298935, -0.010805254802107811, 0.007411813363432884, 0.019043589010834694, 0.0030574570409953594, 0.0006765883299522102, 0.008473523892462254, 0.008930462412536144, 0.006877598352730274, 0.010462550446391106, -0.008863265626132488, -0.010381914675235748, -0.0032153695356100798, -0.005940202157944441, 0.0381946936249733, -0.01619444228708744, -0.013029470108449459, -0.03206634148955345, 0.0010482709622010589, -0.002224216004833579, -0.007371495012193918, 0.012995871715247631, -0.033329639583826065, 0.027792619541287422, 0.009851058945059776, -0.005631096661090851, 0.006581932306289673, -0.008413046598434448, 0.029270950704813004, -0.03359842672944069, -0.0030557771679013968, -0.0069817532785236835, -0.01309666782617569, 0.02158362977206707, -0.0017042802646756172, -0.025346653535962105, -0.0012977393344044685, 0.015468716621398926, 0.001577446237206459, -0.000186261284397915, 0.0035782326012849808, -0.02921719290316105, 0.03736145421862602, 0.018626969307661057, 0.006907836999744177, 0.007949388585984707, 0.006699526682496071, 0.0037630239967256784, -0.013358735479414463, -0.013526727445423603, 0.015562792308628559, 0.01192072220146656, 0.002316611586138606, 0.0035211152862757444, -0.012149191461503506, 0.014689233154058456, -0.0073647755198180676, 0.029082799330353737, -0.038974177092313766, 0.023586096242070198, -0.019379572942852974, 0.0359637588262558, 0.0020226254127919674, 0.013681280426681042, -0.01524024736136198, -0.021435797214508057, -0.017847485840320587, 0.003759664250537753, 0.04125887155532837, 0.005056563299149275, 0.028168922290205956, 0.004219962749630213, -0.013022750616073608, -0.004932249430567026, 0.0015892056981101632, 0.009367241524159908, -0.0009483156609348953, -0.027846377342939377, 0.04512941092252731, -0.0037126264069229364, -0.0104020731523633, 0.006393780931830406, -0.03773775324225426, 0.008339130319654942, 0.04843549430370331, 0.001248181564733386, 0.01687985099852085, 0.013587204739451408, 0.01595253311097622, 0.041097596287727356, -0.0006900277221575379, 0.007862032391130924, -0.008218175731599331, 0.010892610065639019, 0.019944027066230774, 0.007411813363432884, -0.0033867214806377888, 0.03754960373044014, 0.0179818794131279, 0.0059301224537193775, -0.0010801894823089242, -0.018264105543494225, 0.013278098776936531, 0.006010758690536022, 0.014648914337158203, -0.04359732195734978, 0.031985703855752945, 0.02956661768257618, -0.022282477468252182, 0.007754517253488302, -0.0041762846522033215, 0.022013690322637558, 0.029297828674316406, 0.03679699823260307, 0.014514520764350891, 0.012209668755531311, -0.021973371505737305, 0.005358949303627014, -0.006894397549331188, 0.017229273915290833, -0.02483595907688141, -0.03948487341403961, 0.009857778437435627, -0.013775356113910675, -0.04440368339419365, -0.005315271206200123, -0.011524260975420475, 0.026354607194662094, -0.002316611586138606, -0.020750388503074646, 0.026045501232147217, 0.009965294040739536, 0.013204182498157024, -0.013022750616073608, -0.020602555945515633, 0.020374087616801262, -0.0009457957348786294, -0.009011098183691502, -0.019715556874871254, -0.03351779282093048, -0.020064981654286385, -0.011087480932474136, -0.012572531588375568, 0.01327137928456068, 0.04421553015708923, -0.005197676829993725, 0.028061406686902046, -0.007095987908542156, 0.019930588081479073, 0.03281894326210022, -0.019083907827734947, 6.724935519741848e-05, 0.0061283535324037075, -0.010973246768116951, -0.004461871460080147, -0.005164078436791897, -0.024231186136603355, 0.02158362977206707, -0.005063283257186413, 0.0048348139971494675, -0.010576785542070866, -0.015280565246939659, -0.016557306051254272, -0.005684853997081518, -0.009420999325811863, -0.01034831628203392, -0.0058528464287519455, 0.01843881793320179, -0.006524814758449793, -0.0063299438916146755, -0.0011683853808790445, 0.023357627913355827, -0.0011356269242241979, -0.013009311631321907, -0.007371495012193918, -0.015509034506976604, 0.03314149007201195, 0.07332520931959152, 0.00548326363787055, -0.011947601102292538, 0.014877384528517723, -0.0003649629361461848, -0.015011778101325035, -0.021341722458600998, -0.011336109600961208, 0.033786579966545105, 0.00904469657689333, -0.005839406978338957, -0.0024140470195561647, 0.0049120900221168995, -0.006518095266073942, 0.016221322119235992, -0.009474757127463818, 0.018129711970686913, -0.03171691671013832, -0.008110661059617996, -0.0208041463047266, -0.0052279154770076275, -0.0044349925592541695, 0.01315714418888092, 0.00604435708373785, 0.011436904780566692, 0.007358056027442217, 0.026273971423506737, 0.04701092094182968, 0.01236422173678875, -0.021892735734581947, -0.018183469772338867, -0.022000251337885857, 0.011262193322181702, 0.00882294774055481, -0.0031565723475068808, 0.015509034506976604, 0.029082799330353737, -0.0027987491339445114, -0.016947047784924507, -0.02112669125199318, -0.0007236261735670269, -0.011900563724339008, 0.008245054632425308, 0.0033564830664545298, -0.002120060846209526, -0.053703729063272476, 0.010455830954015255, 0.014635475352406502, -0.024715004488825798, -0.01607348769903183, 0.013627522625029087, 0.009360522031784058, -0.020091859623789787, -0.0064811366610229015, 0.033007096499204636, -0.018573211506009102, 0.0022006970830261707, -0.011672094464302063, -0.010388634167611599, -0.011201716028153896, 0.0047440980561077595, 0.01843881793320179, -0.0017857564380392432, -0.006766723468899727, -0.026784667745232582, -0.02406991459429264, -0.009770423173904419, -0.016382593661546707, -0.010005611926317215, 0.01732335053384304, 0.001296059344895184, -0.019540846347808838, -0.007277419790625572, -0.0061619519256055355, 0.012297024950385094, 0.011282352730631828, 0.030319221317768097, -0.018707605078816414, 0.0006018318235874176, 0.002437565941363573, -0.0037630239967256784, -0.001360736321657896, 0.012545653618872166, -0.026730909943580627, 0.01253893319517374, 0.0028911447152495384, -0.0043274774216115475, -0.023250112310051918, -0.007391654420644045, 0.018479134887456894, 0.0036151909735053778, 0.011000125668942928, -0.0029952998738735914, -0.009414279833436012, -0.008943901397287846, 0.0013145385310053825, -0.014917702414095402, 0.019137665629386902, 0.015885336324572563, -0.04004932567477226, -0.010953087359666824, 0.0023888482246547937, 0.005113680846989155, -0.020951980724930763, 0.014286051504313946, -0.0006946474895812571, -0.0060712359845638275, -0.0024846037849783897, 0.0017521580448374152, -0.002199017209932208, 0.005547100678086281, 0.0006862478912808001, 0.000549754302483052, 0.010428952053189278, -0.013963506557047367, 0.02630084939301014, -0.01511929277330637, 0.01421885471791029, -0.021502994000911713, -0.0003903717442881316, -0.007909069769084454, -0.04856988787651062, 0.03685075789690018, -0.0021267805714160204, -0.0241639893501997, 0.0063131446950137615, 0.0033363238908350468, -0.03754960373044014, -0.0028222680557519197, -0.003826861036941409, 0.013587204739451408, 0.01909734681248665, -0.014984899200499058, -0.011840086430311203, -0.038947299122810364, -0.021449236199259758, 0.0017126798629760742, 0.015320883132517338, -0.00036979271681047976, -0.012048396281898022, -0.03338339924812317, -0.005080082453787327, 0.004438352305442095, 0.0010449110995978117, -0.0002028505114139989, -0.03518427535891533, -0.007559646386653185, 0.014366688206791878, -0.003413600381463766, 0.026919061318039894, -0.004858332686126232, -0.009938415139913559, -0.006135073024779558, -0.02686530351638794, 0.0020142258144915104, -0.02822268009185791, 0.00973010528832674, 0.004340916872024536, 0.012048396281898022, 0.024809079244732857, 0.03642069548368454, 0.00792922917753458, -0.007102707866579294, 0.009602431207895279, -0.0025316416285932064, -0.002963381353765726, -0.001464891480281949, -0.021274525672197342, -0.015334323048591614, 0.01651698723435402, 0.004112447611987591, 0.0033984810579568148, -0.0015236887848004699, 0.008762470446527004, -0.018143150955438614, 0.03134061396121979, -0.0058696456253528595, -0.022860370576381683, -0.027255045250058174, -0.0014867304125800729, 0.005909963510930538, -0.007767956703901291, -0.02956661768257618, -0.00048255742876790464, 0.011927441693842411, -0.007915790192782879, 0.000743785232771188, 0.007720918860286474, 0.010946367867290974, 0.014299491420388222, 0.021314842626452446, -0.004062050022184849, 0.026354607194662094, -0.011242033913731575, 0.012780842371284962, -0.003334644017741084, -0.001158305793069303, -0.010361755266785622, 0.002696273848414421, -0.007653722073882818, -0.01343265175819397, -0.0010499508352950215, -0.015656867995858192, -0.015562792308628559, -0.024688124656677246, 0.01113451924175024, -0.0035815925803035498, -0.008869985118508339, 0.018075954169034958, -0.014998338185250759, -0.015670306980609894, -0.0258707907050848, -0.04179644584655762, -0.0033749621361494064, 0.001116307801567018, -0.0014119739644229412, -0.011275632306933403, 0.03303397446870804, -0.019809633493423462, -0.009958574548363686, -0.0033900814596563578, -0.021314842626452446, 0.04004932567477226, 0.019366133958101273, 0.06058468669652939, 0.00328928604722023, 0.001228862558491528, -0.018586650490760803, 0.01102028414607048, 0.019970905035734177, -0.015871897339820862, -0.012787561863660812, 0.007418532855808735, 0.02597830444574356, -0.019419891759753227, -0.019984345883131027, -0.0042602806352078915, -0.005271593574434519, -0.03730769455432892, 0.03281894326210022, -0.0020125459413975477, -0.010220642201602459, -0.026032062247395515, 0.001948708901181817, 0.004925529472529888, -0.0037495845463126898, 0.0021469395142048597, 0.005825967527925968, -0.0062627471052110195, -0.010019050911068916, -0.012008078396320343, 0.049295615404844284, -0.014071022160351276, 0.02101917751133442, 0.003773103468120098, -0.02338450588285923, -0.006151872221380472, -0.023102279752492905, -0.00655169365927577, 0.02295444719493389, 0.011725851334631443, 0.03306085243821144, -0.004135966766625643, 0.01388287078589201, 0.0032691271044313908, 0.010187043808400631, -0.012680047191679478, -0.008366008289158344, 0.01573750376701355, 0.023787686601281166, 0.02179866097867489, -0.014810186810791492, 0.0015060495352372527, -0.008056903257966042, 0.02776574157178402, -0.017941560596227646, -0.007882190868258476, -0.02427150495350361, 0.005516862031072378, -0.013560325838625431, 0.00444171205163002, -0.02584391087293625, -0.010146724991500378, -0.02798077091574669, 0.006272826809436083, -0.0005321151111274958, -0.010764936916530132, 0.0019587883725762367, 0.01628851890563965, -0.007069109473377466, -0.009031257592141628, -0.026032062247395515, -0.028034528717398643, -0.016342274844646454, 0.013016031123697758, -0.022457189857959747, -0.0056613353081047535, 0.009004378691315651, -0.01720239594578743, 0.0034875168930739164, -0.005006165709346533, 0.02427150495350361, -0.0034169601276516914, 0.009152212180197239, -0.005375748500227928, 0.0022527745459228754, 0.0063299438916146755, 0.016611063852906227, -0.013405772857367992, -0.02889464795589447, 0.032308246940374374, 0.010146724991500378, 0.008480243384838104, -0.011322670616209507, 0.0072572603821754456, 0.012444857507944107, 0.003363202791661024, -0.00015990751853678375, -0.0030104191973805428, 0.012438138015568256, -0.030319221317768097, 0.03179755434393883, 0.008836386725306511, -0.02440589852631092, -0.03002355620265007, 0.007203503046184778, 0.029593495652079582, -0.012465016916394234, -0.0219196155667305, 0.019473649561405182, 0.0006963274208828807, 0.009219408966600895, 0.012216388247907162, 0.001611044630408287, -0.00016043249343056232, 0.0042334022000432014, -0.020105300471186638, 0.02945910207927227, -2.8086185920983553e-05, -0.005718452390283346, -0.0016454830765724182, 0.022510947659611702, 0.015159610658884048, 0.009810741059482098, -0.013661121018230915, -0.011094201356172562, 0.005805808585137129, -0.01978275552392006, -0.0037126264069229364, -0.016476668417453766, 0.009454597719013691, 0.04711843654513359, 0.013728317804634571, -0.018815120682120323, -0.014460763894021511, 0.016947047784924507, -0.026730909943580627, -0.022914128378033638, -0.002078062854707241, 0.027792619541287422, 0.011403306387364864, -0.004438352305442095, -0.0058360472321510315, 0.03526490926742554, 0.005977160297334194, 0.01606004871428013, -0.020978858694434166, -0.0069145564921200275, -0.0051707979291677475, -0.008628076873719692, 0.01039535365998745, -0.00792250968515873, -0.02248406782746315, -0.017820606008172035, 0.031770672649145126, -0.003910857252776623, -0.011732570827007294, 0.0054026274010539055, 0.0067398445680737495, 0.005251434165984392, -0.0017219195142388344, 0.02238999307155609, 0.004881851840764284, 0.018613528460264206, 0.03964614495635033, -0.01999778486788273, 0.011611617170274258, -0.003038977738469839, 0.014581717550754547, -0.0010633902857080102, -0.00888342410326004, 0.006692806724458933, 0.0018126352224498987, 0.008896864019334316, 0.03959238901734352, -0.018815120682120323, 0.009830900467932224, -0.018573211506009102, -0.00542614609003067, -0.011430185288190842, 0.03461981937289238, 0.02619333565235138, -0.0022342954762279987, -0.03359842672944069, -0.02462092787027359, -0.014810186810791492, -0.017377106472849846, -0.010173603892326355, 0.0021251006983220577, -0.032657671719789505, -0.000541354704182595, 0.00700191268697381, 0.011766169220209122, 0.0025232420302927494, -0.01821034774184227, 0.0016354034887626767, 0.007935948669910431, -0.010657421313226223, 0.03136749193072319, 0.013459530659019947, 0.029942918568849564, 0.005463104695081711, 0.0073647755198180676, -0.030426736921072006, -0.01833130232989788, -0.025118185207247734, 0.00013418371963780373, -0.026730909943580627, 0.011940881609916687, -0.0070153516717255116, -0.01842537894845009, 3.722075780387968e-05, 0.0062392279505729675, -0.013909749686717987, -0.011940881609916687, -0.0073110181838274, 0.21320219337940216, -0.00596036110073328, 0.0029197034891694784, 0.013217621482908726, 0.01572406478226185, -0.0064878566190600395, 0.012189510278403759, 0.007680600509047508, -0.004458511248230934, 0.0006492896354757249, 0.0007307658088393509, -0.018008757382631302, -0.020199375227093697, 0.0032355287112295628, 0.02519882097840309, -0.016557306051254272, -0.02978164702653885, -0.03104494698345661, -0.001627843827009201, 0.013284818269312382, 0.025010669603943825, -0.01651698723435402, -0.009938415139913559, -0.012008078396320343, 0.027738861739635468, 0.04813982918858528, 0.017739970237016678, 0.0002083102590404451, 0.028491467237472534, -0.01572406478226185, -0.006649129092693329, -0.005029684863984585, -0.00809722114354372, 0.009763703681528568, -0.03349091485142708, 0.017121758311986923, 0.0241639893501997, -0.006524814758449793, 0.02148955501616001, 0.014984899200499058, -0.003924296237528324, 0.021180449053645134, 0.025830471888184547, 0.010630542412400246, -0.011725851334631443, 0.014205415733158588, -0.013735037297010422, 0.011860244907438755, 0.011994639411568642, 0.004465231206268072, -0.00780827458947897, -0.011114359833300114, 0.010187043808400631, 0.006857439409941435, -0.00015245286340359598, 0.018385060131549835, 0.02608582004904747, 0.005268233362585306, -0.007774676196277142, 0.0025182021781802177, -0.007942668162286282, 0.02697281911969185, -0.001965508097782731, 0.036931391805410385, -0.010019050911068916, 0.02608582004904747, -0.009461317211389542, -0.009226128458976746, -0.01628851890563965, 0.01935269497334957, 0.008305531926453114, -0.01819690875709057, -0.0014186936896294355, -0.004011652432382107, -0.009031257592141628, -0.010872451588511467, 0.018828559666872025, 0.022914128378033638, 0.01630195789039135, 0.030749281868338585, -0.0074319723062217236, -0.017108319327235222, -0.034915488213300705, -0.007680600509047508, 0.0012918595457449555, -0.022430310025811195, 0.0179818794131279, -0.015818139538168907, 0.022887248545885086, -0.03303397446870804, -0.011047163046896458, 0.013869430869817734, -0.010415513068437576, -0.015334323048591614, 0.02114013209939003, -0.032093219459056854, -0.021180449053645134, 0.004713859409093857, -0.0011877043871209025, 0.010953087359666824, -0.03451230749487877, 0.010549906641244888, -0.0035446342080831528, 0.015549352392554283, -0.0044013941660523415, -0.02260502241551876, -0.020051542669534683, 0.012780842371284962, 0.012283585034310818, -0.0067129661329090595, -0.0006354302749969065, -0.02843770943582058, -0.0037193461321294308, -0.01129579171538353, 0.0026895541232079268, 0.019970905035734177, -0.0060544367879629135, -0.0123709412291646, -0.001318738330155611, -0.020414404571056366, 0.015025217086076736, -0.03701202943921089, 0.0038738988805562258, -0.014353248290717602, -0.011315951123833656, -0.009515075013041496, -0.014689233154058456, 0.02112669125199318, 0.0033144850749522448, -0.020306890830397606, 0.03510363772511482, -0.01763245463371277, -0.005217835772782564, -0.007559646386653185, 0.006350102834403515, -0.019581163302063942, 0.023518899455666542, 0.014702672138810158, -0.011315951123833656, 0.0034841569140553474, -0.021220767870545387, 0.010442391969263554, -0.008527280762791634, 0.006040997337549925, 0.013990385457873344, -0.01943333074450493, 0.009797302074730396, 0.0074991690926253796, -0.0015740863746032119, -0.008675114251673222, -0.025212259963154793, -0.005741971544921398, -0.006941435392946005, -0.014904262498021126, -0.023021643981337547, -0.012794281356036663, -0.0325770378112793, -0.013593924231827259, -0.01674545742571354, 0.010039210319519043, -0.04066753759980202, 0.0006056116544641554, 0.029593495652079582, 0.006595371291041374, -0.021193888038396835, -0.032227613031864166, -0.16933608055114746, 0.011107640340924263, 0.011517541483044624, -0.028814012184739113, 0.03182443231344223, 0.02304852195084095, 0.009373961947858334, 0.017739970237016678, -0.004737378563731909, -0.002462764736264944, -9.160821355180815e-05, -0.018008757382631302, 0.002204056829214096, -0.0052111162804067135, 0.003860459430143237, -0.012229828163981438, 0.008607917465269566, 0.04359732195734978, 0.04359732195734978, 0.013392333872616291, 0.017565257847309113, -0.010321437381207943, 0.004650022368878126, -0.004199803341180086, 0.005631096661090851, -0.0033867214806377888, 0.005634456407278776, -0.010543187148869038, 0.019258618354797363, -0.01674545742571354, 0.012801000848412514, -0.015992851927876472, 0.017256153747439384, 0.025319775566458702, 0.0025652400217950344, -0.019540846347808838, -0.020857904106378555, -0.009488196112215519, 5.9952195442747325e-05, 0.011033724062144756, 0.004623143933713436, 0.0066289701499044895, -0.0011566259199753404, 0.0066222501918673515, 0.007828433997929096, 0.019285498186945915, 0.0013195782667025924, -0.0014539719559252262, -0.0038638191763311625, 0.0025014029815793037, 0.02238999307155609, -0.010267679579555988, 0.014380127191543579, 0.021879296749830246, 0.002991940127685666, -0.016328835859894753, -0.014407006092369556, 0.016947047784924507, -0.0009995532454922795, 0.023236673325300217, 0.0047608972527086735, -0.046365831047296524, 0.007868751883506775, 0.01564342901110649, -0.009266446344554424, 0.01039535365998745, 0.017175516113638878, 0.01991714909672737, 0.0029348228126764297, -0.0075394874438643456, 0.01991714909672737, 0.009790581651031971, 0.017699651420116425, -0.013546885922551155, -0.030399858951568604, 0.0028273076750338078, -0.02159707061946392, -0.013976946473121643, 0.013089947402477264, 0.0017756769666448236, 0.0005766330286860466, 0.015670306980609894, -0.03797966241836548, -0.004152765963226557, 0.012068555690348148, 0.0017991958884522319, -0.009347083047032356, 0.003618550719693303, -0.039753660559654236, -0.0054194265976548195, -0.0021637387108057737, -0.004737378563731909, -0.042656563222408295, -0.002482923911884427, -0.01654386706650257, 0.020374087616801262, 0.016261639073491096, -0.016920167952775955, 0.01529400423169136, -0.013499848544597626, 0.0005631936364807189, -0.0036118309944868088, -0.025050988420844078, 0.014326370321214199, 0.0460701659321785, 0.008628076873719692, -0.013687999919056892, -0.0053455098532140255, 0.04305974766612053, -0.00244596553966403, -0.03201258182525635, 0.011840086430311203, 0.035479940474033356, 0.006417299620807171, -0.0007786435890011489, 0.017229273915290833, 0.011591457761824131, -0.014151657931506634, 0.027819499373435974, 0.0021335002966225147, 0.010758216492831707, -0.0053690290078520775, 0.01191400270909071, 0.03179755434393883, -0.005372388754040003, -0.02833019569516182, -0.12891045212745667, -0.015482155606150627, 0.009656188078224659, 0.03169003874063492, -0.03308773413300514, -0.024903155863285065, -0.010012331418693066, 0.022551264613866806, -0.006927995942533016, 0.020105300471186638, -0.009414279833436012, -0.03195882588624954, 0.0069481548853218555, 0.001920150127261877, -0.01147050317376852, 0.008480243384838104, -0.0064710574224591255, -0.015267126262187958, -0.027160968631505966, 0.0031364134047180414, 0.009797302074730396, -0.006232508458197117, 0.00030007597524672747, 0.0035748728550970554, 0.0030440175905823708, 0.005573979113250971, -0.012444857507944107, -0.0017126798629760742, 0.0035110358148813248, 0.0022544546518474817, -0.016221322119235992, 0.012888357043266296, -0.014232294633984566, -0.02429838292300701, -0.00016620723181404173, 0.0011818247148767114, 0.017041122540831566, -0.025494487956166267, 0.04104384034872055, -0.0067734429612755775, -0.003005379345268011, 0.009535234421491623, 0.004683620762079954, -0.02216152288019657, 0.0007059869822114706, -0.00020117059466429055, -0.04362419992685318, 0.019298937171697617, 0.004162845201790333, -0.02800765074789524, -0.015146171674132347, -0.022766295820474625, -0.005459744483232498, -0.020857904106378555, 0.022067448124289513, -0.015656867995858192, -0.006840640213340521, -0.014971459284424782, -0.020710071548819542, -0.004314038436859846, -0.02631428837776184, 0.010892610065639019, -0.02699969708919525, 0.0208041463047266, 0.0033128049690276384, -0.011457064189016819, -0.011315951123833656, -0.015374640934169292, 0.0017252792604267597, -0.017619015648961067, 0.023478582501411438, 0.0038201413117349148, -0.0030473775696009398, 0.019648360088467598, -0.024123672395944595, -0.0038806183729320765, -0.013647682033479214, -0.01005936972796917, 0.01557623129338026, -0.009185810573399067, -0.00045903853606432676, -0.018344741314649582, 0.024983791634440422, -0.04354356229305267, 0.03139436990022659, -0.0070153516717255116, 0.01331841666251421, -0.0011347868712618947, -0.0037260656245052814, -0.04937624931335449, 0.004270360339432955, 0.017175516113638878, 0.04056002199649811, 0.022215280681848526, -0.012801000848412514, 0.0038839783519506454, -0.009125333279371262, -0.032093219459056854, 0.017350228503346443, -0.010039210319519043, -0.022457189857959747, -0.0021150209940969944, -0.032765187323093414, 0.012055115774273872, 0.017336789518594742, -0.021059494465589523, -0.0066289701499044895, -0.024983791634440422, -0.01219622977077961, -0.0031112143769860268, -0.007559646386653185, 0.025817032903432846, -0.013446090742945671, 0.023209793493151665, 0.00570837315171957, -0.028383951634168625, -0.02046816237270832, -0.02260502241551876, -0.006642409134656191, -0.0009777143131941557, -0.004700419958680868, -0.014931141398847103, -0.006215709261596203, 0.009481476619839668, -0.0027685104869306087, -0.0013304977910593152, -0.02124764584004879, 0.025696078315377235, -0.011336109600961208, 0.0219196155667305, -0.007216942496597767, -0.0037361453287303448, 0.024096792563796043, -0.01282116025686264, 0.0018747922731563449, 0.04002244770526886, 0.005016245413571596, -0.009394120424985886, 0.008278653025627136, 0.02306196093559265, 0.01388287078589201, -0.011376427486538887, 0.010691019706428051, -0.03359842672944069, -0.01753837987780571, 0.003843660233542323, -0.015656867995858192, -0.014890823513269424, 0.0069817532785236835, 0.010361755266785622, 0.01955428533256054, 0.020831026136875153, 0.02846458926796913, 0.014460763894021511, -0.0008000625530257821, -0.0013674560468643904, -0.015885336324572563, -0.0030238586477935314, 0.019487088546156883, 0.000995353446342051, -0.015791261568665504, -0.02730880305171013, 0.05012885481119156, 0.030883675441145897, 0.01989026926457882, -0.02237655408680439, -0.0010272719664499164, 0.001110428012907505, -0.00764700211584568, -0.013506568036973476, -0.010664140805602074, -0.030749281868338585, 0.001060030423104763, -0.0039578950963914394, 0.02294100634753704, 0.015495595522224903, 0.019513966515660286, -0.014097900129854679, 0.0004653382347896695, 0.001731998985633254, -0.009837619960308075, 0.026824984699487686, -0.011544420383870602, 0.011739291250705719, -0.021758342161774635, 0.02226903848350048, 0.015495595522224903, 0.01101356465369463, -0.013076508417725563, 0.02494347281754017, -0.003146492876112461, 0.01146378368139267, 0.013963506557047367, -0.0025434009730815887, 0.014662354253232479, -0.0038906980771571398, 0.021677706390619278, 0.0073244571685791016, -0.003689107485115528, 0.0005745331291109324, -0.01095980778336525, -0.00825849361717701, 0.011450344696640968, -0.010140005499124527, -0.011786328628659248, 0.0066289701499044895, -0.025212259963154793, 0.008446644991636276, -0.015428398735821247, -0.007586525287479162, -0.0052951122634112835, 0.011006845161318779, 0.01618100330233574, -0.016127245500683784, 0.028733376413583755, 0.006975033786147833, -0.029270950704813004, 0.023303870111703873, -0.02013217844069004, -0.010946367867290974, -0.019083907827734947, 0.01628851890563965, -0.0006807881291024387, -0.008164417929947376, 0.019473649561405182, -0.009373961947858334, -0.008318970911204815, 0.0014951300108805299, 0.039296720176935196, -0.013667840510606766, 0.0051371995359659195, -0.010670861229300499, 0.04822046309709549, -0.007761236745864153, -0.03123309835791588, -0.002466124715283513, -0.014259172603487968, -0.01315042469650507, 0.01776684820652008, 0.05416066572070122, -0.018264105543494225, 0.05187597498297691, 0.018358182162046432, -0.01545527670532465, 0.007888911291956902, 0.02046816237270832, 0.006558413151651621, 0.011954320594668388, -0.022013690322637558, 0.03308773413300514, -0.023236673325300217, 0.001611044630408287, -0.004717219155281782, -0.003584952326491475, -0.0020024662371724844, -0.011598177254199982, 0.008386167697608471, 0.009817460551857948, -0.01006608922034502, -0.01730990968644619, 0.011517541483044624, 0.014595157466828823, 0.012404539622366428, 0.01775340922176838, 0.009817460551857948, 0.005204396788030863, -0.001515289070084691, 0.03607127442955971, -0.009246287867426872, 0.0023888482246547937, -0.04510252922773361, 0.004770976956933737, -0.0010465910891070962, -0.02394896000623703, -0.020374087616801262, 0.03281894326210022, -0.028814012184739113, 0.0034203201066702604, 0.02798077091574669, 0.005701653193682432, 0.02396239899098873, 0.008413046598434448, 0.015535913407802582, -0.022430310025811195, -0.03093743324279785, -0.011638496071100235, 0.0032523279078304768, 0.013567045331001282, -0.02833019569516182, -0.04117823392152786]} +{"id": "test:10000016", "text": "\"Are film trailers spoiling movies?\\nIn April we asked if high ticket prices were ruining your visits to the cinema. Yes, you said overwhelmingly. But something else has put a strain on my relationship with the cinema \u2013 and it\u2019s not the popcorn.\\nHave you ever gone to your local cinema to watch a new film only to feel you\u2019ve seen it before? I know I have. \u2018Spoilerific\u2019 film trailers are now often so detailed it seems hardly worth watching the movie itself.\\nImagine if the trailer for Casablanca told you whether Ingrid Bergman went off with Humphrey Bogart or Paul Henreid at the end. Or if the trailer for Citizen Kane revealed just what the dying man meant when he muttered the word \u2018Rosebud\u2019.\\nAnd it seems that even some film directors agree. Colin Trevorrow, director of new film, Jurassic World, has said he think that trailers have shown far more of the film than he would have wanted.\\nIn the last couple of months, I\u2019ve paid \u00a317.50 a ticket to see two films I\u2019ve been anticipating for some time. OK \u00a317.50 sounds steep, but that was for iMAX 3D and I\u2019d still have paid more than \u00a310 for a \u2018normal\u2019 ticket.\\nBut I left the cinema unfulfilled, because I felt like I\u2019d seen them both six months before.\\nIn both cases the trailer revealed the entire structure of the story, key plot twists and expensive action sequences. Throw in a few character deaths for good measure and you\u2019ve got the basis of a significant chunk of what you\u2019ve paid your money for.\\nI really did feel cheated by what the studio had wanted me to see in advance.\\nSo is this just a modern trend? I checked out trailers for 1981\u2019s Raiders of the Lost Ark, and 1964\u2019s Goldfinger to get a bigger picture.\\nBoth revealed some well-known scenes (including Goldfinger\u2019s iconic laser and dialogue), but the plot basics were instead explained by voiceover, rather than any especially huge visual giveaways.\\nBesides, the chance of actually seeing these trailers was significantly lessened due to the technology available at the time of release. Which got me thinking further.\\n\u2018If you don\u2019t like it, don\u2019t watch it\u2019, I hear you cry. I wish it were that simple. In the age of the internet, exposure has increased tenfold.\\nMarketing campaigns target social media and television, while the days of a simple poster are gone. Why commission a still image when you can display scenes from the film on a screen in a station or other public place?\\nNot only that, but trailers are also forced upon you in the cinema itself before other films. Without a blindfold and a soundproof booth to hide in you have little choice.\\nIt\u2019s not that I have a problem with marketing and ads. But they\u2019re spoiling the experience. I don\u2019t want to see all the best bits wrapped up into two minutes, six months ahead of release.\\nHave you had a film spoiled by its trailer/marketing? Is Hollywood revealing too much in a desperate attempt to put bums on seats?\\nDo ticket prices ruin your film-going experience?\\nSomething George mentioned earlier about Terminator 2 trailer jogged my memory about another film from the same franchise, Terminator Genisys.\\nTo promote the film, a trailer had been released that included a major plot twist ACTUALLY in the trailer. Now, I had heard about this trailer and had been purposely avoiding it. However, even when you try to avoid spoilers, sometimes it\u2019s out of your control. When I went to see Mad Max, they showed the afore-mentioned trailer and before I realised, it was too late. It cannot be unseen.\\nEven the director, Alan Taylor, was unhappy about the spoilers in the trailer.\\nGenisys was [spoiler alert] \u2026.. indeed half of the inspiration for this convo. I saw that article about Alan Taylor\u2019s thoughts the other day, but could only sympathise so much as the film was, in my opinion, a travesty.\\nHave you seen the film now Ryan? If so what did you think of the big \u2018reveal\u2019? For a film that had very little else going for it that really did completely spoil the experience for me.\\nI just can\u2019t understand why that decision was taken. You don\u2019t need to resort to something like that to generate interest \u2013 if said twist is that good then word of mouth will spread.\\nFor anyone interested, the trailer for the new Bond film was released today.\"", "vector_field": [-0.002657292876392603, -0.01792420819401741, 0.024036789312958717, -0.035331495106220245, 0.00737903593108058, -0.013892230577766895, -0.011087938211858273, -0.022447260096669197, -0.024295249953866005, -0.013646692968904972, 0.023028794676065445, 0.043369606137275696, 0.02274448797106743, 0.0123027004301548, 0.01695498265326023, 0.008096262812614441, 0.055103693157434464, 0.02928352914750576, 0.027526000514626503, -0.014008537866175175, -0.016489753499627113, -0.004846126772463322, -0.02527739852666855, -0.009614715352654457, -0.005398585461080074, 0.014835610054433346, 0.01119778398424387, -0.031325362622737885, 0.004451975226402283, -0.0015636836178600788, 0.010848863050341606, 0.032359205186367035, -0.022796180099248886, -0.016761137172579765, -0.027681076899170876, -0.016515599563717842, 0.013659616000950336, -0.014783917926251888, 0.020521732047200203, -0.0037185947876423597, 0.010939323343336582, 0.008613183163106441, -0.014073152095079422, -0.027681076899170876, -0.009608253836631775, 0.022822026163339615, 0.021064497530460358, -0.0019723735749721527, -0.012968235649168491, 0.0003305462305434048, 0.02910260669887066, -0.0038316710852086544, -0.03031736984848976, -0.022757411003112793, -0.021969107910990715, -0.011223630048334599, -0.00732088228687644, 0.043369606137275696, 0.02799122966825962, -0.01141747459769249, 0.016554368659853935, 0.005970428232103586, -0.01700667478144169, -0.0007931494619697332, -0.011456243693828583, 0.00916887167841196, -0.001245454652234912, -0.014396227896213531, -0.017575286328792572, -0.0012179932091385126, 0.027758615091443062, 0.02716415748000145, 0.009291640482842922, -0.014383304864168167, 0.012167009525001049, -0.03388411924242973, -0.010603325441479683, -0.007372574415057898, -0.014602995477616787, 0.017239289358258247, 0.030627522617578506, 0.006971961352974176, -0.004099823534488678, 0.019991889595985413, 0.018027592450380325, 0.019552506506443024, 0.004070746712386608, 0.021167881786823273, -0.015417144633829594, -0.0009336870862171054, -0.021581418812274933, 0.01801466941833496, -0.010583940893411636, 0.03339304402470589, 0.0087101049721241, 0.015094069764018059, 0.0010887631215155125, 0.012192855589091778, -0.03411673381924629, 0.002822061302140355, -0.0136725390329957, 0.019707582890987396, 0.0021726801060140133, -0.017200520262122154, -0.012121778912842274, 0.02995552495121956, 0.02410140447318554, -0.013032850809395313, 0.008451645262539387, -0.011126707307994366, -0.004235514905303717, 0.018208514899015427, -0.006031812634319067, -0.026854004710912704, -5.6841028708731756e-05, -0.0018399128457531333, 0.008600260131061077, -0.013414079323410988, -0.009892560541629791, -0.013943922705948353, 0.02575554884970188, -0.0017349134432151914, 0.01965589076280594, -0.021051574498414993, 0.024980168789625168, 0.00991840660572052, -0.00424520717933774, 0.006568117532879114, 0.012154086492955685, -0.007314420770853758, 0.008542106486856937, 0.008729489520192146, 0.02366202138364315, 0.004720127675682306, -0.0174202099442482, 0.03119613416492939, -0.018428204581141472, 0.005883198231458664, -0.021620187908411026, -0.05427662283182144, 0.028973376378417015, 0.03507303446531296, -0.005485815461724997, -0.010835940018296242, -0.008406414650380611, -0.0033438275568187237, 0.014964839443564415, 0.012399623170495033, 0.03750256076455116, -0.0046232049353420734, 0.006810423452407122, -0.00267829280346632, -0.00506904860958457, -0.003296981565654278, 0.015236223116517067, 0.01854451186954975, 3.238323188270442e-05, -0.0037379791028797626, -0.0077667259611189365, 0.010319019667804241, 0.020767269656062126, 0.015934064984321594, 0.0038058250211179256, 0.005207971204072237, 0.015869449824094772, 0.03409088775515556, 0.005195048172026873, -0.015287915244698524, -0.003032060107216239, -0.014034383930265903, -0.012619314715266228, 0.02301587164402008, -0.010868247598409653, 0.006387195084244013, -0.006022120360285044, 0.023080486804246902, 0.006810423452407122, 0.018622050061821938, -0.01895804889500141, -0.05427662283182144, -0.005715198814868927, -0.011456243693828583, 0.04667789489030838, 0.045773282647132874, -0.0027429077308624983, 0.001364992349408567, 0.017239289358258247, -0.00774734141305089, 0.027448462322354317, -0.03287612646818161, -0.010590402409434319, 0.017562363296747208, -0.012942389585077763, -0.006251503713428974, -0.64511638879776, -0.003473057644441724, 0.020767269656062126, 0.004736281465739012, 0.011268860660493374, 0.02843061089515686, 0.012225163169205189, 0.01518453098833561, -0.018079284578561783, 0.021193727850914, -0.02071557752788067, 0.0013326848857104778, 0.00353121105581522, -0.013866384513676167, -0.004577974323183298, -0.010112251155078411, -0.0027784460689872503, -0.015287915244698524, 0.008845796808600426, 0.0005314585869200528, -0.011094399727880955, 0.05820521339774132, -0.028172150254249573, -0.027060773223638535, 0.01475807186216116, 0.008671336807310581, -0.02174941822886467, -0.026039855554699898, -0.00535658560693264, 0.015145761892199516, -0.02598816342651844, 0.013284849002957344, -0.006997807417064905, 0.0011679165763780475, 0.05748152732849121, 0.0003717383078765124, 0.0203408095985651, 0.02560047246515751, -0.0007616496295668185, 0.038303788751363754, -0.009795637801289558, -0.0290509145706892, 0.017665747553110123, -0.002602370223030448, 0.0014029537560418248, 0.022111261263489723, 0.02089649997651577, 0.0058670444414019585, 0.01686452142894268, -0.007811956573277712, 0.014163613319396973, 0.0314287468791008, -0.012761467136442661, 0.010971630923449993, 0.013711308129131794, 0.012322084978222847, 0.03305704519152641, -0.025535857304930687, 0.014383304864168167, 0.017148828133940697, -0.022020800039172173, 0.018001746386289597, -0.0005241893813945353, 0.006380733568221331, -0.004716896917670965, 0.037011485546827316, 0.0007838610326871276, -0.017846670001745224, 0.00353121105581522, -0.008903950452804565, 0.03225582093000412, 0.020379578694701195, -0.020844807848334312, -0.01810513064265251, -0.005770121701061726, 0.0165672916918993, 0.04197391867637634, -0.0014885686105117202, 0.013071619905531406, 0.025794317945837975, 0.016670675948262215, 0.002663754392415285, -0.025083553045988083, 0.02013404108583927, 0.009272255934774876, -0.028689071536064148, -0.017226366326212883, 0.0065099638886749744, 0.010926400311291218, -0.0020983729045838118, -0.012612853199243546, 0.04091423377394676, -0.00045190134551376104, 0.001648490782827139, -0.01612791046500206, 0.014667610637843609, 0.00689442316070199, 0.006600424647331238, 0.017407286912202835, -0.029619527980685234, -0.010487018153071404, -0.0228995643556118, 0.00512720225378871, 0.0023988327011466026, 0.03354812040925026, 0.01603744924068451, -0.007133498787879944, -0.0037638251669704914, 0.03486626595258713, -0.015210377052426338, 0.023377716541290283, 0.008897488936781883, -0.010002406314015388, -0.02009527198970318, 0.009110718965530396, -0.013414079323410988, 0.019500814378261566, 0.03711486980319023, -0.021930338814854622, -0.012677468359470367, -0.01736851967871189, 0.031532131135463715, 0.0013666077284142375, -0.019229430705308914, 0.004429359920322895, 0.023713713511824608, 0.005954274442046881, 0.0009199564228765666, 0.0007148036966100335, 0.006222426891326904, -0.008264261297881603, -0.017639901489019394, -0.016257140785455704, -0.01727805845439434, -0.023287255316972733, 0.008477491326630116, 0.012593468651175499, -0.007928263396024704, -0.010532248765230179, -0.03928593546152115, -0.014202382415533066, -0.011029784567654133, 1.2544401215563994e-05, -0.010545171797275543, -0.02645339071750641, -0.02239556796848774, -0.021297112107276917, -0.01875128038227558, -0.00748888123780489, -0.023506946861743927, 0.00381874805316329, 0.006396887358278036, 0.03334135189652443, -0.014021460898220539, 0.006707039661705494, 0.0019142201635986567, 0.004981818608939648, -0.017471902072429657, -0.02087065391242504, -0.01256762258708477, 0.009084872901439667, 0.007534111849963665, -0.0006142465863376856, 0.018790049478411674, 0.0034665961284190416, -0.011921471916139126, 0.031919822096824646, -0.011107322759926319, -0.01527499221265316, -0.014706379733979702, 0.014926071278750896, -0.014577149413526058, -0.029180144891142845, 0.04817696288228035, -0.010790709406137466, 0.019914351403713226, -0.003931824117898941, 0.008884565904736519, 0.0074824197217822075, -0.005666737910360098, 0.001599221839569509, -0.017329750582575798, 0.0012002240400761366, -0.002161372685804963, 0.03378073498606682, 0.017704516649246216, 0.01128824520856142, 0.016916213557124138, -0.015081146731972694, 0.016851598396897316, 0.01645098626613617, -0.0054373545572161674, -0.02778446115553379, 0.007049499079585075, -0.030265677720308304, -0.02089649997651577, 0.017316827550530434, 0.006571348290890455, -0.005592430476099253, 0.02548416517674923, 0.04181884601712227, 0.009356255643069744, -0.006978422868996859, -0.012335008010268211, 0.006406579632312059, 0.0039156703278422356, 0.013019927777349949, -0.018143899738788605, 0.013504540547728539, 0.002654062118381262, 0.03119613416492939, -0.022822026163339615, -0.014111921191215515, -0.012186394073069096, 0.010816555470228195, 0.0062967343255877495, -0.011107322759926319, 0.047013893723487854, -0.005333970300853252, 0.01736851967871189, -0.01205070223659277, -0.02172357216477394, 0.0028462917543947697, 0.010590402409434319, -0.03481457382440567, 0.02080603875219822, 0.017678670585155487, 0.00949194747954607, 0.006881500128656626, -0.03982870280742645, -0.013879307545721531, 0.02884414792060852, -0.01580483466386795, 0.03266935795545578, 0.021801110357046127, 0.005153048317879438, 0.019668813794851303, -0.013982691802084446, 0.0439123697578907, -0.01912604831159115, 0.0028608301654458046, 0.01278731320053339, 0.013310695067048073, -0.025057706981897354, 0.02045711688697338, 0.0008698797901161015, 0.009640561416745186, -0.005531046073883772, -0.027422616258263588, 0.008477491326630116, -0.02283494919538498, 0.03540903329849243, -0.00015659048222005367, -0.0055536613799631596, 0.015998680144548416, -0.026673082262277603, 0.02154264971613884, 0.006707039661705494, 0.030033063143491745, 0.010293173603713512, 0.008406414650380611, -0.026026932522654533, 0.01789836212992668, -0.009950714185833931, -0.019966043531894684, 0.012257469817996025, 0.007146421819925308, -0.009356255643069744, 0.009330409578979015, 0.020948190242052078, -0.011469166725873947, -0.011087938211858273, 0.0028317535761743784, 0.005918736103922129, 0.004506898112595081, 0.002473140135407448, 0.01769159361720085, 0.015081146731972694, 0.011249476112425327, 0.02801707573235035, -0.005379200913012028, -0.030627522617578506, 0.017988823354244232, 0.013220233842730522, -0.022770334035158157, -0.05649937689304352, 0.013439925387501717, -0.02421771176159382, -0.004203207325190306, 0.009078411385416985, -0.03127367049455643, 0.016438063234090805, -0.032385051250457764, -0.01497776247560978, -0.003540903329849243, -0.006991345901042223, 0.024579554796218872, 0.008794104680418968, 0.025329090654850006, -0.013737154193222523, -0.019798044115304947, 0.007275651674717665, -0.026492159813642502, -0.027060773223638535, 0.022020800039172173, 0.016515599563717842, 0.0008811873849481344, -0.008147954940795898, -0.02967122010886669, -0.02951614372432232, 0.03551241755485535, -0.020470039919018745, -0.03579672425985336, 0.017730362713336945, 0.01645098626613617, -0.006335503421723843, 0.01654144562780857, -0.012490084394812584, -0.0022292183712124825, 0.009795637801289558, -0.025083553045988083, 0.0010685710003599524, -0.005747506394982338, -0.0025442165788263083, 0.11351567506790161, 0.017756208777427673, 0.003143521025776863, 0.021180804818868637, -0.010118712671101093, -0.00493012648075819, -0.01636052504181862, -0.038949936628341675, 0.004403513856232166, -0.006474425550550222, -0.001906143268570304, -0.015262069180607796, 0.007307959254831076, 0.011734088882803917, 0.010894093662500381, -0.003825209569185972, 0.008387030102312565, -0.015649758279323578, 0.0006069773808121681, -0.017019597813487053, -0.017084212973713875, -0.007540573365986347, -0.009388563223183155, 0.011598397046327591, -0.007760264445096254, -0.013401156291365623, 0.022033723071217537, 0.02469586208462715, 0.002356833079829812, -0.010590402409434319, -0.00839995313435793, -0.014383304864168167, -0.0070107304491102695, -0.008632567711174488, -0.011863318271934986, -0.017433132976293564, 0.010441788472235203, 0.009472562931478024, 0.017639901489019394, 0.006816884968429804, -0.01594698801636696, -0.0017009904840961099, 0.00575396791100502, -0.00852272193878889, -0.02363617531955242, -0.019578352570533752, 0.001490991679020226, 0.0061965808272361755, 0.008626106195151806, -0.02465709298849106, 0.024411557242274284, -0.0024602171033620834, -0.03393581137061119, 0.010467634536325932, 0.009278717450797558, -0.0018560666358098388, -0.008813489228487015, -0.0061190431006252766, -0.020702654495835304, 0.039337627589702606, -0.01912604831159115, -0.04432590678334236, 0.006387195084244013, -0.001891604857519269, 0.003931824117898941, -0.041870538145303726, 0.015262069180607796, 0.012011933140456676, 0.011527320370078087, 0.006254734471440315, 0.007107652723789215, -0.025212783366441727, -0.03776102140545845, -0.029412759467959404, 0.009362717159092426, -0.008354722522199154, 0.02995552495121956, -0.007146421819925308, 0.017911285161972046, 0.004044900648295879, -0.009879637509584427, -0.02030204050242901, 0.01594698801636696, -0.03931178152561188, -0.010041174478828907, -0.005954274442046881, -0.025561703369021416, 0.00618042703717947, -0.022912487387657166, 0.014680533669888973, 0.005411508493125439, 0.0039124395698308945, -0.009291640482842922, 0.0036345950793474913, 0.029619527980685234, 0.023765405640006065, 0.003989977762103081, 0.02156849578022957, 0.037683483213186264, -0.00821257010102272, 0.007850725203752518, -0.013956845737993717, -0.03272105008363724, -0.015791911631822586, -0.00695257680490613, -0.009634099900722504, -0.012638699263334274, -0.01321377232670784, 0.018350666388869286, -0.005543969105929136, -0.01875128038227558, -0.010351327247917652, -0.014783917926251888, 0.003867209190502763, 0.004138592164963484, 0.0069073461927473545, 0.001485337852500379, 0.02089649997651577, 0.0054405853152275085, -0.010454711504280567, -0.014783917926251888, -0.01928112283349037, 0.012800236232578754, 0.00849041435867548, -0.023028794676065445, -0.01601160317659378, 0.003277597250416875, -0.03241089731454849, -0.020379578694701195, -0.008807027712464333, -0.001227685483172536, 0.0029141376726329327, -0.002137142000719905, -0.026143239811062813, -0.047660041600465775, -0.013724231161177158, 0.008012263104319572, -0.01769159361720085, -0.02322264015674591, 0.01356915570795536, -0.018725434318184853, 0.004574743565171957, 0.00663596298545599, -0.019474968314170837, 0.03328965976834297, -0.030394908040761948, -0.0045456672087311745, -0.004784742835909128, 0.013000543229281902, 0.0023439100477844477, -0.008807027712464333, -0.006648886017501354, -0.038949936628341675, -0.008781181648373604, 0.03411673381924629, -0.04776342585682869, -0.012070086784660816, -0.027732769027352333, 0.0004660358827095479, 0.01953958347439766, 0.01921650767326355, -0.016295909881591797, 0.010202712379395962, -0.005379200913012028, 0.0055504306219518185, 0.020030658692121506, 0.0045456672087311745, 0.009201179258525372, -0.03269520401954651, 0.015119915828108788, 0.02154264971613884, 0.007753802929073572, 0.020909423008561134, 0.005162740591913462, 0.03739917650818825, 0.016412217170000076, -0.0026185237802565098, -0.009582407772541046, -0.0006057658465579152, -0.024850938469171524, -0.016076218336820602, 0.009084872901439667, -0.015533451922237873, -0.020108195021748543, -0.024682939052581787, 0.004313053097575903, 0.0077667259611189365, 0.002880214713513851, -0.005514892283827066, 0.0041612074710428715, 0.028714917600154877, -0.007773187477141619, 0.013827615417540073, -0.02799122966825962, 0.02242141403257847, 0.005792737007141113, -0.028301380574703217, -0.024127250537276268, 0.011223630048334599, 0.013401156291365623, 0.0108876321464777, -0.007876571267843246, 0.019113125279545784, 0.000991032924503088, -0.015817757695913315, 0.015701450407505035, -0.002500601578503847, 0.0032388281542807817, -0.003783209715038538, 0.0033664428628981113, -0.018001746386289597, -0.016942059621214867, -0.007682726718485355, -0.014060229063034058, -0.014318689703941345, 0.01610206440091133, -0.0010322249727323651, 0.010842401534318924, -0.0165672916918993, -0.02313217893242836, 0.011869779787957668, 0.0031742132268846035, 0.02050880901515484, 0.022847872227430344, 0.011514397338032722, 0.014047306030988693, -0.011947317980229855, -0.012619314715266228, 0.017782054841518402, -0.006655347533524036, 0.0007527650450356305, 0.011352860368788242, 0.008425799198448658, -0.00010409076639916748, -0.010215635411441326, -0.004745973739773035, -0.013129773549735546, -0.03463365510106087, -0.045773282647132874, 0.020146964117884636, 0.005692583508789539, -0.0023939867969602346, -0.012806697748601437, -0.010797170922160149, -0.0009781098924577236, 0.02384294383227825, -0.003002983285114169, 0.01739436388015747, -0.002733215456828475, -0.04029392823576927, 0.0015967987710610032, 0.0038639784324914217, 0.0136725390329957, 0.023183871060609818, 0.001526530017144978, -0.05238986387848854, -0.0029351375997066498, -0.011275322176516056, 0.026854004710912704, -0.004057823680341244, 0.007954109460115433, 0.012192855589091778, -0.04667789489030838, 0.005527815315872431, 0.02946445159614086, 0.010583940893411636, 0.012322084978222847, -0.010629171505570412, 0.0038058250211179256, 0.022964179515838623, -0.020211579278111458, -0.004180592019110918, -0.022847872227430344, -0.0030013679061084986, -0.035745032131671906, -0.009336871095001698, -0.010396557860076427, -0.01668359898030758, 0.014796840958297253, 0.00024513324024155736, 0.02719000168144703, 0.007398420479148626, -0.025264475494623184, 0.003032060107216239, -0.014202382415533066, 0.003443980822339654, 0.006235349923372269, 0.0036798256915062666, 0.007566419430077076, -0.016140833497047424, -0.022473106160759926, -0.006102889310568571, -0.0052370475605130196, 0.01442207396030426, -0.004238745663315058, -0.007463035639375448, 0.0064776563085615635, 0.012929466553032398, -0.025212783366441727, -0.005227355752140284, -0.004329206887632608, 0.011953779496252537, -0.010661479085683823, 0.0015515682753175497, -0.0017316826852038503, 0.0074824197217822075, 0.016580214723944664, -0.034943804144859314, -0.012063625268638134, 0.009395024739205837, 0.005608584266155958, -0.002486063167452812, 0.0009377255337312818, -0.015520528890192509, 0.011075015179812908, 0.026362929493188858, -0.008884565904736519, -0.021671880036592484, -0.017148828133940697, 0.010260866023600101, -0.00450366735458374, 0.010125174187123775, -0.00472981994971633, -0.009420870803296566, 0.00885871984064579, -0.01561099011451006, 0.01071317121386528, -0.008322414942085743, 0.009214102290570736, -0.02121957391500473, 0.0203408095985651, 0.02513524517416954, -0.0024747555144131184, -0.004280745517462492, -0.014383304864168167, -0.0040481314063072205, -0.019888505339622498, 0.03398750349879265, -0.015417144633829594, 0.0019222969422116876, 0.017782054841518402, 0.008671336807310581, 0.01356915570795536, 0.013530386611819267, 0.004380898550152779, 0.004203207325190306, -0.011708242818713188, -0.011217168532311916, -0.0013431848492473364, -0.01636052504181862, -0.02080603875219822, 0.01241900771856308, 0.0299038328230381, -0.002985214116051793, -0.01895804889500141, -0.013284849002957344, -0.04538559541106224, 0.007553496398031712, 0.006600424647331238, 0.010648556053638458, 0.05176955834031105, 0.027526000514626503, -0.004739512223750353, 0.023080486804246902, -4.780502422363497e-05, 0.009705176576972008, -0.002180757001042366, -0.003032060107216239, -0.0044455137103796005, -0.013452848419547081, -0.0016735291574150324, 0.01804051548242569, -0.029361067339777946, 0.01043532695621252, 0.006765193305909634, -0.005611815024167299, -0.0013698384864255786, 0.0290509145706892, -0.015882372856140137, 0.004248437937349081, 0.006816884968429804, 0.0027186772786080837, 0.019526660442352295, 0.0004139399970881641, 0.03282443434000015, -0.008729489520192146, 0.0042968993075191975, 0.007740879897028208, 0.020999882370233536, 0.007650419138371944, 0.00812210887670517, 0.009265794418752193, 0.01709713600575924, -0.015520528890192509, 0.001385992276482284, -0.03016229346394539, -0.014745148830115795, -0.023442331701517105, 0.029025068506598473, 0.021736495196819305, -0.016347602009773254, 0.016076218336820602, 0.022770334035158157, -0.02050880901515484, 0.005094894673675299, 0.008690720424056053, 0.003624902805313468, -0.0017962977290153503, 0.014577149413526058, 0.01518453098833561, 0.017730362713336945, -0.01104916911572218, 0.01965589076280594, -0.01004763599485159, 0.008193185552954674, -0.01898389495909214, 0.00010802824544953182, -0.0072239600121974945, 0.03119613416492939, -0.013775923289358616, -0.01679990626871586, 0.012457776814699173, -0.03496965020895004, -0.009743945673108101, -0.011837472207844257, -0.00363136432133615, 0.0023019101936370134, -0.009815022349357605, 0.015817757695913315, -0.0019287584582343698, 0.024670016020536423, -0.030084755271673203, -0.014344535768032074, -0.008561491034924984, -0.047194816172122955, 0.011604858562350273, 0.20800869166851044, -0.004972126334905624, -0.005970428232103586, 0.03931178152561188, -0.0019012971315532923, -0.011708242818713188, 0.0210128054022789, -0.005117509979754686, -0.025148168206214905, -0.009104257449507713, -0.006535809952765703, 0.03920839726924896, -0.020999882370233536, -0.0037412098608911037, 0.03117028810083866, -0.02633708342909813, -0.010079943574965, -0.019138971343636513, 0.005837967619299889, -0.008018724620342255, 0.0015976064605638385, 0.0016816060524433851, -0.016670675948262215, 0.005818583071231842, 0.019940197467803955, 0.0030805212445557117, -0.015856526792049408, -0.00300621404312551, 0.015106992796063423, 0.010396557860076427, -0.011036246083676815, -0.02757769264280796, 0.0045133596286177635, 0.004067515954375267, -0.02207249216735363, 0.023196794092655182, 0.002513524377718568, 0.019526660442352295, 0.009569485671818256, 0.006745808757841587, 0.02151680365204811, 0.03372904285788536, -0.0038284403271973133, -0.008309491910040379, 0.0061158123426139355, -0.016257140785455704, -0.009511332027614117, -0.01134639885276556, -0.005046433303505182, 0.004238745663315058, -0.04311114549636841, -0.009479024447500706, 0.012503007426857948, 0.013401156291365623, -0.001292300526984036, -0.011165476404130459, -0.004752435255795717, 0.0016912983264774084, -0.025380782783031464, 0.0009894175454974174, -0.0005621507298201323, 0.01838943548500538, -0.012793774716556072, 0.010028251446783543, -0.03600349277257919, 0.039156705141067505, -0.012257469817996025, 0.01071317121386528, 0.020961113274097443, -0.022059569135308266, 0.00011883107072208077, -0.018738357350230217, -0.025833087041974068, -0.006393656600266695, -0.000987802166491747, 0.0024036788381636143, 0.010926400311291218, -0.012399623170495033, 0.03241089731454849, -0.0006449387292377651, -0.011301168240606785, -0.005708737298846245, 0.0051562790758907795, 0.005993043538182974, -0.018143899738788605, -0.04432590678334236, 0.006322580389678478, -0.015908218920230865, 0.019048510119318962, -0.012057163752615452, -0.0067587317898869514, -0.012696852907538414, -0.011223630048334599, -0.005941351410001516, -0.014434996992349625, -0.0028495225124061108, 0.011540243402123451, 0.015430067665874958, -0.019526660442352295, -0.024967245757579803, -0.018324820324778557, 0.028353072702884674, 0.005909043829888105, 0.01780790090560913, -0.009627638384699821, 0.004594128113240004, -0.001222031656652689, 0.022214645519852638, 0.02025034837424755, -0.018673742190003395, 0.007165806367993355, -0.03755425289273262, 0.010758401826024055, -0.009750407189130783, -0.006564886774867773, 0.001707452000118792, -0.005027048755437136, -0.01583068072795868, 0.028533995151519775, -0.03905332088470459, -0.002263141330331564, 0.014628841541707516, 0.02103865146636963, -0.003508595982566476, 0.01642514020204544, -0.008109185844659805, -0.042154841125011444, 0.03820040449500084, 0.0012898774584755301, -0.007055960595607758, 0.011727627366781235, -0.004135361406952143, -0.00578304473310709, 0.004009362310171127, -0.012767928652465343, -0.023739559575915337, 0.006167504005134106, -0.007398420479148626, 0.014370381832122803, 0.007004268933087587, 0.014163613319396973, -0.0013981076190248132, 0.01086178608238697, 0.011831010691821575, 0.011637166142463684, -0.02013404108583927, -0.006303195841610432, -0.022124184295535088, -0.018970971927046776, -0.011275322176516056, 0.01580483466386795, 0.0018802972044795752, -0.02575554884970188, -0.014564226381480694, 0.02451493963599205, -0.03297950699925423, -0.025006014853715897, -0.027965383604168892, -0.0008367645787075162, 0.002318063983693719, -0.012968235649168491, 0.02027619443833828, 0.013097465969622135, 0.00043615142931230366, -0.0022389106452465057, 0.00780549505725503, -0.16127909719944, 0.012522391974925995, -0.007941186428070068, -0.02080603875219822, 0.03962193429470062, -0.01495191641151905, 0.019888505339622498, 0.00116064737085253, -0.007908878847956657, -0.004985049366950989, -0.011617781594395638, -0.024359865114092827, -0.022563567385077477, -0.02363617531955242, -0.006810423452407122, -0.01119778398424387, -0.017782054841518402, 0.04882311448454857, 0.004910741932690144, -0.00806395523250103, 0.03354812040925026, -0.005511661525815725, 0.04099177196621895, -0.011062092147767544, 0.014396227896213531, 0.006713501177728176, -2.715850314416457e-05, 0.0068750386126339436, -0.0038930552545934916, -0.004397052340209484, -0.012115317396819592, -0.007443651091307402, -0.00839995313435793, -0.021465111523866653, 0.036158569157123566, 0.005883198231458664, -0.0009143025963567197, -0.017872516065835953, -0.010377173312008381, 0.017084212973713875, 0.028482303023338318, -0.0011453012702986598, 0.007592265494167805, -0.006719962693750858, 0.0040481314063072205, 0.021852800622582436, 7.042028300929815e-06, -0.00326305883936584, -0.0030482138972729445, -0.022150030359625816, 0.014357458800077438, -0.019087279215455055, 0.027629384770989418, 0.009911945089697838, -0.01847989670932293, -0.003389058168977499, -0.010919938795268536, -0.010583940893411636, -0.019823890179395676, -0.004435821436345577, -0.023998020216822624, -0.0003049021470360458, -0.012561161071062088, -0.009246409870684147, -0.017407286912202835, -0.011598397046327591, 0.007236883044242859, 0.0002120180579368025, -0.016140833497047424, 0.020754346624016762, -0.00874241255223751, 0.0019949888810515404, 0.020793115720152855, -0.011882702820003033, -0.007036576047539711, 0.017730362713336945, -0.005779813975095749, -0.031118595972657204, -0.01495191641151905, 0.005527815315872431, -0.02368786744773388, 0.01099101547151804, -0.025949394330382347, 0.0066941166296601295, 0.0020741422194987535, 0.008483952842652798, 0.001227685483172536, 0.009414409287273884, -0.02315802499651909, -0.011695319786667824, 0.017471902072429657, 0.0004874395963270217, -0.037063177675008774, 0.014357458800077438, 0.0041288998909294605, 0.016916213557124138, 0.006186888553202152, 0.010706709697842598, -0.0025393704418092966, -0.02407555840909481, 0.005537507589906454, 0.012884235940873623, -0.007773187477141619, 0.01898389495909214, -0.0015451067592948675, 0.02951614372432232, 0.01857035793364048, 0.01659313775599003, 0.005892890505492687, -0.015145761892199516, -0.004904280416667461, -0.016515599563717842, 0.0065131946466863155, -0.010648556053638458, -0.009750407189130783, 0.017794977873563766, 0.02160726487636566, -0.02174941822886467, 0.026776466518640518, -0.00438412930816412, 0.03649456799030304, -0.025652164593338966, -0.009466101415455341, 0.008277184329926968, -0.02404971234500408, -0.024618323892354965, -0.11082769185304642, -0.013246079906821251, 0.022020800039172173, -0.008891027420759201, 0.0030546754132956266, 0.025148168206214905, -0.0009029950015246868, 0.016373448073863983, -0.0025102936197072268, 0.026324160397052765, -0.023713713511824608, -0.02483801543712616, -0.0003693152393680066, -0.012134701944887638, 0.021620187908411026, -0.013982691802084446, 0.00965348444879055, -0.022473106160759926, -0.022705720737576485, 0.02416601963341236, -0.007883032783865929, -0.01187624130398035, 0.00381874805316329, -0.007954109460115433, -0.012774390168488026, -0.005282278172671795, -0.013504540547728539, 0.015520528890192509, 0.003418134758248925, 0.004471359774470329, 0.009815022349357605, -0.019823890179395676, -0.009323948062956333, -0.01603744924068451, -0.010635633021593094, 0.0035182880237698555, -0.0007337843999266624, -0.021193727850914, 0.002397217322140932, -0.03791609779000282, -0.02580724097788334, 0.02324848622083664, 0.0008698797901161015, -0.017846670001745224, -0.005249970592558384, -0.00245052482932806, -0.007081806659698486, 0.04225822538137436, 0.005153048317879438, -0.02154264971613884, -0.012638699263334274, -0.016903290525078773, -0.00498828012496233, -0.018880510702729225, 0.025626318529248238, 0.00498828012496233, 0.0046942816115915775, -0.005185355897992849, 0.021271266043186188, 0.015391298569738865, -0.015701450407505035, -0.01131409127265215, -0.018001746386289597, -0.0020596040412783623, 0.009130103513598442, -0.0034342885483056307, 0.021969107910990715, -0.008438722230494022, -0.004519821144640446, -0.02713831141591072, -0.02628539316356182, -0.004791203886270523, -0.0017704516649246216, 0.017239289358258247, -0.02719000168144703, -0.019823890179395676, -0.02351986989378929, -0.010144558735191822, 0.02598816342651844, -0.026776466518640518, 0.009149487130343914, -0.004490744322538376, 0.009724561125040054, -0.034659501165151596, 0.02313217893242836, 0.02156849578022957, -0.012761467136442661, 0.003154828678816557, -0.023119255900382996, -0.03822625055909157, -0.006739347241818905, 0.009336871095001698, 0.005873505957424641, 0.003350289072841406, -0.01433161273598671, 0.019048510119318962, 0.03034321591258049, -0.007521188817918301, 0.0353831872344017, 0.015507605858147144, 0.007236883044242859, -0.009556562639772892, -0.03486626595258713, 0.011598397046327591, 0.008748874068260193, -0.031325362622737885, 0.004991510882973671, 0.014770994894206524, 0.001142878201790154, 0.0014578765258193016, 0.004603820387274027, 0.012554699555039406, -0.014021460898220539, 0.020534655079245567, -0.01356915570795536, -0.02760353870689869, -0.011747011914849281, -0.018815895542502403, -0.00695257680490613, -0.01636052504181862, -0.009634099900722504, 0.013439925387501717, -0.02580724097788334, -0.004251668695360422, 0.0013884153449907899, -0.007475958205759525, 0.0032226743642240763, 0.008587337099015713, -0.00102657126262784, 0.030446600168943405, 0.00464258948341012, -0.020909423008561134, 0.04419667646288872, -0.016257140785455704, -0.0016533369198441505, -0.001801143866032362, -0.002087065251544118, -0.02569093368947506, -0.014926071278750896, 0.02071557752788067, -0.00780549505725503, 0.03013644739985466, -0.013620846904814243, -0.0153783755376935, 0.01278731320053339, -0.008651952259242535, -0.040009625256061554, -0.008051032200455666, -0.016735291108489037, -0.004655512515455484, 0.02112911269068718, 0.00561504578217864, 0.026905696839094162, 0.020612193271517754, -0.02066388539969921, -0.005240278318524361, -0.006025351118296385, -0.025626318529248238, 0.01610206440091133, 0.002733215456828475, -0.005789506249129772, -0.018841741606593132, 0.04734988883137703, 0.0020596040412783623, -0.012380238622426987, -0.019022664055228233, 0.025471242144703865, -0.01924235373735428, -0.015042377635836601, 0.011152553372085094, 0.025148168206214905, 0.010984553955495358, -0.008328876458108425, 0.015998680144548416, 0.034892112016677856, 0.046135127544403076, 0.03158382326364517, 0.01028671208769083, 0.005014125723391771, -0.014021460898220539, -8.894661732483655e-05, 0.013982691802084446, 0.03117028810083866, -0.0024133711121976376, -0.011068553663790226, -0.0167482141405344, 0.01921650767326355, 0.016489753499627113, -0.006700578145682812, -0.0020579886622726917, 0.007133498787879944, 0.007527650333940983, -0.009220563806593418, 0.006290272809565067, -0.006668270565569401, 0.007385497447103262, 0.01210885588079691, 0.01165655069053173, -0.019798044115304947, -0.019991889595985413, -0.0006962268962524831, -0.00441320613026619, 0.028921686112880707, 0.0009288410074077547, -0.008748874068260193, 0.005233817268162966, -0.015119915828108788, 0.009795637801289558, -0.012406084686517715, -0.050864946097135544, 0.03054998442530632, 0.02454078570008278, 0.0077990335412323475, -0.01847989670932293, 0.007359651383012533, -0.002728369552642107, -0.01944912225008011, 0.010273789055645466, 0.010409480892121792, -0.00880056619644165, -0.01574021950364113, 0.011294706724584103, 0.014887302182614803, 0.019345737993717194, 0.022524798288941383, -0.014008537866175175, 0.03964778035879135, -0.006086735520511866, 0.026905696839094162, -0.02824968844652176, -0.007540573365986347, 0.00024937361013144255, -0.01580483466386795, 0.013879307545721531, -0.021529726684093475, 0.010247942991554737, 0.009311025030910969, -0.020237425342202187, 0.01707128994166851, 0.018182668834924698, -0.02174941822886467, 0.029438605532050133, 0.011152553372085094, 0.010965169407427311, 0.025910625234246254, -0.03098936565220356, 0.024682939052581787, 0.012393161654472351, 0.0049753570929169655, -0.009395024739205837, -0.003206520574167371, 0.00017567210306879133, 0.02492847666144371, 0.04564405232667923, -0.00037961328052915633, -0.027939537540078163, -0.0042645917274057865, 0.004968895576894283, -0.004720127675682306, -0.0002990464272443205, -0.012283315882086754, -0.0025038323365151882, -0.006448579486459494, 0.014861456118524075, 0.001239800825715065, 0.0025716780219227076, 0.007036576047539711, 0.020935269072651863, 0.0034924421925097704, -0.008819950744509697, -0.018854664638638496, 0.0005940544069744647, -0.0281979963183403, 0.012664545327425003, -0.001975604332983494, 0.024915553629398346, 0.004390590824186802, -0.021232496947050095, -0.006338734179735184, -0.001978835090994835, -0.015934064984321594, -0.008076878264546394, 0.0050561255775392056, -0.030213985592126846, -0.026905696839094162, -0.0145254572853446, 0.01601160317659378, 0.009388563223183155, -0.010151020251214504, -0.015623913146555424]} +{"id": "test:10000017", "text": "\"UNStudio has joined forces with HPP Architects to create a consortium (UNS + HPP) to carry out the next phases of their winning project at the architectural design competition for FOUR Frankfurt. Take a look at the complete story after the jump.\\nFrom the architects: The centrally located 16,000 square meter site was purchased by Gro\u00df & Partner real estate development company \u2013 who will be carrying out the development of the project \u2013 back in 2015. Situated in the very core of the city, the site has been completely inaccessible for the last 45 years. Now four new high-rise towers will change Frankfurt\u2019s skyline from the air, while cultivating its liveliness on the ground. The development of these towers, reaching heights of 228 meters, will open up new streets to create a multi-use, vibrant inner-city quarter, bringing together a healthy mix of work, living, relaxation and recreation.\\nThe choice of programmes allows for a smooth transition from Frankfurt\u2019s shopping district to the east of Ro\u00dfmarkt, to the high-rise office towers clustered around Park Taunusanlage. With a development concept that is unique to Europe, the FOUR Frankfurt project brings together facilities that will establish a lively new neighbourhood for Frankfurt, its visitors, and its (future) residents.\\nThe new high-rise complex will be integrated into the expanding city structure by incorporating the heritage-listed facades of the Junghofstra\u00dfe into the design and by making a multi-storey base building the connecting element of the entire site.\\nAccording to the assessment by the competition jury, the unique quality of the quarter can be found in its public development and amenity value. The new development will therefore create connected spaces, accessible rooftops, pathways and passages. The existing block on the Junghofstra\u00dfe will be opened up to strengthen the surrounding pathways and ensure a high level of accessibility. This will create a multi-use, diverse quarter comprising 50% office spaces, 30% living accommodation (including subsidized housing), in addition to retail spaces, restaurants and hotels.\\nFour Frankfurt is expected to be completed in 2023.\"", "vector_field": [0.0010291831567883492, -0.0011480656685307622, 0.0052308314479887486, -0.02014889195561409, 0.006175098475068808, 0.01040052343159914, -0.03274364769458771, -0.003882364137098193, -0.009626088663935661, -0.03195562586188316, 0.009965753182768822, -0.002924510510638356, -0.0115214167162776, -0.02277110144495964, -0.001973450183868408, -0.017635375261306763, 0.009388323873281479, -0.0023317961022257805, 0.004306944552809, -0.05570496246218681, -0.0034577834885567427, 0.017159845679998398, -0.005940730217844248, -0.02014889195561409, 0.007126159034669399, 0.030162198469042778, 0.009564949199557304, -0.01755385659635067, -0.01980922743678093, 0.01660279557108879, -0.008681821636855602, -0.00019286568567622453, -0.011202131398022175, 0.014143625274300575, 0.0075405496172606945, -0.011202131398022175, -0.0015692495508119464, -0.005825244355946779, 0.011297238059341908, -0.0034441968891769648, 0.00908941961824894, -0.01660279557108879, 0.014428943395614624, 0.0004003794165328145, 0.008009286597371101, -0.0004269156779628247, 0.024754740297794342, -0.010115205310285091, -0.010094826109707355, 0.017363643273711205, 0.004908150527626276, 0.05956354737281799, -0.022512955591082573, 0.016820181161165237, -0.005383680574595928, -0.002978856675326824, -0.004908150527626276, 0.01315859891474247, -0.0072824046947062016, -0.014985993504524231, 0.00796852633357048, -0.018980447202920914, -0.03184693306684494, 0.023314563557505608, -0.045949798077344894, 0.008844860829412937, -0.04690086096525192, 0.008206292055547237, -0.008987519890069962, -0.00781907420605421, 0.03600442409515381, 0.01980922743678093, 0.01396699994802475, 0.0012287358986213803, 0.041112978011369705, -0.005091569386422634, 0.020216824486851692, -0.0030145214404910803, -0.006715164985507727, 0.006297377869486809, 0.012907247059047222, -0.02570580132305622, -0.024944953620433807, 0.023300977423787117, 0.025583522394299507, 0.016629967838525772, -0.0034509901888668537, 0.036520715802907944, 0.011745594441890717, -0.0007302784360945225, -0.005791278090327978, 0.03293386101722717, -0.015176204964518547, 0.01919783093035221, -0.021657001227140427, -0.01053638942539692, -0.008566335774958134, 0.025787319988012314, -0.008097599260509014, 0.010665462352335453, -0.011528209783136845, 0.007581309415400028, -0.025434069335460663, -0.0077851079404354095, -0.019007619470357895, 0.009286424145102501, 0.005781088024377823, -0.005054206121712923, -0.004171078559011221, -0.0029958400409668684, -0.009530982933938503, 0.04005322605371475, -0.01558380201458931, -0.01945597678422928, 0.005920350551605225, -0.034591421484947205, 0.016752248629927635, 0.00010784344340208918, 0.015298483893275261, 0.004493759945034981, 0.023165112361311913, 0.020271170884370804, 0.0207738745957613, -0.001637182431295514, 0.024442249909043312, 0.032634954899549484, -0.00034178729401901364, -0.02074670046567917, -0.0226895809173584, -0.019388044252991676, 0.005101758986711502, 0.003953693434596062, 0.03662940859794617, 0.011541795916855335, -0.028015518561005592, 0.023260217159986496, -0.01053638942539692, 0.015597389079630375, -0.040759727358818054, -0.04831386357545853, 0.008342157118022442, 0.019917920231819153, -0.024822674691677094, -0.011412723921239376, -0.005006653256714344, 0.00908262562006712, 0.013416743837296963, 0.009143765084445477, 0.005679188761860132, -0.020638009533286095, -0.02635795623064041, -0.012071672827005386, -0.01661638170480728, 0.011616522446274757, 0.036520715802907944, 0.011358377523720264, -0.001888534054160118, 0.03706417977809906, -0.021032018586993217, 0.011738801375031471, 0.009870647452771664, -0.012336610816419125, -0.007126159034669399, -0.009857060387730598, 0.0027037286199629307, -0.005145915783941746, 0.019374456256628036, 0.005149312317371368, -0.015108272433280945, -0.0018273944733664393, 0.00793456006795168, 0.010923607274889946, -0.03749895095825195, 0.020597249269485474, 0.028015518561005592, 0.023817267268896103, 0.013090665452182293, 0.010685841552913189, -0.02956438809633255, -0.03086869977414608, 0.012526823207736015, -0.0032947445288300514, 0.01725495047867298, 0.018369050696492195, -0.017676135525107384, 0.0005727590760216117, 0.01509468536823988, 0.00908941961824894, -0.002538991393521428, 0.0032539849635213614, 0.016113677993416786, 0.006480796728283167, 0.011467070318758488, -0.010054065845906734, -0.6421559453010559, 0.0015327355358749628, 0.005713155027478933, -0.03282516822218895, 0.005278384778648615, 0.028423115611076355, 0.008946760557591915, 0.008675028569996357, -0.02338249608874321, 0.023803681135177612, -0.007839454337954521, 0.008416883647441864, 0.004734921734780073, 0.009300011210143566, -0.011698042042553425, -0.012587962672114372, 0.007757934741675854, -0.02563786879181862, -0.0003997425374109298, 0.0004925133544020355, 0.01748592220246792, 0.028450289741158485, -0.011942599900066853, -0.0115553829818964, 0.018939686939120293, 0.02339608408510685, -0.0033439958933740854, -0.012438510544598103, -0.04157492145895958, -0.012825727462768555, -0.02372216060757637, 0.021521136164665222, -0.036276157945394516, -0.0019530702847987413, 0.04836820811033249, 0.006307567935436964, -0.028531808406114578, 0.017784828320145607, 0.019401630386710167, 0.04035212844610214, 0.004262788221240044, -0.005054206121712923, 0.022526541724801064, -0.00028637953801080585, -0.005563702899962664, 0.0004428374522831291, 0.044319409877061844, -0.04342269524931908, 0.0024778516963124275, -0.006046026013791561, -0.003685358678922057, -0.022811859846115112, -0.007995699532330036, 0.014673502184450626, -0.018939686939120293, -0.0010334289399906993, 0.01232981774955988, -0.00633474113419652, -0.01172521524131298, -0.011582556180655956, 0.010441283695399761, 0.004201648756861687, -0.013437123037874699, -0.020705942064523697, -0.0027088236529380083, 0.012526823207736015, -0.014034932479262352, -0.006664215587079525, 0.004996463190764189, -0.012880073860287666, -0.016657141968607903, 0.033259935677051544, -0.014130039140582085, -0.002173852175474167, 0.019388044252991676, 0.01953749544918537, -0.0010020099580287933, -0.02434714324772358, 0.010115205310285091, 0.03521640598773956, 0.0038144311401993036, -0.022961312904953957, -0.011813527904450893, 0.02338249608874321, 0.014170798473060131, -0.022295570001006126, -0.03472728654742241, -0.005635032430291176, 0.023776507005095482, -0.014700675383210182, 0.011935806833207607, -0.034944672137498856, 0.005210451781749725, 0.0034034373238682747, -0.010121999308466911, 0.04676499217748642, -0.007031052839010954, 0.02442866377532482, 0.0007014069706201553, -0.04315096512436867, -0.005315747577697039, -0.015760427340865135, 0.0185049157589674, -0.0004593961057253182, 0.015706080943346024, 0.008552749641239643, 0.003476465120911598, -0.001541227218694985, 0.033912092447280884, -0.012784968130290508, -0.029020925983786583, -0.0031130241695791483, -0.022811859846115112, -0.01756744273006916, 0.01847774349153042, -0.029808947816491127, 0.01475502084940672, -0.005828640889376402, 0.012486062943935394, -0.003532509785145521, 0.019034793600440025, -0.01759461499750614, 0.023844441398978233, 0.0030637728050351143, -0.00908941961824894, 0.022580888122320175, 0.011738801375031471, -0.02858615480363369, -0.02539331093430519, 0.004870787262916565, 0.001177786267362535, 0.005791278090327978, 0.04255315661430359, -0.008158738724887371, 0.032390397042036057, 0.016426170244812965, 0.017350057139992714, -0.00957174226641655, 0.026738381013274193, -0.03051544912159443, -0.029401350766420364, -0.021344510838389397, 0.01479578111320734, -0.022254811599850655, 0.0006254070904105902, -0.015787601470947266, -0.014999579638242722, -0.012275471352040768, -0.005400663707405329, -0.017064739018678665, -0.026086226105690002, 0.002328399335965514, -0.03241756930947304, 0.016969632357358932, -0.0020923325791954994, -0.012553996406495571, 0.02597753331065178, -0.03543378785252571, -0.01218036562204361, -0.012771381065249443, 0.021575482562184334, 0.010787741281092167, -0.026575341820716858, 0.007771521341055632, -0.007221265230327845, -8.252783300122246e-05, 0.011969773098826408, 0.006830650847405195, -0.014551222324371338, -0.023817267268896103, -0.017975039780139923, -0.03146651014685631, -0.005750518292188644, 0.025597108528017998, -0.01718701794743538, -0.01332163717597723, 0.0039061405695974827, -0.018627194687724113, 0.02142602950334549, -0.009041866287589073, 0.00022205559071153402, 0.007764728274196386, -0.030977392569184303, -0.005916953552514315, 0.004670385271310806, 0.03440120816230774, -0.0030892477370798588, 0.01820601150393486, 0.003434007056057453, 0.013484676368534565, 0.020624421536922455, 0.02891223318874836, -0.010339383967220783, 0.05940051004290581, -0.017472336068749428, 0.007526963017880917, 0.0019411820685490966, 0.0076492419466376305, -0.013124632649123669, 0.01302273292094469, 0.01821959763765335, -0.01661638170480728, 0.02597753331065178, -0.011765974573791027, 0.02658892795443535, -0.030352409929037094, 0.021616242825984955, -0.004517536610364914, 0.019279351457953453, -0.001191372866742313, 0.0015403780853375793, -0.00635172426700592, -0.04429223760962486, 0.004670385271310806, 0.006684595253318548, 0.013430329971015453, -0.009945373050868511, -0.007914179936051369, -0.019619015976786613, 0.0048775807954370975, 0.005536529701203108, 0.017350057139992714, 0.02043421007692814, 0.015380003489553928, -0.0025695611257106066, -0.004245805088430643, 0.024184104055166245, 0.01748592220246792, 0.001987036783248186, -0.008539162576198578, -0.01332843117415905, 2.088405381073244e-05, 0.014904473908245564, 0.006912170443683863, 0.02798834629356861, 0.009558156132698059, 0.006212461739778519, -0.01138555072247982, 0.02040703594684601, 0.0018715508049353957, 0.03858587518334389, 0.010264658369123936, 0.006603075657039881, -0.0041303192265331745, 0.010339383967220783, -0.014184385538101196, 0.02312435209751129, 0.013206151314079762, -0.002578052692115307, 0.024170517921447754, -0.021928733214735985, 0.022580888122320175, -0.006310964468866587, -0.014890886843204498, 0.015774015337228775, 0.015108272433280945, 0.007262024562805891, -0.011439897119998932, 0.03858587518334389, 0.00991820078343153, -0.009721195325255394, 0.0040420060977339745, 0.02535255067050457, -0.0006466360646300018, -0.011636902578175068, 0.010719808749854565, 0.0041269222274422646, -0.00746582355350256, 0.0002651505055837333, -0.0075405496172606945, 0.00025156393530778587, 0.002496533328667283, -0.0026952370535582304, 0.0005969601916149259, 0.006868014112114906, -0.0011472165351733565, 0.003238700097426772, 0.00255767279304564, 0.015733255073428154, 0.01953749544918537, -0.014836540445685387, -0.00974836852401495, 0.013185772113502026, 0.025284618139266968, 0.0014970707707107067, -0.009503809735178947, -0.013036319054663181, 0.0023895390331745148, -0.0016872829291969538, 0.00377367134205997, -0.004650005605071783, -0.014659915119409561, -0.018423397094011307, -0.004639815539121628, -0.01982281357049942, -0.03228170424699783, 0.02760792151093483, -0.020678767934441566, -0.005889780819416046, -0.017947867512702942, -0.00022502765932586044, -0.004198251757770777, 0.011861080303788185, -0.02345043048262596, 0.033993612974882126, 0.006474003195762634, 0.005957713350653648, 0.01104588620364666, 0.00675592478364706, -0.012744207866489887, -0.0037566882092505693, 0.001384132425300777, -0.0018970256205648184, 0.03225453197956085, -0.015515869483351707, -0.016331063583493233, -0.027879653498530388, -0.005808261223137379, 0.029401350766420364, -0.0266568623483181, 0.00447677681222558, -0.018042972311377525, -0.011073059402406216, 0.01411645207554102, 0.1019536629319191, 0.02505364641547203, 0.0003638654889073223, 0.007071812637150288, -0.0027563765179365873, 0.0051832785829901695, -0.01597781293094158, -0.03456424921751022, -0.009469843469560146, 0.015040338970720768, 0.012221124954521656, -0.03518922999501228, -0.0018817407544702291, 0.014850127510726452, 0.019985852763056755, -0.002251975005492568, 0.01189504750072956, -0.01398058608174324, 0.009877440519630909, 0.001114948419854045, -0.0007982113165780902, -0.01270344853401184, -0.00737751042470336, 0.04192817211151123, 0.014890886843204498, -0.0029720633756369352, 0.017934279516339302, -0.0004065358079969883, -0.005648619029670954, -0.03695548698306084, 0.011154578998684883, 0.016480516642332077, 0.01751309633255005, 0.003501939820125699, -0.03056979551911354, -0.030596967786550522, 0.005635032430291176, -0.021697761490941048, 0.01220753788948059, 0.002630700822919607, -0.009870647452771664, 0.023233044892549515, 0.019034793600440025, 0.0038891572039574385, 0.002858275780454278, 0.00398766016587615, -0.014130039140582085, 0.017390817403793335, -0.011568969115614891, -0.0005370943108573556, 0.020570075139403343, -0.011480656452476978, -0.006817064248025417, 0.0025016283616423607, -0.0038076378405094147, -0.015026752837002277, -0.001579439383931458, -0.02177928015589714, -0.0035732693504542112, -0.022309157997369766, -0.020420623943209648, -0.030162198469042778, 0.011915426701307297, -0.005057602655142546, -0.013606955297291279, -0.009714401327073574, -0.005006653256714344, -0.0055874790996313095, -0.03440120816230774, -0.00957853626459837, 0.020080959424376488, -0.004099749028682709, -0.02790682576596737, 0.0231107659637928, 0.0357326939702034, -0.018070146441459656, 0.01884458027780056, 0.029048098251223564, 0.010869260877370834, 0.006059612613171339, -0.02698293887078762, -0.01136517059057951, 0.007642448879778385, -0.0224721971899271, -0.011915426701307297, 0.02441507577896118, 0.0002509270852897316, -0.021262990310788155, 0.0006113959243521094, 0.015787601470947266, 0.025515589863061905, -0.0033541859593242407, 0.0057607083581388, 0.016113677993416786, -0.021262990310788155, -0.005730138160288334, 0.03831414505839348, -0.0018019196577370167, -0.0002991169458255172, -0.015162618830800056, 0.01122930459678173, -0.009802713990211487, 0.008131565526127815, -0.015380003489553928, 0.01979564130306244, 0.004870787262916565, 0.02047497034072876, -0.004378274083137512, -0.030705660581588745, -0.007153332233428955, 0.0272954311221838, -0.002732600085437298, -0.0011930711334571242, 0.020339103415608406, -0.005648619029670954, 0.0013433726271614432, -0.0020991258788853884, 0.01720060408115387, 0.01630389131605625, 0.017635375261306763, -0.0143202506005764, -0.020013026893138885, 0.03146651014685631, 0.02143961749970913, 0.000562993751373142, 0.01948314905166626, 0.023898785933852196, -0.002111014211550355, -0.017798414453864098, 0.00810439232736826, -0.034999020397663116, 0.039183683693408966, 0.006046026013791561, -0.018939686939120293, 0.005424440372735262, -0.020692354068160057, -0.02180645428597927, -0.001485182554461062, -0.003885760670527816, 0.009714401327073574, -0.03407513350248337, 0.01752668246626854, -0.01880382001399994, 0.018749475479125977, -0.010054065845906734, -0.025800907984375954, -0.00624982500448823, -0.001922500436194241, -0.013049906119704247, 0.026113398373126984, -0.03209149092435837, 0.008342157118022442, 0.011691248044371605, 0.027458470314741135, -0.026751967146992683, -0.035569656640291214, 0.007513376418501139, -0.04907471314072609, 0.016426170244812965, 0.004656798671931028, 0.018042972311377525, 0.029102444648742676, -0.0016618079971522093, 0.004799457732588053, -0.012390957213938236, -0.003831414273008704, -0.006124149076640606, -0.006946136709302664, -0.012873280793428421, 0.028015518561005592, 0.014415357261896133, 0.02308359183371067, 0.007418270222842693, -0.00895355362445116, -0.011439897119998932, 0.032037146389484406, -0.01332843117415905, -0.005044016055762768, -0.014062105678021908, -0.034618593752384186, -0.007139745634049177, 0.0017645565094426274, -0.019659774377942085, -0.01789352111518383, -0.03415665030479431, -0.004198251757770777, 0.013117838650941849, -0.01493164710700512, 0.010957573540508747, -0.0014045123243704438, 0.032009974122047424, -0.023572709411382675, -0.006494383327662945, 0.0034068338572978973, 0.0016295398818328977, 0.005227434914559126, -0.004942116793245077, -0.03543378785252571, 0.00036874815123155713, 0.03744460269808769, 0.008661442436277866, -0.007071812637150288, -0.010848880745470524, -0.006463813595473766, 0.01234340388327837, 0.01821959763765335, -0.008090806193649769, -0.005519546568393707, -0.017648961395025253, -0.005149312317371368, 0.013199358247220516, -0.008185911923646927, 0.010319004766643047, -0.03089587390422821, -0.003388152224943042, 0.011813527904450893, -0.013708855025470257, 0.009544569067656994, -0.025828080251812935, -0.010312210768461227, 0.018314704298973083, -0.005128932185471058, 0.02734977751970291, 0.02082822099328041, 0.011358377523720264, 0.00874296110123396, 0.007628862280398607, -0.02891223318874836, -0.010108412243425846, 0.013824340887367725, -0.022540129721164703, 0.022023839876055717, 0.01821959763765335, -0.008545956574380398, -0.012560789473354816, 0.00019127351697534323, 0.01626313105225563, -0.019945092499256134, -0.03279799222946167, 0.018654368817806244, 0.026779141277074814, 0.008321777917444706, -0.011827114038169384, 0.013131425715982914, -0.015814773738384247, -0.0028565775137394667, -0.013715648092329502, -0.01849132962524891, -0.009680435061454773, -0.014700675383210182, -0.02434714324772358, 0.014727848581969738, -0.027458470314741135, 0.007526963017880917, 0.00845764297991991, -0.027404123917222023, 0.0014775401214137673, -0.023599881678819656, -0.0008712391718290746, 0.014102865941822529, -0.00036938503035344183, 0.032037146389484406, -0.009157352149486542, -0.006772907916456461, 0.01717343181371689, -0.014972406439483166, -0.018450569361448288, -0.01023748517036438, -0.005390474107116461, 0.025936773046851158, -0.0048266309313476086, -0.016793007031083107, -0.01172521524131298, 0.027091631665825844, 0.0015675511676818132, -0.014836540445685387, -0.0046432120725512505, -0.010393730364739895, 0.001942880335263908, -0.009938579984009266, 0.0001858601171988994, -0.005733535159379244, -0.010509216226637363, -0.012486062943935394, 0.024931367486715317, -0.01184070110321045, -0.001168445567600429, -0.007126159034669399, 0.013912653550505638, -0.03518922999501228, -0.010617909021675587, 0.0018070145742967725, -0.020923325791954994, 0.007459030020982027, 0.008579922839999199, -0.01430666446685791, -0.023599881678819656, 0.03421099856495857, -0.02345043048262596, 0.010210311971604824, 0.01654844917356968, 0.016045745462179184, -0.017676135525107384, 0.0220510121434927, -0.0033219177275896072, -0.017051152884960175, 0.013043113052845001, -0.006100372411310673, -0.015719668939709663, 0.03535227105021477, 0.011188545264303684, -0.006494383327662945, -0.0017331375274807215, -0.00252540479414165, 0.03513488546013832, -0.0032658730633556843, -0.014048519544303417, 0.016371823847293854, -0.011956186965107918, 0.03051544912159443, 0.024306384846568108, 0.007982113398611546, 0.015597389079630375, 0.0005430384771898389, 0.03149368241429329, 0.002175550442188978, -0.0048911673948168755, 0.011460276320576668, -0.03453707695007324, -0.009021486155688763, 0.011677661910653114, 0.01688811369240284, -0.03777068108320236, -0.0010249372571706772, -0.044971566647291183, -0.019714120775461197, -0.019252177327871323, 0.010244278237223625, 0.0047790780663490295, -0.0041269222274422646, 0.007805487606674433, 0.030950220301747322, -0.003055281238630414, 0.021276578307151794, -0.01792069338262081, -0.018015800043940544, 0.008566335774958134, -0.007683208677917719, -0.0207738745957613, -0.025203097611665726, 0.020692354068160057, 0.039129339158535004, 0.010488836094737053, -0.04089559242129326, 0.003488353220745921, -0.0010368255898356438, -0.02245860919356346, -0.011956186965107918, -0.01315180491656065, -0.00020454164769034833, 0.023137938231229782, 0.019089138135313988, -0.008022872731089592, 0.025569936260581017, -0.015678908675909042, 0.01151462271809578, -0.022512955591082573, -0.02010813169181347, 0.013953412882983685, -0.01978205516934395, 0.0279339998960495, 0.0027784546837210655, -0.03945541754364967, 0.006219255272299051, 0.018980447202920914, 0.012377370148897171, -0.002871862379834056, 0.018708715215325356, -0.01315859891474247, 0.0031418956350535154, 0.010950780473649502, 0.015841947868466377, -0.021657001227140427, 0.009558156132698059, 0.048612769693136215, -0.024238450452685356, 0.005387077108025551, 0.02826007828116417, 0.02243143692612648, -0.00942229013890028, -0.018097318708896637, -0.00486399419605732, 0.006592886056751013, 0.019075552001595497, 0.02470039390027523, -0.019428802654147148, 0.03497184440493584, -0.02951004169881344, 0.03306972607970238, 0.0018273944733664393, 0.0011964677833020687, 0.011691248044371605, 0.020352691411972046, -0.007248437963426113, -0.016725074499845505, -0.004751904867589474, -0.024184104055166245, -0.014727848581969738, 0.008410090580582619, -0.024170517921447754, 0.0009807809256017208, -0.03220018371939659, 0.005291971378028393, -0.007071812637150288, -0.005872797220945358, -0.0005612954264506698, -0.023531949147582054, -0.011147785000503063, 0.03249908983707428, 0.006609869189560413, 0.014401770196855068, 0.012268678285181522, -0.015352830290794373, 0.0012499649310484529, 0.004819837864488363, -0.00297376187518239, 0.005383680574595928, -0.016779420897364616, 0.0024065221659839153, 0.0113447904586792, -0.022186879068613052, -0.02726825699210167, 0.007961733266711235, -0.004520933143794537, -0.023654228076338768, 0.019619015976786613, 0.20325517654418945, 0.007873420603573322, 0.019252177327871323, 0.01563814841210842, -0.0038144311401993036, 0.010631495155394077, 0.02182004041969776, 0.005978093482553959, -0.011779561638832092, 0.016996806487441063, -0.025515589863061905, 0.02018965221941471, 0.0061886850744485855, -7.132952305255458e-05, 0.028477462008595467, -0.025529175996780396, -0.031140431761741638, -0.021983079612255096, -0.020855393260717392, 0.020352691411972046, 0.009721195325255394, -0.004106542561203241, -0.0076016890816390514, -0.012900453992187977, 0.009320391342043877, -0.03146651014685631, -0.009333977475762367, 0.011677661910653114, 0.030107852071523666, 0.019985852763056755, -0.010006513446569443, 0.015529456548392773, 0.017404403537511826, -0.006395880598574877, 0.01139913685619831, -0.015216965228319168, -0.0012015628162771463, 0.007560929283499718, 0.005305557977408171, 0.022947726771235466, 0.009632882662117481, 0.006209065206348896, -0.021235818043351173, -0.004181268624961376, 0.014252318069338799, -0.006976706441491842, -0.01089643407613039, -0.006864617578685284, -0.005315747577697039, -0.02597753331065178, -0.03418382629752159, -0.0016100092325359583, 0.019659774377942085, 0.031384989619255066, -0.0008640213054604828, 0.020733114331960678, 0.005312351044267416, 0.009829887188971043, 0.011962980031967163, 0.002083841012790799, -0.03434686362743378, 0.033613190054893494, -0.025080818682909012, 0.027404123917222023, -0.011453483253717422, -0.003084152704104781, -0.0027563765179365873, 0.016181612387299538, 0.011324411258101463, 0.009510602802038193, -0.014238731004297733, -0.007425063755363226, -0.0053327311761677265, -0.009415497072041035, 0.0020634611137211323, -0.020026613026857376, 0.02270316891372204, -0.019578255712985992, 0.046275876462459564, 0.04320530965924263, -0.002552577992901206, -5.790216528112069e-05, -0.0020396846812218428, 0.013090665452182293, -0.007839454337954521, -0.014293077401816845, 0.04260750114917755, -0.006728751584887505, -0.011976566165685654, -0.03410230576992035, -0.0185049157589674, -0.003333806060254574, -0.009619295597076416, -0.01726853847503662, 0.020026613026857376, 0.00396728003397584, 0.01979564130306244, -0.0007977867498993874, -0.018763061612844467, 0.0319012813270092, -0.015216965228319168, 0.03157520294189453, 0.0030569795053452253, -0.006660818587988615, -0.015991399064660072, -0.0006755075883120298, 0.004609245806932449, 0.011154578998684883, 0.020040199160575867, -0.014034932479262352, 0.004266184754669666, -0.02929265797138214, -0.014170798473060131, -0.019320109859108925, 0.016738660633563995, 0.03320559114217758, -0.0069393436424434185, -0.02209177240729332, 0.011691248044371605, -0.01479578111320734, -0.01687452755868435, -0.0028124211821705103, 0.0018732491880655289, 0.0033253144938498735, -0.007982113398611546, 0.008240258321166039, 0.004612642340362072, 0.006725355051457882, -0.01586912013590336, -0.026235677301883698, 0.032716475427150726, -0.01624954491853714, 0.027784546837210655, -0.01249285601079464, -0.002705426886677742, 0.00830819085240364, -0.017105499282479286, -0.02116788551211357, -0.01396699994802475, 0.029645908623933792, -0.005991680081933737, -0.004232218489050865, -0.009340770542621613, -0.007520169485360384, -0.023885199800133705, -0.022580888122320175, 0.005852417554706335, -0.006110562477260828, -0.011317617259919643, -0.02499930001795292, -0.005098362453281879, -0.008702201768755913, -0.001435931189917028, -0.016453342512249947, 0.0143202506005764, -0.008240258321166039, -0.008654648438096046, -0.014836540445685387, 0.03959128260612488, -0.011528209783136845, -0.015244138427078724, -0.008817687630653381, 0.00586260762065649, -0.00042903859866783023, -0.011820320971310139, 0.004201648756861687, -0.17358209192752838, 0.029482869431376457, 0.015366417355835438, -0.032689303159713745, 0.014034932479262352, 0.0017085119616240263, 0.004432620480656624, -0.0020091149490326643, -0.01756744273006916, 0.01509468536823988, 0.022309157997369766, -0.004323927685618401, -0.01948314905166626, -0.02336890995502472, -0.00577429449185729, 0.009660054929554462, -0.01726853847503662, 0.009170938283205032, 0.027879653498530388, 0.0046296254731714725, 0.003977470099925995, -0.02732260338962078, 0.009211698547005653, -0.004599055740982294, 0.003957089968025684, -0.0037940512411296368, 0.001112400903366506, 0.012302644550800323, -0.028749193996191025, -0.011283650994300842, -0.01038693729788065, -0.008090806193649769, 0.02671120874583721, -0.008620682172477245, -0.014048519544303417, -0.016195198521018028, 0.011467070318758488, 0.002985649975016713, -0.009096212685108185, 0.022906966507434845, 0.015081099234521389, -7.51507468521595e-05, 0.01091002020984888, -0.012676275335252285, -0.01915707252919674, 0.002768264850601554, 0.029347004368901253, 0.0010198423406109214, 0.005679188761860132, -0.004704352002590895, 0.0042763748206198215, -0.013674888759851456, -0.0020532712806016207, 0.022023839876055717, -0.00814515259116888, -0.002595036057755351, -0.019238591194152832, 0.014836540445685387, 0.010495630092918873, 0.0042763748206198215, -0.014021346345543861, -0.013559402897953987, 0.013994173146784306, -0.010597528889775276, 0.005027032922953367, -0.04361290857195854, 0.006256618071347475, 0.025135165080428123, -0.018110904842615128, -0.011901840567588806, -0.018953273072838783, -0.009232078678905964, 0.022839033976197243, -0.012064879760146141, -0.005502562969923019, 0.00923887174576521, 0.0039333137683570385, -0.0036581854801625013, -0.005787881091237068, -0.0016346349148079753, -0.012261884286999702, 0.020583663135766983, -0.007751141674816608, -0.004690765403211117, -0.0010291831567883492, -0.02074670046567917, -0.02440148964524269, 0.008009286597371101, 0.007241644896566868, 0.008403297513723373, 0.0009722893009893596, -0.026806313544511795, -0.022947726771235466, 0.015543042682111263, 0.020243998616933823, 0.012689861468970776, 0.005458406638354063, 0.015610975213348866, -0.03342297673225403, -0.012608341872692108, 0.0021127124782651663, -0.017363643273711205, -0.03546096384525299, 0.01270344853401184, 0.011161372065544128, 0.010203517973423004, -0.019741294905543327, 0.03228170424699783, 0.03244474157691002, -0.021616242825984955, -0.016643555834889412, 0.013776787556707859, 0.028097039088606834, 0.0151354456320405, 0.006783097982406616, 0.005071189254522324, 0.010699428617954254, -0.011623315513134003, 0.00737751042470336, 0.020040199160575867, 0.04885732755064964, 0.010719808749854565, -0.03315124660730362, 0.015610975213348866, -0.003203035332262516, 0.009863854385912418, -0.09602991491556168, -0.005145915783941746, 0.021548308432102203, 0.026072638109326363, -0.01527131162583828, 0.021290164440870285, 0.0026154157239943743, 0.012132812291383743, -0.023477602750062943, 0.020977672189474106, -0.012934420257806778, -0.040460821241140366, 0.01851850189268589, -0.0005927143502049148, 0.043368350714445114, -0.010271451435983181, 0.01979564130306244, -0.003270968096330762, -0.009992926381528378, 0.0013679983094334602, -0.0017492716433480382, -0.000641541148070246, 0.010604321956634521, -0.02986329421401024, -0.03584138676524162, -0.004745111800730228, -0.021344510838389397, -0.0024489802308380604, 0.015312070958316326, 0.009755161590874195, 0.02696935273706913, -0.011439897119998932, 0.02339608408510685, -0.02245860919356346, 0.02076028846204281, -0.008281017653644085, 0.0020957293454557657, -0.016521276906132698, 0.01854567602276802, -0.0402706116437912, -0.0039808666333556175, -0.00023436843184754252, 0.023233044892549515, -0.04575958847999573, -0.005128932185471058, 0.0012737414799630642, -0.023803681135177612, 0.015203378163278103, 0.012139605358242989, -0.007866627536714077, -0.025814494118094444, 0.0017832380253821611, 0.002620510756969452, -0.00874296110123396, 0.01684735342860222, 0.011290444061160088, -0.0191434845328331, 0.004982876591384411, -0.014659915119409561, -0.004918340593576431, 0.012642309069633484, -0.005991680081933737, -0.01562456227838993, 0.005662205629050732, 0.022241225466132164, 0.012540409341454506, -0.01543434988707304, -0.013905860483646393, 0.021303750574588776, -0.028395943343639374, -0.020964086055755615, 0.007092192769050598, 0.0008402448147535324, -0.017947867512702942, -0.0073910970240831375, -0.020257584750652313, -0.027241084724664688, -0.03287951275706291, 0.016480516642332077, -0.006810271181166172, -0.03152085468173027, -0.0207738745957613, 0.03739025816321373, 0.0028090246487408876, -0.007336751092225313, 0.02009454555809498, 0.020923325791954994, -0.01509468536823988, 0.03635767847299576, -0.017023978754878044, 0.017798414453864098, 0.02796117216348648, 0.00031843536999076605, 0.019605427980422974, 0.007017466239631176, 0.012227918021380901, 0.012255091220140457, 0.010176345705986023, -0.03540661558508873, 0.019578255712985992, -0.04317813739180565, -0.020230410620570183, -0.06336779147386551, 0.00927283801138401, 0.022934140637516975, -0.011433103121817112, -0.012560789473354816, 0.019890746101737022, -0.014972406439483166, 0.0286405012011528, -0.007031052839010954, 0.013009145855903625, -0.021983079612255096, 0.0034221187233924866, 0.007092192769050598, -0.011983360163867474, -0.035270750522613525, 0.01053638942539692, 0.01221433188766241, -0.010094826109707355, 0.005529736168682575, 0.0066947853192687035, 0.001453763572499156, -0.009544569067656994, 0.018966859206557274, 0.00976195465773344, -0.009992926381528378, -0.013736028224229813, -0.04429223760962486, 0.014021346345543861, 0.005825244355946779, -0.008613889105618, 0.01657562330365181, -0.01597781293094158, -0.022173291072249413, 0.016004987061023712, 0.006942740175873041, -0.029020925983786583, -0.02627643756568432, 0.027689442038536072, -0.006969913374632597, 0.02603187970817089, -0.028504636138677597, -0.025203097611665726, -0.009986133314669132, -0.0279339998960495, 0.0007803789339959621, -0.009605709463357925, 0.008179118856787682, -0.00471114506945014, 0.020379863679409027, -0.016358237713575363, 0.016344651579856873, 0.01720060408115387, -0.022499369457364082, -0.00405219616368413, -0.007370717357844114, -0.012975179590284824, -0.0003137649910058826, 0.009014693088829517, -0.010196724906563759, -0.023504776880145073, 0.04377594590187073, 0.005254608113318682, -0.001110702520236373, 0.02858615480363369, 0.020013026893138885, -0.03345014899969101, -0.018450569361448288, -0.00280392961576581, 0.0032896497286856174, -0.009870647452771664, -0.005196865182369947, -0.0029584767762571573, 0.008362537249922752, 0.033586014062166214, -0.006429846864193678, -0.015026752837002277, 0.0208418071269989, -0.014225144870579243, -0.028450289741158485, 0.01661638170480728, 7.032114081084728e-05, -0.014007759280502796, -0.002712220186367631, 0.03086869977414608, 0.014619155786931515, -0.012920833192765713, -0.0022621648386120796, 0.011745594441890717, -0.00844405684620142, 0.030678488314151764, 0.008314983919262886, 0.01912989839911461, -0.008321777917444706, 0.005621445830911398, 0.007757934741675854, 0.006198875140398741, 0.009564949199557304, -0.01690169982612133, 0.009782334789633751, 0.023246631026268005, 0.0016949253622442484, 0.01542076375335455, 0.009884233586490154, -0.020882567390799522, -0.019360870122909546, -1.4926657740943483e-06, -0.020909739658236504, -0.01787993311882019, -0.004969289992004633, -0.005006653256714344, 0.022214051336050034, 0.027485642582178116, 0.015475110150873661, -0.004381670616567135, -0.027472056448459625, 0.007656035479158163, -0.012445303611457348, -0.011446690186858177, -0.027730200439691544, 0.017023978754878044, 0.010699428617954254, 0.002538991393521428, 0.02180645428597927, -0.003158878767862916, 0.018056558445096016, 0.005410853773355484, 0.006983499974012375, -0.015678908675909042, -0.0015089591033756733, -0.008994312956929207, 5.8910543884849176e-05, 0.009924993850290775, -0.01823318377137184, -0.021290164440870285, -0.02404823899269104, -0.0027037286199629307, -0.01688811369240284, -0.0004394408024381846, 0.025936773046851158, 0.06146566942334175, 0.021371683105826378, -0.014361010864377022, -0.015841947868466377, -0.020067373290657997, -0.003186051966622472, -0.01008123904466629, -0.014687088318169117, 0.001486880937591195, -0.044645488262176514, 0.011331204324960709, -0.0015471712686121464, -0.000767216959502548, -0.024863433092832565, 0.011154578998684883, -0.006528349593281746, -0.024591702967882156, 0.003620822448283434, -0.03279799222946167, 0.015257724560797215, 0.02214611880481243, -0.020013026893138885, 0.013640922494232655, 0.016032159328460693, -0.034917499870061874, 0.009279631078243256, 0.006443433463573456, 0.0075881024822592735, -0.007336751092225313, -0.010346177034080029, -0.010713014751672745, 0.025461243465542793, -0.031113257631659508, -0.025583522394299507, 0.011541795916855335, 0.0064977798610925674, -0.004999859724193811, 0.0019258970860391855, 0.005495769903063774, -0.01445611659437418, 0.015257724560797215, 0.025841666385531425, 0.0034068338572978973, -0.013566195964813232, 0.0009179430198855698, -0.011541795916855335, -0.004449603613466024, -0.00665402552112937, -0.02660251595079899]} +{"id": "test:10000018", "text": "\"\u25b2Students are shouting several slogans to guarantee the school's autonomy in front of Jogyesa Temple.\\nOn April 15th, 2016, the General Student Council and Student Council of Postgraduate held the 4.15 Jogye rally. The rally was also held in the last year with the same objectives. Goals of the rally are to criticize Jogye Order for its responsibility on Dongguk University\u2019s present conflicts related to President Han Tae-sik (Bogwang) and to guarantee the school\u2019s autonomy from the order.\\nAt 11:00 A.M., approximately 200 people, about 150 Donggukians and 50 people who are related to Dongguk University, gathered together in Manhae Hall. As soon as the march began, the marchers were obstructed by police officers because of a cow, which was prepared by the GSC, was not reported to participate in the rally. Scuffles broke out between the police and the GSC, and the cow was not able to go with the marchers. The parade was started from Chungmu-ro where Dongguk University is located, passed Myeong-dong and finally to Jogyesa Temple.\\nAfter several speeches from students, the GSC and the gathered students tried to deliver request proposal to Jogye Order together, but police officers blocked the way. Therefore, only ten representatives were allowed to go to deliver the request proposal to the persons concerned of the Jogye Order at the temple gate. The request proposal involves following three demands: Jogye Order has to stop intervening in the school matters and should guarantee the school\u2019s autonomy, every sunim who spoiled the school needs to apologize and resign, and board of directors should be reformed in a more democratic way.\"", "vector_field": [-0.023073215037584305, -0.0076410407200455666, -0.00497349863871932, -0.012177908793091774, -0.043499354273080826, 0.00390921114012599, -0.017888221889734268, -0.008180007338523865, -0.02971819043159485, -0.01226659957319498, 0.000516793632414192, 0.009326162748038769, -0.002556678606197238, -0.008111783303320408, -0.01419050432741642, -0.003435057122260332, 0.020344270393252373, -0.004550512880086899, 0.005945684853941202, 0.02310050278902054, 0.014354241080582142, -0.01255313865840435, -0.03610391914844513, -0.005567044019699097, 0.024110212922096252, 0.017860932275652885, 0.032092370092868805, 8.654534212837461e-06, 0.02035791613161564, 0.0378231517970562, -0.003095644759014249, -0.002735765418037772, -0.032392553985118866, 0.014381530694663525, -0.010465497151017189, -0.021217532455921173, -0.0019204937852919102, 0.008609815500676632, 0.015227503143250942, 0.010922594927251339, 0.005065600853413343, -0.005485175643116236, 0.010567832738161087, -0.015609554946422577, -0.007149830926209688, 0.018734194338321686, -0.0012757809599861503, -0.017970090731978416, -0.020398849621415138, 0.0357491560280323, 0.023769095540046692, 0.006880348082631826, -0.027139339596033096, -0.013255841098725796, -0.025501973927021027, 0.0010625822469592094, 0.0034828137140721083, 0.012675940990447998, -0.022541070356965065, -0.050540026277303696, 0.010676990263164043, -0.025283658877015114, -0.033156659454107285, 0.0011887958971783519, 0.007845711894333363, 0.00994699727743864, -0.02355077862739563, -0.023564424365758896, -0.025310948491096497, -0.007156653329730034, 0.023578068241477013, 0.033238526433706284, -0.0008689978276379406, -0.02346891164779663, 0.03146471455693245, -0.016005251556634903, -0.018993444740772247, -0.015923382714390755, 0.005420363508164883, 0.01052007544785738, 0.008964578621089458, -0.029308849945664406, -0.0024219369515776634, -0.0034282347187399864, -0.006822357885539532, 0.0028278673999011517, -0.007374968845397234, 0.031219109892845154, -0.004540279041975737, -0.01906166784465313, 0.01405405718833208, 0.031410135328769684, -0.01063605584204197, 0.010588299483060837, -0.01693309284746647, 0.007054318208247423, -0.013249019160866737, 0.03173760697245598, 0.013078459538519382, 0.005979796871542931, 0.0043083191849291325, 0.002607846399769187, -0.023687226697802544, -0.019416430965065956, -0.04082499071955681, -0.022459201514720917, 0.0015060354489833117, -0.002167804166674614, 0.0078047774732112885, -0.0008058910025283694, -0.004588035866618156, 0.041643671691417694, -0.03282918781042099, -0.04235319793224335, 0.0089986901730299, -0.009817373007535934, 0.00552611006423831, 0.025474684312939644, -0.006177645176649094, -0.011488850228488445, 0.026811866089701653, 0.01272369734942913, 0.0393240712583065, 0.006027553230524063, 0.017819998785853386, -0.02629336714744568, -0.014367885887622833, -0.0050041996873915195, -0.011591185815632343, 0.0005752100259996951, 0.017860932275652885, 0.01829756423830986, 0.02183154597878456, 0.0125736054033041, 0.008814486674964428, -0.001639924244955182, 0.005693257786333561, 0.024860672652721405, -0.018092893064022064, -0.027671484276652336, 0.014408819377422333, 0.0065835751593112946, -0.004666492808610201, -0.030345847830176353, 0.0024151147808879614, 0.024260304868221283, 0.026811866089701653, 0.030946215614676476, -0.004721071571111679, -0.0004831082187592983, 0.01955287717282772, -0.0056113894097507, 0.005014433059841394, -0.003112700767815113, 0.0035066918935626745, -0.005011021625250578, 0.0028500400949269533, 0.035967469215393066, -0.01967567950487137, 0.010745213367044926, 0.009899240918457508, -0.010097089223563671, 0.010683812201023102, 0.0011802678927779198, -0.013187617994844913, 0.0278488639742136, 0.011079508811235428, 0.01780635491013527, 0.001053201500326395, -0.009373920038342476, 0.014176859520375729, 0.0251062773168087, -0.026375235989689827, 0.0019648391753435135, -0.014436108991503716, 0.008739440701901913, 0.005413541104644537, 0.010513253509998322, -0.02308685891330242, -0.002541328314691782, -0.03689531236886978, 0.0026436636690050364, 0.015718713402748108, 0.024669645354151726, -0.01810653693974018, -0.009783261455595493, 0.0024304650723934174, 0.003112700767815113, -0.00025221402756869793, 0.0066347429528832436, 0.03135555610060692, -0.006423249840736389, -0.006044609006494284, 0.014258728362619877, -0.6331148147583008, -0.023427976295351982, 0.03640410304069519, -0.005635267589241266, 0.0034162956289947033, 0.02236368879675865, 0.010356339626014233, 0.005396484863013029, 0.008330099284648895, 0.0242330152541399, 0.004577802028506994, 0.024628711864352226, -0.003980845678597689, -0.007886645384132862, -0.013760696165263653, -0.02438310720026493, -0.0039467341266572475, -0.009885596111416817, 0.02587038092315197, 0.0024884550366550684, -0.029226981103420258, 0.010472319088876247, 0.00798898097127676, 0.011256890371441841, 0.007661507930606604, 0.033347684890031815, 0.007504593580961227, -0.009510367177426815, -0.006621098145842552, 0.024096567183732986, -0.02731672115623951, -0.007306745275855064, -0.012778276577591896, 0.026075052097439766, 0.03700447082519531, 0.010997640900313854, -0.04052480682730675, 0.013958544470369816, 0.0036977180279791355, 0.054988205432891846, -0.021695097908377647, -0.0180383138358593, 0.022090794518589973, -0.0027494102250784636, -0.005369195714592934, 0.010110734030604362, 0.03427552431821823, -0.02325059473514557, -0.01021306961774826, -0.0021029917988926172, -0.00800944771617651, 0.007156653329730034, 0.014436108991503716, 0.01966203562915325, -0.006539229769259691, -0.012314355932176113, 0.018843352794647217, -0.004086592234671116, -0.011877724900841713, 0.00428102957084775, 0.019266339018940926, 0.028517456725239754, 0.01462713535875082, 0.00014422890671994537, -0.012307533994317055, 0.005867227911949158, -0.033429551869630814, 0.004369720350950956, 0.011427449062466621, -0.007183942943811417, 0.008473368361592293, 0.013685650192201138, -0.023346109315752983, -0.0008370180148631334, 0.033238526433706284, 0.02310050278902054, 0.024887962266802788, -0.010949884541332722, -0.004523223266005516, 0.011905014514923096, -0.019648389890789986, 0.0009747443837113678, -0.012600895017385483, 0.013822097331285477, 8.858404908096418e-05, -0.0066347429528832436, 0.009749148972332478, 0.006235634908080101, -0.004444766324013472, -0.014081346802413464, 0.023291530087590218, 0.040879569947719574, 0.025856737047433853, 0.012177908793091774, 0.01806560344994068, 0.03012753278017044, -0.003987668082118034, 0.001507741049863398, 0.029800059273838997, -0.010881660506129265, -0.002785227494314313, -0.011004462838172913, -0.031437426805496216, -0.010492786765098572, 0.011352403089404106, 0.027780640870332718, -0.06271111220121384, -0.0054305968806147575, 0.034903183579444885, -0.01963474601507187, 0.0081254281103611, 0.018024669960141182, 0.009694570675492287, -0.02221359685063362, 0.002045001834630966, -0.024137502536177635, 0.016005251556634903, 0.02320966124534607, 0.017547104507684708, -0.0016186044085770845, 0.0068939924240112305, -0.017356079071760178, 0.024601422250270844, 0.006300447508692741, 0.01108633168041706, 0.012866967357695103, -0.0014250200474634767, -0.017779065296053886, -0.010445029474794865, -0.000244752096477896, -0.011666231788694859, -0.010997640900313854, 0.009817373007535934, -0.0019801894668489695, 0.04953031614422798, 0.015705067664384842, 0.00775019871070981, -0.010199424810707569, 0.017478881403803825, -0.023632647469639778, -0.03457570821046829, 0.007374968845397234, 0.003228680929169059, -0.0251472108066082, 0.004397009499371052, 0.0024304650723934174, -0.0278079304844141, -0.006999738980084658, 0.005676201544702053, 0.0034640522208064795, -0.002172921085730195, 0.00510653480887413, -0.016537396237254143, -0.009182893671095371, -0.007402258459478617, -0.0016160460654646158, -0.023987410590052605, -0.004134349059313536, -0.0078047774732112885, -0.006846236065030098, 0.021313045173883438, 0.014736292883753777, -0.015841515734791756, -0.006491473410278559, 0.012655474245548248, -0.02089005894958973, -0.01386985369026661, 0.0066142757423222065, -0.009510367177426815, -0.0322561077773571, -0.01061558909714222, 0.00487116351723671, 0.022199952974915504, 0.02359171397984028, 0.012150619179010391, -0.0036499614361673594, -0.0008562059374526143, -0.005294149741530418, -0.010185780003666878, -0.009612701833248138, 0.0035032806918025017, 0.009940175339579582, -0.027030181139707565, -0.010083445347845554, 0.01734243333339691, 0.004912097472697496, -0.0016228683525696397, 0.00585017167031765, -0.0054817646741867065, 0.013133038766682148, -0.003735241014510393, 0.013098927214741707, -0.0020381794311106205, -0.024287594482302666, 0.010322227142751217, -0.015732357278466225, 0.0032815542072057724, 0.00916924886405468, 0.01697402633726597, 0.0323379747569561, 0.018556812778115273, -0.024014700204133987, 0.02925427071750164, -0.012969302013516426, -0.005335083696991205, -0.010144846513867378, 0.013535558246076107, -0.021599585190415382, 0.009599057026207447, 0.018161116167902946, -0.0005040016840212047, -0.029909217730164528, -0.016223566606640816, -0.004311730153858662, -0.021244822070002556, 0.027944378554821014, 0.0009645108366385102, 0.009496722370386124, -0.02561113052070141, 0.005730780772864819, -0.0085620591416955, -0.014709003269672394, 0.027262141928076744, -0.009155604057013988, -0.008746262639760971, 0.005877461284399033, -0.014067701995372772, 0.02876306138932705, -0.010656522586941719, -0.004011546261608601, -0.03776857256889343, 0.015268436633050442, -0.007463659625500441, 0.040688540786504745, 0.02625243365764618, 0.00914878211915493, 0.011536607518792152, -0.012075573205947876, 0.02289583347737789, -0.01833849772810936, 0.017738129943609238, 0.012846499681472778, 0.03001837432384491, -0.018215695396065712, 0.004407243337482214, -0.023987410590052605, -0.002599318278953433, -4.479837298276834e-05, -0.010881660506129265, 0.02663448452949524, -0.009442143142223358, 0.013208084739744663, 0.0014659541193395853, 0.004032013472169638, 0.012648651376366615, -0.012846499681472778, 0.01829756423830986, -0.001146156108006835, 0.006610864773392677, 0.024410396814346313, 0.0028381007723510265, -0.0028125171083956957, 0.024014700204133987, -0.010151668451726437, -0.00247822143137455, 0.019989509135484695, 0.017888221889734268, 0.012082396075129509, -0.020221468061208725, -0.0359947606921196, -0.0042571513913571835, 0.01548675261437893, 0.02522907964885235, -0.03318394720554352, 0.0009508661460131407, -0.009851484559476376, 0.008807663805782795, 0.003252559108659625, 0.0036431390326470137, 0.027453167364001274, -0.017669906839728355, -0.03896930813789368, 0.0015026243636384606, 0.001884676399640739, 0.004758594557642937, -0.010792969726026058, -0.015050121583044529, 0.00045837718062102795, -0.026989247649908066, 0.022350044921040535, -0.028517456725239754, 0.0014156393008306623, -0.011741277761757374, -0.0009022568119689822, -0.013017058372497559, -0.016141697764396667, -0.006754134315997362, 0.008780374191701412, 0.00407976983115077, 0.005000788252800703, 0.015172923915088177, 0.022377334535121918, -0.03381160646677017, -0.028435587882995605, 0.0251062773168087, 0.02289583347737789, -0.014709003269672394, -0.024151146411895752, 0.01863868162035942, -0.009844662621617317, -0.014722648076713085, -0.02210444025695324, -0.021640518680214882, -0.014859095215797424, -0.008794018998742104, -0.0064164274372160435, -0.00871215108782053, -0.014381530694663525, 0.0376594141125679, -0.0005645501078106463, -0.0120960408821702, -0.012955657206475735, -0.030618742108345032, -0.012682762928307056, 0.1112317219376564, 0.010554187931120396, -0.0011256890138611197, -0.008357387967407703, 0.019798481836915016, 0.014313306659460068, -0.029308849945664406, -0.0377412848174572, -0.0034180013462901115, -0.0033514832612127066, -0.009025979787111282, -0.020289693027734756, 0.022077150642871857, -0.0062390463426709175, 0.014859095215797424, -0.0044549996964633465, -0.011447916738688946, 0.00277328840456903, 0.002744293538853526, 0.025351881980895996, 0.012976124882698059, -0.016537396237254143, 0.006822357885539532, 0.018706904724240303, -0.006549463607370853, 0.00687693664804101, 0.02325059473514557, 0.023196017369627953, 0.00022407181677408516, -0.02503805421292782, -0.028926797211170197, 0.014722648076713085, -0.004263973794877529, 0.005468119867146015, -0.005877461284399033, 0.0112227788195014, 0.02659355103969574, 0.0018710315926000476, -0.0139244319871068, 0.016141697764396667, 0.020248757675290108, 0.021504072472453117, 0.011338758282363415, -0.01155025139451027, 0.01989399455487728, 0.003714773803949356, -0.008480191230773926, 0.020726323127746582, 0.005290738306939602, -0.010247181169688702, 0.03452112898230553, -0.03430281579494476, -0.01135922595858574, 0.00015840661944821477, 6.0661686802632175e-06, -0.034029919654130936, 0.0016092236619442701, 0.008016270585358143, 7.16880604159087e-05, 0.009974286891520023, -0.010526898317039013, -0.043963272124528885, 0.005648912396281958, 0.005062189418822527, -0.014231438748538494, -0.018393076956272125, -0.01746523566544056, -0.00760010676458478, -0.005311205517500639, 0.001954605570062995, -0.014449753798544407, -0.018092893064022064, -0.038096047937870026, 0.016823934391140938, 0.011993705295026302, 0.028135403990745544, 0.02610234171152115, 0.005246393382549286, 0.00019784837786573917, 0.009633169509470463, -0.0020023619290441275, -0.031328268349170685, -0.013822097331285477, -0.0011435977648943663, -0.014081346802413464, 0.042817119508981705, 0.030345847830176353, 0.019034378230571747, -0.014749937690794468, 0.010363161563873291, 0.022377334535121918, 0.005154291167855263, -0.0031092895660549402, -0.03667699545621872, -0.02651168219745159, -0.017124118283391, 0.004502756055444479, 0.004356075543910265, -0.013760696165263653, -0.0035885602701455355, 0.00842561200261116, -0.018856996670365334, 0.015595910139381886, 0.004267384763807058, -0.014490688219666481, 0.040988724678754807, 0.0008280636975541711, 0.0005573013331741095, 0.00977643858641386, 0.029881928116083145, 0.023905541747808456, -0.03675886243581772, -0.009176071733236313, -0.0014582789735868573, 0.006017319392412901, -0.007258988916873932, 0.013078459538519382, 0.024560488760471344, 0.005901339463889599, -0.03274731710553169, 0.008002625778317451, -0.02089005894958973, 0.027862509712576866, 0.018584102392196655, -0.007661507930606604, 0.02031698077917099, -0.014668069779872894, -0.025747578591108322, -0.011468383483588696, -0.014995542354881763, -0.005703491158783436, 0.006075309589505196, -0.013330887071788311, -0.01521385833621025, -0.021353980526328087, -0.011925481259822845, -0.012355290353298187, 0.026607194915413857, 0.005335083696991205, -0.01627814583480358, -0.01110679842531681, -0.008377855643630028, -0.02651168219745159, -0.028653902933001518, 0.01063605584204197, -0.02617056481540203, -0.009373920038342476, -0.03694989159703255, 0.005904750898480415, 0.01636001467704773, -0.007634218316525221, -0.005266860127449036, -0.0027647605165839195, 0.0010736685944721103, -0.004134349059313536, -0.02233639918267727, -0.027589615434408188, -0.015186568722128868, 0.009094202890992165, 0.0055636330507695675, 0.01738336868584156, -0.007115719374269247, 0.011816323734819889, 0.014326951466500759, -0.011570719070732594, -0.0025583840906620026, -0.011741277761757374, 0.00023366576351691037, -0.032283395528793335, 0.003035949310287833, 0.01897979900240898, 0.013794807717204094, 0.008814486674964428, -0.004130937624722719, -0.018611392006278038, 0.022622939199209213, 0.012730519287288189, -0.0170831847935915, 0.021026507019996643, -0.039733413606882095, -0.003191157942637801, 0.004458410665392876, -0.01902073435485363, -0.01225295476615429, -0.03744110092520714, -0.02765783853828907, 0.0038034645840525627, 0.024628711864352226, 0.005195225588977337, -0.021353980526328087, 0.02058987505733967, -0.024369461461901665, -0.0017388485139235854, -0.0028142225928604603, 0.0098719522356987, 0.0014864212134853005, -0.008336921222507954, -0.014545266516506672, -0.017369722947478294, 0.011741277761757374, 0.023236950859427452, 0.012901078909635544, 0.011638942174613476, 0.0006639006896875799, 0.0006059106672182679, 0.0003609453560784459, -0.028162693604826927, -0.022964056581258774, 0.012430336326360703, -0.0376594141125679, -0.0016493049915879965, -0.0099128857254982, 0.008070848882198334, -0.004369720350950956, 0.011625297367572784, 0.003174101933836937, -0.02408292330801487, 0.014681713655591011, -0.004008135292679071, -0.011154554784297943, 0.006525585427880287, 0.007265811320394278, 0.015896093100309372, 0.002921674633398652, -0.0004886513925157487, 0.02617056481540203, -0.003936500288546085, -0.009708215482532978, 0.03446655347943306, 0.022759385406970978, -0.014531622640788555, -0.011024930514395237, -0.00428102957084775, -0.0233188197016716, -0.01765626296401024, -0.013269485905766487, 0.027521392330527306, -0.02016689069569111, -0.01917082630097866, 0.025774868205189705, 0.02849016711115837, -0.01810653693974018, -0.0021115196868777275, 0.0025805567856878042, -0.022281819954514503, 0.021094730123877525, 0.0008996984688565135, 0.016632908955216408, -0.004567568656057119, -0.006931515410542488, -0.0037898197770118713, 0.01746523566544056, -0.02540646120905876, 0.018747840076684952, 0.0056591457687318325, -0.010035688057541847, -0.005676201544702053, 0.0013968778075650334, 0.004144582431763411, 0.014558911323547363, -0.02016689069569111, 0.02952716499567032, -0.005382840521633625, -0.009114669635891914, 0.006058253813534975, -0.010711101815104485, 5.543165752897039e-05, -0.011379692703485489, -0.007354501634836197, 0.027112049981951714, -0.017219631001353264, 0.0052839163690805435, 0.02263658307492733, 0.017124118283391, 0.017697196453809738, -0.00038290483644232154, -0.0103495167568326, -0.005297560710459948, -0.013515090569853783, -0.006904226262122393, 0.013699294999241829, 0.005505642853677273, -0.01601889543235302, -0.0024117035791277885, -0.015609554946422577, -0.024055633693933487, -0.01959381252527237, -0.0027170039247721434, 0.021313045173883438, 0.005846760701388121, -0.002346891211345792, 0.01167987659573555, -0.0054612974636256695, 0.006883759051561356, -0.023564424365758896, -0.004270796198397875, -0.017601683735847473, 0.025174500420689583, -0.03594018146395683, 0.006549463607370853, -0.029008666053414345, 0.012430336326360703, 0.002265022834762931, -0.0012561666080728173, 0.009005512110888958, -0.029090533033013344, 0.02195434831082821, -0.0019443719647824764, -0.029772769659757614, -0.03255629166960716, -0.01286014448851347, 0.010704279877245426, -0.012559960596263409, -0.0321742407977581, 0.005403307266533375, 0.001379821915179491, 0.009858307428658009, 0.00020083316485397518, -0.05878143385052681, 0.000944043742492795, 0.0017431124579161406, -0.00380005338229239, 0.028462877497076988, -0.02754868008196354, 0.010772502981126308, 0.02746681310236454, 0.017301499843597412, 0.007490948773920536, -0.0325017124414444, 0.013556024990975857, -0.0008928760653361678, 0.014545266516506672, -0.01447704341262579, 0.004993965849280357, -0.01959381252527237, -0.0188706424087286, -0.017519814893603325, 0.021067440509796143, 0.0009610996348783374, -0.020098665729165077, 0.013665182515978813, 0.011761744506657124, -0.01182996854186058, 0.03735923022031784, -0.013037526048719883, -0.019389141350984573, -0.004758594557642937, -0.015964318066835403, -0.014845450408756733, -0.02008502185344696, -0.015091056004166603, 0.042817119508981705, -0.0028295728843659163, -0.014286017045378685, -0.013719761744141579, -0.00351351429708302, -0.02457413263618946, -0.006065076217055321, 0.01727421022951603, 0.0050349002704024315, -0.0008988456684164703, 0.008616638369858265, 0.0007798807928338647, -0.00775019871070981, -0.018802417442202568, 0.022854898124933243, -0.02004408650100231, -0.0037591191940009594, 0.013951721601188183, -0.03282918781042099, 0.008787197060883045, 0.010103912092745304, -0.00738179124891758, 0.013133038766682148, 0.00973550509661436, 0.019143536686897278, -0.00025093485601246357, 0.023223306983709335, 0.011536607518792152, 0.021326690912246704, 0.006682499311864376, -0.010949884541332722, -0.021695097908377647, -0.003356599947437644, 0.015473107807338238, -0.03359328955411911, 0.011857258155941963, -0.006013908423483372, -0.004557334817945957, 0.0037522967904806137, 0.0013039231998845935, 0.004018368665128946, -0.008521124720573425, 0.009653636254370213, -0.014831805601716042, -0.019866706803441048, 0.013214907608926296, 0.011666231788694859, 0.03894202038645744, 0.014859095215797424, -0.021244822070002556, 0.011720811016857624, 0.00960587989538908, 0.0016885335789993405, 0.007907113060355186, -0.006460772827267647, -0.012826032936573029, 0.0016092236619442701, 0.0047858841717243195, -0.04158909246325493, 0.00975597184151411, 0.00019934076408389956, 0.03119182027876377, -0.03495776280760765, -0.004482289310544729, -0.015363950282335281, -0.007716086693108082, -0.052177391946315765, 0.0359947606921196, 0.024887962266802788, -0.024669645354151726, -0.004328785929828882, -0.029226981103420258, -0.038096047937870026, 0.006276569329202175, -0.012853322550654411, 0.0022087383549660444, -0.02465600147843361, -0.01363789290189743, -0.0007854239665903151, -0.003656783839687705, -0.020371560007333755, -0.021258467808365822, 0.008623460307717323, -0.01154342945665121, 0.008132250979542732, 0.21493156254291534, 0.007306745275855064, 0.014736292883753777, 0.0357491560280323, -0.0043731313198804855, 0.006102599203586578, 0.02346891164779663, -0.002821044996380806, -0.028790351003408432, 0.0013141566887497902, -0.043090011924505234, -0.00657334178686142, -0.006818946450948715, -0.006303858477622271, 0.0023161903955042362, -0.030263978987932205, -0.037686705589294434, -0.03733194246888161, -0.020685389637947083, -0.0038375763688236475, 0.001453162170946598, 0.009203360415995121, -0.012177908793091774, 0.005341906100511551, 0.010513253509998322, 0.0036738396156579256, 0.005038311239331961, 0.019525587558746338, 0.032856475561857224, 0.024492263793945312, -0.01346733421087265, 0.011891369707882404, 0.006252691149711609, -0.0066961441189050674, 0.01951194368302822, 0.007108896970748901, -0.0033327217679470778, -0.011577541008591652, 0.0029370251577347517, 0.009128314442932606, -0.003344660857692361, -0.016264501959085464, -0.004151404835283756, 0.009694570675492287, -0.004608502611517906, -0.0038171091582626104, -0.0188706424087286, -0.012300711125135422, -0.006668854970484972, -0.004141170997172594, -0.054988205432891846, 0.0018642093054950237, 0.01715140789747238, 0.027262141928076744, 0.004618736449629068, -0.01678300090134144, 0.02971819043159485, 0.01734243333339691, 0.0005619917064905167, 0.007477304432541132, -0.020330626517534256, 0.03263815864920616, 0.0015793758211657405, 0.021449493244290352, -0.0032593815121799707, 0.0016578329959884286, -0.003283259691670537, 0.01447704341262579, 0.011884547770023346, -0.005932040046900511, -0.006979272235184908, 0.008630282245576382, -0.01288061123341322, -0.010513253509998322, -0.01447704341262579, -0.027030181139707565, 0.05741696432232857, 0.017642617225646973, 0.009373920038342476, 0.028599323704838753, -0.02655261754989624, 0.020030442625284195, 0.0007798807928338647, -0.003793230978772044, -0.005536343436688185, -0.014790872111916542, 0.01356284786015749, -0.008971400558948517, -0.03926949203014374, 0.0033821838442236185, 0.00028163546812720597, -0.006164000369608402, -0.004976910073310137, -0.007668330334126949, -0.013133038766682148, 0.0011086331214755774, 0.029172401875257492, 0.009428498335182667, 0.004840462934225798, 0.012048283591866493, 0.005058778449892998, 0.03823249414563179, 0.018843352794647217, -0.008186829276382923, 0.005376018118113279, 0.01822933927178383, 0.007279455661773682, 0.0061571779660880566, -0.013051170855760574, -0.014081346802413464, -0.013303597457706928, -0.040033597499132156, 0.004444766324013472, -0.0014446342829614878, -0.017096828669309616, 0.025065341964364052, 0.025351881980895996, -0.009994754567742348, 0.02453319914638996, -0.014709003269672394, -0.003769352799281478, -0.01390396524220705, -0.015145634301006794, 0.0018079248256981373, 0.004014957696199417, 0.0024151147808879614, -0.009121492505073547, 0.02312779240310192, 0.009448966011404991, -0.034220948815345764, 0.019498297944664955, -0.041452646255493164, 0.021640518680214882, -0.004393598530441523, 0.009182893671095371, -0.013249019160866737, -0.008323276415467262, 0.005133824422955513, -0.02461506798863411, -0.015882449224591255, -0.006447128020226955, 0.0036806620191782713, 0.008057205006480217, -0.008568881079554558, 0.030182112008333206, -0.008364210836589336, 0.009217005223035812, -0.001503477105870843, -0.028135403990745544, -0.01259407214820385, 0.0026010239962488413, -0.01491367444396019, -0.013542380183935165, -0.023564424365758896, 0.018734194338321686, -0.010438207536935806, -0.023632647469639778, -0.03252900391817093, -0.0014949492178857327, 0.03359328955411911, -0.034330103546381, 0.008070848882198334, 0.00815953966230154, -0.01521385833621025, -0.026525327935814857, -0.004789295140653849, -0.17476151883602142, 0.011864080093801022, 0.026375235989689827, -0.010431385599076748, -0.00670637795701623, 0.010090267285704613, 0.03446655347943306, 0.010376806370913982, -0.03329310566186905, -0.008418789133429527, 0.022227242588996887, -0.0013124510878697038, -0.04633745551109314, 0.002778405323624611, -0.0014633957762271166, 0.012873789295554161, -0.02499711886048317, 0.009101025760173798, 0.012798743322491646, 0.022568359971046448, 0.027644194662570953, -0.010629233904182911, -0.006211756728589535, -0.00644030561670661, -0.015909738838672638, -0.004796117544174194, -0.014613490551710129, -0.0138834984973073, 0.0024219369515776634, -0.008070848882198334, -0.029418006539344788, 0.019266339018940926, 0.01859774813055992, 0.002462871139869094, 0.0016100764041766524, 0.012184730730950832, -0.02561113052070141, -0.009128314442932606, -0.012177908793091774, 0.008950933814048767, 0.007613751571625471, 0.011161377653479576, 0.0019307272741571069, -0.0016296907560899854, -0.019266339018940926, 0.015227503143250942, -0.002169509883970022, 0.016100764274597168, 0.019457364454865456, -0.011059042066335678, 0.0057921819388866425, -0.02723485231399536, 0.010485963895916939, 0.019416430965065956, 0.0011163082672283053, 0.0071634757332503796, 0.0295544546097517, 0.005007610656321049, -0.013692472130060196, -0.0030683553777635098, -0.007197587750852108, -0.004628969822078943, 0.02381002902984619, -0.008896354585886002, -0.021012861281633377, -0.023387042805552483, -0.023441622033715248, 0.001216938137076795, -0.006160588935017586, -0.011557074263691902, -0.002404881175607443, -0.020862771198153496, 0.026115985587239265, -0.004441354889422655, 0.00748412637040019, 6.172101711854339e-05, -0.0011359226191416383, 0.006757545284926891, -0.018938865512609482, -0.008548414334654808, 0.010704279877245426, 0.02110837586224079, -0.022991346195340157, 0.002195093547925353, 0.014954608865082264, 0.014395175501704216, 0.020439784973859787, -0.011727632954716682, -0.011318291537463665, 0.007688797079026699, 0.01303070317953825, 0.0009431910002604127, -0.0007705000462010503, -0.01375387329608202, 0.006013908423483372, 0.010417740792036057, -0.012716875411570072, 0.0024458153638988733, -0.0295544546097517, -0.03206508234143257, 0.002384413965046406, -0.00026607196195982397, -0.03419365733861923, -0.0016603913391008973, 0.014790872111916542, 0.004178693983703852, 0.002389530884101987, 0.02720756269991398, 0.021845189854502678, -0.014108636416494846, -0.038450807332992554, 0.0022019159514456987, 0.01818840578198433, -0.004417476709932089, 0.02095828391611576, 0.026866445317864418, 0.002583967987447977, -0.021203888580203056, 0.01535030547529459, 0.006559696979820728, 0.07673788070678711, 0.011870902962982655, 0.009196538478136063, 0.009128314442932606, -0.01434059627354145, 0.0020961693953722715, -0.09998847544193268, 0.0009619524353183806, 0.02522907964885235, 0.029854638502001762, -0.009837839752435684, 0.010861193761229515, 0.006337970495223999, 0.055151939392089844, -0.04399056360125542, 0.0224455576390028, -0.009851484559476376, -0.03452112898230553, 0.014094991609454155, -0.009994754567742348, 0.024014700204133987, -0.004182105418294668, 0.022008925676345825, -0.018625037744641304, 0.008248230442404747, -0.0016800055745989084, 0.012061928398907185, -0.013269485905766487, 0.0008800841751508415, -0.021545005962252617, -0.007613751571625471, -0.0008621754823252559, -0.030072953552007675, 0.051304131746292114, 0.009080558083951473, 0.041643671691417694, -0.002089346991851926, 0.01948465406894684, 0.014886384829878807, -0.012955657206475735, 0.008650749921798706, -0.012621361762285233, -0.03727736324071884, -0.038259781897068024, 0.03503962978720665, 0.004410654306411743, -0.004731304943561554, 0.006788245867937803, 0.018652325496077538, -0.028353719040751457, -0.02472422458231449, -0.002677775453776121, -0.017751775681972504, -0.0008067438029684126, 0.005471530836075544, -0.026675419881939888, -0.021981636062264442, -0.021585939452052116, -0.018843352794647217, -0.008630282245576382, 0.03457570821046829, 0.02080819196999073, 0.01963474601507187, 0.02050800807774067, -0.034166369587183, 0.015022831968963146, 0.015718713402748108, 0.012907900847494602, 0.010022043250501156, -0.02301863580942154, 0.003458935534581542, 0.018365787342190742, -0.011038575321435928, -0.012907900847494602, 0.014163214713335037, -0.011250068433582783, -0.008950933814048767, 0.010663345456123352, -0.01857045851647854, 0.0032184473238885403, -0.00369089562445879, 0.001006297767162323, -0.032010503113269806, -0.002295723417773843, -0.0029677257407456636, -0.032856475561857224, -0.008480191230773926, -0.015814226120710373, 0.003018893301486969, -0.011536607518792152, 0.022309109568595886, 0.004860930144786835, 0.02952716499567032, -0.0037011292297393084, 0.005723958369344473, -0.0036499614361673594, -0.001978483749553561, 0.033156659454107285, -0.005454475060105324, -0.040497515350580215, -0.01636001467704773, 0.02944529615342617, 0.0010779325384646654, 0.018365787342190742, -0.008657571859657764, 0.011175022460520267, -0.02305956929922104, -0.014736292883753777, -0.06947889178991318, 0.01047914195805788, 0.013651537708938122, -0.007954869419336319, -0.00945578794926405, 0.0029830760322511196, 0.01700131595134735, 0.0035544484853744507, 0.008903177455067635, -0.007606929168105125, -0.02393283136188984, 0.01481816079467535, -0.005795592907816172, -0.005665968172252178, -0.03359328955411911, -0.031328268349170685, 0.0466376394033432, -0.00011384809477021918, 0.009080558083951473, 0.00578194810077548, 0.009633169509470463, 0.004482289310544729, 0.01567777805030346, 0.024601422250270844, -0.020835481584072113, 0.0017124118749052286, -0.027944378554821014, 0.042516935616731644, -0.0016254268120974302, -0.02016689069569111, -0.011331936344504356, -0.030373137444257736, -0.004819995723664761, 0.012819210067391396, 0.010642877779901028, -0.011918659321963787, 0.0031826298218220472, 0.03348413109779358, 0.02172238752245903, 0.04257151111960411, -0.02971819043159485, -0.0008105813758447766, -0.0005679613095708191, -0.016346368938684464, -0.005665968172252178, 0.006720022298395634, -0.012109684757888317, 0.00554657680913806, 0.006992916576564312, -0.02572028897702694, 0.022609293460845947, 0.02076725661754608, 0.009551300667226315, -0.001651863451115787, 0.010342694818973541, -0.00876672938466072, 0.007231699302792549, 0.00760010676458478, 0.0032116249203681946, -0.030345847830176353, 0.004605091642588377, -0.01019260287284851, 0.024219369515776634, -0.0072248768992722034, 0.046091850847005844, -0.021026507019996643, -0.001597284572198987, -0.014463398605585098, -0.002210443839430809, 0.008036737330257893, -0.03820520266890526, -0.012048283591866493, 0.028135403990745544, 0.01255313865840435, -0.011120443232357502, -0.0032934932969510555, 0.002242850139737129, 0.0047551835887134075, -0.005045133642852306, 0.017710840329527855, -0.004495933651924133, -0.011754922568798065, -0.00436630891636014, 0.014354241080582142, 0.013999477960169315, 0.004332197364419699, -0.011386515572667122, -0.010513253509998322, 0.010117556899785995, 0.012218843214213848, -0.03602204844355583, 0.012191553600132465, -0.011741277761757374, -0.021981636062264442, 0.004171872045844793, 0.021967992186546326, 0.007470482029020786, 0.012007350102066994, 0.015882449224591255, 0.007859356701374054, 0.006474417634308338, 0.0038682769518345594, -0.003438468324020505, -0.012198375537991524, -0.012505382299423218, 0.01363789290189743, -0.02305956929922104, -0.005335083696991205, 0.0009517189464531839, 0.009094202890992165, 0.007409080397337675, -0.0005956771201454103, 0.01251902710646391, 0.009530833922326565, -0.01814747229218483, 0.0018778539961203933, 0.03211966156959534, -0.003006954211741686, -0.022854898124933243, 0.04006088525056839, -0.01006297767162323, 0.004472055472433567, 0.02312779240310192, 0.004420887678861618, 0.036431390792131424, -0.015527686104178429, 0.012443981133401394, 0.002824456198140979, -0.021790610626339912, 0.004673315212130547, 0.021135665476322174, 0.019730258733034134, -0.013685650192201138, -0.011843613348901272, -0.002945553045719862, -0.007293100468814373, 0.005860405508428812, 0.020112311467528343, -0.022418268024921417, 0.05253215506672859, 0.028708482161164284, 0.006402782630175352, 0.0039262669160962105, 0.015323015861213207, 0.03506691753864288, 0.01608712039887905, 0.01787457801401615, 0.006030964199453592, -0.030809767544269562, 0.022732095792889595, -0.016673842445015907, -0.0007815863937139511, -0.03222882002592087, 0.0026675418484956026, -0.009865129366517067, 0.009107847698032856, 0.017451591789722443, 0.002824456198140979, -0.003742063185200095, 0.04439990594983101, -0.019689325243234634, 0.012853322550654411, 0.017055895179510117, -0.0061571779660880566, -0.013692472130060196, -0.004069536458700895, -0.03127368912100792, -0.020412495359778404, 0.004188927821815014, 0.0043731313198804855, 0.030673321336507797, -0.03708633780479431, -0.02750774659216404, -0.025433750823140144, -0.006249279715120792, 0.017096828669309616, 0.007013383787125349, 0.011802678927779198, -0.011693521402776241, -0.0052634491585195065, 0.002048413036391139, -0.00974232703447342, -0.011127266101539135, -0.007272633258253336, -0.004349253140389919, -0.029800059273838997, 0.006982683204114437, -0.015595910139381886]} +{"id": "test:10000019", "text": "\"Quartz is one of the most common and varied minerals on earth, and its abundant colors produce many gemstone types. Amethyst and Citrine are the most popular and valuable gem varieties of Quartz, but other forms also make important gemstones. Chalcedony describes any form of Quartz that is microcrystalline, in compact form without any visible crystals. Chalcedony also has several varieties used as gemstones, most notably Agate, Carnelian, Tiger's Eye, and Chrysoprase.\\nPure Quartz, which is also known as Rock Crystal, is colorless. Various impurities are responsible for the extensive range of colors. The main crystalline Quartz varieties used as gemstones are described below.\\nAmethyst, the purple variety, is the most popular and valuable Quartz gemstone. Amethyst ranges from light to dark purple. See the Amethyst gemstone page for more details.\\nCitrine is the yellow, orange, or reddish-brown variety of Quartz. It is usually colored by heat treatment of Amethyst or Smoky Quartz. Light yellow or lemon yellow Citrine is often called Lemon Quartz in the gem trade. See the Citrine gemstone page for more details.\\nSmoky Quartz is the brown \\\"smoky\\\" variety of Quartz. It ranges in color from light brown to black. Despite its dark color, it is rarely opaque. See the Smoky Quartz gemstone page for more details.\\nThe rosy pink variety of Quartz is known as Rose Quartz, and its color is usually soft, ranging from very light pink to medium pink in intensity. Rose Quartz is often milky or hazy, and it may lack good transparency. See the Rose Quartz gemstone page for more details.\\nThe colorless, transparent variety of Quartz, free of any impurities, is known as \\\"Rock Crystal\\\". Flawless and very large cuts may be cut from Rock Crystal.\\nMilky Quartz is the white, translucent to opaque variety of Quartz. Though very common in nature, it is not used as a gemstone.\\nColorless Quartz with golden yellow Rutile inclusions, as hairlike growths within the gemstone, are known as Rutilated Quartz. See the Rutilated Quartz gemstone page for more details.\\nAmetrine is an interesting, color-zoned combination of purple Amethyst and brownish-yellow Citrine. See the Ametrine gemstone page for more details.\\nPrasiolite, or Green Quartz, describes a light green Quartz artificially colored by heat treatment of certain types of Amethyst. May also be called \\\"Green Amethyst\\\" by some jewelers.\\nThe blue variety of Quartz, which is uncommon in nature, is seldom used as a gemstone. Most \\\"Blue Quartz\\\" is clear Rock Crystal irradiated with gold to from a deep sky blue color. Blue Quartz may also refer to a dull grayish-blue Quartz in massive form with Crocidolite inclusions.\\nColorless Quartz with Tourmaline inclusions, often as thin long black crystals, is known as \\\"Tourmalinated Quartz\\\".\\nCat's Eye Quartz is Quartz with dense, tiny Rutile inclusions that cause a cat's eye effect. It is not common, and the chatoyant effect is usually weak. Cat's Eye Quartz is usually grayish in color and translucent.\\nAll forms of Quartz are used as gemstones, and they are all affordable. They are cut into various gemstone cuts and cabochons, and used in all forms of jewelry. Lesser quality stones are often tumbled for use in bracelets, necklaces, and as costume jewelery. Large spheres and carvings are also cut from all the Quartz forms. Due to its abundance and lack of luster, Rock Crystal is not commonly cut into gemstones, although some very large spheres and sculptures are carved from it. Small crystals of Rock Crystal are sometime worn as pendants, sometimes being polished and smoothed, and sometimes in their entirely natural crystal form.\\nVarieties specific to Amethyst, Citrine, Smoky Quartz, Rose Quartz, Rutilated Quartz, and Chalcedony are listed separately.\\nAmethyst\\t- Purple variety of Quartz, and its most popular and valuable gemstone variety. (See the Amethyst gemstone page for more details.) Tumbled Amethyst with white Milky Quartz is sometimes known as Amethyst Quartz.\\nAventurine\\t- Opaque, compact Quartz / Chalcedony containing small Mica, Hematite, or Goethite scales which cause a glistening effect. Aventurine is most often green but may also be other colors such as gray, orange, and brown.\\nBlue Quartz - Rare natural blue variety of Quartz. It is caused by inclusions of blue minerals, especially Dumortierite. Most \\\"Blue Quartz\\\" is what is popularly known as \\\"Aqua Aura\\\", essentially clear Rock Crystal synthetically irradiated with gold to form a deep sky blue color. Blue Quartz may also refer to a dull grayish-blue Quartz in massive form with Crocidolite inclusions.\\nCat's Eye Quartz - Quartz with dense, tiny Rutile inclusions that cause a cat's eye effect. Cat's Eye Quartz is not common, and the chatoyant effect is usually weak. Cat's Eye Quartz is usually grayish in color and translucent.\\nLemon Quartz - Lemon Quartz is a light to dark yellow Citrine, distinguished from most Citrine by lacking orange, brown, or reddish tints. More often though it is clear Quartz that is irradiated to produce an intensely colored yellow gemstone. Lemon Quartz has recently experienced a popularity increase in the gemstone market.\\nMilky Quartz - White, translucent to opaque variety of Quartz. It is not commonly used as a gemstone.\\nPrase - Light to emerald green, transparent to translucent Quartz / Chalcedony, with coloring caused from inclusions of green minerals, such as Actinolite, Hedenbergite, Chlorite, or Malachite.\\nPrasiolite - Light green gem form Quartz artificially colored by heat treatment of certain types of Amethyst. May also be called Green Amethyst by some jewelers.\\nRock Crystal - The colorless, transparent variety of Quartz, free of impurities is called \\\"Rock Crystal\\\".\\nTourmalinated Quartz - Quartz with splintery Tourmaline inclusions.\\nAmethyst may be heat treated to deepen the purple color. Most gem Citrine is produced by heat treating Amethyst, and the green Quartz known as Prasiolite or \\\"Green Amethyst\\\" is also produced by heating Amethyst from specific localities.\\nCertain colorful Quartz types not found in nature are produced through irradiation. Some forms of Quartz with a multicolored rainbow effect are synthetically treated to produce their color effect using film deposition. The process involves bonding an extremely thin metallic film layer over the top of the gemstone, so that the interesting color effects are reflected from the crown. Some vividly colorful forms of Quartz are synthetic grown using the hydrothermal method.\\nQuartz is extremely common and is found in numerous localities throughout the world. The important sources are far too numerous to mention, though in general the most prolific countries that produce Quartz gemstones are Brazil, Madagascar, India, and the U.S. (Arkansas). Specific sources for the popular Quartz varieties are described on their dedicated pages.\\nSee the individual variety pages for specific variety similarities.\\nRock crystal is similar to glass, but the softness of glass usually lends it to scratches and soft etches which are lacking on Rock Crystal. Rock Crystal is rarely cut into small facets, so it usually is not a concern of confusion to other colorless gems such as Diamond, White Topaz, and White Sapphire. These white gemstones will also have a greater dispersion and exhibit more fire.\\nAdditional images for the varieties Amethyst, Citrine, Smoky Quartz, Rose Quartz, Rutilated Quartz, and Chalcedony are listed separately.\\nHave a question about Quartz? Visit our Q&A Community and ask the experts!\"", "vector_field": [0.005358927417546511, 0.0186565313488245, 0.003535090945661068, -0.0370042622089386, -0.013715766370296478, 0.02931005321443081, -0.020354919135570526, -0.04516167193651199, -0.02039351873099804, -0.039912108331918716, -0.013960232026875019, 0.006671317853033543, -0.005262428428977728, -0.02601621113717556, -0.028486592695116997, 0.010460523888468742, 0.031960565596818924, 0.01637914776802063, 0.011650681495666504, -0.012950205244123936, -0.023082632571458817, 0.007887208834290504, 0.014757959172129631, 0.0014233646215870976, -0.016842344775795937, 0.04328314960002899, 0.006269237492233515, -0.009746428579092026, -0.02173164114356041, 0.017588606104254723, -0.016906676813960075, 0.022079039365053177, 0.004329601768404245, 0.0008652770193293691, -0.021487176418304443, 0.020354919135570526, -0.0020731266122311354, 0.010177459567785263, 0.002229133853688836, -0.006471886299550533, 0.013921631500124931, 0.004570850171148777, 0.01314963772892952, 0.010100259445607662, -0.016031749546527863, 0.0059700896963477135, -0.01798746921122074, 0.000708867737557739, -0.025360016152262688, 0.01312390435487032, 0.036849863827228546, 0.01905539445579052, -0.023044032976031303, 0.001213478622958064, 0.005188445560634136, -0.0371071957051754, 0.02563021332025528, 0.0303651113063097, 0.006754950620234013, -0.004728465341031551, 0.03610360249876976, 0.012827972881495953, -0.003292234381660819, 0.011515582911670208, -0.0018978194566443563, -0.016070349141955376, 0.005288161337375641, -0.0031474854331463575, 0.014063164591789246, -0.024716686457395554, -0.005506893154233694, 0.011116718873381615, 0.005937923211604357, -0.0038374552968889475, 0.009592030197381973, -0.02745726704597473, -0.02034205198287964, -0.0012890697689726949, 0.0023996159434318542, -0.008054474368691444, 0.004963280633091927, -0.006095538847148418, -0.04356621578335762, 0.01989172212779522, 0.011226085014641285, 0.03826518729329109, -0.04184209555387497, 0.0018431366188451648, -0.013677166774868965, -0.03476547822356224, -0.01520828902721405, 0.014346228912472725, 0.014088897034525871, -0.0067163510248064995, -0.01766580529510975, 0.014745092950761318, 0.009958727285265923, 0.022490769624710083, -0.028152061626315117, -0.020792381837964058, 0.012795806862413883, 0.017254075035452843, -0.040246639400720596, -0.002463948680087924, -0.030004847794771194, -0.02407335862517357, 0.012294010259211063, 0.0015086056664586067, -0.0012432326329872012, -0.017086809501051903, -0.010261092334985733, 0.012300443835556507, 0.028383659198880196, -0.08085354417562485, 0.01989172212779522, 0.011090985499322414, 0.055686526000499725, -0.02189890667796135, -0.007301779929548502, -0.020766649395227432, 0.0027630964759737253, 0.020380651578307152, 0.03371042013168335, -0.01709967479109764, 0.03214069828391075, -0.00478958198800683, -0.029824716970324516, -0.019402792677283287, -0.005715975072234869, -0.017369873821735382, 0.019981788471341133, 0.027097001671791077, 0.01893959566950798, 0.018579332157969475, -0.011116718873381615, -0.006208121310919523, -0.030262179672718048, 0.014513494446873665, 0.005191662348806858, -0.00028366773040033877, 0.019132593646645546, 0.020933914929628372, 0.0049890135414898396, 0.016353413462638855, 0.0062595875933766365, 0.04474994167685509, 0.0035608240868896246, 0.009083800949156284, -0.011348317377269268, 0.014153230004012585, 0.01525975577533245, -0.0018656530883163214, 0.021126912906765938, 0.015143956057727337, 0.00896800123155117, -0.008305372670292854, 0.01028039213269949, 0.03270682692527771, -0.03280976042151451, -0.007128081284463406, 0.005455426871776581, 0.02959311753511429, -0.0010389757808297873, 0.003380692098289728, 0.01525975577533245, 0.0134584354236722, 0.03142017126083374, 0.0024462572764605284, -0.0012094578705728054, 0.0029705699998885393, -0.00846620462834835, 0.010441224090754986, -0.020933914929628372, 0.028332194313406944, 0.008138107135891914, 0.003943604417145252, 0.02056078426539898, 0.008311806246638298, -0.013419835828244686, -0.015439887531101704, 0.008305372670292854, -0.024317823350429535, 0.0034385917242616415, -0.001416127197444439, -0.015439887531101704, -0.0023690578527748585, 0.03152310475707054, 0.018347732722759247, -0.010241791605949402, -0.018463531509041786, -0.02579747885465622, 0.014423428103327751, 0.010299691930413246, -0.024999750778079033, -0.6237714290618896, -0.01872086338698864, 0.005641992203891277, -0.010318991728127003, 0.009347565472126007, 0.015787284821271896, 0.01917119324207306, 0.008022308349609375, -0.025257082656025887, -0.004117303527891636, 0.004348901566118002, 0.024137690663337708, -0.007687777280807495, -0.007449745666235685, 0.00658768555149436, -0.022812433540821075, -0.022709501907229424, -0.0186565313488245, 0.001252078334800899, 0.003940388094633818, -0.003782772459089756, -0.01314963772892952, -0.0036315901670604944, -0.0055712261237204075, 0.005329977720975876, 0.010897987522184849, -0.007951541803777218, 0.008080207742750645, 0.030931241810321808, 0.041250232607126236, -0.024472221732139587, 0.01922265999019146, -0.010827220976352692, 0.014667892828583717, 0.043823547661304474, 0.0004101220110896975, -0.006298187654465437, 0.02507694996893406, 0.0036669732071459293, 0.02061224915087223, -0.005358927417546511, 0.00933469831943512, 0.030493777245283127, -0.023545827716588974, 0.003831022186204791, -0.0017627205234020948, 0.0030992357060313225, -0.001371094142086804, 0.0036380235105752945, -0.014256162568926811, -0.0013445569202303886, -0.014449161477386951, 0.019209792837500572, -0.004779931623488665, -0.002747013233602047, -0.022246304899454117, 0.021821707487106323, -0.0033967753406614065, 0.00444218423217535, -0.004519383888691664, -0.011464116163551807, 0.0021889256313443184, 0.0003679035580717027, -0.023751692846417427, -0.0067999837920069695, 0.034456681460142136, -0.007276047021150589, -0.015980282798409462, 0.010286824777722359, 0.005651642102748156, 0.0027775713242590427, 0.03548600897192955, -0.013612833805382252, -0.0005094358930364251, -0.016507813706994057, -0.009842928498983383, 0.0011129585327580571, -0.009714262560009956, 0.006941515952348709, 0.0016026925295591354, -0.006249937694519758, -0.014552094042301178, -0.00838257186114788, -0.003853538539260626, 0.022104771807789803, -0.006198471412062645, -0.021435709670186043, -0.027714598923921585, 0.02283816784620285, 0.008170274086296558, -0.004586933180689812, 0.025643080472946167, -0.02745726704597473, -0.03504854440689087, -0.014307629317045212, 0.012885873205959797, 0.03674693405628204, -0.010878687724471092, -0.003168393624946475, -0.007018715608865023, -0.032114967703819275, -0.010389757342636585, -0.018965328112244606, 0.02306976541876793, 0.0012014162493869662, 0.013831566087901592, -0.00944406446069479, -0.03098270855844021, 0.029773250222206116, -0.022825300693511963, 0.01559428684413433, -0.007642744109034538, -0.017408473417162895, -0.018965328112244606, 0.029773250222206116, -0.017086809501051903, 0.023983292281627655, -0.003538307501003146, -0.004992230329662561, -0.036798398941755295, 0.01239694282412529, -0.00609875563532114, 0.002085993066430092, -0.011875846423208714, 0.017755869776010513, -0.025192750617861748, 0.0030236446764320135, 0.008234607055783272, -0.005934706889092922, -0.02584894560277462, -0.0020747347734868526, -0.030879775062203407, 0.011637815274298191, -0.0022870332468301058, -0.025167016312479973, -0.012049545533955097, 0.007604144513607025, 0.011193918064236641, -0.011727881617844105, -0.021577242761850357, 0.02640220709145069, -0.00041132824844680727, -0.003818155499175191, -0.014230430126190186, -0.013921631500124931, -0.03170323744416237, -0.0608331561088562, 0.01559428684413433, -0.018965328112244606, -0.01582588441669941, 0.003289017593488097, -0.022156238555908203, -0.02056078426539898, 0.0405554361641407, -0.002550798002630472, 0.005635559093207121, -0.02240070328116417, -0.013085304759442806, 0.017871670424938202, -0.013188237324357033, 0.01682947762310505, 0.02807486243546009, -0.03157456964254379, -0.02941298484802246, 0.0029850448481738567, -0.021860307082533836, -0.033401623368263245, 0.0031764351297169924, -0.02111404575407505, -0.017562871798872948, -0.004245969001203775, 0.03458534926176071, -0.001191766350530088, 0.03157456964254379, -0.015156823210418224, 0.03185763582587242, -0.019685856997966766, -0.018566465005278587, -0.0003874044632539153, 0.010807921178638935, 0.018579332157969475, -0.010878687724471092, -0.02740580029785633, -0.03767332434654236, 0.018141867592930794, 0.0186565313488245, 0.010878687724471092, 0.01932559162378311, 0.000512250408064574, 0.010557022877037525, 0.022722367197275162, 0.031677503138780594, -0.008266773074865341, -0.009296098724007607, -0.006008689757436514, -0.015851618722081184, 0.005397527478635311, 0.006992982234805822, 0.006365736946463585, 0.02725140191614628, 0.024497954174876213, 0.012345477007329464, 0.019544323906302452, -0.006709917914122343, 0.014603559859097004, -0.0355374738574028, 0.017691537737846375, -0.008627037517726421, 0.007121648173779249, 0.0009175474988296628, 0.001539967954158783, 0.001149949966929853, 0.009546997025609016, -0.030493777245283127, 0.008543404750525951, 0.01922265999019146, 0.0033195759169757366, -0.0027807881124317646, -0.002994694747030735, -0.0034675414208322763, 0.019763056188821793, 0.008086641319096088, 0.009566296823322773, -0.013934498652815819, -0.010216059163212776, 0.031059907749295235, 0.003206993220373988, -0.03849678486585617, -0.0005822124076075852, 0.0007961192168295383, -0.019853122532367706, -0.009308965876698494, 0.011798647232353687, 0.01317537110298872, -0.00840830523520708, 0.039757709950208664, 0.01726694032549858, -0.04168769717216492, 0.03592346981167793, 0.0004447009414434433, -0.023365696892142296, 0.00958559662103653, -0.008048041723668575, -2.8949789339094423e-05, 0.03764759376645088, -0.011882279999554157, 0.018785197287797928, 0.011805080808699131, 0.0038085056003183126, 0.01300810556858778, -0.007443312555551529, 0.02355869486927986, -0.03515147790312767, 0.016417747363448143, -0.002636039163917303, 0.006748517509549856, 0.03510000929236412, 0.01900392770767212, 0.02049645036458969, 0.006179171614348888, 0.004570850171148777, 0.0019187276484444737, -0.0004712382215075195, -0.013844432309269905, 0.006877183448523283, 0.04915031045675278, 0.009701396338641644, -0.02807486243546009, -0.026312140747904778, -0.019132593646645546, -0.016237614676356316, -0.026762472465634346, -0.009180299937725067, -0.017228340730071068, -3.5810284316539764e-05, -0.016250481829047203, -0.025475814938545227, -0.007578411605209112, 0.00863347016274929, -0.006600551772862673, -0.023198431357741356, -0.031677503138780594, -0.004869997967034578, 0.026376474648714066, -0.01245484221726656, -0.01253847498446703, -0.025771746411919594, 0.017961734905838966, -0.015607153065502644, 0.014230430126190186, 0.0025733145885169506, -0.003782772459089756, -0.0023899658117443323, 0.005867157131433487, -0.026209209114313126, -0.01520828902721405, 0.03260389715433121, 0.0008242648327723145, -0.000180333066964522, 0.008614170365035534, 0.004728465341031551, -0.0006847429322078824, -0.01309173833578825, 0.019261259585618973, 0.044981539249420166, -0.0009601680212654173, -0.016147548332810402, 0.005815690848976374, -0.010544156655669212, -0.015053890645503998, -0.03286122903227806, -0.0134841687977314, -0.008009441196918488, -0.03129150718450546, 0.012184644117951393, -0.01917119324207306, 0.021152645349502563, 0.014140363782644272, 0.03389055281877518, -0.021757375448942184, 0.017202608287334442, -0.02830646000802517, -0.020264852792024612, -0.024047624319791794, 0.07416292279958725, -0.015632886439561844, -0.016842344775795937, 0.017640070989727974, 0.023880358785390854, -0.0006674534524790943, -0.023584427312016487, -0.00036629525129683316, -0.007771410048007965, -0.02730286866426468, 0.0036380235105752945, -0.013226836919784546, -0.009868661873042583, 0.011258251033723354, -0.0018238367047160864, -0.005982956383377314, -0.0020264852792024612, 0.013020971789956093, -0.013329769484698772, 0.007340379990637302, 0.02133277803659439, -0.006793550681322813, 0.02897552214562893, 0.020071852952241898, -0.004091570153832436, 0.019209792837500572, 0.03530587628483772, 0.007237447425723076, 0.03453388065099716, -0.007147381082177162, -0.002989869797602296, 0.02333996258676052, -0.0004724444879684597, -0.018064668402075768, 0.0006835366948507726, -0.0033453090582042933, -0.0069801160134375095, 0.0027647048700600863, -0.01843779906630516, -0.03160030394792557, -0.009263932704925537, 0.026711005717515945, 0.003367825411260128, -0.026427941396832466, 0.011251818388700485, -0.02144857682287693, 0.017035342752933502, -0.005937923211604357, -0.00501474691554904, -0.009000168181955814, -0.004091570153832436, 0.017537139356136322, -0.013265436515212059, -0.006468669511377811, 0.008601304143667221, 0.017421340569853783, -0.005651642102748156, 0.006571602076292038, -0.01219751127064228, -0.030159246176481247, 0.015079623088240623, 0.005535842850804329, -0.006198471412062645, 0.007604144513607025, -0.05743638053536415, -0.015053890645503998, 0.009765729308128357, -0.011650681495666504, -0.029104188084602356, 0.01677801087498665, 0.021126912906765938, -0.016687944531440735, -0.0016613962361589074, -0.0022548669949173927, 0.02066371589899063, 0.015967417508363724, -0.005703108385205269, 0.02150004357099533, 0.0269168708473444, 0.02345576323568821, -0.008009441196918488, -0.017241207882761955, -0.010466957464814186, -0.027585932984948158, -0.012010945938527584, -0.02840939350426197, 0.033401623368263245, -0.01293090544641018, -0.03453388065099716, 0.03955184668302536, 0.00946979783475399, -0.0002114942908519879, 0.021101180464029312, 0.009013034403324127, -0.007835743017494678, 0.015864484012126923, 0.012680008076131344, -0.03324722498655319, 0.012203944846987724, -0.010923719964921474, 0.0021953589748591185, 0.01278294064104557, 0.013934498652815819, -0.02512841671705246, -0.007552678231149912, 0.009617763571441174, 0.002393182599917054, 0.01805180124938488, 0.0019927104003727436, 0.005384660791605711, 0.009321832098066807, -0.017073942348361015, 0.00980432890355587, -0.014063164591789246, -0.007970841601490974, 0.0017434206092730165, -0.012654274702072144, 0.010203192010521889, -0.03744172677397728, -0.015697218477725983, 0.03576907142996788, -0.0639725998044014, 0.03594920411705971, 0.013355502858757973, -0.006124488543719053, 0.027585932984948158, -0.013239703141152859, -0.02840939350426197, -0.02201470546424389, -7.172109326347709e-05, 0.005117679480463266, 0.024523688480257988, -0.009598463773727417, -0.028538059443235397, -0.01203024573624134, -0.005754574667662382, -0.0043263849802315235, 0.025334281846880913, -0.0028579875361174345, 0.02088244818150997, -0.005616259295493364, 0.03275829553604126, -0.008980867452919483, 0.006153438705950975, -0.019145460799336433, -0.038522519171237946, -0.002304724883288145, -0.017408473417162895, 0.0008668853552080691, 0.004895730875432491, -0.01237120945006609, 0.009366865269839764, -0.02139711007475853, 0.0005701500340364873, 0.030493777245283127, -0.019930321723222733, -0.02735433354973793, -0.0029770033434033394, 0.029438719153404236, 0.023468628525733948, 0.011110285297036171, 0.023262763395905495, 0.013844432309269905, 0.02077951468527317, -0.015092490240931511, -0.0029737865552306175, 0.006040855776518583, 0.0045740664936602116, -0.005568009335547686, -0.00916100014001131, -0.01537555456161499, 0.004548333585262299, 0.013201103545725346, 0.003007561434060335, -0.008530537597835064, 0.04328314960002899, -0.005635559093207121, -0.010344725102186203, -0.023327097296714783, -0.020200518891215324, -0.023700227960944176, 0.048712845891714096, 0.010750021785497665, 0.0097206961363554, -0.008929401636123657, -0.005153062287718058, 0.03013351373374462, -0.016572145745158195, -0.011547748930752277, -0.028898322954773903, 0.01984025537967682, 0.004915030673146248, 0.01932559162378311, -0.013754366897046566, 0.03165176883339882, -0.014860891737043858, -0.007687777280807495, -0.029773250222206116, -0.015529953874647617, -0.010704988613724709, 0.008832902647554874, -0.003654106752946973, 0.007662044372409582, -0.013432702049612999, -0.003859971882775426, -0.010203192010521889, -0.01362570095807314, 0.007481912150979042, 0.006909349467605352, -0.025939011946320534, -0.028563791885972023, 0.0028065212536603212, -0.02352009527385235, -0.01917119324207306, 0.024330688640475273, -0.003699139691889286, -0.023031165823340416, 0.023301362991333008, 0.001807753462344408, -0.013201103545725346, 0.025617346167564392, -0.011232518590986729, 0.04037530720233917, 0.004445401020348072, 0.03597493842244148, -0.012737907469272614, -0.01239051017910242, -0.013136770576238632, 0.0013469693949446082, 0.014745092950761318, -0.003486841218546033, 0.0318833664059639, -0.0006646388792432845, 0.010248225182294846, -0.004403584636747837, 0.007276047021150589, -0.004660916049033403, 0.0002480836119502783, -0.019531458616256714, -0.014359095133841038, 0.007018715608865023, 0.016687944531440735, -0.02205330692231655, 0.0036734065506607294, 0.013728633522987366, -0.005494026467204094, -0.021152645349502563, 0.02624780870974064, 0.0006839387933723629, -0.010351157747209072, -0.013818698935210705, 0.030262179672718048, -0.019685856997966766, 0.01577441766858101, 0.032398030161857605, 0.011045953258872032, -0.01637914776802063, -0.0008282856433652341, -0.03113710694015026, 0.016186147928237915, -0.014719359576702118, -0.0048603480681777, -0.007925809361040592, -0.017421340569853783, 0.004226669203490019, 0.0011893538758158684, -0.0035608240868896246, -0.023044032976031303, -0.005590525921434164, 0.027817530557513237, -0.012461275793612003, -0.0004390717949718237, -0.005799607839435339, 0.004484000615775585, 0.002103684702888131, -0.0041205198504030704, -0.004188069608062506, -0.0010454090079292655, -0.019081126898527145, 0.013278303667902946, 0.022194838151335716, -0.003950037993490696, -0.013117470778524876, -0.027894729748368263, 0.011579915881156921, 0.0027389717288315296, -0.012905173003673553, 0.014835158362984657, 0.016559278592467308, -0.009765729308128357, -0.021487176418304443, 0.0010574714979156852, 0.015581419691443443, -0.007353246212005615, 0.003747389419004321, -0.04032383859157562, -0.01292447280138731, 0.010988052934408188, 0.011830814182758331, 0.010923719964921474, 0.016186147928237915, 0.02635074220597744, -0.018116135150194168, -0.014822292141616344, 0.029901916161179543, -0.030339378863573074, 0.008183140307664871, -0.0194413922727108, -0.037801992148160934, -0.010318991728127003, 0.006375386845320463, 0.013947364874184132, -0.01206241175532341, -0.0040787034668028355, 0.0010968752903863788, 0.014616427011787891, 0.019029662013053894, 0.02261943556368351, -0.00046239246148616076, 0.017871670424938202, -0.006452586501836777, -0.016070349141955376, -0.002724496880546212, -0.023134097456932068, 0.001981452340260148, -0.03787919133901596, -0.006999415811151266, 0.008498371578752995, -0.004561200272291899, -0.002549189841374755, -0.015143956057727337, 0.00983649492263794, -0.012885873205959797, -0.019763056188821793, -0.003921088296920061, -0.011875846423208714, -0.018141867592930794, 0.01711254194378853, -0.023970425128936768, 0.015169689431786537, 0.010885120369493961, 0.02223343774676323, 0.005748141556978226, 0.031008441001176834, -0.022863900288939476, -0.001605105004273355, -0.01805180124938488, -0.0067292177118361, -0.03098270855844021, -0.012242544442415237, 0.011972346343100071, 0.04266555607318878, 0.0303651113063097, -0.0019444607896730304, -0.035846270620822906, 0.00491824746131897, -0.011271118186414242, -0.021152645349502563, 0.021088313311338425, -0.0032777595333755016, 0.013805832713842392, 0.0026408641133457422, 0.00989439431577921, 0.040195174515247345, 0.0071988473646342754, 0.0001614352804608643, -0.017472805455327034, 0.028666725382208824, 0.0017739786999300122, 0.012641407549381256, 0.01699674315750599, -0.0028885456267744303, -0.023417163640260696, 0.010415490716695786, 0.0037859890144318342, -0.0029930865857750177, -0.015722952783107758, 3.887614730047062e-05, 0.007276047021150589, 0.013368369080126286, 0.02121697925031185, 0.017691537737846375, 0.014140363782644272, 0.006684184540063143, 0.023430028930306435, -0.03968051075935364, -0.017408473417162895, 0.004252402577549219, -0.005799607839435339, -0.011496283113956451, 0.030210712924599648, 0.0088586350902915, -0.005950789898633957, 0.00304294447414577, 0.004567633382976055, -0.008170274086296558, 0.001814186805859208, -0.0034353749360889196, 0.006870749872177839, 0.01520828902721405, 0.0008596479310654104, 0.005580876022577286, -0.014153230004012585, -0.0056130425073206425, 0.01192087959498167, 0.005317111033946276, -0.0005769853596575558, -0.0022050088737159967, -0.0006264412659220397, 0.008163840509951115, 0.008054474368691444, -0.02138424478471279, 0.02188604138791561, -0.013883031904697418, -0.008504805155098438, 0.01258994173258543, -0.013998831622302532, -0.04300008714199066, 0.0210111141204834, 0.021924640983343124, 0.026029076427221298, 0.012139611877501011, -0.024922551587224007, -0.014950958080589771, -0.005963656585663557, -0.00899373460561037, -0.0009690137812867761, -0.05820837616920471, -0.0018383115530014038, -0.001764328801073134, -0.016893809661269188, -0.011618515476584435, -0.010248225182294846, 0.008305372670292854, 0.00459658307954669, 0.0032214683014899492, 0.18517571687698364, 0.0246137548238039, 0.002298291539773345, 0.04539326950907707, 0.010595622472465038, -0.011554182507097721, 0.014577827416360378, -0.004310301970690489, 0.008922968059778214, 0.03175470232963562, -0.00047445486416108906, 0.008948701433837414, -0.02563021332025528, 0.010241791605949402, 0.010557022877037525, -0.007945109158754349, -0.029181387275457382, -0.001889777951873839, -0.01760147139430046, 0.0025636646896600723, 0.01270574051886797, -0.018875261768698692, 0.004207369405776262, 0.009373298846185207, 0.051286160945892334, 0.0033517421688884497, 0.0048346146941185, -0.006883616559207439, 0.025257082656025887, -0.005786741152405739, -0.021757375448942184, -0.014860891737043858, -0.02552727982401848, -0.003451458178460598, -0.011419083923101425, -0.010692122392356396, 0.019351325929164886, -0.007623444311320782, 0.016224749386310577, 0.028795389458537102, -0.029721783474087715, -0.014513494446873665, -0.007153814658522606, -0.02099824696779251, 0.004924680572003126, 0.04269129037857056, -0.004976146854460239, 0.005230261944234371, 0.015298355370759964, -0.00877500232309103, -0.018579332157969475, 0.01571008563041687, 0.027894729748368263, 0.004069053567945957, -0.014950958080589771, 0.01256420835852623, 0.026505140587687492, 0.007996574975550175, -0.01716400869190693, -0.0028065212536603212, -0.0064847529865801334, 0.018141867592930794, 0.004545116797089577, 0.010737155564129353, -0.003917871508747339, 0.015607153065502644, -0.021744508296251297, 0.005124112591147423, 0.023301362991333008, 0.00043022603495046496, -0.0126993078738451, 0.0056805917993187904, -0.009392598643898964, 0.011534882709383965, 0.028203528374433517, -0.03870265185832977, 0.004731682129204273, 0.0037087895907461643, 0.01203024573624134, 0.010550590232014656, -0.007134514860808849, -0.0013550110161304474, -0.0011676414869725704, 0.004339251667261124, -0.0009601680212654173, 0.00941833108663559, 0.0007313842652365565, -0.018566465005278587, 0.01821906678378582, 0.005223828833550215, -0.0004177615337539464, -0.007076615002006292, 0.01203024573624134, -0.03152310475707054, 0.032835494726896286, 0.006552302278578281, 0.020406384021043777, -0.00857557076960802, -0.020985379815101624, -0.011187485419213772, 0.007185981143265963, 0.014423428103327751, 0.0186565313488245, 0.04004077613353729, 0.004734898917376995, -0.000645339023321867, -0.002219483722001314, -0.019132593646645546, -0.0009481055894866586, 0.014423428103327751, -4.098706995137036e-05, -0.013960232026875019, 0.006851450074464083, -0.01726694032549858, -0.0059282733127474785, -0.0037666892167180777, -0.0003045758930966258, -0.051286160945892334, 0.0002890957985073328, 0.015504220500588417, -0.017459940165281296, -0.04629392921924591, -0.0028129545971751213, -0.008131674490869045, -0.008202440105378628, -0.03782772272825241, -0.0066069853492081165, 0.024716686457395554, 0.003650889964774251, -0.0389857143163681, 0.002045785076916218, -0.011026653461158276, 0.018540730699896812, -0.005867157131433487, 0.009907261468470097, 0.02869245782494545, 0.0057352748699486256, -0.01949285715818405, 0.006780683994293213, -0.013033838011324406, -4.0836290281731635e-05, -0.0068964832462370396, 0.02812632918357849, 0.017138274386525154, 0.007893642410635948, -0.014217562973499298, -0.02350722812116146, 0.0016179715748876333, -0.027766063809394836, -0.028769657015800476, -0.016855210065841675, -0.020457850769162178, 0.008337539620697498, 0.0009649929706938565, 0.01666221208870411, 0.01537555456161499, -0.03672119975090027, 0.01231331005692482, 0.009926561266183853, 0.02959311753511429, -0.025540146976709366, 0.009353998117148876, 0.01984025537967682, 0.0016026925295591354, 0.01228114403784275, 0.0011845289263874292, -0.15872204303741455, 0.02484535239636898, 0.02931005321443081, -0.007205280940979719, -0.014577827416360378, -0.024266356602311134, 0.0032118181698024273, -0.008363272063434124, 0.002907845424488187, -0.02111404575407505, 0.019518591463565826, -0.01211387850344181, -0.01727980747818947, -0.01534982118755579, 0.005542276427149773, -0.0016597879584878683, -0.01189514622092247, 0.010524856857955456, 0.04840404912829399, 0.02273523434996605, -0.00950196385383606, -0.021075446158647537, 0.03293842822313309, -0.0002758271584752947, 0.03826518729329109, 0.023880358785390854, 0.02240070328116417, 0.016893809661269188, 0.023700227960944176, -0.031960565596818924, 0.016224749386310577, -0.03286122903227806, -0.008247473277151585, -0.01540128793567419, 0.005432910285890102, -0.013664300553500652, -0.010569890029728413, -0.00026798658655025065, -0.0011957871029153466, 0.003118535503745079, 0.00944406446069479, 0.0004877235332969576, 0.003084760857746005, 0.001770762144587934, 0.001377527485601604, 0.03561467304825783, 0.019081126898527145, -0.017923135310411453, 0.03427654877305031, -0.008138107135891914, 0.031445905566215515, 0.0031523103825747967, -0.016842344775795937, 0.0054200440645217896, 0.027097001671791077, 0.040246639400720596, 0.008170274086296558, 0.026427941396832466, 0.011013786308467388, -0.017627205699682236, -0.0006598139298148453, -0.02061224915087223, 0.010685688816010952, -0.015735818073153496, 0.0030767193529754877, -0.023816026747226715, -0.0018415282247588038, 0.0043263849802315235, -0.04155902937054634, 0.016572145745158195, -0.007874342612922192, -0.0062595875933766365, 0.008163840509951115, -0.01239694282412529, -0.007674910593777895, 0.009180299937725067, -0.00932826567441225, 0.014719359576702118, -0.005043696612119675, -0.006163088604807854, -0.025951877236366272, 0.029953381046652794, -0.01838633231818676, 0.003824588842689991, -0.010158159770071507, 0.011226085014641285, -0.015272621996700764, 0.0062595875933766365, -0.027045536786317825, -0.003824588842689991, 0.01278937328606844, -0.00880073569715023, -0.016057483851909637, -0.015143956057727337, 0.001814186805859208, 0.01275720726698637, -0.015362688340246677, -0.0044100177474319935, 0.0012641408247873187, -0.02864099107682705, 0.01738273911178112, 0.0016983876703307033, -0.0028177795466035604, 0.008093073964118958, 0.014127496629953384, -0.00434568477794528, 0.029850449413061142, 0.010164592415094376, 0.06567098945379257, 0.010518423281610012, -0.010499123483896255, 0.02586181089282036, 0.022477902472019196, 0.03391628712415695, -0.02462662011384964, -0.0033195759169757366, 0.018257666379213333, -0.0011193918762728572, 0.021139780059456825, 0.006902916356921196, 0.029001254588365555, -0.002110118046402931, -0.0021985757630318403, 0.007539811544120312, -0.00013037456665188074, -0.023082632571458817, -0.09510970860719681, -0.007919375784695148, 0.005497243255376816, 0.02050931751728058, -0.0012930906377732754, 0.01716400869190693, -0.012667140923440456, 0.03587200492620468, 0.005658075679093599, 0.05620118975639343, -0.009000168181955814, -0.017858803272247314, 0.005532626528292894, -0.014410561881959438, -0.005490810144692659, -0.023146964609622955, -0.014757959172129631, -0.007411146070808172, -0.043025821447372437, 0.014719359576702118, -0.021770240738987923, -0.010524856857955456, -0.013638567179441452, 0.0024205241352319717, 0.00015550458920188248, -0.008562704548239708, -0.03911438211798668, 0.0024912902154028416, 0.00619525508955121, -0.004586933180689812, 0.0009649929706938565, -0.007507645525038242, 0.002298291539773345, -0.03165176883339882, -0.014590693637728691, 0.006838583387434483, -0.021358510479331017, 0.001417735475115478, 0.03870265185832977, -0.03739026188850403, -0.01304027158766985, 0.026839671656489372, -0.0028531625866889954, 0.004577283281832933, 0.023700227960944176, -0.00877500232309103, -0.04130169749259949, 0.020457850769162178, 0.005494026467204094, -0.0038438886404037476, -0.023108365014195442, -0.009855794720351696, -0.017228340730071068, -0.007070181891322136, 0.020470717921853065, 0.006330353673547506, -0.00518201244994998, 0.027225667610764503, -0.026762472465634346, 2.7140427846461535e-05, -0.020740915089845657, -0.002245216863229871, 0.004229885991662741, 0.011206785216927528, -0.0004189677711110562, 0.018013201653957367, -0.0028563791420310736, -0.007835743017494678, 0.018630797043442726, -0.05599532648921013, -0.00682571716606617, 0.013239703141152859, -0.025424348190426826, 0.0053621442057192326, -0.005510109942406416, 0.02424062415957451, -0.012737907469272614, -0.007398279383778572, 0.017858803272247314, -0.01681661047041416, -0.02712273597717285, -0.003895354922860861, -0.014320495538413525, -0.021191244944930077, 0.010383324697613716, 0.011026653461158276, 0.02233637124300003, -0.027714598923921585, 0.02974751591682434, -0.03968051075935364, -0.001014046836644411, 0.030931241810321808, 0.01317537110298872, -0.00036368172732181847, -0.002311158226802945, 0.02056078426539898, -0.005214178469032049, 0.013973098248243332, 0.021860307082533836, 0.016404880210757256, -0.04639686271548271, -0.031111372634768486, -0.04199649393558502, 0.024935418739914894, 0.0046319663524627686, -0.026711005717515945, 0.0007458591717295349, 0.011998079717159271, -0.02211763896048069, -0.026222076267004013, 0.0006626285030506551, 0.0016613962361589074, -0.030262179672718048, 0.016790878027677536, -0.014449161477386951, -0.007764976937323809, -0.01837346702814102, -0.023700227960944176, 0.018630797043442726, 0.0013646609149873257, 0.028486592695116997, -0.015722952783107758, -0.027997663244605064, 0.002915887162089348, 0.00575135787948966, 0.019685856997966766, -0.025424348190426826, 0.0027293218299746513, 0.0030300780199468136, -0.018360599875450134, -0.017537139356136322, -0.01743420585989952, 0.015787284821271896, -0.005465076770633459, 0.017292674630880356, 0.010190325789153576, -0.0018704780377447605, -0.004648049362003803, -0.0210111141204834, 0.034173619002103806, -0.002755054971203208, 0.0031587437260895967, -0.020535049960017204, -0.015053890645503998, 0.03957757726311684, -0.002708413638174534, -0.027920464053750038, -0.01244840957224369, -0.01230687741190195, -0.012094578705728054, 0.024948284029960632, 0.020766649395227432, 0.024266356602311134, 0.015195422805845737, -0.0026858970522880554, -0.031497370451688766, -0.02251650206744671, -0.009321832098066807, 0.01599314995110035, 0.01287300605326891, -0.004175202921032906, -0.0033903419971466064, 0.03126577287912369, 0.017588606104254723, 0.004876431077718735, -0.02740580029785633, 0.012216811068356037, -0.0011660332093015313, -0.022040439769625664, -0.019016794860363007, 0.010428356938064098, -0.01699674315750599, -0.00918673351407051, -0.009508397430181503, 0.0303651113063097, 0.02166730910539627, 0.006089105736464262, 0.01236477680504322, 0.020483583211898804, 0.0005605000769719481, 0.01643061451613903, 0.016404880210757256, -0.007617011200636625, 0.029670316725969315, -0.06500192731618881, 0.01966012269258499, 0.023211296647787094, -0.005410393700003624, -0.04305155202746391, 0.008105941116809845, -0.014513494446873665, 0.015735818073153496, 0.024652354419231415, 0.005677375476807356, -0.025939011946320534, -0.013053137809038162, -0.008260339498519897, 0.013792966492474079, -0.012725040316581726, 0.006902916356921196, 0.010183892212808132, 0.044981539249420166, 0.021525776013731956, -0.0029014120809733868, 0.01910686120390892, -0.001586609287187457, -0.003740956075489521, -0.015735818073153496, -0.019235527142882347, -0.0038471051957458258, -0.02089531533420086, 0.005863940808922052, 0.012184644117951393, -0.0017530705081298947, 0.013432702049612999, -0.001412910525687039, -0.014127496629953384, 0.002264516893774271, -0.017472805455327034, -0.011489849537611008, 0.002914278768002987, 0.02355869486927986, -0.00665845163166523, 0.008929401636123657, -0.001474830904044211, 0.0018962111789733171, 0.032449498772621155, -0.0007120844093151391, 0.024755286052823067, -0.03371042013168335, 0.007314646616578102, -0.0023240246810019016, 0.005375010892748833, -0.03733879327774048, 0.0037087895907461643, 0.0016179715748876333, 0.0035125743597745895, -0.013278303667902946, 0.014204696752130985, -0.0006300599779933691, -0.012776507064700127, 0.05604679137468338, 0.011927313171327114, 0.019865987822413445, 0.02858952432870865, -0.0019766271580010653, 0.028872590512037277, 0.014667892828583717, 0.01540128793567419, 0.009617763571441174, -0.04495580494403839, -0.003422508481889963, -0.004323168657720089, 0.020354919135570526, -0.014642160385847092, -0.00921889953315258, -0.004294218495488167, -0.011489849537611008, 0.009617763571441174, -0.039294514805078506, 0.020046120509505272, 0.03538307547569275, -0.01033829152584076, 0.01551708672195673, 0.02551441453397274, 0.0088586350902915, -0.03203776478767395, 0.03414788469672203, 0.0031828684732317924, -0.03103417344391346, -0.011277550831437111, 0.023635894060134888, -0.013033838011324406, -0.056458521634340286, -0.03795639052987099, -0.01917119324207306, -0.02434355579316616, 0.008511237800121307, -0.002335282973945141, 0.004336034879088402, 0.022593701258301735, -0.01671367883682251, 0.015954550355672836, -0.003573690541088581, -0.01659787818789482, -0.004850698169320822, -0.020187653601169586, -0.014655026607215405, -0.019904587417840958, -0.012326177209615707]} +{"id": "test:10000020", "text": "\"A full time vegan, poet, artist and woman.\\nFrontal and thorough when inspired.\\nOur memories: childhood memories, the language we dearly speak and nature's colorful and tasteful palette are a constant and renewable source of wonder and inspiration.\\nI am just getting through finishing two special orders of my Xocolate Bears new edition, which are going to Costa Rica. Both orders are reconnecting me with childhood friends and grade school teachers.\\nI sell my projects on Etsy. In local shops and in our hometown San Jose, Costa Rica in La Tienda E\u00f1\u00e9 a shop supporting local indie and urban designers, artisans and artists.\\nHandmade over mass production, and over all those other products largely available. Handmade by choice to keep our creative and critical side active, to balance and renovate while innovating ourselves every day. Handmade is more a legacy and tradition worth the while to take back and pass down, its techniques connected to our earlier days and years. The things we made, the stuff we broke. Handmade is the milestones we will all like to share. Going back and supporting handmade will keep creativity at edge and sharp.\"", "vector_field": [-0.010974211618304253, 0.004552758298814297, -0.015299506485462189, 0.005924534518271685, -0.006110004149377346, -0.006113503593951464, -0.021052569150924683, 0.00015047547640278935, -0.0023726134095340967, -0.03188680484890938, 0.01271692756563425, 0.0009081020252779126, -0.014256676658987999, -0.036478057503700256, 8.480649557895958e-05, 0.004710232373327017, 0.045240629464387894, 0.011884063482284546, -0.003849372733384371, -0.01595740020275116, -0.025027921423316002, 0.04059338569641113, 0.007775732781738043, -0.010526284575462341, -0.0060470146127045155, -0.010008368641138077, 0.020842604339122772, -0.007037353236228228, -0.0029500192031264305, -0.008258653804659843, -0.0027033095248043537, 0.006019019056111574, -0.022158389911055565, -0.012856905348598957, -0.03796181455254555, -0.013444809243083, -0.006606923416256905, 0.004230810794979334, 0.004762724041938782, -0.000958843738771975, 0.01700722798705101, 0.017791101709008217, 0.006284975912421942, 0.0013087866827845573, -0.012800914235413074, 0.011758084408938885, -0.01721719466149807, 0.008216661401093006, -0.012695930898189545, 0.009721416048705578, 0.050923701375722885, 0.021164551377296448, -0.011373146437108517, 0.004825713578611612, -0.012534957379102707, -0.009784406051039696, -0.020940586924552917, 0.024677978828549385, 0.02771548368036747, -0.012206011451780796, 0.002388360910117626, 0.025475848466157913, -0.015523470006883144, 0.011100191622972488, 0.0003643781237769872, -0.028037430718541145, -0.011485128663480282, -0.013822747394442558, 0.01375975739210844, 0.013969723135232925, -0.0017812097212299705, 0.003480182960629463, -0.02227037027478218, 0.0060050212778151035, 0.02395009808242321, 0.018420998007059097, -0.0018879423150792718, -0.009826398454606533, 0.013521797023713589, 0.00827965047210455, -0.0009002283331938088, -0.032642681151628494, -0.005004184320569038, 0.026791634038090706, -0.0006202739314176142, 0.019904756918549538, -0.017105212435126305, 0.01703522354364395, -0.012912895530462265, -0.0023971092887222767, -0.005637581460177898, 0.008517611771821976, 0.01006435975432396, 0.0024933437816798687, -0.017483150586485863, 0.011380145326256752, -0.006585926748812199, 0.023572158068418503, -0.009889388456940651, -0.001653480576351285, -0.01268893200904131, 0.011583112180233002, 0.006512438878417015, -0.01480958703905344, -0.03510627895593643, 0.0015861165011301637, 0.00641095545142889, 0.013668772764503956, 0.022102398797869682, -0.002237885259091854, -0.01804305985569954, 0.010113351978361607, 0.007023355457931757, -0.036254093050956726, -0.007684747688472271, -0.0011863067047670484, 0.024062078446149826, -0.004066337365657091, 0.011870065703988075, -0.021976418793201447, 0.012779917567968369, 0.004843210801482201, 0.006347965449094772, -0.024272045120596886, 0.014242678880691528, 0.01931685209274292, -0.012094029225409031, -0.02040867507457733, -0.006120502483099699, -0.01909288950264454, -0.007817725650966167, -0.011947053484618664, 0.015817422419786453, 0.002890528878197074, 0.005431115161627531, 0.010085356421768665, -0.031634844839572906, 0.017721112817525864, 0.006732902955263853, -0.00883256085216999, 0.022830279543995857, 0.007002358790487051, -0.005004184320569038, -0.014543630182743073, 0.005791556090116501, 0.027463525533676147, -0.002213389379903674, 0.03345454856753349, 0.009840396232903004, 0.0228582751005888, 0.026931611821055412, -0.021584482863545418, 0.020086726173758507, -0.013675771653652191, 0.018434995785355568, 0.03185880929231644, -0.005833549425005913, 0.03043104149401188, -0.026133742183446884, -0.02227037027478218, 0.004636744502931833, 0.001638607936911285, 0.007418791297823191, 0.017735110595822334, 0.029311222955584526, 0.0365060530602932, 0.02900327369570732, 0.002295625861734152, 0.005746063776314259, 0.012031039223074913, -0.0034574365708976984, 0.04535260796546936, -0.03902564197778702, 0.010729251429438591, 0.005098669324070215, 0.01048429124057293, -0.017945075407624245, 0.031410880386829376, -0.020422672852873802, -0.012555954046547413, -0.002302624750882387, 0.0059875245206058025, 0.0030025106389075518, 0.009917384013533592, -0.023908104747533798, -0.00467173894867301, 0.013164854608476162, -0.00720182666555047, 0.007943705655634403, 0.0067224046215415, -0.027603501453995705, 0.01678326539695263, -0.00037881327443756163, -0.004797718022018671, -0.6642757654190063, -0.012248003855347633, 0.01805705763399601, -0.04297299683094025, 0.02917124703526497, 0.009504451416432858, 0.026203729212284088, -0.01489357277750969, -0.013395817019045353, -0.006501940544694662, -0.00645994720980525, 0.03502229228615761, 0.011807075701653957, -0.01029532216489315, 0.0077687338925898075, -0.014711602590978146, 0.012520959600806236, -0.03821377456188202, -0.016685280948877335, 0.004076835699379444, -0.0034329406917095184, 0.020478662103414536, -0.015061546117067337, 0.009140510112047195, 0.006809890270233154, 0.014697604812681675, 0.010568277910351753, 0.006452948320657015, -0.010155345313251019, 0.016489313915371895, -0.0029832639265805483, 0.03717794269323349, -0.006774895824491978, 0.0027418031822890043, 0.04890802875161171, -0.0007169456803239882, -0.023824118077754974, 0.031382884830236435, -0.011912059038877487, 0.021094562485814095, -0.05139962583780289, 0.01703522354364395, 0.02880730666220188, 0.006753899622708559, 0.0071283383294939995, 0.008853557519614697, 0.006529936101287603, -0.004836211912333965, 0.006106504704803228, -0.022550325840711594, 0.01744115725159645, -0.014137696474790573, 0.0015170028200373054, -0.02077261544764042, 0.012730925343930721, -0.013906734064221382, 0.014935566112399101, -0.019666794687509537, 0.02600776217877865, 0.02560182847082615, 0.003529174951836467, 0.008083682507276535, -0.003681400092318654, -0.015061546117067337, -0.0352182611823082, 0.007026854902505875, -0.002734804293140769, -0.003957855049520731, 0.003394446801394224, -0.023866111412644386, -0.007397794630378485, 0.026385700330138206, -0.03401445597410202, -0.0064284526742994785, -0.0033349564764648676, -0.0024968432262539864, 0.01889692060649395, -0.017343174666166306, 0.005035679321736097, 0.023600153625011444, 0.015187525190412998, -0.01636333391070366, -0.021066566929221153, -0.001582617056556046, 0.036422066390514374, -0.026091748848557472, -0.008055686950683594, -0.0076707499101758, 0.007131837774068117, 0.00880456529557705, -0.0002148868516087532, 0.015467479825019836, -0.0025650819297879934, -0.030095096677541733, -0.0015642450889572501, -0.01070125587284565, -0.008888551965355873, 0.024062078446149826, -0.00614849803969264, -0.02623172476887703, 0.008160670287907124, -0.0046647400595247746, 0.028037430718541145, 0.010379308834671974, -0.006323469802737236, 0.007026854902505875, -0.024649983271956444, 0.01082023698836565, 0.03110293112695217, -0.021500496193766594, -0.013381819240748882, -0.003992849495261908, -0.007061849348247051, -0.014865577220916748, -0.007880715653300285, -0.014347662217915058, 0.018742945045232773, -0.0012178015895187855, -0.0032929633744060993, -0.0053156339563429356, -0.0010069608688354492, 0.008545607328414917, -0.014060708694159985, -0.001445264439098537, -0.003550171386450529, 0.01991875469684601, -0.012108027003705502, -0.006050514057278633, -0.004668239504098892, 0.0005332255968824029, 0.001158311264589429, -0.00294127082452178, 0.008223660290241241, -0.007460784167051315, 0.019442832097411156, 0.005966527853161097, -0.0020681628957390785, -0.004374287091195583, 0.0015957399737089872, -0.04854409024119377, -0.011114188469946384, 0.00249159405939281, -0.014949563890695572, -0.003221224993467331, -0.02224237471818924, -0.020478662103414536, 0.021500496193766594, -0.006393458228558302, -0.036198101937770844, 0.015859415754675865, -0.011177178472280502, -0.01868695579469204, -0.01280791312456131, 0.027673490345478058, 0.0072998106479644775, -0.008825561963021755, -0.02141650952398777, -0.013605782762169838, -0.008139673620462418, -0.010554280132055283, 0.005588589236140251, 0.030571019276976585, -0.018197035416960716, -0.01930285431444645, -0.026749640703201294, -0.006764397490769625, -0.01849098689854145, 0.023796122521162033, 0.002092659007757902, -0.03798981010913849, -0.01993275247514248, -0.0009290985763072968, -0.004699734039604664, 0.013857741840183735, 0.003105744021013379, 0.0008293648716062307, -0.0021434007212519646, -0.016307342797517776, 0.0026595667004585266, -0.00880456529557705, -0.004766223486512899, -0.00456325663253665, 0.004171320237219334, 0.011394143104553223, 0.015019552782177925, -0.008258653804659843, 0.009014531038701534, 0.015201522968709469, -0.00799269787967205, 0.04286101460456848, 0.021164551377296448, 0.0026508180890232325, -0.0064179543405771255, -0.005494104698300362, -0.005392621271312237, -0.011044200509786606, -0.025755802169442177, 0.0032439713831990957, 0.011303158476948738, 0.01703522354364395, -0.003933358937501907, -0.015271511860191822, 0.01048429124057293, 0.010736250318586826, 0.009581438265740871, -0.05293937399983406, 0.02707158774137497, -0.0069288709200918674, 0.04073336347937584, 0.005963028408586979, 0.011569114401936531, -0.03468634933233261, -0.012912895530462265, -0.011800076812505722, -0.01240197941660881, 0.016447320580482483, -0.008692583069205284, 0.033930469304323196, -0.012506961822509766, -0.00033769497531466186, 0.009651427157223225, 0.0034784332383424044, -0.006228984799236059, -0.003187980502843857, 0.00341544346883893, 0.019414836540818214, -0.018630964681506157, 0.006382959894835949, -0.028695324435830116, -0.025951771065592766, 0.004556257743388414, 0.012569951824843884, 0.012003043666481972, 0.012800914235413074, -0.005697071552276611, 0.008132674731314182, 0.007131837774068117, -0.012422976084053516, 0.01301087997853756, 0.015677444636821747, 0.02645568922162056, 0.03994949162006378, -0.002622822532430291, 0.01249996293336153, -1.864198225121072e-06, -0.0023078739177435637, 0.00504617765545845, -0.012737924233078957, 0.0066979085095226765, 0.012653937563300133, -0.010722252540290356, 0.006389958783984184, -0.02581179328262806, -0.017931077629327774, 0.006694409064948559, -0.011002207174897194, 0.003866869956254959, -0.002822290174663067, 0.028919287025928497, 0.01933084987103939, 0.002888779155910015, 0.012094029225409031, 0.011709092184901237, 0.01028132438659668, -0.011898061260581017, 0.0003033568209502846, -0.0031004948541522026, -0.01910688728094101, -0.01532750204205513, 0.0057950555346906185, -0.003441689070314169, -0.017707115039229393, -0.007705744355916977, -0.0052491445094347, 0.011807075701653957, -0.003508178284391761, -0.0014793839072808623, -0.011534120887517929, 0.0037688857410103083, 0.019232865422964096, -0.03524625673890114, -0.01018334086984396, 0.01636333391070366, 0.01218501478433609, 0.0005887790466658771, 0.008972537703812122, -0.0003877805720549077, 0.012317992746829987, -0.005879042204469442, 0.00796470232307911, 0.001984176691621542, -0.0017987069441005588, 0.018588971346616745, 0.013640777207911015, -0.0027610501274466515, 0.003600913332775235, 0.023250211030244827, -0.023138228803873062, -0.007432788610458374, -0.01955481432378292, -0.01019733864814043, -0.0019124383106827736, -0.008286649361252785, 0.007845721207559109, 0.00965842604637146, 0.009735413827002048, -0.023446179926395416, 0.0034031954128295183, 0.0007370673702098429, -0.02959117852151394, 0.012835908681154251, -0.012220008298754692, -0.00014796026516705751, -0.011954052373766899, 0.02560182847082615, 0.001478509046137333, -0.00186344631947577, -0.0026945609133690596, 0.019778776913881302, 0.005459110252559185, 0.014781591482460499, -0.025111908093094826, -0.022634312510490417, 0.01972278580069542, 0.10599073022603989, -0.005312134511768818, -0.029563182964920998, 0.021122558042407036, 0.016027389094233513, -0.001472385018132627, -0.0172031968832016, 0.003716394305229187, 0.01786108873784542, -0.010974211618304253, -0.014683607034385204, -0.01762312836945057, 0.008230659179389477, 0.00830764602869749, 0.010260327719151974, -0.001433016499504447, -0.014417650178074837, -0.004731229040771723, 0.006617421749979258, -0.018393002450466156, -0.004048840142786503, -0.016125371679663658, -0.0002834538172464818, -0.006526436656713486, 0.01196805015206337, -0.011296159587800503, 0.032194752246141434, 0.012996882200241089, 0.02877931110560894, -0.009581438265740871, 0.019400838762521744, 0.018630964681506157, -0.01804305985569954, -0.0013149107107892632, -0.007985698990523815, 0.020716624334454536, 0.01763712614774704, 0.01891091838479042, 0.03239072114229202, -0.004034842364490032, 0.03085097298026085, 0.025335870683193207, 0.0037758846301585436, -0.020002741366624832, -0.00267881341278553, -0.043140970170497894, -0.017371170222759247, 0.01970878802239895, -0.010596273466944695, -0.012996882200241089, 0.02416006289422512, -0.0022431344259530306, -0.01343081146478653, -0.010358312167227268, 0.010953214950859547, 0.010568277910351753, 0.011030202731490135, 0.018183037638664246, -0.0165593009442091, 0.012059034779667854, 0.015677444636821747, -0.022732295095920563, -0.013353823684155941, -0.004076835699379444, -0.004500266630202532, -0.012667935341596603, -0.012317992746829987, 0.036030128598213196, -0.014739598147571087, -0.012359986081719398, -0.017161203548312187, -0.03166284039616585, -0.03605812415480614, 0.00228337780572474, -0.025993764400482178, 0.005847547203302383, -0.0009002283331938088, 0.013493801467120647, 0.007012857124209404, 0.026119744405150414, 0.00765675213187933, -0.0056060864590108395, -0.011198175139725208, -0.018113048747181892, -0.012248003855347633, 0.005508102476596832, 0.0042938003316521645, -0.006715405732393265, -0.006883378140628338, 0.014326665550470352, 0.017749108374118805, 0.0040523395873606205, 0.024328036233782768, 0.004318296443670988, -0.0010445797815918922, 0.00504617765545845, 0.02984313666820526, 0.0220464076846838, 0.0028870294336229563, -0.023068241775035858, 0.02270429953932762, -0.012541956268250942, -0.0058265505358576775, -0.028751315549016, 0.006078509613871574, 0.012024040333926678, -0.01700722798705101, -0.009329480119049549, -0.010729251429438591, 0.011835071258246899, 0.00818166695535183, -0.012583949603140354, 0.018197035416960716, 0.02772948145866394, -0.020478662103414536, 0.010071358643472195, -0.018113048747181892, 0.023572158068418503, 0.020212706178426743, -0.005847547203302383, 0.0048852041363716125, -0.04896401986479759, 0.027561508119106293, 0.018141044303774834, -0.0006920122541487217, 0.011436136439442635, 0.021724460646510124, -0.017063219100236893, 0.009770408272743225, 0.0008083682623691857, -0.015817422419786453, 0.030123092234134674, -0.014837582595646381, -0.015649449080228806, -0.02726755663752556, -0.012898897752165794, -0.02287227287888527, 0.005781057756394148, -0.00319847883656621, -0.017917079851031303, -0.02922723814845085, 0.0016954736784100533, 0.0057740588672459126, -0.027561508119106293, -0.010127349756658077, -0.029899127781391144, 0.0005756561877205968, -0.015411488711833954, -0.0010104603134095669, 0.04176919534802437, -0.023586155846714973, -0.007180829998105764, -0.0018581971526145935, 0.01682525873184204, -0.015789426863193512, 0.00698836101219058, -0.02897527813911438, -0.01406770758330822, -0.0042238119058310986, 0.031018944457173347, 0.01996074803173542, -0.0003228223940823227, -0.011058198288083076, -0.016069380566477776, 0.0030340056400746107, 0.007866717875003815, 0.004605249501764774, 0.015355497598648071, -0.021794447675347328, 0.01219901256263256, 0.0028485357761383057, 0.028639333322644234, -0.01745515502989292, -0.007019856013357639, -0.00944146141409874, 0.007051351014524698, -0.036450061947107315, -0.029507191851735115, -0.020114721730351448, -0.031130926683545113, 0.01187706459313631, 0.029983114451169968, -0.008440624922513962, 0.013115862384438515, 0.004650742281228304, -0.008216661401093006, 0.04050939902663231, 0.013269837945699692, 0.014326665550470352, -0.003186230780556798, 0.026819629594683647, -0.009210499003529549, 0.01592940464615822, -0.01091822050511837, -0.023432182148098946, 0.00582305109128356, -0.031158922240138054, -0.013675771653652191, -0.014739598147571087, 0.009973375126719475, -0.014095703139901161, -0.015299506485462189, -0.010785242542624474, 0.012115025892853737, -0.003362952033057809, 0.026413695886731148, -0.004440776538103819, -0.00796470232307911, -0.008573602885007858, -0.009385470300912857, -0.017049221321940422, -0.018532980233430862, -0.039361584931612015, -0.015103538520634174, 0.00016983170644380152, -0.0038983647245913744, -0.028835300356149673, 0.022368354722857475, -0.006543933413922787, -0.025699812918901443, -0.0021119059529155493, -0.009231495670974255, 0.03045903705060482, -0.012961887754499912, 0.001268543303012848, 0.007551769260317087, 0.01352879498153925, -0.010043363086879253, 0.020744619891047478, 0.013997718691825867, -0.01928885653614998, 0.021752456203103065, 0.02519589476287365, 0.013787752948701382, 0.005812552757561207, 0.01996074803173542, 0.01805705763399601, -0.009896387346088886, -0.028485357761383057, 0.02206040546298027, 0.01829501800239086, -0.004164321348071098, -0.03155085816979408, -0.00222913664765656, -0.01447364129126072, 0.01637733168900013, -0.021962421014904976, 0.01640532724559307, -0.02374013140797615, -0.01260494627058506, -0.012961887754499912, 0.012262001633644104, 0.004808216355741024, 0.024691976606845856, -0.012513960711658001, -0.012220008298754692, -0.03191480040550232, -0.008993534371256828, 0.014613619074225426, 0.004608748946338892, 0.0004404907231219113, 0.014529632404446602, 0.002754051238298416, 0.0014531381893903017, 0.007121339440345764, -0.01678326539695263, -0.01206603366881609, 0.007831723429262638, 0.018309015780687332, 0.025937773287296295, -0.009539445862174034, -0.02057664655148983, -0.02499992586672306, -0.00015561527106910944, 0.000370720837963745, -0.011093192733824253, 0.006068011280149221, -0.0033244583755731583, -0.022116396576166153, -0.014781591482460499, 0.014235679991543293, -0.0007598137017339468, -0.01765112392604351, -0.00033725754474289715, 0.010526284575462341, -0.015355497598648071, -0.022816281765699387, -0.028457362204790115, 0.01847698912024498, 0.001636858214624226, -0.017343174666166306, -0.03143887594342232, -0.002568581374362111, 0.009889388456940651, 0.008244656957685947, -0.008153671398758888, -0.021878434345126152, 0.017553139477968216, -0.03572217747569084, 0.025475848466157913, 0.006491442210972309, -0.00762175815179944, -0.014445645734667778, -0.006309472024440765, -0.0070688482373952866, -0.014669609256088734, 0.0005196653073653579, -0.002167896833270788, -0.03997748717665672, 0.0038423738442361355, 0.012912895530462265, 0.01594340242445469, -0.015173527412116528, -0.01870095171034336, 0.005021681543439627, -0.00327721587382257, 0.010708254761993885, -0.01676926761865616, -0.04638843983411789, 0.011142184026539326, -0.015537467785179615, -0.012464968487620354, 0.007019856013357639, -0.0028607838321477175, -0.004237809218466282, -0.021332524716854095, -0.005949030630290508, -0.00624298257753253, -0.018532980233430862, -0.0006981362239457667, -0.01765112392604351, 0.028205404058098793, -0.0024303540121763945, -0.010351313278079033, -0.009588437154889107, -0.0013306582113727927, -0.028401371091604233, 0.03247470781207085, 0.0071178399957716465, -0.024859948083758354, -0.004468771629035473, 0.045464590191841125, 0.0161533672362566, 0.009728414937853813, 0.018099050968885422, -0.006610422860831022, -0.006561430636793375, 0.005707569885998964, -0.02582579106092453, -0.017315179109573364, -0.014165692031383514, 0.04879605025053024, 0.0034539371263235807, -0.023432182148098946, -0.029031269252300262, -0.013857741840183735, -0.046136483550071716, -0.0159853957593441, 0.007831723429262638, 0.004538760520517826, 0.005830049980431795, -0.0034766835160553455, -0.016895247623324394, 0.010505287908017635, -0.02222837693989277, 0.032866645604372025, -0.013269837945699692, -0.022186385467648506, 0.012338989414274693, -0.0014627616619691253, 0.0016377330757677555, 0.00588254164904356, -0.03463035821914673, -0.005483606364578009, 0.025321872904896736, 0.01702122576534748, -0.014375657774508, 0.02162647619843483, -0.0208286065608263, 0.020660633221268654, -0.01765112392604351, 0.045688554644584656, 0.015873413532972336, 0.004818714689463377, 0.02732354775071144, -0.020002741366624832, -0.013087866827845573, -0.008860556408762932, -0.007320806849747896, -0.007782731670886278, 0.0016272348584607244, 0.015145531855523586, -0.003518676618114114, 0.0007151959580369294, 0.007614759262651205, -0.012338989414274693, 0.01091822050511837, 0.007936706766486168, 0.03143887594342232, 0.0022343858145177364, -0.0014032713370397687, 0.027379538863897324, -0.0002628946676850319, -0.008349639363586903, 0.0009570940164849162, 0.007502777501940727, -0.008342640474438667, 0.02228436805307865, -0.008034690283238888, -0.027589503675699234, 0.02250833250582218, 0.004213313572108746, 0.03213876113295555, -0.02646968699991703, 0.0026210728101432323, -0.02984313666820526, 0.019260860979557037, -0.011688095517456532, 0.0026298214215785265, 0.016951236873865128, -0.028023432940244675, 0.011611107736825943, 0.0020454167388379574, -0.006767896935343742, -0.005683073773980141, -0.019036898389458656, -0.024244049564003944, -0.014158693142235279, 0.0017505896976217628, -5.5526106734760106e-05, -0.02880730666220188, -0.026175733655691147, -0.011219171807169914, -0.007257817313075066, -0.01490757055580616, 0.008475618436932564, 0.206718310713768, -0.01240197941660881, -0.00608550850301981, 0.015117536298930645, -0.0005634081899188459, 0.01826702244579792, 0.021108560264110565, 0.002008672570809722, 0.0010113351745530963, 0.0209965780377388, -0.008321643806993961, 0.0004501141665969044, 0.0007606885628774762, -0.007929707877337933, -0.006120502483099699, -0.03236272558569908, -0.050307802855968475, -0.021570485085248947, -0.01637733168900013, -0.02096858248114586, 0.009539445862174034, -0.017329176887869835, 0.002269380260258913, -0.008972537703812122, 0.01991875469684601, 0.0033122103195637465, -0.009868391789495945, 0.005137162748724222, 0.016671283170580864, -0.0018337011570110917, -0.019806772470474243, 0.005781057756394148, 0.01228299830108881, -0.01829501800239086, -0.014753595925867558, -0.008097680285573006, 0.009693420492112637, 0.006718905176967382, 0.01954081654548645, 0.00514066219329834, 0.02604975551366806, 0.019190872088074684, 0.011821073479950428, 0.001819703495129943, 0.0017435909248888493, 0.03944557160139084, -0.02207440324127674, -0.015677444636821747, 0.00619399081915617, 0.028751315549016, -0.012017041444778442, -0.006295474246144295, 0.018980907276272774, 0.024062078446149826, 0.007320806849747896, 0.005900038406252861, -0.0017392166191712022, 0.002106656786054373, -0.02478996105492115, 0.027981439605355263, 0.007782731670886278, 0.020842604339122772, -0.01739916391670704, 0.026147739961743355, -0.002622822532430291, -0.002416356233879924, -0.007156333886086941, -0.01637733168900013, -0.003151236567646265, -0.008552606217563152, -0.003165234113112092, -0.012108027003705502, -0.005329631734639406, 0.007733739912509918, -0.015803424641489983, -0.03191480040550232, 0.015859415754675865, 0.009602434933185577, 0.016517307609319687, 0.027813468128442764, -0.005182655528187752, -0.007474781945347786, -0.005007683765143156, -0.022984255105257034, -0.017889084294438362, -0.049859873950481415, 0.008062685839831829, -0.014095703139901161, -0.01218501478433609, -0.018658960238099098, 0.019050896167755127, -0.016111373901367188, 0.006795892491936684, -0.0030130089726299047, 0.003011259250342846, 0.006162495817989111, 0.013927730731666088, -0.00925249233841896, -0.011737087741494179, 0.0037198937498033047, -0.018546978011727333, 0.005599087569862604, 0.025923775508999825, 0.03233473002910614, 0.002832788275554776, 0.015439484268426895, -0.018155042082071304, -0.0037898824084550142, -0.004006847273558378, -0.011394143104553223, 0.014529632404446602, -0.03742989897727966, -0.003635907545685768, -0.008881553076207638, -0.01868695579469204, -0.002348117297515273, -0.01418668869882822, -0.018309015780687332, 0.022648310288786888, 0.0015773680061101913, 0.01321384683251381, -0.011058198288083076, -0.00325271999463439, 0.011527121998369694, -0.0018774440977722406, -0.028009435161948204, -0.01888292282819748, 0.024076076224446297, 0.00262457225471735, -0.024034082889556885, 0.015467479825019836, 0.005581590346992016, 0.013766756281256676, -0.012968886643648148, -0.006288475356996059, -0.006813389714807272, 0.008433626033365726, -0.005879042204469442, 0.003315709764137864, 0.015173527412116528, -0.015201522968709469, 0.009182503446936607, -0.008888551965355873, -0.015649449080228806, 0.01599939353764057, -0.030599012970924377, 0.0008162420126609504, -0.010449296794831753, -0.007411792408674955, -0.02246633917093277, 0.0008306771633215249, 0.021374516189098358, -0.013969723135232925, -0.010771244764328003, 0.015369495376944542, 0.00945545919239521, -0.03751388564705849, -0.010211336426436901, 0.010974211618304253, -0.006368962116539478, -0.03174682706594467, -0.0018756943754851818, 0.03966953605413437, 0.016209358349442482, -0.020884597674012184, -0.016475316137075424, -0.18208232522010803, 0.010330316610634327, 0.00012576075096148998, -0.03236272558569908, 0.01683925651013851, -0.0030147586949169636, 0.023404186591506004, 0.01844899356365204, -0.004993686452507973, -0.00023052493634168059, 0.007845721207559109, 0.002701559802517295, -0.014697604812681675, -0.03986550495028496, 0.011576113291084766, -0.0045247627422213554, -0.00862959399819374, 0.029255231842398643, 0.044624727219343185, 0.022186385467648506, 0.016041386872529984, -0.022984255105257034, 0.009623431600630283, -0.02120654471218586, 0.010043363086879253, 0.00461224839091301, 0.026735642924904823, -0.004069836810231209, 0.0044722710736095905, 0.002160897944122553, -0.0028800307773053646, -0.006967364810407162, 0.04305698350071907, 0.007593762595206499, 0.015523470006883144, -0.006001521833240986, -0.025699812918901443, 0.0038738688454031944, -0.016545303165912628, 0.0018232029397040606, 0.011093192733824253, 0.006799391936510801, 0.004549258854240179, 0.025685815140604973, -0.013654774986207485, 0.0015196274034678936, 0.007572765927761793, -0.011492127552628517, -0.0161533672362566, -0.012073032557964325, 0.013773755170404911, -0.008552606217563152, -0.010960213840007782, 0.01912088505923748, 0.003908862825483084, 0.024607989937067032, 0.017707115039229393, 0.007271815091371536, 0.011044200509786606, -0.002724305959418416, 0.01184906903654337, -0.021472500637173653, 0.0163913294672966, -0.010848232544958591, 0.00913351122289896, -0.01910688728094101, 0.007376797962933779, 3.939592352253385e-05, -0.0042833019979298115, 0.015117536298930645, -0.028667328879237175, 0.006617421749979258, 0.01892491616308689, 2.0709516320494004e-05, 0.0037093956489115953, -0.0051651583053171635, -0.005875542759895325, -0.00723682064563036, 0.0002790795115288347, -0.00630247313529253, -0.01395572628825903, 0.00839863158762455, -0.043364934623241425, 0.006578927859663963, -0.01681126095354557, 0.006029517389833927, 0.008153671398758888, 0.0011163180461153388, -0.018840929493308067, -0.010491290129721165, 0.01594340242445469, -0.012653937563300133, -0.0057950555346906185, -0.03171883150935173, -0.01703522354364395, 0.02208840101957321, 0.008706580847501755, 0.013920731842517853, -0.013745760545134544, -0.0030252570286393166, 0.019694790244102478, -0.01828102022409439, -0.018169039860367775, 0.015817422419786453, 0.027561508119106293, 0.019218867644667625, 0.00619399081915617, 0.021038571372628212, 0.028415368869900703, -0.008314644917845726, -0.022606316953897476, 0.03510627895593643, 0.017301181331276894, 0.04109730198979378, -0.00881856307387352, 0.02537786401808262, 0.003954355604946613, -0.01721719466149807, 0.045912519097328186, -0.007026854902505875, 0.032446712255477905, -0.01301087997853756, 0.0017164703458547592, 0.004986687563359737, 0.017553139477968216, -0.00562008423730731, -0.1230679452419281, -0.004815215244889259, 0.002526588272303343, 0.006106504704803228, -0.025475848466157913, 0.011387144215404987, -0.008447623811662197, 0.029703158885240555, -0.006862381938844919, 0.02919924259185791, 0.0010953214950859547, -0.02564382180571556, -0.0008936668746173382, -0.010666262358427048, 0.029983114451169968, -0.010771244764328003, -0.01956881210207939, -0.011730088852345943, -0.022424345836043358, 0.024845952168107033, 0.0033122103195637465, -0.011128186248242855, -0.01871494948863983, -0.008006695657968521, -0.031130926683545113, 0.0055955881252884865, -0.03376249969005585, 0.01301087997853756, 0.012667935341596603, 0.0025318374391645193, -0.03605812415480614, 0.004237809218466282, 0.014326665550470352, -0.01447364129126072, -0.0009850894566625357, 0.0075587681494653225, -0.005515101365745068, -0.0014050210593268275, 0.008034690283238888, -0.01007835753262043, -0.007292811758816242, 0.0070898449048399925, 0.024915939196944237, -0.0032474708277732134, 0.006263979244977236, 0.0011836821213364601, -0.015103538520634174, 0.02243834361433983, -0.005532598588615656, -0.021304529160261154, -0.026343706995248795, -0.0077687338925898075, 0.0011609358480200171, -0.016475316137075424, 0.026371702551841736, -0.0218084454536438, -0.011653101071715355, 0.014375657774508, 0.003912362270057201, -0.010939217172563076, 0.008391632698476315, 0.006585926748812199, -0.023152226582169533, 0.010372309945523739, 0.00307424901984632, 0.002589578041806817, -0.03342655301094055, -0.003639406990259886, 0.014669609256088734, -0.011618106625974178, -0.005074173212051392, -0.0019001903710886836, -0.012359986081719398, 0.007775732781738043, -0.021570485085248947, 0.011632104404270649, -0.007663751021027565, -0.0016456068260595202, 0.012303994968533516, -0.018770940601825714, 0.001079574110917747, -0.02308223955333233, -0.00026464436086826026, -0.023446179926395416, 0.007002358790487051, 0.004311297554522753, 0.012933892197906971, -0.0026945609133690596, -0.018323013558983803, -0.024202056229114532, -0.009987372905015945, 0.014865577220916748, 0.017539141699671745, -0.025153901427984238, -0.00467173894867301, 0.008132674731314182, 0.007313807960599661, 0.018351009115576744, 0.0018931914819404483, -0.005630582571029663, -0.0340704470872879, -0.025937773287296295, -0.056186843663454056, 0.018155042082071304, 0.008363637141883373, -0.002185393823310733, -0.0017024725675582886, -0.006158996373414993, 0.008895550854504108, 0.014179689809679985, -0.0027505517937242985, 0.012255002744495869, -0.01175108551979065, 0.03692598268389702, 0.021892432123422623, -0.014375657774508, -0.01744115725159645, -0.004741727374494076, 0.006750400178134441, -0.004657741170376539, 0.013395817019045353, 0.011310157366096973, 0.0075097763910889626, 0.02581179328262806, 0.011919057928025723, 0.02143050730228424, -0.018099050968885422, -0.0004951693117618561, -0.025867784395813942, 0.014585623517632484, -0.005763560999184847, -0.012863904237747192, 0.028891291469335556, -0.02644169144332409, 0.022942261770367622, 0.04417680203914642, -0.014004717580974102, -0.01721719466149807, 0.0032264741603285074, 0.05571092292666435, 0.01702122576534748, 0.016335338354110718, -0.030403045937418938, -0.03356653079390526, 0.012108027003705502, -0.01870095171034336, -0.0014408902497962117, -0.005179156083613634, -0.027827465906739235, 0.005186154972761869, 0.040649376809597015, 0.011807075701653957, 0.012017041444778442, 0.004559757187962532, -0.0014680108288303018, -0.008937543258070946, 0.01016934309154749, -0.010372309945523739, 0.022578321397304535, 0.007180829998105764, 0.017721112817525864, -0.009924382902681828, 0.01573343575000763, -0.008573602885007858, 0.017959073185920715, -0.011548117734491825, 0.006449448876082897, 0.0019439331954345107, 0.002379612298682332, 0.0006316470680758357, -0.007831723429262638, 0.006732902955263853, -0.010428301058709621, -0.007023355457931757, -0.0045037660747766495, -0.001543248537927866, 0.013115862384438515, -0.007803728338330984, -0.021164551377296448, 0.014459643512964249, -0.02227037027478218, 0.03552621230483055, -0.021290531381964684, -0.007313807960599661, -0.01892491616308689, 0.040397416800260544, 0.022774288430809975, 0.004801217466592789, 0.003614910878241062, 0.007187828887254, 0.0014242678880691528, -0.0006837010732851923, 0.0006955116405151784, 0.007908711209893227, -0.005060175433754921, -0.012856905348598957, 0.011821073479950428, 0.016881249845027924, -0.022382352501153946, -0.00914750900119543, 0.009280487895011902, 0.021276533603668213, -0.009392469190061092, 0.014004717580974102, 0.005074173212051392, -0.019274858757853508, -0.011310157366096973, 0.02017071284353733, -0.01661529205739498, -0.013017878867685795, 0.00333145703189075, 0.025349868461489677, 0.0018057057168334723, -0.013731762766838074, 0.029507191851735115, 0.002558083040639758, -0.007754736114293337, 0.020492659881711006, 8.825124677969143e-05, -0.009182503446936607, -0.025839788839221, 0.03975352272391319, 0.0017155954847112298, -0.026525678113102913, 0.013066871091723442, -0.021948423236608505, 0.03110293112695217, 0.01956881210207939, 0.014865577220916748, -0.009518449194729328, 0.021976418793201447, 0.012989883311092854, 0.011905060149729252, -0.004545759409666061, -0.01909288950264454, 0.0001329783262917772, -0.02060464210808277, 0.0036779006477445364, 0.0007733739912509918, 0.035582203418016434, 0.006536934990435839, 0.06405356526374817, 0.0007226322195492685, 0.007950704544782639, 0.00723682064563036, -0.004377786535769701, 0.023432182148098946, 0.006774895824491978, -0.010260327719151974, 0.0020331686828285456, 0.007285812869668007, 0.03334256634116173, -0.0007309434004127979, 0.01532750204205513, -0.03110293112695217, -0.0352182611823082, 0.00030663752113468945, -0.021094562485814095, 0.033902473747730255, -0.003639406990259886, 0.003011259250342846, 0.034322407096624374, 0.014641613699495792, 0.02017071284353733, 0.0033279575873166323, -0.012646939605474472, 0.011345150880515575, 0.022186385467648506, 0.0012589198304340243, -0.022760290652513504, -0.03087896853685379, 0.004990187007933855, 0.00851061288267374, -0.03885766863822937, 0.010841233655810356, 0.008552606217563152, -0.021990416571497917, -0.016895247623324394, -0.0027313048485666513, -0.005917535629123449, 0.02687562070786953, -0.013206847943365574, 0.0070793465711176395, -0.008202663622796535, -0.019652796909213066, -6.791080522816628e-05, -0.010253328830003738, 0.008083682507276535, -0.005007683765143156, -0.05559894070029259]} +{"id": "test:10000021", "text": "\"Hurrah! A cooperative worldwide effort to rescue Thailand children trapped in a flooded cave rescued them all in less than 3 weeks from the time they entered the cave to the time of their rescue.\\nIt should be much easier, shouldn\u2019t even take a heroic effort, to rescue children trapped in separation from their families at the Mexican border. These things are possible, but this week, the administration did not even meet the first deadline to get all the children below 5 years old reunited with their families.\\nIt should even be logistically possible with a cooperative world wide effort to develop economic systems that could rescue all the hungry children everywhere living in poverty.\\nIn the U.S. alone, 1 in 5 children live in poverty, according to a recently released United Nations report.\\nSuch inequality and poverty is a direct result of economic policies that are chosen by elected leaders. The solution is to elect different leaders than the ones in power now. Please consider carefully when you vote this November.\"", "vector_field": [-0.020046241581439972, -0.007147691678255796, 0.003870246699079871, -0.018410926684737206, 0.004016743507236242, -0.013566304929554462, 0.0052909268997609615, -0.012959875166416168, -0.01685737632215023, -0.022349311038851738, -0.00019238379900343716, 0.024148158729076385, -0.01787944883108139, 0.007256712298840284, 0.00475263549014926, -0.0013048454420641065, 0.03257003426551819, -0.039465613663196564, -0.017320716753602028, -0.020932037383317947, -0.016012463718652725, 0.011351815424859524, -0.002406979911029339, -0.005859880708158016, -0.015017647296190262, -0.007856328040361404, 0.028318213298916817, -0.00882389023900032, 0.022526470944285393, -0.014513424597680569, -0.010861220769584179, -0.01552186906337738, -0.013014385476708412, -0.010649992153048515, -0.019187701866030693, -0.008939724415540695, 0.0006430537323467433, -0.025156604126095772, 0.019419372081756592, -0.016571197658777237, 0.011917361989617348, -0.016598451882600784, 0.007508823648095131, 0.011331373825669289, -0.027241630479693413, 0.0006558296736329794, -0.0019845233764499426, -0.02574259042739868, -0.0311255045235157, 0.014826860278844833, 0.020659485831856728, 0.01353904977440834, -0.027813991531729698, -0.016789238899946213, -0.026301324367523193, -0.005243230145424604, -0.02176332287490368, 0.011092890053987503, 0.010929358191788197, -0.027991149574518204, 0.01459519099444151, 0.006810407619923353, -0.010077632032334805, 0.02031879499554634, -0.004820773843675852, -0.011678878217935562, -0.005427203141152859, 0.0166938453912735, 0.02848174422979355, 0.014281755313277245, 0.039710912853479385, 0.01807023584842682, 0.021954109892249107, -0.014336265623569489, 0.044180773198604584, 0.00488209817558527, 0.005910984240472317, 0.028536254540085793, 0.009375808760523796, 0.0021395378280431032, -0.0012281900271773338, -0.02616504766047001, -0.00327574135735631, 0.011351815424859524, 0.014472542330622673, -0.004221158102154732, 0.0034699351526796818, 0.019065052270889282, -0.015985209494829178, -0.013355076313018799, -0.030471378937363625, 0.017770428210496902, -0.003129244316369295, 5.509607944986783e-06, -0.007099994923919439, 0.017852194607257843, -0.019869083538651466, 0.022417450323700905, 0.015808049589395523, -0.042654480785131454, -0.013123407028615475, 0.002061178907752037, -0.004810553044080734, -0.013293752446770668, -0.015658145770430565, -0.005198940634727478, 0.0028141054790467024, -0.013123407028615475, 0.009859589859843254, 0.01595795340836048, -0.013913809321820736, 0.022076759487390518, 0.0020100753754377365, -0.024856794625520706, 0.005420389585196972, -0.008380992338061333, -0.00685810437425971, -0.011324559338390827, -0.0006805297452956438, 0.008019859902560711, 0.013416401110589504, 0.01571265608072281, -0.00535225123167038, -0.015767166391015053, 0.006377730518579483, 0.0012767384760081768, -0.020346051082015038, -0.0030832511838525534, -0.00441194511950016, -0.012108148075640202, -0.00022230070317164063, 0.027963895350694656, 0.026710152626037598, 0.004200716502964497, -0.006343661341816187, 0.03417809307575226, -0.008619475178420544, -0.008864772506058216, -0.015140295960009098, -0.012128589674830437, 0.03505026176571846, 0.00844912976026535, 0.007597403135150671, -0.01905142515897751, -0.002621615072712302, 0.00980507954955101, 0.008912469260394573, -0.0012929212534800172, -0.012537418864667416, 0.012251238338649273, 0.027091726660728455, 0.002299662446603179, 0.01638041064143181, 0.003413720987737179, 0.034014564007520676, 0.008435502648353577, 0.008585406467318535, 0.007856328040361404, 0.0004228823527228087, -0.030089804902672768, -0.002330324612557888, -0.00010454946459503844, -0.018397299572825432, -0.0031394651159644127, 0.007590589579194784, 0.029217636212706566, 0.02754143811762333, 0.012114962562918663, 0.007331664673984051, -0.006398171652108431, -0.011011124588549137, 0.03344220295548439, -0.00821064691990614, 0.015113040804862976, -0.0027936638798564672, -0.008973794057965279, 0.003209306625649333, 0.032597288489341736, -0.018301906064152718, -0.02035967819392681, -0.03128903731703758, 0.01332100760191679, 0.0032024928368628025, 0.019310349598526955, -0.014199989847838879, -0.007617844734340906, 0.03586791828274727, 0.0060711088590323925, 0.028127426281571388, -0.008851145394146442, 0.008864772506058216, 0.019746433943510056, -0.0014249390223994851, 0.016993653029203415, -0.6541261672973633, -0.012830412946641445, 0.0028243260458111763, -0.0007465385715477169, -0.016680218279361725, 0.006585551891475916, 0.016707472503185272, 0.006885359529405832, -0.014377148821949959, 0.019582903012633324, -0.017034536227583885, 0.018479064106941223, -0.0275823213160038, -0.03712166100740433, -0.01367532555013895, -0.0007358919829130173, -0.009355367161333561, -0.017688661813735962, 0.022213036194443703, 0.008994235657155514, -0.02215852402150631, 0.02063222974538803, -0.007999418303370476, 0.018410926684737206, 0.0010169618763029575, 0.010418322868645191, 0.021327240392565727, -0.020291538909077644, 0.01019346620887518, -0.006299371365457773, 0.011058821342885494, 0.00913732498884201, -0.011235980316996574, 0.0053181820549070835, 0.05941646173596382, -0.007829072885215282, -0.003505707485601306, 0.00685810437425971, 0.02294892817735672, 0.03398730605840683, -0.014090968295931816, 0.01186966523528099, 0.01320517249405384, 0.0034324589651077986, 0.004531186539679766, 0.01571265608072281, 0.023330500349402428, -0.007140877656638622, -0.00586328748613596, -0.014213616959750652, 0.0007380212773568928, 0.017511503770947456, -0.03684910759329796, -0.0122376112267375, -0.015222061425447464, 0.0036045080050826073, -0.005130802281200886, -0.027050843462347984, 0.006735455710440874, -0.013518608175218105, -0.015739912167191505, -0.016871005296707153, -0.013368704356253147, -0.0077881901524960995, -0.044262539595365524, 0.03363298997282982, -0.008122066967189312, 0.005345437675714493, 0.01783856563270092, 0.0066094002686440945, 0.013906995765864849, 0.0006149467662908137, -0.01212177611887455, -0.0008985718013718724, 0.00165064656175673, 0.013055268675088882, 0.02409364841878414, -0.007018228992819786, -0.018956031650304794, 0.023412266746163368, -0.021231846883893013, -0.02228117361664772, 0.017688661813735962, 0.01056822668761015, 0.03654929995536804, -0.0033762450329959393, -0.022621864452958107, 0.007392989005893469, 0.006272116210311651, -0.010145769454538822, -0.016707472503185272, 0.025565432384610176, 0.004197309724986553, -0.010690875351428986, -0.0029163125436753035, 0.0035329628735780716, -0.014758721925318241, 0.008517268113791943, 0.04677002131938934, -0.026614759117364883, -0.009110069833695889, -0.015494613908231258, 0.015726283192634583, -0.006527634337544441, 0.0029725267086178064, 0.022090386599302292, -0.0006779745453968644, 0.008892027661204338, 0.05513738840818405, -0.03979267552495003, -0.013586746528744698, -0.003573845839127898, 0.010268418118357658, -0.017593268305063248, 0.020332422107458115, -0.02459787018597126, -0.00874212384223938, -0.01693914271891117, 0.00903511792421341, -0.008115253411233425, 0.007011414971202612, 0.023126086220145226, 0.017743173986673355, -0.017865821719169617, -0.011065634898841381, 0.003412017598748207, 0.015004019252955914, -0.019773690029978752, -0.001694936421699822, -0.012373887002468109, 0.0026284288614988327, 0.0008674837881699204, 0.0029503817204385996, -0.00972331315279007, 0.02989901788532734, 0.017988471314311028, 0.018410926684737206, -0.035459090024232864, 0.022649118676781654, -0.016748355701565742, 0.0021463516168296337, 0.010125328786671162, 0.02432531863451004, -0.014881370589137077, 0.0028566918335855007, -0.013130220584571362, -0.00410532345995307, -0.006561703514307737, -0.029163125902414322, -0.03262454271316528, -0.018574457615613937, -0.011385884135961533, -0.007474754471331835, 0.006418613251298666, 0.01913319155573845, -0.008878400549292564, -0.016652962192893028, -0.015535497106611729, -0.010445578023791313, -0.030280591920018196, 0.002689753193408251, 0.02330324612557888, -0.01886063814163208, -0.008530896157026291, -0.002246855292469263, 0.00044971174793317914, -0.0063266269862651825, 0.0019623786211013794, 0.011106518097221851, -0.03665832057595253, 0.02086389996111393, -0.030716676265001297, -0.0067661176435649395, 0.004922980908304453, 0.008653544820845127, 0.01708904653787613, -0.021204590797424316, -0.02726888656616211, -0.03603145107626915, 0.012639625929296017, 0.0002403998951194808, -0.006813814397901297, -0.027364280074834824, -0.011419952847063541, 0.007931279949843884, -0.009123697876930237, 0.018642596900463104, 0.0026028770953416824, -0.0066230278462171555, 0.03126177936792374, 0.0027221189811825752, 0.012578302063047886, 0.011992313899099827, -0.002427421510219574, 0.006762710865586996, -0.008401433005928993, 0.004541407339274883, 0.030662165954709053, 0.014554307796061039, 0.042109373956918716, 0.015617262572050095, -0.005778114777058363, 0.02318059653043747, -0.003040664829313755, 0.015194806270301342, -0.013034827075898647, -0.0004688756016548723, -0.010609108954668045, 0.0271462369710207, 0.01186966523528099, 0.007243084721267223, -0.033496711403131485, 0.007604217156767845, 0.0035465904511511326, 0.0011779381893575191, 0.028781551867723465, 0.0007980680093169212, 0.00758377555757761, -0.0168982595205307, 0.010690875351428986, 0.010152583941817284, -0.02082301676273346, 0.002153165405616164, 0.004991119261831045, 0.0008687613299116492, -0.008326481096446514, 0.008762565441429615, 0.020741252228617668, 0.023085203021764755, -0.008428688161075115, 0.003101989161223173, -0.00646290322765708, 0.009314484894275665, 0.0280184056609869, 0.012503349222242832, -0.008237902075052261, 0.030198825523257256, -0.013641256839036942, -0.006203978322446346, 0.005921205040067434, 0.005468086339533329, 0.012857668101787567, 0.02553817629814148, 0.012796343304216862, -0.004183682147413492, -0.005001340061426163, 0.03902953118085861, 0.005982529371976852, -0.0015015944372862577, 0.028972338885068893, -0.00943031907081604, 0.020414188504219055, -0.004698125179857016, 0.0052261957898736, -0.005307961720973253, -0.012394328601658344, 0.0028566918335855007, 0.021313611418008804, 0.021313611418008804, 0.026274068281054497, 0.02121821790933609, 0.02153165452182293, 0.01792033202946186, -0.002958898898214102, 0.002991264685988426, -0.0012196728494018316, -0.009334925562143326, -0.0020237029530107975, -0.005113767925649881, -0.03156159073114395, -0.03529556095600128, -0.016339527443051338, 0.015699028968811035, -0.021313611418008804, 0.014213616959750652, -0.000613669166341424, -0.012537418864667416, -0.011808340437710285, 0.005321589298546314, -0.002229820704087615, -0.013859299011528492, -0.021708812564611435, 0.01850632019340992, 0.009927728213369846, -0.02899959497153759, -0.023194225504994392, -0.02585161291062832, 0.01255786046385765, -0.022104013711214066, 0.00548171391710639, -0.00355681125074625, -0.0029248299542814493, -0.0003483562613837421, 0.01748424768447876, -0.004452827852219343, -0.014240872114896774, 0.011958244256675243, -0.003914536442607641, 0.01599883660674095, -0.012530605308711529, -0.0006967125227674842, 0.007502009626477957, -0.006820628419518471, -0.011331373825669289, 0.01512666791677475, -0.012898551300168037, 0.00878300704061985, -0.01583530567586422, 0.001619984395802021, -0.012700950726866722, 0.015317454934120178, -0.029163125902414322, -0.0034188313875347376, 0.021354494616389275, -0.01259192917495966, -0.02161341905593872, -0.02282627858221531, 0.010247977450489998, 0.02927214652299881, -0.007243084721267223, 0.008101625367999077, -0.019978104159235954, -0.026614759117364883, -0.009157766588032246, 0.12668244540691376, 0.00043863931205123663, -0.00698415981605649, 0.013818415813148022, 0.01241477020084858, -0.017729545012116432, -0.007931279949843884, -0.0090010492131114, 0.02397099882364273, -0.013368704356253147, 0.0015126668149605393, -0.0019317164551466703, 0.01414547860622406, 0.0012077486608177423, 0.014268127270042896, 0.03096197359263897, -0.0037101220805197954, 0.0038021085783839226, -0.017061792314052582, -0.008142508566379547, -0.002137834206223488, -0.008755751885473728, 0.014199989847838879, 0.005764487199485302, 0.009614292532205582, -0.009668802842497826, 0.03006255067884922, 0.025470038875937462, -5.9035315643996e-05, -0.016434920951724052, -0.01459519099444151, 0.008183390833437443, 0.012918991968035698, 0.0015194806037470698, -0.013089337386190891, -0.004350620787590742, -0.01139269769191742, 0.0010186652652919292, 0.02409364841878414, -0.006735455710440874, 0.006122212391346693, 0.030444122850894928, 0.001227338332682848, -0.01721169613301754, 0.013430028222501278, 0.004902539774775505, 0.0008163801394402981, 0.029817253351211548, -0.011126958765089512, -0.015971580520272255, 0.04262722283601761, -0.029163125902414322, -0.0386752113699913, -0.00740661658346653, -0.007358919829130173, 0.011147400364279747, -0.015590007416903973, -0.010486460290849209, 0.010254791006445885, -0.008006231859326363, -0.014840488322079182, -0.018519947305321693, -0.009069187566637993, -0.015590007416903973, 0.0012477798154577613, -0.02845449000597, -0.0034222383983433247, 0.015303827822208405, -0.029326658695936203, -0.01606697402894497, 0.006919428706169128, -0.03649479150772095, -0.04322683811187744, 0.009920913726091385, 0.020018987357616425, 0.0015859153354540467, 0.009859589859843254, 0.011617553420364857, 0.015930699184536934, 0.00030002076528035104, -0.008217460475862026, -0.03316964954137802, -0.026301324367523193, -0.013041640631854534, 0.020523209124803543, -0.0031394651159644127, -0.004926387686282396, -0.018642596900463104, -0.019078681245446205, 0.016925515606999397, 0.005529410671442747, 0.02785487473011017, 0.027064470574259758, 0.012816784903407097, 0.016612078994512558, 0.013082523830235004, 0.016216877847909927, 0.01357311848551035, -0.009082814678549767, 0.0072090160101652145, 0.023684820160269737, -0.0264784824103117, -0.0373397022485733, 0.01831553317606449, -0.0030491820070892572, 0.007195387966930866, -0.009968610480427742, 0.010540971532464027, -0.004166647791862488, -0.01944662630558014, 0.03567713126540184, -0.028536254540085793, 0.01827464997768402, -0.00565887289121747, -0.0029384575318545103, 0.02011438086628914, 0.028236446902155876, 0.006585551891475916, 0.02739153429865837, -0.02848174422979355, -0.01316428929567337, -0.011133773252367973, -0.00040733834612183273, 0.037394214421510696, -0.0009028304484672844, -0.0033217344898730516, 0.011876478791236877, -0.04303605109453201, 0.01704816333949566, -0.003716935869306326, -0.010643178597092628, 0.016952769830822945, 0.014227245002985, -0.008721682243049145, -0.03336043655872345, -0.021940482780337334, -0.011971872299909592, 0.0036624253261834383, -0.02935391291975975, -0.009580222889780998, 0.023957371711730957, 0.0007605920545756817, -0.018370043486356735, -0.038838744163513184, 0.010575040243566036, -0.031616099178791046, 0.004991119261831045, -0.026042399927973747, -0.001933419844135642, 0.0206731129437685, -0.007890397682785988, -0.009225904941558838, -0.018138375133275986, 0.0011089483741670847, -0.002923126332461834, -0.03905678540468216, -0.0032825551461428404, -0.01538559328764677, 0.009811893105506897, 0.025415528565645218, 0.033851031213998795, -0.0031769410707056522, 0.01041150838136673, -0.011617553420364857, 0.024461593478918076, 0.008428688161075115, 0.016830122098326683, 0.005195533391088247, -0.010636364109814167, -0.003921350464224815, 0.0009096442372538149, 0.014554307796061039, -0.014090968295931816, -0.021150080487132072, -0.0004256504762452096, 0.029517443850636482, 0.016843749210238457, -0.019950848072767258, 0.0012034899555146694, -0.038130104541778564, -0.0035942872054874897, -0.008353736251592636, -0.0009897066047415137, -0.0027919604908674955, -0.016830122098326683, -0.017784055322408676, 0.035976942628622055, -0.004548221360892057, 0.011242793872952461, -0.013096151873469353, 0.023098831996321678, -0.009246346540749073, 0.005597548559308052, 0.00821064691990614, -0.007461126893758774, -0.02393011748790741, -0.026301324367523193, -0.02117733471095562, 0.002735746558755636, 0.004439200274646282, 0.024379828944802284, -0.009266788139939308, -0.01102475170046091, -0.0036385769490152597, -0.01568540185689926, 0.00936899520456791, -0.013464097864925861, 0.0005331810098141432, 0.016216877847909927, -0.013818415813148022, 0.01740248315036297, -0.011556229554116726, 0.007536078803241253, -0.015903443098068237, -0.0031070993281900883, 0.0029946714639663696, -0.02856351062655449, 0.010445578023791313, -0.0037714464124292135, -0.022213036194443703, -0.01685737632215023, -0.01418636180460453, 0.005171685013920069, 0.0047662630677223206, 0.03079844079911709, 0.009954983368515968, -0.0006703090039081872, -0.011440394446253777, 0.0005221085157245398, 0.011079262010753155, -0.003306403523311019, 0.02522474154829979, 0.0029282367322593927, -0.0115698566660285, -0.012707764282822609, -0.013048455119132996, 0.01610785722732544, -0.0015339599922299385, -0.041155438870191574, 0.015617262572050095, 0.00698415981605649, -0.01027523260563612, -0.015222061425447464, -0.022976182401180267, -0.001841433346271515, 0.009975424036383629, 0.004926387686282396, -0.013757091946899891, 0.0004501376242842525, -0.030553143471479416, -0.027841245755553246, 0.02042781561613083, -0.02499307133257389, 0.019637413322925568, 0.009185021743178368, -0.012660067528486252, 0.017947588115930557, -0.014622446149587631, -0.01178789883852005, -0.00988003145903349, -0.018438182771205902, 0.019187701866030693, -0.003054292406886816, 0.00574745237827301, 0.004319958388805389, 0.008646730333566666, -0.019473882392048836, -0.016175996512174606, -0.013804788701236248, 0.02094566635787487, -0.018288278952240944, -0.012428397312760353, -0.0027664087247103453, 0.014758721925318241, 0.0042654480785131454, -0.02349403314292431, -0.015603635460138321, -0.011031566187739372, -0.02848174422979355, -0.019773690029978752, 0.023098831996321678, -0.0046776835806667805, -0.02935391291975975, -0.02251284383237362, -0.0022110827267169952, -0.009423505514860153, -0.0007158764055930078, -0.02050958201289177, -0.011331373825669289, 0.009791451506316662, 0.013941064476966858, -0.004510745406150818, -0.0016617190558463335, -0.010077632032334805, -0.009055559523403645, -0.008994235657155514, -0.007570147980004549, 0.028045659884810448, -0.004888911731541157, 0.01310296542942524, 0.009307670406997204, 0.015535497106611729, -0.017157185822725296, -0.003103692550212145, -0.01685737632215023, -0.0007111919112503529, 0.0010561412200331688, -0.012176286429166794, -0.012844040058553219, 0.02102743089199066, 0.0015791015466675162, 0.011045193299651146, -0.0319976732134819, -0.006507192738354206, -0.0010697689140215516, 0.020972920581698418, 0.004875284153968096, -0.016298644244670868, -0.018301906064152718, 0.05391089990735054, 0.003911129664629698, 0.005750859621912241, 0.013361889868974686, -0.00891928281635046, -0.005491934716701508, -0.016530314460396767, 0.026464855298399925, -0.022022249177098274, -0.03987444192171097, 0.01314384862780571, 0.010813524015247822, 0.03619498386979103, -0.0035499974619597197, -0.00960747804492712, -0.01396831963211298, -0.01062955055385828, -0.011351815424859524, 0.02376658469438553, -0.013225614093244076, -0.017320716753602028, 0.013300566002726555, 0.0166938453912735, 0.024761401116847992, -0.007392989005893469, -0.007651913911104202, -0.014268127270042896, 0.015153924003243446, -0.010970241390168667, -0.02899959497153759, -0.021150080487132072, -0.007720051798969507, 0.03630400449037552, -0.002688049804419279, -0.01796121522784233, -0.0006634952151216567, 0.013491353020071983, -0.0168982595205307, 0.007161319255828857, -0.017675034701824188, 0.021817835047841072, 0.010772640816867352, 0.018710734322667122, 0.02050958201289177, 0.030389612540602684, -0.01606697402894497, 0.043717432767152786, -0.00776093453168869, 0.004698125179857016, 0.008013046346604824, -0.01873799040913582, -0.00565887289121747, 0.009791451506316662, -0.0315888449549675, 0.011092890053987503, 0.011290490627288818, 0.008871586993336678, -0.010295674204826355, 0.03237924724817276, -0.012844040058553219, -0.004016743507236242, -0.0072635263204574585, 0.008306039497256279, 0.032679054886102676, -0.014063713140785694, -0.005348844453692436, -0.023671191185712814, -0.011331373825669289, 0.022567354142665863, 0.01902417093515396, -0.014431659132242203, -0.010438763536512852, 0.00560776935890317, -0.013375517912209034, -0.0004957049968652427, -0.03232473507523537, -0.002088434062898159, 0.006868325173854828, -0.01440440397709608, 0.01796121522784233, 0.004238192457705736, -0.017238950356841087, 0.015249316580593586, 0.021749695762991905, -0.0005353103042580187, -0.006789966020733118, 0.001739226165227592, 0.0014470838941633701, -0.005025188438594341, 0.013579932041466236, -0.014077341184020042, 0.016448548063635826, -0.009907286614179611, -0.004599324893206358, -0.013743463903665543, 0.022403821349143982, -0.00026062841061502695, 0.018056608736515045, -0.011304118670523167, 0.027173493057489395, 0.009893658570945263, -0.019228585064411163, -0.012775902636349201, 0.01811111904680729, -0.01568540185689926, -0.0017579641425982118, -0.012673694640398026, -0.005648652091622353, -0.012782716192305088, 0.00111235526856035, -0.01418636180460453, -0.008224274031817913, -0.012203541584312916, -0.001761371036991477, 0.005045629572123289, -0.016952769830822945, 0.027991149574518204, 0.21673381328582764, -0.0020049649756401777, 0.009055559523403645, 0.026805546134710312, -0.00047781874309293926, 0.013607187196612358, 0.023507660254836082, 0.006592365447431803, -0.015590007416903973, 0.0014990392373874784, -0.017511503770947456, 0.00998223852366209, 0.009239532053470612, 0.000432677217759192, -0.020564092323184013, -0.014295383356511593, -0.039465613663196564, -0.018056608736515045, -0.009062373079359531, -0.009389436803758144, -0.002577325329184532, -0.002604580717161298, -0.0064833443611860275, -0.00998223852366209, -0.001188158872537315, 0.008994235657155514, -0.010295674204826355, 0.012619184330105782, 0.00031854581902734935, 0.0043949102982878685, 0.0006771227926947176, 0.0004854842845816165, 0.0004463048535399139, 0.017238950356841087, -0.027337023988366127, -0.00882389023900032, 0.0021122824400663376, -0.01255104597657919, 0.001425790716893971, 0.02330324612557888, -0.006510599981993437, 0.027214374393224716, 0.007781376130878925, -0.005362472031265497, -0.003067919984459877, 0.028399979695677757, 0.009859589859843254, -0.02015526406466961, -0.013600373640656471, 0.014636073261499405, -0.03077118657529354, -0.007815444841980934, 0.014213616959750652, 0.021068314090371132, -0.013470911420881748, 0.0004352324176579714, -0.0028362502343952656, 0.025156604126095772, 0.0027170085813850164, 0.0311255045235157, -0.005921205040067434, 0.03235199302434921, -0.005454458761960268, 0.0324610136449337, -0.02612416446208954, 0.03663106635212898, -0.00903511792421341, 0.001889130100607872, 0.010350184515118599, -0.025470038875937462, -0.015331082977354527, 0.001964082010090351, -0.012510163709521294, 0.011747016571462154, -0.01965104043483734, -0.02935391291975975, 0.029408423230051994, 0.02334412932395935, 0.016353154554963112, 0.0028106984682381153, -0.007099994923919439, -0.002393352333456278, -0.015848932787775993, 0.008994235657155514, 0.001752853742800653, -0.02459787018597126, -0.010261604562401772, 0.012939433567225933, -0.037421468645334244, -0.00497067766264081, 0.011079262010753155, -0.014022829942405224, -0.008585406467318535, -0.0004569514421746135, -0.009852775372564793, 0.038729723542928696, 0.008530896157026291, 0.016257761046290398, -0.029953528195619583, -0.013041640631854534, -0.006401578895747662, 0.0249113067984581, 0.021190963685512543, 0.015044902451336384, 0.002579028718173504, -0.010002680122852325, -0.007549706380814314, 0.01005719043314457, 0.013763905502855778, -0.017824938520789146, 0.014554307796061039, 0.006687758956104517, 0.00462658004835248, -0.005621396936476231, 0.0032382654026150703, 0.0064697167836129665, -0.011188283562660217, -0.022063132375478745, 0.0018243988743051887, -0.000966709922067821, 0.0005242378683760762, -0.0578356571495533, -0.007706424221396446, 0.027786735445261, -0.021190963685512543, -0.010799895972013474, -0.029762741178274155, 0.024311689659953117, -0.0019129784777760506, -0.03802108392119408, -0.008953352458775043, -0.045407261699438095, -0.006411799695342779, -0.008803448639810085, -0.01310296542942524, -0.0008402285166084766, -0.013866112567484379, -0.000626870954874903, 0.008817075751721859, 0.017416110262274742, -0.005566886626183987, -0.008803448639810085, 0.009464388713240623, -0.0012707763817161322, 0.0017545572482049465, -0.026710152626037598, 0.026655642315745354, -0.004319958388805389, -0.017238950356841087, 0.0038021085783839226, -0.004922980908304453, 0.012973503209650517, 0.01940574310719967, -0.03156159073114395, 0.019187701866030693, -0.01186966523528099, -0.03156159073114395, -0.01436352077871561, 0.007358919829130173, 0.021013803780078888, -0.03276082128286362, 0.021000176668167114, 0.004473269451409578, -0.008360550738871098, -0.009995865635573864, -0.0008244715863838792, -0.17465169727802277, 0.01948750950396061, 0.00698415981605649, -0.017620524391531944, 0.01988271065056324, -0.0014905219431966543, 0.0382663831114769, 0.0021599791944026947, -0.017974842339754105, -0.016802866011857986, 0.009130511432886124, 0.010356998071074486, -0.030743930488824844, -0.000918161531444639, -0.0078018177300691605, 0.005233009345829487, -0.017865821719169617, 0.021190963685512543, 0.02196773886680603, 0.004347213543951511, 0.03194316104054451, -0.0024972630199044943, 0.018996914848685265, 0.0029980784747749567, 0.016707472503185272, 0.0177976842969656, 0.00609154999256134, 0.009287228807806969, 0.002732339547947049, -0.019828200340270996, -0.032242968678474426, 0.0037816669791936874, 0.015099412761628628, 0.007270340342074633, -0.01418636180460453, 0.014077341184020042, -0.021940482780337334, -0.010874847881495953, -0.0025705115403980017, 0.023793840780854225, 0.02612416446208954, 0.008483199402689934, 0.02432531863451004, -0.002202565548941493, -0.01314384862780571, 0.01873799040913582, -0.01298713032156229, -0.023562170565128326, -0.004118951037526131, -0.005893949419260025, -0.009989052079617977, -0.016598451882600784, 0.0027749259024858475, 0.0011191690573468804, -0.0073044090531766415, 0.015194806270301342, 0.0010203687706962228, 0.0017954400973394513, 0.0057747079990804195, 0.0010280342539772391, -0.007965349592268467, 0.005123988725244999, 0.008701241575181484, -0.0088988421484828, -0.02699633315205574, -0.034995753318071365, 0.00574745237827301, 0.0017187846824526787, -0.0302533358335495, 0.002820919267833233, -0.012953061610460281, 0.00021559334709309042, -0.016216877847909927, -0.030226081609725952, 0.003955419175326824, 0.014854115433990955, 0.0061392467468976974, -0.015113040804862976, -0.01662570796906948, 0.020018987357616425, -0.007454313337802887, 0.03316964954137802, 0.004827587399631739, 0.011747016571462154, 0.00947120226919651, 0.021940482780337334, 0.002379724755883217, 0.0009948168881237507, 0.003151389304548502, -0.021981365978717804, 0.008680799975991249, -0.025688080117106438, -0.028154682368040085, -0.010145769454538822, 0.005941646173596382, 0.012714577838778496, 0.010561412200331688, 0.013566304929554462, 0.00642542727291584, -9.432874503545463e-05, -0.0008700389298610389, 0.016121484339237213, -0.014704211615025997, 0.010765827260911465, 0.021667929366230965, -0.0011643106117844582, 0.021068314090371132, 0.014826860278844833, 0.03802108392119408, 0.006016598083078861, 0.009709686040878296, 0.01349816657602787, -0.0036590183153748512, 0.021245473995804787, 0.00852408166974783, 0.013866112567484379, -0.014240872114896774, 0.005365878809243441, 0.016830122098326683, -0.0005037963856011629, 0.03382377699017525, -0.02125910110771656, -0.013484538532793522, -0.003897501854225993, -0.0001148766532423906, 0.0057883355766534805, -0.14368972182273865, 0.008421874605119228, 0.0063811372965574265, 0.01216265931725502, -0.00968924444168806, -0.0017111191991716623, -0.0024870422203093767, 0.020128007978200912, -0.009886845014989376, 0.03336043655872345, -0.009266788139939308, -0.02530650794506073, 0.008633103221654892, 0.0071136225014925, 0.05658191442489624, -0.006268709432333708, -0.017429737374186516, -0.024652380496263504, -0.015549125149846077, 0.033660244196653366, 0.006282337009906769, -0.027609577402472496, 0.017702290788292885, -0.01771591790020466, -0.00896016601473093, -0.0005225344211794436, -0.024175414815545082, 0.022839905694127083, 0.00539313443005085, 0.013702580705285072, -0.0013755387626588345, 0.008878400549292564, 0.01708904653787613, -0.03088020719587803, 0.007059111725538969, 0.01516755111515522, -0.047723956406116486, -0.014581562951207161, 0.002347359200939536, -0.0529569648206234, 0.007910838350653648, 0.008966979570686817, 0.011419952847063541, -0.029135871678590775, 0.0033251415006816387, -0.018601713702082634, 0.0005727862589992583, -0.004987712018191814, 0.01477234996855259, -0.010309301316738129, -0.016952769830822945, -0.012762274593114853, -0.0164621751755476, 0.01092254463583231, -0.005781521555036306, -0.004006523173302412, 0.010575040243566036, 0.010295674204826355, 0.019473882392048836, -0.01276908814907074, 0.026737408712506294, -0.007174946833401918, -0.012646439485251904, 0.03976542130112648, 0.036167725920677185, -0.0026846430264413357, -0.030471378937363625, -0.01503127533942461, 0.016789238899946213, -0.016761984676122665, -0.02962646633386612, 0.03556811064481735, -0.013729835860431194, 0.0211364533752203, -0.034477900713682175, -0.01229893509298563, -0.022690001875162125, -0.02294892817735672, 0.027677714824676514, 0.003686273703351617, -0.010547785088419914, -0.01902417093515396, 0.0011983796721324325, -0.010261604562401772, 0.007358919829130173, 0.015140295960009098, 0.017320716753602028, -0.0004905946552753448, -0.0040099299512803555, -0.011167841963469982, 0.011481277644634247, 0.012544232420623302, -0.016748355701565742, 0.00024316800408996642, 0.017116302624344826, 0.03870246559381485, 0.004895725753158331, 0.001841433346271515, 0.0019300129497423768, -0.015058530494570732, -0.009886845014989376, -0.02196773886680603, -0.04565255716443062, 0.02271725796163082, -0.0036964945029467344, -0.0271462369710207, 0.005485120695084333, -0.013314194045960903, 0.00896016601473093, -0.0067286416888237, -0.0005020929384045303, -0.0031190235167741776, -0.011454022489488125, 0.03712166100740433, 0.011222352273762226, -0.01717081293463707, -0.013307379558682442, 0.020741252228617668, 0.024938561022281647, -0.008483199402689934, 0.00929404329508543, 0.01499039214104414, 0.0025432563852518797, -0.004299516789615154, 0.010309301316738129, 0.016707472503185272, 0.010827151127159595, 0.01302801351994276, -0.00462658004835248, 0.023139713332057, 0.02499307133257389, -0.0072090160101652145, 0.0010382549371570349, -0.03603145107626915, 0.010813524015247822, 0.002534738974645734, -0.013579932041466236, 0.015985209494829178, 0.010213907808065414, -0.00330129312351346, 0.03578615561127663, 0.06410437077283859, -0.009123697876930237, -0.0209047831594944, -0.021954109892249107, -0.048269063234329224, -0.0067116073332726955, 0.006176722701638937, -0.0120059410110116, 0.015113040804862976, 0.0032995897345244884, 0.005853066686540842, 0.013709395192563534, 0.014717839658260345, -0.006302778609097004, 0.0013568007852882147, 0.01375027745962143, -0.023834723979234695, 0.001866985228843987, 0.026151420548558235, 0.010425136424601078, -0.016407664865255356, 0.0037101220805197954, -0.014131851494312286, 0.024039138108491898, -0.01400920283049345, 0.015699028968811035, -0.013702580705285072, 0.003570438828319311, 0.0038021085783839226, -0.023003438487648964, -0.030008038505911827, -0.0011310932459309697, -0.0008402285166084766, -0.00591439101845026, 0.029490189626812935, -0.0002225136267952621, 0.016285017132759094, -0.009089628234505653, 0.008714868687093258, -0.0004829291137866676, 0.014090968295931816, 0.00531477527692914, -0.023671191185712814, -0.04747866094112396, 0.024461593478918076, -0.0040712542831897736, 0.0069398703053593636, 0.008013046346604824, 0.013682140037417412, -0.03175237402319908, 0.012101334519684315, -0.01823376677930355, 0.0018703921232372522, -0.020918410271406174, 0.016353154554963112, -0.017579641193151474, 0.03614047169685364, -0.0175660140812397, -0.006013191305100918, 0.020645858719944954, 0.020291538909077644, 0.0036658323369920254, -0.012442025355994701, -0.0033575070556253195, -0.008237902075052261, -0.02365756407380104, -0.0065957726910710335, -0.018124746158719063, -0.01681649498641491, -0.004003115929663181, 0.02507483772933483, 0.0006383692380040884, 0.01807023584842682, 0.020754879340529442, 0.00625848863273859, -0.03142531216144562, 0.015371965244412422, 0.001023775665089488, -0.007726865820586681, -0.02173606865108013, 0.03442339226603508, -0.005253450945019722, -0.004418758675456047, 0.0215725377202034, -0.03663106635212898, 0.05044948309659958, 0.0037816669791936874, 0.008673985488712788, -0.013784347102046013, 0.010827151127159595, -0.008905655704438686, 0.0018925369950011373, 0.012094520963728428, -0.025565432384610176, -0.010534157045185566, -0.0004927239497192204, -0.018887894228100777, 0.010023120790719986, 0.01575353927910328, -0.016216877847909927, 0.03311513736844063, 0.0010714723030105233, -0.021599791944026947, 0.006786559242755175, 0.02243107743561268, 0.030989227816462517, 0.0004846325609833002, 0.012837226502597332, -0.000529774057213217, -0.017770428210496902, 0.026628388091921806, -0.005454458761960268, 0.011890106834471226, -0.019664669409394264, -0.023017065599560738, -0.010840779170393944, -0.0052261957898736, 0.018383672460913658, -0.023316873237490654, 0.017729545012116432, -0.00040371849900111556, -0.008142508566379547, 0.03104373812675476, 0.02334412932395935, -0.021150080487132072, 0.0032604101579636335, 0.017770428210496902, -0.01921495608985424, -0.006350475363433361, -0.023330500349402428, 0.005294333677738905, 0.023507660254836082, -0.01764778047800064, -0.003686273703351617, 0.028945084661245346, -0.009934541769325733, -0.000430547894211486, 0.014227245002985, -0.011249607428908348, 0.022771768271923065, 0.01165162306278944, 0.004984305240213871, -0.022213036194443703, -0.03575889766216278, -0.0026556842494755983, -0.004974084440618753, -0.009859589859843254, -0.005253450945019722, -0.00976419635117054]} +{"id": "test:10000022", "text": "\"Learn the fundamentals of Flash as we guide you through this comprehensive introduction to the leading interactive-design program. We will show you how to draw and animate your own graphics. This is the perfect starting point for anyone interested in creating enhanced web output.\\nAfter we give you a complete tour of the Flash environment, we will take you from a blank canvas to being able to produce your own graphics and animations. You will see how it easy it is to import your own audio and video, add simple interactivity, and then publish your projects in a variety of formats.\\nThis 2-day Introduction course helps you learn about the tools and features available in Adobe Flash CC. We will help you understand how this industry-leading program takes interactive design to a new level. We will guide you in learning the work area and the best practices for creating Flash content. Plus, you will get the hands-on experience you need to create your own projects.\\nOur goal is for you to be able to create simple visual projects that are ready to use for the web.\\nThis course is intended for designers and developers who are new to Flash and who want to learn the interface and commonly used features.\\nIf you are considering this Flash CC Design Introduction 2 Days Course but would like further information simply send us your query quickly & easily with the form below. Whether you would like to check date availability, pricing, course suitability or just need further information about a course send us your query and we will get you a swift reply, usually within 2 business hours!\\nYour course covered everything I needed to know and also gave really useful tips and advice making me feel more confident about using the program.\"", "vector_field": [-0.016736609861254692, 0.014633101411163807, 0.001984287751838565, -0.028952637687325478, -0.0030507403425872326, 0.006336655467748642, -0.02468029409646988, -0.028299372643232346, -0.005386157426983118, -0.01772957108914852, -0.0007075659814290702, 0.020303428173065186, -0.008120064623653889, -0.007878357544541359, 0.0055266087874770164, -0.014319535344839096, 0.015769779682159424, 0.004265810362994671, 0.0032777495216578245, -0.037105366587638855, -0.011079348623752594, 0.04554552957415581, 0.004050233401358128, -0.010497943498194218, -0.014776819385588169, -0.006209269165992737, -0.012379342690110207, -0.023034069687128067, -0.018199920654296875, -0.013470293022692204, 0.012575321830809116, 0.005660527851432562, 0.00572585454210639, -0.026012951508164406, -0.022315479815006256, -0.018931575119495392, -0.0031650615856051445, -0.0036811395548284054, 0.0029576504603028297, -0.005092188715934753, 0.015573801472783089, 0.014149686321616173, -0.010693922638893127, -0.011523567140102386, -0.01608334667980671, -0.009341667406260967, 0.0029249871149659157, -0.010687390342354774, -0.014698428101837635, 0.039117418229579926, 0.002735540736466646, 0.027123499661684036, 0.005608266685158014, -0.0306249912828207, 0.009080362506210804, -0.016540631651878357, -0.021531563252210617, 0.028351634740829468, 0.019754687324166298, -0.008453229442238808, 0.008825589902698994, 0.012921551242470741, -0.007662780582904816, -0.0161748044192791, -0.011615024879574776, 0.005738919600844383, -0.012869290076196194, 0.0023027537390589714, -0.016331586986780167, 0.008753730915486813, 0.016749674454331398, 0.017246155068278313, -0.0002141479344572872, 0.014149686321616173, 0.03438778966665268, 0.027463195845484734, -0.025777775794267654, -0.01259491965174675, 0.03237573802471161, 0.01458084024488926, -0.018056202679872513, -0.02329537458717823, 0.0037529985420405865, 0.012823562137782574, -0.015586866065859795, 0.03655662387609482, 0.02437979355454445, 0.005510277580469847, -0.03545914217829704, 0.014136621728539467, -0.018866248428821564, -0.002018584171310067, 0.024510445073246956, 0.013418031856417656, -0.02100895345211029, 0.005644196178764105, 0.01531249564141035, 0.020904431119561195, -0.01698485016822815, -0.014750689268112183, 0.0039718421176075935, 0.00441932724788785, 0.0049941991455852985, -0.011713013984262943, -0.02389637753367424, -0.011699948459863663, 0.021139604970812798, -0.003860787022858858, 0.014672297984361649, 0.004079630598425865, -0.018683334812521935, 0.027332544326782227, 0.0062941936776041985, -0.022916482761502266, -0.0020332825370132923, -0.013019541278481483, -0.006607760209590197, -0.012973812408745289, 0.00045034350478090346, -0.03705310449004173, 0.008048205636441708, 0.03342095762491226, -0.0030670720152556896, -0.0043180715292692184, -0.011138142086565495, 0.032480258494615555, -0.005905501544475555, -0.009322069585323334, -0.003367573255673051, -0.00968136452138424, 0.003638677531853318, 0.0033087795600295067, 0.015965759754180908, -0.00597409438341856, -0.007799965795129538, 0.027332544326782227, -0.015978824347257614, 0.03164408355951309, -0.038516413420438766, -0.01698485016822815, 0.04191338270902634, 0.004461789503693581, 0.00813313014805317, -0.02031649462878704, 0.0231124609708786, 0.01350948866456747, 0.010275834240019321, 0.02230241522192955, 0.007545193191617727, 0.005800979677587748, 0.01911448873579502, -0.001675620791502297, 0.017180828377604485, -0.002621219726279378, 0.024693358689546585, 0.025268230587244034, -0.002774736611172557, 0.027724500745534897, 0.022028043866157532, -0.008440163917839527, 0.018526552245020866, 0.008035141043365002, 0.01531249564141035, -0.004703497048467398, 0.02089136652648449, 0.024301400408148766, 0.023256178945302963, -0.026862194761633873, 0.02179287001490593, 0.00487661175429821, 0.02086523547768593, 0.01776876673102379, -0.028456157073378563, 0.02407929114997387, 0.019401924684643745, 0.017298417165875435, 0.009276341646909714, -0.0029511176981031895, -0.005647462792694569, -0.016658218577504158, -0.0008267865632660687, 0.011667285114526749, -0.012000449933111668, 0.0071793654933571815, -0.032297346740961075, -0.017546657472848892, 0.007434138096868992, -0.005853240843862295, -0.008668806403875351, -0.015743650496006012, -0.008492425084114075, 0.02500692568719387, 0.021753674373030663, 0.012359744869172573, -0.6359127759933472, 0.01772957108914852, 0.03634757921099663, -0.045702312141656876, -0.009798952378332615, -0.005281635094434023, -0.0011758742621168494, -0.021779803559184074, 0.006078616715967655, 0.04400382563471794, 0.008420566096901894, -0.006209269165992737, 0.026940586045384407, -0.011817536316812038, -0.010589401237666607, -0.011386382393538952, 0.004422593861818314, -0.03543300926685333, -0.00729042012244463, 0.0023272510152310133, -0.008329109288752079, 0.015142647549510002, -0.006989919114857912, 0.007969814352691174, 0.013940642587840557, -0.008139662444591522, -0.009746691212058067, -0.01791248470544815, 0.006153741851449013, 0.02239387109875679, -0.038673195987939835, 0.009126090444624424, 0.018173789605498314, -0.00781303085386753, 0.03483200818300247, -0.006068817805498838, -0.005618066061288118, 0.02029036357998848, 0.016788871958851814, 0.016070282086730003, -0.030546599999070168, 0.013019541278481483, 0.027672240510582924, 0.002577124396339059, -0.012020047754049301, -0.012294418178498745, 0.006284394767135382, 0.006578363012522459, 0.004020836669951677, -0.013431097380816936, 0.01782102696597576, -0.013208987191319466, -0.017063241451978683, -0.013189389370381832, -0.0011464774142950773, 0.010902967303991318, 0.026836063712835312, 0.02666621468961239, 0.012104972265660763, 0.01079191267490387, -0.007989412173628807, 0.012542658485472202, -0.026744605973362923, -0.0026816464960575104, -0.021296389400959015, 0.008126597851514816, -0.015142647549510002, 0.003511291230097413, 0.02070845291018486, 0.002243960043415427, 0.012777833268046379, -0.00016086612595245242, -0.03543300926685333, 0.015234104357659817, -0.003057273104786873, -0.002435039496049285, 0.011601959355175495, -0.013418031856417656, -0.015965759754180908, 0.019049162045121193, 0.012967280112206936, 0.006470574531704187, 0.008218054659664631, -0.019545642659068108, 0.010491411201655865, -0.013522554188966751, -0.00904769916087389, -0.030154641717672348, -0.020329559221863747, -0.013522554188966751, -0.0006491805543191731, -0.008048205636441708, -0.016697414219379425, -0.026613954454660416, -0.0002884566492866725, 0.04826310649514198, 0.006555499043315649, 0.016723545268177986, -0.016004955396056175, -0.02227628417313099, 0.01818685419857502, -0.024144617840647697, 0.0021459704730659723, 0.0029935799539089203, 0.021805934607982635, 0.015469279140233994, -0.004664300940930843, 0.005239172838628292, 0.0607796348631382, -0.01929740235209465, -0.01881398819386959, -0.024667229503393173, -0.00865574087947607, -0.017533591017127037, 0.0032075236085802317, -0.02449738048017025, 0.002464436460286379, -0.007257757242769003, 0.027123499661684036, -0.01839589886367321, 0.002972348826006055, 0.005467815324664116, 0.002632651710882783, -0.006944190710783005, -0.013561749830842018, 0.016488369554281235, -0.006669819820672274, 0.005533141549676657, -0.03658275306224823, -0.003576617455109954, 0.018709465861320496, 0.004380131606012583, 0.014541644603013992, -0.008525088429450989, -0.006891929544508457, 0.004376865457743406, 0.0018764992710202932, -0.0029135551303625107, 0.0005385340773500502, -0.050092242658138275, -0.02317778766155243, 0.002425240585580468, 0.016945654526352882, -0.008570816367864609, -0.009322069585323334, -0.00954417884349823, -0.003156895749270916, 0.0031960916239768267, 0.004478121176362038, 0.007074843160808086, -0.02221095748245716, -0.041312381625175476, -0.005258771125227213, 0.025059185922145844, 0.0017654445255175233, 0.0002788618439808488, -0.0026587822940200567, -0.01770344004034996, -0.004115559626370668, -0.027933545410633087, 0.005235906690359116, 0.012647180818021297, -0.02509838342666626, 0.011386382393538952, -0.018408965319395065, -0.011660752817988396, 0.006444443948566914, -0.0024693359155207872, 0.0034002363681793213, -0.04517970234155655, -0.005640930030494928, 0.008845187723636627, -0.015586866065859795, 0.0413646437227726, -0.00730348564684391, -0.007434138096868992, -0.03078177385032177, 0.014424057677388191, -0.013718532398343086, -0.01638384722173214, 0.007473334204405546, 0.02461496740579605, -0.0047459593042731285, -0.008701469749212265, 0.003491693176329136, 0.008361772634088993, -0.002015317790210247, 0.0024007433094084263, -0.045754574239254, 0.003367573255673051, 0.009139155969023705, 0.009426591917872429, -0.01420194748789072, -0.005448217503726482, -0.027541587129235268, 0.014946668408811092, -0.03206217288970947, -0.003478627884760499, 0.002350115217268467, 0.011543165892362595, 0.027463195845484734, 0.00329571426846087, 0.02266824245452881, 0.014541644603013992, 0.028090329840779305, -0.034701354801654816, 0.019284337759017944, 0.0014363630907610059, 0.052522383630275726, -0.005484146997332573, 0.0008557751425541937, -0.019192880019545555, 0.019153684377670288, -0.013052203692495823, 0.0019254940561950207, 0.022446133196353912, -0.020512472838163376, 0.010275834240019321, -0.026026016101241112, 0.006241932511329651, -0.019375793635845184, 0.004706763196736574, 0.008825589902698994, -0.017964744940400124, -0.0015882467851042747, 0.021322520449757576, -0.011974319815635681, 0.006173339672386646, -0.026221996173262596, -0.03898676484823227, 0.007904487662017345, -0.007035647518932819, -0.005023595876991749, -0.001394717488437891, 0.0042037502862513065, 0.01601801998913288, -0.009322069585323334, 0.0007067494443617761, 0.04361186921596527, 0.011458241380751133, 0.018827052786946297, 0.026169734075665474, 0.03219282254576683, -0.019258206710219383, 0.020760713145136833, 0.02761998027563095, 0.012020047754049301, 0.0002478318347129971, 0.004409528337419033, 0.025268230587244034, -0.012810496613383293, -0.004118826240301132, -0.017572786659002304, 0.012483865022659302, 0.0018209719564765692, -0.004105760715901852, 0.00886478554457426, 0.021531563252210617, 0.013169791549444199, 0.024654163047671318, -0.019911469891667366, 0.009001970291137695, -0.0048994761891663074, -0.011863264255225658, -0.009655234403908253, 0.0028286308515816927, 0.002096975687891245, -0.033290307968854904, -0.019493382424116135, -0.013084867037832737, -0.008668806403875351, -0.022563720121979713, -0.003563552163541317, 0.014972798526287079, 0.005307765677571297, 0.006457509472966194, 0.01737680844962597, -0.007780367974191904, 0.004435658920556307, 0.03425713628530502, -0.030677253380417824, -0.030964689329266548, 0.0034720953553915024, -0.018291376531124115, -0.00865574087947607, -0.027881285175681114, 0.014737623743712902, -0.009282873943448067, -0.0009447822812944651, 0.00859694741666317, -0.01827831193804741, 0.006944190710783005, 0.025895364582538605, 0.00820498913526535, 0.00332511099986732, -0.029266202822327614, 0.02001599222421646, -0.0001153418343164958, -0.015730584040284157, -0.015835106372833252, -0.004696964286267757, 0.001593962893821299, 0.00012524286285042763, -0.02666621468961239, 0.016135606914758682, 0.006865798961371183, -0.0008557751425541937, -0.010471813380718231, -0.007473334204405546, 0.0007826912915334105, -0.00552334263920784, -0.017834093421697617, -0.008322576992213726, -0.00487661175429821, 0.01936272904276848, -0.01989840529859066, 0.015221038833260536, -0.01259491965174675, 0.010857238434255123, 0.012065776623785496, -0.0073884096927940845, -0.015168777666985989, -0.0187225304543972, 0.0017376808682456613, 0.07807804644107819, -0.018591878935694695, -0.015325561165809631, 0.006748211570084095, -0.014437122270464897, -0.015573801472783089, -0.01388838142156601, -0.024876272305846214, 0.029475247487425804, -0.009779354557394981, -0.009348200634121895, 0.011262262240052223, 0.0009831615025177598, -0.008152727968990803, 0.024118486791849136, 0.023909442126750946, 0.021596889942884445, -0.004948470741510391, 0.02038181945681572, 0.006695950403809547, -0.01060899905860424, 0.004409528337419033, 0.03357774391770363, 0.00572585454210639, 0.0004113518516533077, -0.01608334667980671, 0.02470642514526844, -0.00149352359585464, -0.0021345384884625673, -0.0328199565410614, 0.02398783527314663, 0.011536632664501667, 0.018330572172999382, 0.005255504511296749, -0.019310468807816505, 0.040894292294979095, -0.005578869953751564, 0.03336869925260544, -0.003831390291452408, -0.004050233401358128, 0.030494337901473045, 0.010328095406293869, -0.008113532327115536, -0.006722080986946821, 0.021805934607982635, 0.003338176291435957, -0.012967280112206936, 0.0198722742497921, -0.02100895345211029, -0.007512529846280813, 0.016187869012355804, 0.029919466003775597, -0.007545193191617727, 0.028534548357129097, -0.012712507508695126, -0.02191045694053173, -0.007447203621268272, -0.0034035027492791414, -0.0051085203886032104, -0.01587430201470852, -0.006891929544508457, -0.03587723150849342, 0.017442135140299797, -0.007982879877090454, 0.0022619247902184725, -0.013378836214542389, -0.012810496613383293, 0.024301400408148766, -0.013587879948318005, 0.016997914761304855, 0.00018311791063752025, -0.030494337901473045, -0.017781831324100494, 0.0005107703618705273, -0.0004376865108497441, 0.00188793137203902, 0.013326575048267841, 0.00026559241814538836, -0.005239172838628292, -0.004380131606012583, 0.0022063974756747484, -0.02257678471505642, 0.010706988163292408, -0.015547670423984528, -0.008283380419015884, 0.030520468950271606, -0.012307483702898026, 0.0007308385102078319, 0.008244184777140617, 0.03399582952260971, 0.006807005498558283, -0.0009504983318038285, 0.026862194761633873, -0.005634397268295288, 0.007917553186416626, 0.01740293949842453, 0.006307258736342192, 0.01310446485877037, 0.023047134280204773, -0.017899418249726295, 0.007564791012555361, -0.023517485707998276, -0.01793861575424671, -0.003948977682739496, 0.03300287202000618, 0.001990820514038205, -0.003860787022858858, 0.006666553672403097, -0.0036158133298158646, -0.023099396377801895, -0.010380356572568417, -0.017951680347323418, 0.02149236761033535, 0.0114974370226264, 0.022406937554478645, 0.026313452050089836, -0.017233090475201607, 0.021152671426534653, 0.003736667102202773, -0.021636085584759712, 0.03261091187596321, -0.05233946815133095, -0.018343638628721237, 0.011654220521450043, -0.0273848045617342, 0.03355161100625992, -0.005092188715934753, -0.024275271221995354, 0.001956524094566703, -0.020159710198640823, -0.008381370455026627, 0.015469279140233994, -0.010857238434255123, -0.0010444049257785082, -0.02362200617790222, -0.006607760209590197, -0.019101424142718315, 0.015234104357659817, -0.005458016414195299, -0.010576335713267326, -0.02840389497578144, -0.004553246311843395, 0.013444161973893642, -0.0295797698199749, 0.030964689329266548, -0.023582810536026955, -0.0065032378770411015, 0.016396913677453995, 0.015756715089082718, 0.012222559191286564, -0.014045164920389652, 0.017259221524000168, -0.039274200797080994, -0.0017850424628704786, 0.0003252027090638876, -0.019702427089214325, -0.02218482829630375, -0.017690375447273254, 0.022642111405730247, 0.0402148999273777, 0.02110040932893753, 0.012477332726120949, -0.012314015999436378, -0.008727599866688251, 0.006911527365446091, 0.01478988490998745, 0.012091906741261482, -0.013006475754082203, -0.028299372643232346, 0.016880327835679054, 0.008113532327115536, 0.015077320858836174, -0.009910007007420063, -0.003142197383567691, 0.02897876687347889, 0.025451144203543663, -0.013914511539041996, -0.012562256306409836, -0.0015278198989108205, -0.01881398819386959, 0.01124266441911459, 0.01686726324260235, -0.014724558219313622, 0.0022602917160838842, -0.022942613810300827, -0.0031079009640961885, 0.031147602945566177, 0.009452722035348415, -0.008740665391087532, -0.0011317789321765304, 0.02671847678720951, -0.0016739876009523869, -0.008165793493390083, 0.004958269651979208, 0.009295939467847347, 0.010648194700479507, -0.015704452991485596, -0.041077207773923874, 0.011085880920290947, 0.019911469891667366, 0.023661203682422638, -0.012013515457510948, -0.0011154473759233952, -0.002196598332375288, -0.01782102696597576, 0.02211950160562992, -0.013809990137815475, -0.015822041779756546, 0.02070845291018486, 0.02029036357998848, -0.021335585042834282, -0.023282309994101524, -0.009583375416696072, 0.014384862035512924, -0.005634397268295288, 0.015704452991485596, -0.007623584475368261, 0.0194149911403656, 0.005817311350256205, -0.012438136152923107, 0.017964744940400124, -0.05665100738406181, 0.03624305874109268, 0.003720335429534316, -0.0006471391534432769, 0.0027224754448980093, 0.011673818342387676, -0.010184377431869507, -0.006274595856666565, 0.005987159907817841, -0.002593456069007516, 0.023726528510451317, 0.02997172810137272, -0.011615024879574776, -0.013378836214542389, -0.0039391787722706795, 0.02336070127785206, 0.007545193191617727, -0.032271213829517365, 0.012692908756434917, 0.024275271221995354, 0.012751703150570393, -0.029083289206027985, -0.008897448889911175, -0.01797781139612198, 0.02978881448507309, 0.002041448373347521, -0.005020329728722572, -0.0006781691336072981, 0.004896209575235844, -0.03133051469922066, 0.03177473694086075, -0.02512451261281967, 0.02191045694053173, 0.0046218391507864, -0.005173846613615751, 0.0075582582503557205, 0.015116516500711441, -0.007669313345104456, -0.008564284071326256, -0.012660246342420578, 0.016057215631008148, -0.01587430201470852, 0.021714476868510246, 0.040110375732183456, 0.012777833268046379, -0.008975840173661709, 0.0016886860830709338, 0.015586866065859795, 0.030520468950271606, -0.016122542321681976, -0.01857881247997284, -0.022237088531255722, 0.02500692568719387, -0.008505490608513355, 0.0005369008868001401, -0.00781303085386753, -0.032715436071157455, -0.00879292655736208, -0.021440107375383377, 0.0075582582503557205, -0.004945204593241215, -0.016135606914758682, -0.026888323947787285, 0.021936587989330292, 0.00619293749332428, 0.00730348564684391, 0.010733119212090969, 0.02119186706840992, -0.02329537458717823, -0.0007512529846280813, -0.027254151180386543, 0.007884889841079712, -0.013013008050620556, 0.00903463363647461, -0.005794446915388107, 0.010282367467880249, 0.01989840529859066, -0.03645210340619087, -0.012523060664534569, -0.0005283268401399255, 0.03598175197839737, -0.02897876687347889, 0.02049940824508667, -0.019062228500843048, -0.018748661503195763, 0.036896321922540665, 0.001535985735245049, -0.03101694956421852, -0.03365613520145416, -0.017925549298524857, 0.00014820914657320827, -0.010504476726055145, -0.0004042067739646882, 0.018866248428821564, 0.00046871654922142625, -0.019584838300943375, -0.0009047699277289212, -0.014881341718137264, -0.029292333871126175, -0.006833136081695557, -0.004572844132781029, 0.008107000030577183, -0.012327081523835659, 0.025294361636042595, -0.025777775794267654, 0.005507010966539383, 0.02323004975914955, -0.036321450024843216, -0.0052032433450222015, -0.002537928521633148, -0.01712856814265251, -0.0002572224766481668, -0.019349664449691772, -0.02296874299645424, -0.013326575048267841, -0.024445118382573128, 0.002746972953900695, -0.04180886223912239, -0.018552683293819427, 0.004690431524068117, 0.006787407677620649, -0.00864267535507679, 0.03036368638277054, -0.020721517503261566, -0.011020555160939693, -0.030076250433921814, -0.035093314945697784, -0.036504361778497696, 0.0032630509231239557, -0.017664244398474693, 0.027463195845484734, -0.011745677329599857, -0.014241144061088562, -0.03404809162020683, -0.025634057819843292, -0.060309283435344696, -0.0027224754448980093, -0.0038803850766271353, 0.019349664449691772, -0.0011758742621168494, 0.020068254321813583, 0.01830444298684597, 0.02876972407102585, -0.008479359559714794, 0.03020690195262432, -0.018226051703095436, 0.0005818127538077533, -0.004664300940930843, 0.007826096378266811, 0.00904769916087389, -0.016788871958851814, -0.009158753789961338, 0.004458523355424404, 0.02221095748245716, 0.027123499661684036, 0.0027600382454693317, 0.0015678323106840253, -0.017990875989198685, -0.006820070557296276, 0.023334570229053497, 0.022825025022029877, 0.024536576122045517, -0.014463253319263458, 0.01800394058227539, -0.01290848571807146, 0.012170298025012016, -0.005196711048483849, -0.017847158014774323, -0.01105975080281496, -0.019558709114789963, 0.010138649493455887, -0.001255082432180643, -0.004099228419363499, -0.0026555159129202366, -0.010922565124928951, -0.00329571426846087, -5.304576006892603e-06, 0.015835106372833252, -0.01015824731439352, 0.012921551242470741, 0.019885340705513954, 0.0063497209921479225, 0.0005797713529318571, -0.0041645546443760395, -0.0019646899309009314, -0.01065472699701786, -0.003664808114990592, -0.011321055702865124, -0.01728535071015358, 0.0049223401583731174, -0.010883369483053684, 0.0342048741877079, -0.022080305963754654, -0.019166750833392143, -0.021832065656781197, 0.017246155068278313, -0.031905386596918106, 0.01957177370786667, -0.006774342153221369, 0.0029266204219311476, 0.01193512324243784, -0.00994920264929533, -0.01712856814265251, -0.014123556204140186, -0.021779803559184074, -0.01797781139612198, -0.018382834270596504, 0.01280396431684494, 0.03344709053635597, 0.011112011969089508, 0.007715041749179363, 0.0008194373804144561, 0.03854254633188248, 0.0018422029679641128, -0.01283662673085928, 0.22054174542427063, -0.005879371426999569, 0.012810496613383293, 0.026143604889512062, -0.0023615474347025156, 0.025046121329069138, 0.024249140173196793, 0.017873289063572884, -0.019519511610269547, 0.010099452920258045, -0.007414540275931358, 0.015887368470430374, -0.04781888425350189, -0.0021051415242254734, -0.011458241380751133, -0.03036368638277054, -0.027123499661684036, -0.023817986249923706, -0.007793433032929897, 0.004647969733923674, 0.03357774391770363, 0.0041318912990391254, -0.031304385513067245, -0.01549540925770998, 0.026522496715188026, 0.00253629544749856, -0.029318464919924736, 0.008407500572502613, 0.025869233533740044, 0.02038181945681572, -0.011575828306376934, 0.007897955365478992, 0.006137410178780556, -0.0011080981930717826, -0.017559722065925598, 0.004135157912969589, 0.03799380362033844, -0.008662273176014423, 0.014228078536689281, 0.014254208654165268, 0.008832122199237347, 0.007205496076494455, -0.013372302986681461, -0.014528580009937286, 0.01136025134474039, 0.028508417308330536, -0.023661203682422638, -0.01920594647526741, -0.02858680859208107, 0.0002804950054269284, -0.02038181945681572, -0.014228078536689281, 0.02512451261281967, 0.017259221524000168, 0.00506605813279748, 0.016632087528705597, 0.010125583969056606, -0.003439432242885232, 0.010112518444657326, 0.0011056484654545784, -0.01812152937054634, 0.03036368638277054, 0.00484068226069212, 0.028560679405927658, 0.004549980163574219, 0.02185819484293461, 0.02699284628033638, 0.01622706465423107, -0.019441120326519012, 0.01620093360543251, -0.010798444971442223, 0.005134650971740484, -0.0035896827466785908, 0.008420566096901894, -0.011196935549378395, -0.01719389483332634, -0.009335135109722614, 0.029553638771176338, 0.030755644664168358, 0.04037168249487877, -0.024601902812719345, -0.0015278198989108205, -0.031304385513067245, -0.0034753617364913225, 0.003334909910336137, -0.04220081865787506, 0.013770793564617634, -0.016605956479907036, -0.00015596665616612881, -0.032480258494615555, 0.005409021396189928, 0.007532127667218447, 0.0114974370226264, -0.016605956479907036, 0.00397510826587677, 0.0018520018784329295, 0.014254208654165268, 0.018591878935694695, -0.003044207813218236, 0.026940586045384407, -0.0281948521733284, 0.04659074917435646, 0.02410542219877243, 0.028534548357129097, -0.018343638628721237, 0.014123556204140186, -0.010824576020240784, 0.018644139170646667, 0.015769779682159424, -0.02858680859208107, -0.0006553049315698445, -0.012137635610997677, 0.018173789605498314, -0.011268794536590576, -0.02010744996368885, 0.016971785575151443, 0.011164273135364056, -0.015612997114658356, 0.01299994345754385, -0.0009562143823131919, -0.017781831324100494, -0.0077673024497926235, 0.003899982897564769, 0.012000449933111668, -0.003935912624001503, -0.026130538433790207, -0.029109420254826546, 0.012941149063408375, -0.02509838342666626, -0.045101311057806015, 0.0020741114858537912, 0.0009243678068742156, 0.01638384722173214, 0.004866812843829393, -0.0023615474347025156, 0.0018111729295924306, 0.004873345606029034, 0.0119024608284235, 0.014776819385588169, 0.010844173841178417, 0.006963788531720638, 0.005912034306675196, 0.005611533299088478, 0.029684292152523994, -0.001043588388711214, -0.02780289389193058, 0.005846708081662655, 0.0025150643195956945, 0.005866305902600288, -0.0012575321597978473, 0.0010460381163284183, 0.015351691283285618, -0.007956748828291893, -0.029736552387475967, 0.020747648552060127, 0.011804470792412758, -0.0055592721328139305, -0.038333501666784286, -0.0018552682595327497, -0.015612997114658356, -0.01770344004034996, 0.006019822787493467, 0.015351691283285618, 0.007270822301506996, -0.011098946444690228, -0.0229948740452528, -0.164099782705307, 0.0017164497403427958, 0.037105366587638855, -0.0012142534833401442, 0.02251145988702774, -0.013013008050620556, 0.011210001073777676, 0.011601959355175495, -0.012170298025012016, -0.001946725184097886, 0.0010101086227223277, -0.0176773089915514, -0.02437979355454445, -0.008120064623653889, 7.139965600799769e-05, 0.015730584040284157, -0.02320391871035099, 0.025294361636042595, 0.03752345219254494, 0.012960746884346008, 0.03376065567135811, -0.011000956408679485, -0.00442586001008749, 0.0006875597755424678, -0.0077281068079173565, -0.0007443120703101158, -0.0014036998618394136, -0.010772314853966236, 0.006127611268311739, -0.016945654526352882, 0.008917046710848808, -0.013431097380816936, 0.012869290076196194, -0.0008014726336114109, 0.018356703221797943, -0.025202903896570206, 0.0007308385102078319, -0.013444161973893642, -0.02362200617790222, 0.04233147203922272, 0.013013008050620556, -0.008871317841112614, 0.013339639641344547, -0.0008157627307809889, -0.00659142853692174, 0.02606521174311638, 0.006052486132830381, -0.0162923913449049, 0.017389873042702675, -0.00943965744227171, 0.055501263588666916, -0.008805991150438786, 0.0012632482685148716, -0.0022913215216249228, 0.005039927549660206, 0.010726585984230042, -0.027646109461784363, -7.752400415483862e-05, 0.009988398291170597, -0.020721517503261566, -0.00352109014056623, 0.0011187136406078935, 0.0043735988438129425, 0.001156276324763894, 0.005849974229931831, -0.053306300193071365, 0.0055037448182702065, 0.030102381482720375, 0.001562932855449617, 0.009687896817922592, 0.024850143119692802, -0.02699284628033638, -0.02110040932893753, -0.017154699191451073, 0.003130765166133642, -0.008218054659664631, -0.022798895835876465, -0.0057291206903755665, 0.03519783541560173, 0.0010729852365329862, -0.0043572671711444855, 0.021361716091632843, -0.004958269651979208, 0.0007904488011263311, 0.007577856071293354, -0.0184873566031456, 0.004932139068841934, 0.007564791012555361, -0.00015331276517827064, 0.004402995575219393, 0.0014322801725938916, -0.043089259415864944, -0.023831050843000412, -0.004765557125210762, 0.0071075065061450005, 0.01806926727294922, 0.023282309994101524, -0.006904995068907738, 0.026522496715188026, -0.0032450861763209105, 0.031304385513067245, 0.01015824731439352, -0.020146645605564117, 0.0053175645880401134, 0.0018209719564765692, -0.01151050254702568, -0.003504758467897773, 0.012072308920323849, 0.023073265329003334, -0.011353719048202038, 0.0013587879948318005, 0.019284337759017944, 0.009942670352756977, 0.019859209656715393, 0.0007782001048326492, 0.0073165507055819035, 0.005255504511296749, -0.0061798724345862865, 0.0003813425428234041, -0.018774792551994324, 0.038307368755340576, -0.009269808419048786, -0.025503406301140785, 0.014933602884411812, -0.011373316869139671, -0.016553696244955063, -0.09898248314857483, -0.034649092704057693, 0.029109420254826546, 0.03734054043889046, 0.006826603319495916, 0.022080305963754654, -0.0016699047992005944, -0.001308976672589779, 0.0031732271891087294, 0.030337555333971977, -0.02717575989663601, -0.010811510495841503, -0.014959733001887798, 0.001086050528101623, 0.017259221524000168, -0.007701976224780083, 0.020695386454463005, 0.01788635365664959, -0.03182699531316757, 0.005556005984544754, -0.01610947772860527, -0.01566525734961033, 0.005634397268295288, -0.014293404296040535, -0.01971549168229103, 0.008035141043365002, -0.03261091187596321, 0.012248690240085125, 0.0162923913449049, 0.01812152937054634, -0.012196429073810577, -0.0004568761505652219, 0.017089372500777245, -0.024157682433724403, -0.005650728940963745, 0.001943458802998066, -0.004690431524068117, -0.021871261298656464, 0.02761998027563095, -0.04165207967162132, 0.0071467021480202675, 0.0036190797109156847, -0.0017082840204238892, -0.04473548382520676, 0.0028220980893820524, -0.0007157318177632987, -0.0140974260866642, 0.004236413631588221, -0.009981865994632244, -0.051111333072185516, -0.03357774391770363, 0.008531620725989342, -0.038098324090242386, -0.023373767733573914, 0.007381877396255732, 0.006039420608431101, -0.016488369554281235, -0.01728535071015358, -0.02040795050561428, 0.0015474178362637758, -0.01181100308895111, -0.010426084510982037, -0.02997172810137272, 0.029867205768823624, 0.0031454635318368673, -0.004383397754281759, -0.03119986318051815, 0.02417074888944626, 0.020525537431240082, -0.007636649999767542, 0.010524074546992779, 0.018618008121848106, -0.014829080551862717, 0.014972798526287079, -0.023426027968525887, -0.0015098551521077752, -0.025634057819843292, -0.01408436056226492, 0.003423100570216775, 0.001508222077973187, -0.013019541278481483, -0.015547670423984528, 0.006362786050885916, -0.020211972296237946, 0.011177337728440762, -0.0140974260866642, 0.02181899920105934, -0.007669313345104456, -0.0047851549461483955, -0.014763754792511463, 0.016749674454331398, 0.016423042863607407, 0.01779489777982235, 0.026156669482588768, 0.0055200764909386635, -0.00041339328163303435, -0.008721067570149899, -0.009041166864335537, 0.011014021933078766, 0.02248532883822918, 0.006111279595643282, -0.022289348766207695, -0.060466066002845764, 0.002131272107362747, 0.014463253319263458, -0.002448104787617922, 0.014567775651812553, 0.015286365523934364, -0.008714534342288971, -0.02038181945681572, -0.0036158133298158646, -0.004517316818237305, -0.022250153124332428, -0.007323083467781544, -0.003088303143158555, 0.017455199733376503, -0.004210283048450947, -0.017638113349676132, 0.011954721063375473, -0.006695950403809547, -0.0032712167594581842, 0.014554710127413273, 0.014920537360012531, -0.015325561165809631, 0.021231062710285187, 0.019493382424116135, 0.015560735948383808, 0.012052711099386215, -0.011281860060989857, 0.003169961040839553, -0.011797938495874405, -0.013979838229715824, 0.013267780654132366, -0.005562538281083107, 0.006477107293903828, 0.02389637753367424, -0.0051248520612716675, -0.031121471896767616, -0.014463253319263458, 0.03731440752744675, -0.003439432242885232, 0.0253074262291193, -0.008283380419015884, 0.005415554158389568, 0.0009815283119678497, -0.03514557331800461, -0.015639128163456917, -0.03059886023402214, -0.015469279140233994, 0.005059525370597839, 0.025594862177968025, 0.003860787022858858, 0.013783859089016914, -0.0027437065728008747, 0.007904487662017345, -0.01689339242875576, -0.00749946478754282, -0.004167820792645216, 0.02077377773821354, -0.016632087528705597, -0.006055752281099558, -0.019166750833392143, 0.04342895373702049, 0.00820498913526535, -0.004572844132781029, 0.0023909443989396095, 0.014737623743712902, 0.003057273104786873, -0.045519400388002396, 0.0005805878899991512, -0.006996451877057552, -0.0007451286655850708, -0.005082389805465937, 0.009870811365544796, -0.00459244241937995, 0.02191045694053173, 0.01932353340089321, 0.01966322958469391, -0.005062791984528303, 0.0049713351763784885, -0.02061699517071247, 0.03376065567135811, 0.036295317113399506, 0.013444161973893642, -0.008773328736424446, 0.023831050843000412, 0.02362200617790222, 0.01348335761576891, 0.013130595907568932, 0.0009749957243911922, -0.031095340847969055, -0.0019369261572137475, 0.00684620114043355, 0.0017507460433989763, -0.007721574045717716, 0.013535618782043457, -0.011425578035414219, 0.02461496740579605, 0.024236075580120087, -0.015011994168162346, 0.014724558219313622, 0.009459255263209343, 0.017024045810103416, 0.01173914410173893, -0.008708002045750618, -0.028116459026932716, -0.01668434962630272, 0.0029380524065345526, -0.021871261298656464, -0.02170141227543354, 0.01566525734961033, 0.014633101411163807, 0.0022194625344127417, -0.0003633777960203588, 0.006904995068907738, 0.006793939974159002, -0.034518443048000336, 0.0017834092723205686, 0.003913048189133406, 0.008479359559714794, -0.025895364582538605, 0.017990875989198685, -0.008649208582937717, 0.016514500603079796, 0.03318578377366066, -0.003948977682739496, 0.0209174957126379, 0.00459244241937995, -0.005418820306658745, 0.015848170965909958, 0.024458184838294983, 0.013836120255291462, -0.016436109319329262, 0.017559722065925598, -0.02780289389193058, -0.009452722035348415, 0.004402995575219393, -0.017324546352028847, 0.00026947117294184864, 0.024837076663970947, -0.002519963774830103, 0.0781303122639656, 0.032114431262016296, -0.013346172869205475, -0.008048205636441708, -0.01863107457756996, 0.012732105329632759, 0.006944190710783005, -0.013953708112239838, -0.002911921823397279, -0.018173789605498314, 0.014894407242536545, 0.0022488594986498356, -0.010177845135331154, -0.023491354659199715, -0.01867027021944523, -0.016945654526352882, -0.014215013012290001, 0.014868276193737984, -0.011144674383103848, 0.005810778588056564, 0.029475247487425804, -0.025620993226766586, 0.00859694741666317, 0.003282648976892233, -0.025660188868641853, 0.010275834240019321, 0.022237088531255722, 0.013927577063441277, -0.03773249685764313, -0.0073165507055819035, -0.018800921738147736, 0.009785886853933334, -0.031722474843263626, 0.0021100409794598818, -0.010497943498194218, 0.00928940623998642, 0.0065424335189163685, -0.02699284628033638, 0.002942951861768961, 0.016762740910053253, 0.010445683263242245, 0.014881341718137264, -0.03801993280649185, -0.050902288407087326, -0.023399896919727325, 0.013346172869205475, -0.02516370825469494, 0.0018960972083732486, -0.03219282254576683]} +{"id": "test:10000023", "text": "\"Earn monthly interest on our Citibank Time Deposits (also known as Fixed Deposits). What's more, you get to enjoy the flexibility of making partial withdrawals before maturity date of your Time Deposit.\\nPartial withdrawals in multiples of RM5,000 before the maturity date.\\nOption to pledge your Time Deposit for overdraft facility.\\nContinue to earn the original interest rate with your remaining balance.\\nView your Citibank\u00ae Time Deposit accounts across participating countries with a single login via Online Global View.\\nOnline account inquiries and banking transactions via Citibank\u00ae or Citigold\u00ae Online.\\nMake banking inquiries anytime, anywhere via live e-Chat, 'Call Me' function and 20 seconds1 call pickup.\\nEligible for protection by PIDM**.\\n* Monthly interest is only available for minimum deposit of RM50,000 and for a minimum tenure of 6 months.\\n** Click here for more details on PIDM protection.\\n1From the moment you press 0 to speak to a CitiPhone officer.\\nFor the Citibank Account Terms and Conditions, click here.\"", "vector_field": [-0.01122037973254919, -0.013122820295393467, 0.018327737227082253, -0.02939404547214508, -0.029956739395856857, 0.02207902818918228, -0.047802165150642395, -0.02325800433754921, -0.03689662739634514, -0.03271661698818207, 0.009793548844754696, 0.027679169550538063, -0.025790126994252205, 0.01448266301304102, -0.019011007621884346, 0.012848172336816788, 0.015929589048027992, -0.008694957010447979, 0.027732759714126587, -0.004853232763707638, -0.025790126994252205, 0.009726561605930328, -0.02700929529964924, 0.007087260484695435, -0.005094387102872133, 0.010403133928775787, 0.0019543557427823544, -0.0029876353219151497, -0.0036709061823785305, 0.016693245619535446, 0.0015055404510349035, -0.016264526173472404, -0.024035057052969933, -0.011957240290939808, -0.006008764263242483, 0.006839407607913017, -0.009873934090137482, -0.0008218509610742331, 0.01985504850745201, 0.009250951930880547, 0.02341877482831478, 0.007154248189181089, -0.0020665593910962343, 0.010851949453353882, -0.01870286464691162, 0.006564759183675051, -0.0009043291211128235, -0.006648493465036154, -0.035422906279563904, 0.023726915940642357, 0.0048800278455019, 0.003486691042780876, -0.0033493670634925365, 0.005121182184666395, -0.028670581057667732, -0.022065630182623863, -0.013417564332485199, 0.010128485970199108, -0.007107357028871775, -0.009003098122775555, 0.009820343926548958, 0.012318971566855907, 0.006236521527171135, -0.00944521464407444, -0.018367929384112358, -0.007810723967850208, 0.006521217525005341, -0.006675288546830416, -0.01737651601433754, -0.003891964443027973, 0.017657862976193428, 0.025267625227570534, 0.023271402344107628, 0.015206126496195793, 0.016880810260772705, -0.004863280802965164, -0.017751645296812057, -0.004424513783305883, 0.02548198401927948, 0.016961194574832916, 0.001442739856429398, -0.005328842904418707, -0.007830820046365261, 0.01024236436933279, 0.012781184166669846, -0.006136040203273296, 0.02343217097222805, 0.02628583274781704, 0.0005296186427585781, -0.009405022487044334, 0.0058212000876665115, 0.017416708171367645, 0.008065275847911835, 0.023284800350666046, 0.016291320323944092, 0.012245286256074905, -0.008487296290695667, 0.00372114684432745, -0.016331514343619347, -0.004890075884759426, 0.0007937999907881021, 0.024544160813093185, -0.010744769126176834, -0.011361053213477135, -0.02662076987326145, 0.000919401238206774, -0.0001247639156645164, -0.030010327696800232, 0.02368672378361225, -0.016063563525676727, -0.009652875363826752, 0.013350577093660831, 0.009793548844754696, -0.05584064871072769, 0.02262832410633564, -0.005703972186893225, -0.013866379857063293, 0.0007414661231450737, -0.023191018030047417, -0.0017634417163208127, -0.0014134328812360764, 0.01069787796586752, 0.035771239548921585, 0.002679493511095643, 0.002061535371467471, -0.014013751409947872, 0.011106501333415508, -0.007663351949304342, 0.0019459822215139866, 0.007562870625406504, -0.029179684817790985, 0.0038752176333218813, 9.032824164023623e-05, 0.0057877060025930405, -0.01914498209953308, 0.020337356254458427, -0.01638510264456272, -0.0038819164037704468, -0.03344007954001427, -0.03510136529803276, 0.03804881125688553, 0.03266302868723869, 0.0044948505237698555, -0.005328842904418707, 0.016344910487532616, 0.015701832249760628, 0.0037747365422546864, 0.009250951930880547, 0.0025555670727044344, -0.014469265937805176, 0.014228111132979393, 0.01647888496518135, 0.02067229337990284, -0.007234632968902588, -0.021074216812849045, -0.002836913801729679, -0.010563903488218784, 0.017497092485427856, -0.014469265937805176, -0.007120754104107618, 0.0057240682654082775, 0.024182429537177086, -0.004277141764760017, 0.013183108530938625, 0.02040434442460537, 0.02805429883301258, -0.01701478473842144, -0.0005848832079209387, 0.01879664696753025, 0.02341877482831478, 0.03384200483560562, 0.01655927114188671, -0.02939404547214508, 0.023458966985344887, -0.011642400175333023, 0.011334258131682873, 0.0077772303484380245, -0.016438692808151245, -0.01996222697198391, -0.00939162541180849, 0.017778439447283745, 0.0018019594717770815, 0.020015817135572433, 0.02912609651684761, -0.007998288609087467, 0.005841296166181564, 0.016612861305475235, 0.021342167630791664, 0.02537480555474758, -0.01527311373502016, -0.020457934588193893, 0.023458966985344887, 0.021074216812849045, -0.015259716659784317, -0.6409348845481873, -6.955170101718977e-05, -0.0051446277648210526, -0.01051031332463026, -0.0021553176920861006, -0.00034707816666923463, -0.0005472028278745711, 0.001547407591715455, -0.0019007658120244741, -0.00249360385350883, -0.005308746825903654, 0.008668161928653717, -0.006343701388686895, -0.027732759714126587, -0.007536076009273529, -0.024303007870912552, -0.025803523138165474, -0.015433883294463158, -0.0344046987593174, 0.012111310847103596, -0.010597397573292255, 0.017966004088521004, -0.024195827543735504, 0.010677781887352467, 0.022601528093218803, -0.03606598451733589, -0.023110631853342056, -0.011963939294219017, 0.002326135290786624, 0.03820957988500595, -0.001873970846645534, 0.000764911703299731, 0.010845250450074673, 0.004648921545594931, 0.04177330434322357, -0.013919969089329243, 0.0024333151523023844, 0.009686369448900223, 0.007382004987448454, 0.0344046987593174, -0.00224910001270473, -0.0038417240139096975, 0.009927524253726006, -0.02672794833779335, 0.029956739395856857, 0.004618776962161064, 8.127448381856084e-05, 0.008004987612366676, -0.018032992258667946, 0.002959165722131729, 0.011448136530816555, -0.004863280802965164, -0.014589842408895493, -0.027866734191775322, 0.009056688286364079, -0.005044146906584501, 0.014951574616134167, 6.013160600559786e-05, 0.015179331414401531, 0.0004276722902432084, -0.013531442731618881, -0.009994511492550373, -0.02893853187561035, -0.023673325777053833, -0.026326024904847145, 0.010965827852487564, -0.008674860931932926, 0.0015457328408956528, 0.029286865144968033, -0.010128485970199108, 0.003821627702564001, 0.027947118505835533, -0.006879599764943123, -0.021931655704975128, 0.00394555414095521, 0.018367929384112358, 0.04407766833901405, -0.003851772053167224, 0.007469088304787874, 0.016358308494091034, 0.003657508874312043, -0.029983533546328545, 0.012613716535270214, 0.009786850772798061, 0.01728273369371891, 0.017162157222628593, 0.005405878182500601, -0.008306430652737617, 0.026580575853586197, 0.0036709061823785305, 0.015728628262877464, -0.009378227405250072, 0.018247351050376892, -0.01925216242671013, -0.025428393855690956, 0.009197361767292023, -0.0018321038223803043, -0.016425296664237976, 0.005811151582747698, -0.003381185932084918, -0.011160090565681458, 0.004936967045068741, -0.0218378733843565, 0.018314339220523834, 0.027290642261505127, -0.009096880443394184, 0.0011128272162750363, -0.017497092485427856, 0.04493510723114014, -0.025106854736804962, -0.0002304783120052889, -0.026955705136060715, 0.009036592207849026, -0.017041578888893127, -0.014348688535392284, -0.026500191539525986, 0.023941274732351303, -0.009458612650632858, 0.005643683485686779, -0.02103402465581894, -0.027545195072889328, 0.008668161928653717, 0.011253872886300087, 0.002761553041636944, -0.013518045656383038, 0.018247351050376892, 0.007073862943798304, 0.003972349222749472, -0.0012225189711898565, 0.006390592083334923, -0.014630035497248173, 0.015567857772111893, 0.007670050486922264, -0.020739281550049782, 0.002331159543246031, 0.009585888125002384, 0.017858825623989105, -0.009143771603703499, -0.004303936846554279, -0.04003163427114487, -0.006631746888160706, -0.018287543207406998, 0.009672972373664379, -0.011816566810011864, 0.009317939169704914, -0.038263168185949326, 0.00819255132228136, -0.0011973987566307187, 0.0024550859816372395, -0.0023345088120549917, -0.010443326085805893, -0.0065480126067996025, -0.009130374528467655, 0.011146693490445614, -0.018850237131118774, -0.03670906275510788, 0.00868155900388956, -0.01702818274497986, -0.03349367156624794, 0.013283589854836464, -0.012118009850382805, 0.0022206304129213095, -0.016974592581391335, -0.029822764918208122, -0.009063387289643288, 0.006243220064789057, 0.016076961532235146, 0.0043474785052239895, 0.004685764666646719, -0.028456222265958786, -0.002232353202998638, 0.01286826841533184, -0.018046390265226364, 0.01207111869007349, 0.010342845693230629, 0.006608301308006048, -0.025535574182868004, -0.010758167132735252, -0.008460501208901405, -0.00677242036908865, 0.006186280865222216, -0.024999676272273064, -0.005804453045129776, -0.03242187201976776, 0.031216099858283997, -0.0033141986932605505, -0.01576882041990757, 0.034699443727731705, -0.03234148770570755, 0.02530781738460064, 0.0009595936280675232, 0.009405022487044334, -0.00019551930017769337, 0.016344910487532616, 0.0004927756381221116, 0.000312537798890844, 0.007167645264416933, -2.757994479907211e-05, 0.013069230131804943, 0.008172455243766308, 0.019399533048272133, 0.006775769405066967, 0.01709516905248165, 0.008058576844632626, -0.00032719128648750484, -0.0001232985669048503, 0.006099197547882795, -0.023217812180519104, 0.020685691386461258, 0.022601528093218803, 0.016371706500649452, -0.015380293130874634, -0.0022624973207712173, 0.008165757171809673, 0.017497092485427856, 9.864932508207858e-05, -0.014321893453598022, -0.0024215923622250557, -0.03081417642533779, 0.01851530186831951, 0.004746052902191877, -0.011012719012796879, 0.010590698570013046, -0.012332369573414326, -0.00405273400247097, 0.018863635137677193, 0.023901082575321198, -0.008393513970077038, -0.0022407264914363623, -0.023646531626582146, -0.030867766588926315, -0.003935506101697683, -0.016344910487532616, 0.011649098247289658, 0.02254793979227543, 0.03788803890347481, 0.032019950449466705, -0.012399356812238693, 0.02636621706187725, -0.004936967045068741, -0.0053053973242640495, 0.03448508307337761, 0.02305704355239868, -0.0297691747546196, 0.006521217525005341, -0.007408800069242716, 0.007281524129211903, 0.03472623601555824, -0.008165757171809673, -0.0030579720623791218, -0.014107533730566502, 0.02280249074101448, -0.027196859940886497, -0.012533331289887428, 0.014509458094835281, -0.0042369491420686245, 0.010979224927723408, -0.004749402403831482, 0.019734470173716545, 0.028188273310661316, 0.019814854487776756, 0.012941954657435417, 0.03199315443634987, -0.01881004497408867, 0.031671613454818726, 0.02102062664926052, 0.03378841280937195, 0.006973382085561752, -0.001359843066893518, -0.005904933903366327, -0.004377622622996569, 0.0009562443010509014, 0.017175553366541862, -0.009907427243888378, 0.024209225550293922, -0.031242895871400833, 0.011300764046609402, 0.009465311653912067, 0.0017466949066147208, -0.014938176609575748, -0.007656652946025133, -0.052705638110637665, 0.011468232609331608, 0.010905538685619831, 0.0028620341327041388, -0.0201363954693079, -0.020377548411488533, -0.023740313947200775, 0.005352288484573364, 0.02770596370100975, -0.017229143530130386, -0.014830997213721275, 0.015728628262877464, 0.012365862727165222, -0.021154602989554405, -0.02191825769841671, 0.0013363974867388606, -0.015969781205058098, 0.004719258286058903, -0.012084516696631908, 0.006531265564262867, 0.004983858205378056, 0.011769675649702549, -0.01755068264901638, 0.00819925032556057, -0.01887703314423561, -0.02896532602608204, -0.01745690032839775, -0.006360447965562344, -0.001847175881266594, 0.0011521822307258844, -0.02207902818918228, -0.003989096265286207, 0.012499838136136532, 0.0071006580255925655, -0.007194440346211195, -0.013598429970443249, -0.006383893545717001, 0.011917048133909702, 0.00449820002540946, 0.0026526986621320248, 0.011428040452301502, -0.039334967732429504, 0.007020273245871067, 0.08472558856010437, -0.0003414261154830456, -0.0030914656817913055, 0.00724133150652051, -0.028027502819895744, -0.025495382025837898, 0.008641366846859455, -0.04051394388079643, 0.010202172212302685, 0.03413674980401993, 0.0069666835479438305, -0.030974946916103363, -0.02308383770287037, 0.014496060088276863, -0.021154602989554405, 0.0018689468270167708, -0.015326703898608685, 0.0082930326461792, 0.013283589854836464, 0.003952253144234419, 0.009860536083579063, 0.014656830579042435, 0.013946764171123505, 0.02893853187561035, 0.025696344673633575, 0.007120754104107618, 0.038879454135894775, 0.01621093600988388, 0.021261781454086304, -0.016505680978298187, 0.010262460447847843, -0.006661891005933285, 0.006584855727851391, 0.007670050486922264, 0.002232353202998638, 0.022320181131362915, -0.006377195008099079, -0.008614571765065193, -0.0032656327821314335, -0.012406055815517902, -0.00106342404615134, 0.03177879378199577, -0.007187741808593273, -0.017430106177926064, 0.009612683206796646, -0.013424263335764408, 0.030760586261749268, 0.0112270787358284, -0.006249919068068266, 0.006249919068068266, 0.02449057251214981, 0.0011747904354706407, -0.027571989223361015, -0.013852981850504875, 0.013163012452423573, 0.030519433319568634, -0.02094024233520031, -0.03778085857629776, 0.010007908567786217, -0.030358662828803062, -0.03073379211127758, -0.03196635842323303, 0.015232921577990055, 0.0016931050922721624, 0.006447531748563051, -0.00178688729647547, -0.008875822648406029, 0.0028737569227814674, -0.009163868613541126, 0.0013188133016228676, -0.002980936551466584, 0.012941954657435417, -0.012218491174280643, -0.005482913926243782, -0.00034812482772395015, 0.016907604411244392, 0.010490217246115208, 0.00576426088809967, -0.007502581924200058, 0.05123191699385643, -0.0065614101476967335, -0.008219346404075623, -0.014308496378362179, -0.01158211100846529, -0.028375837951898575, 0.00966627337038517, 0.017966004088521004, -0.01170938741415739, -0.00724133150652051, 0.029447635635733604, -0.015139139257371426, -0.003503437852486968, 0.02128857746720314, -0.03065340779721737, -0.014040546491742134, 0.0009185639210045338, 0.03293097764253616, -0.007609761785715818, 0.012359164655208588, 0.006092498544603586, -0.008152359165251255, 0.014777407050132751, -0.020779473707079887, -0.010034703649580479, 0.009364830330014229, 0.017751645296812057, 0.02493268810212612, 0.01950671337544918, 0.024276211857795715, 0.01505875401198864, -0.004357526544481516, -0.01747029833495617, 0.024128839373588562, 0.010590698570013046, -0.02039094641804695, 0.023820698261260986, 0.0012694101314991713, 0.024611148983240128, 0.010597397573292255, 0.0009445215109735727, 0.0026259038131684065, -0.010396434925496578, 0.02023017778992653, -0.007207837887108326, 0.005606840364634991, 0.004136468283832073, 0.006872901227325201, -0.06216425076127052, -0.009230855852365494, -0.01941293105483055, -0.004364225082099438, 0.0033929087221622467, -0.02005600929260254, -0.02707628346979618, -0.00945191364735365, 0.01527311373502016, -0.013176409527659416, -0.0028653834015130997, -0.03349367156624794, 0.014777407050132751, -0.003691002493724227, 0.001442739856429398, -0.004109673202037811, 9.488128853263333e-05, 0.016076961532235146, -0.02671455219388008, -0.012888364493846893, 0.0028687329031527042, 0.009780151769518852, 0.034243930131196976, 0.012573523446917534, 0.006487723905593157, -0.0054159266874194145, 0.009927524253726006, 0.020069407299160957, -0.020283766090869904, 0.0027079633437097073, -0.00549966050311923, 0.030626611784100533, 0.03713778033852577, 0.011421341449022293, -0.002121824072673917, 0.024276211857795715, 0.010630890727043152, 0.028081092983484268, 0.007991589605808258, 0.016331514343619347, -0.008473898284137249, 0.002766577061265707, 0.012339068576693535, -0.0022775696124881506, -0.0039020124822854996, 0.012124708853662014, -0.016170743852853775, -0.03263623267412186, 0.011917048133909702, -0.014402278698980808, -0.012091214768588543, -0.04520305618643761, -0.01914498209953308, -0.010235665366053581, 0.017872221767902374, -0.008125564083456993, -0.016505680978298187, -0.026768140494823456, -0.021784283220767975, 0.011461533606052399, 0.0026459998916834593, 0.020471330732107162, -0.004893425386399031, 0.01950671337544918, -0.016545873135328293, 0.034779828041791916, -0.0014996790559962392, 0.0040125418454408646, 0.038879454135894775, -0.013826186768710613, -0.028670581057667732, -0.006769070867449045, 0.0315108448266983, 0.012680703774094582, 0.01889042928814888, 0.010249063372612, 0.000660243968013674, -0.02458435483276844, -0.02396807074546814, -0.034243930131196976, -0.0021737392526119947, -0.00877534132450819, 0.007013574708253145, -0.03169840946793556, -0.013296986930072308, -0.023030247539281845, 0.024249417707324028, 0.03402956947684288, 0.008440405130386353, -0.014107533730566502, 0.028188273310661316, -0.016050167381763458, -0.009297843091189861, -0.009679670445621014, -0.016452090814709663, 0.040996253490448, -0.011850060895085335, 0.001124550006352365, 0.007315017748624086, 0.02505326457321644, -0.010309351608157158, 0.016505680978298187, -0.011823265813291073, -0.0014209689106792212, 0.01421471405774355, 0.03925458341836929, -0.01301563996821642, -0.020511522889137268, 0.016264526173472404, 0.009003098122775555, -0.028402632102370262, -0.003011080902069807, 0.0069398884661495686, 0.012305574491620064, -0.008748546242713928, -0.005523106083273888, 0.01417452096939087, 0.005372384563088417, 0.011876855045557022, -0.016720039770007133, 0.011046212166547775, -0.022949863225221634, -0.0065748076885938644, -0.006638445425778627, 0.04268433526158333, -0.012600318528711796, 0.024008262902498245, -0.01674683578312397, 0.004380972124636173, -0.027397822588682175, -0.011428040452301502, -0.03298456594347954, 0.019734470173716545, -0.013611827977001667, 0.01578221656382084, 0.0025589163415133953, -0.01335727609694004, 0.011059610173106194, -0.010309351608157158, -0.011421341449022293, -0.0006066540954634547, 0.007857615128159523, 0.0221996046602726, 0.005801103543490171, -0.019359340891242027, 0.007382004987448454, 0.008427007123827934, 0.0038149291649460793, 0.0011513449717313051, -0.012821377255022526, 0.0024718327913433313, -0.0035536785144358873, -0.0181803647428751, 0.016961194574832916, 0.02012299746274948, 1.9337361663929187e-05, -0.023365184664726257, 0.00043123101931996644, -0.003865169594064355, -0.007000177167356014, -0.014053944498300552, 0.026218844577670097, -0.02717006579041481, -0.02138235978782177, 0.029340455308556557, -0.02182447537779808, 0.014094136655330658, -0.013524743728339672, -0.007864314131438732, 0.02816147729754448, 0.029367249459028244, -0.0032823795918375254, 0.007294921204447746, 0.0053690355271101, 0.01417452096939087, -0.013839584775269032, -0.008661462925374508, -0.00188234425149858, -0.0261116661131382, 0.014830997213721275, -0.0046556200832128525, -0.003253909992054105, 0.02139575593173504, -0.005804453045129776, 0.001429342431947589, -0.018032992258667946, 0.007723640184849501, -0.002667770953848958, 0.02950122393667698, 1.835610055422876e-05, 0.018850237131118774, -0.03156443312764168, 0.024651341140270233, -0.002622554311528802, -0.010215569287538528, 0.022842682898044586, -0.008386814966797829, 0.030090713873505592, -0.02734423242509365, 0.020364152267575264, 0.011970637366175652, -0.029715584591031075, -0.02834904193878174, -0.013069230131804943, 0.008916014805436134, -0.016170743852853775, 0.0017718151211738586, -0.007187741808593273, 0.002883804962038994, -0.02442358434200287, 0.043354205787181854, -0.010798359289765358, -0.001342258881777525, 0.0022206304129213095, 0.022025438025593758, 0.0024115443229675293, 0.0014017100911587477, 0.0247719194740057, 0.005298698786646128, -0.010074895806610584, -0.02005600929260254, -0.018207158893346786, -0.004608728922903538, -0.00823274441063404, 0.054045386612415314, 0.024450378492474556, -0.027947118505835533, -0.03510136529803276, 0.00574416434392333, -0.01783202961087227, 0.0047527519054710865, -0.009592587128281593, 0.03003712370991707, 0.03038545697927475, -0.0015390341868624091, 0.011394546367228031, 0.03019789233803749, -0.007214536424726248, 0.030144304037094116, -0.0375932939350605, 0.009619382210075855, 0.012519934214651585, -0.024637943133711815, 0.01408073864877224, -0.011260571889579296, -0.043675746768713, 0.019493315368890762, 0.0025739886332303286, 0.011950541287660599, 0.01567503809928894, 0.01603676937520504, 0.010543807409703732, -0.012365862727165222, 0.024369994178414345, 0.004859931766986847, 0.006166184786707163, 0.011206981725990772, -0.025348009541630745, -0.03861150145530701, 0.005610189866274595, 0.019011007621884346, -0.014549650251865387, -0.01006819773465395, 0.003607268212363124, 0.0069800810888409615, -0.00801168568432331, 0.0029692137613892555, -0.01389317400753498, -0.010463422164320946, 0.00434077950194478, -0.015728628262877464, 0.015474075451493263, 0.02209242433309555, 0.008708354085683823, -0.0029641897417604923, -0.0052350605838000774, -0.007609761785715818, -0.025080060586333275, 0.03057302162051201, -0.014777407050132751, -0.029903149232268333, 0.0079313013702631, -0.03464585170149803, 0.04697152227163315, -0.012332369573414326, 0.025160444900393486, 0.003560377052053809, 0.010202172212302685, -0.013906572014093399, -0.02920648083090782, -0.0008289683610200882, 0.014549650251865387, 0.005362336523830891, -0.01791241578757763, 0.01587599888443947, 0.00783751904964447, -0.012171600013971329, 0.004826437681913376, 0.0029742380138486624, -0.02182447537779808, -0.030974946916103363, 0.003600569674745202, 0.005660430062562227, 0.014871189370751381, 0.006645143963396549, -0.0030043821316212416, 0.01914498209953308, -0.005322144366800785, 9.247392881661654e-05, 0.21039384603500366, 0.0168406181037426, 0.016251128166913986, 0.024570956826210022, 0.004756101407110691, 0.017577478662133217, 0.011883554048836231, 0.0004684927116613835, -0.0014963296707719564, 0.0043977187015116215, -0.011166789568960667, 0.01576882041990757, 0.0038618200924247503, -0.00394555414095521, -0.01843491569161415, -0.020538318902254105, -0.01676023192703724, -0.03820957988500595, -0.027732759714126587, 0.008065275847911835, -0.020551716908812523, -0.009492105804383755, -0.01931914873421192, 0.010831853374838829, -0.0018689468270167708, -0.0030529480427503586, 0.022775696590542793, -0.021181397140026093, 0.042443178594112396, 0.007964794524013996, -0.011461533606052399, -0.005442721303552389, -0.004153215326368809, -0.005781007464975119, -0.01914498209953308, -0.003778086043894291, -0.02128857746720314, -0.0168406181037426, 0.014804202131927013, -0.002496953122317791, -0.0032957771327346563, 0.013826186768710613, -0.034699443727731705, -0.03196635842323303, -0.001045839861035347, 0.03571765124797821, 0.01594298705458641, -0.03628034517168999, -0.005911632906645536, 0.006192979868501425, -0.0322343073785305, -0.019989022985100746, 0.006641794927418232, -0.012225189246237278, 0.004474754445254803, 0.001800284837372601, -0.014496060088276863, -0.0013916620519012213, -0.01666644960641861, 0.006430784706026316, -0.013571634888648987, 0.009920825250446796, -0.0059819696471095085, -0.003212043084204197, -0.0315108448266983, 0.017751645296812057, -0.007522678468376398, 0.013919969089329243, 0.023767108097672462, -0.0023026899434626102, 0.011716085486114025, -0.003255584742873907, -0.009746657684445381, 0.0038718683645129204, -0.02138235978782177, -0.003948903642594814, 0.01170938741415739, 0.034431494772434235, 0.02904571034014225, 0.012513235211372375, -0.0065748076885938644, 0.004079529084265232, -0.006725529208779335, 0.004119721241295338, 0.013022338971495628, -0.03555687889456749, 0.02439679019153118, -0.024704931303858757, -0.01649228297173977, -0.007649954408407211, 0.0024718327913433313, -0.02939404547214508, -0.005285301245748997, 0.007582967169582844, -0.006537964567542076, 0.019814854487776756, 0.015608049929141998, 0.011448136530816555, -0.006109245587140322, -0.010222268290817738, -0.00224910001270473, 0.03820957988500595, 0.010456724092364311, 0.008092070929706097, 0.00392210902646184, 0.009672972373664379, -0.021945053711533546, -0.0026041329838335514, -0.007214536424726248, -0.03724496066570282, 0.013732404448091984, -0.018327737227082253, 0.011461533606052399, -0.010490217246115208, -0.031162511557340622, 0.0034799922723323107, -0.007522678468376398, -0.005992017686367035, -0.0003433101228438318, 0.008795437403023243, 0.013397468253970146, -0.030626611784100533, 0.012774486094713211, 0.006162835285067558, -0.015902794897556305, -0.029715584591031075, -0.013029037974774837, -0.0031082124914973974, -0.007737037725746632, -0.05168743059039116, -0.012854870408773422, -0.019493315368890762, 0.02529441937804222, -0.016277924180030823, 0.00015700157382525504, 0.01546067837625742, 0.028000708669424057, 0.0010073221055790782, -0.0037747365422546864, 0.021489538252353668, -0.011106501333415508, 0.0035670758225023746, -0.01323669869452715, -0.0195201113820076, 0.014094136655330658, -0.019386136904358864, -0.006521217525005341, 0.015045356936752796, -0.020699087530374527, -0.0018856936367228627, -0.00685950368642807, -0.045738957822322845, 0.014308496378362179, -0.005442721303552389, 0.01763106882572174, 0.010751468129456043, -0.027571989223361015, -0.012205093167722225, 0.011474931612610817, -0.024356596171855927, -0.024249417707324028, -0.04855242371559143, 0.002095028990879655, 0.004401068203151226, 0.002217280911281705, 0.024075251072645187, -0.16998708248138428, 0.020806267857551575, 0.01381278969347477, -0.012111310847103596, 0.018863635137677193, 0.016907604411244392, 0.01131416205316782, -0.013189807534217834, -0.01816696673631668, -0.013672116212546825, 0.020203381776809692, -0.00859447568655014, -0.022320181131362915, -0.021958449855446815, 0.01051031332463026, 0.031242895871400833, -0.020993832498788834, 0.016157345846295357, -0.0018923924071714282, 0.014254906214773655, 0.023552749305963516, -0.00828633364289999, 0.006099197547882795, -0.03177879378199577, 0.029822764918208122, 0.034163542091846466, -0.021502936258912086, 0.0107380710542202, 0.0148845873773098, -0.03662867844104767, 0.002374701201915741, 0.02360633946955204, 0.01359173096716404, 0.0324486680328846, 0.0036876529920846224, 0.005429323762655258, -0.0028436125721782446, -0.027839938178658485, 0.002131872111931443, 0.012995543889701366, 0.023472364991903305, 0.006698734126985073, -0.005975270643830299, -0.002687867032364011, 0.002761553041636944, 0.035047776997089386, 0.012372561730444431, -0.020900050178170204, -0.001712363911792636, -0.008440405130386353, 0.006166184786707163, -0.015353498049080372, 0.019345944747328758, 0.00392210902646184, 0.010717974044382572, -0.0035067873541265726, 0.014643432572484016, -0.01189025305211544, 0.022213002666831017, -0.004350828006863594, 0.009498804807662964, -0.005077640525996685, -0.0009252626332454383, -0.0297691747546196, 0.014268303290009499, -0.03057302162051201, 0.003093140432611108, -0.005268554203212261, -0.01595638506114483, 0.02628583274781704, -0.010845250450074673, -0.019935432821512222, 0.007408800069242716, 0.004689113702625036, 0.009907427243888378, -0.00706046586856246, -0.03654829412698746, 0.0043910201638937, 0.0017232493264600635, 0.007234632968902588, -0.0241556353867054, 0.04019240289926529, -0.02112780697643757, 0.03175199776887894, -0.014897984452545643, -0.005580045282840729, 0.02139575593173504, -0.002989310072734952, 0.011481630615890026, -0.020739281550049782, 0.010684480890631676, 0.004792944062501192, -0.017711453139781952, 0.007020273245871067, -0.020632101222872734, 0.015742024406790733, -0.018381325528025627, -0.0038819164037704468, -0.009378227405250072, 0.001803634106181562, 0.031108921393752098, -0.01674683578312397, -0.015125741250813007, 0.012814678251743317, 0.027491604909300804, 0.022655118256807327, 0.011783073656260967, 0.009157169610261917, 0.02664756402373314, -0.002676144242286682, -0.016050167381763458, 0.013745802454650402, 0.007562870625406504, 0.028804557397961617, -0.007067164406180382, 0.013116121292114258, -0.01791241578757763, -0.026500191539525986, 0.009137073531746864, -0.0012895063264295459, 0.025535574182868004, -0.028027502819895744, -0.032019950449466705, 0.01925216242671013, -0.013397468253970146, -0.024825507774949074, -0.1069117933511734, -0.013946764171123505, 0.008480597287416458, 0.025254227221012115, 0.0015616423916071653, 0.008581078611314297, 0.005727417767047882, 0.007710243109613657, 0.024436982348561287, 0.008400212973356247, -0.02505326457321644, -0.014375483617186546, -0.010610794648528099, -0.002597434213384986, 0.01536689605563879, -0.018113376572728157, 0.023137427866458893, -0.0028771061915904284, -0.008319827727973461, 0.024530764669179916, 0.006712131667882204, -0.028750967234373093, -0.024008262902498245, -0.012412753887474537, -0.02139575593173504, -0.029072506353259087, -0.040219198912382126, 0.006809263024479151, 0.006986779626458883, 0.02628583274781704, 0.01674683578312397, 0.022748900577425957, -0.0069398884661495686, 0.010302652604877949, -0.019841650500893593, 0.008694957010447979, -0.024892495945096016, -0.012928556650876999, 0.031323280185461044, -0.020189983770251274, -0.0013933366863057017, 0.020283766090869904, 0.02227998897433281, -0.023834096267819405, 0.021958449855446815, -0.008493994362652302, -0.014911382459104061, -0.010275858454406261, -0.01702818274497986, -0.02885814569890499, -0.009947620332241058, -0.017443504184484482, -0.03038545697927475, -0.01578221656382084, 0.019278956577181816, 0.012044323608279228, -0.013585032895207405, 0.02227998897433281, -0.014991766773164272, -0.016987988725304604, -0.019225366413593292, -0.025013072416186333, -0.012988844886422157, -0.009505503810942173, 0.009043291211128235, -0.019828252494335175, -0.02557576633989811, 0.007683448027819395, 0.01613055169582367, -0.011474931612610817, 0.0003384954179637134, 0.01994883082807064, -0.006872901227325201, -0.013933367095887661, -0.021261781454086304, 0.0165860652923584, -0.002043113810941577, -0.035583674907684326, -0.019654085859656334, 0.00706046586856246, -0.021596718579530716, -0.021690500900149345, -0.014911382459104061, -0.031242895871400833, -0.001451113261282444, 0.006075751967728138, 0.02950122393667698, 0.0034699442330747843, 0.01897081546485424, -0.04426523298025131, 0.012325670570135117, 0.014804202131927013, 0.0002631346578709781, 0.007576268166303635, 0.010865346528589725, 0.024128839373588562, 0.009425118565559387, -0.015822410583496094, 0.023740313947200775, 0.01296205073595047, -0.028670581057667732, -0.021958449855446815, -0.05868091061711311, 0.024718329310417175, -0.006762371864169836, -0.005660430062562227, 6.740601384080946e-05, -0.010845250450074673, -0.015353498049080372, -0.0002861615503206849, -0.01674683578312397, 0.017577478662133217, -0.030412252992391586, 0.0024165683425962925, -0.0006372170755639672, 0.007582967169582844, -0.02778634801506996, -0.03644111379981041, 0.025723138824105263, 0.006008764263242483, 0.044586773961782455, 0.01657266728579998, -0.02734423242509365, -0.018220556899905205, 0.026500191539525986, 0.024825507774949074, -0.006728878244757652, 0.021087614819407463, -0.023124029859900475, 0.012660607695579529, 0.0036273645237088203, -0.007462389767169952, 0.028724171221256256, -0.0018505252664908767, 0.012111310847103596, -0.0030228036921471357, -0.003131658071652055, -0.020779473707079887, 0.006199678406119347, 0.020618703216314316, 0.007917903363704681, -0.016277924180030823, -0.016693245619535446, -0.01780523546040058, 0.01665305346250534, -0.011267270892858505, -0.022132616490125656, -0.014871189370751381, -0.0130558330565691, 0.01130746304988861, 0.010322749614715576, -0.00984044000506401, 0.031591229140758514, -0.015621447935700417, -0.004156564362347126, -0.020377548411488533, 0.007308318745344877, -0.004933617543429136, 0.013973559252917767, 0.002540494780987501, 0.009003098122775555, -0.025615958496928215, 0.029903149232268333, 0.0031400315929204226, 0.016545873135328293, -0.03788803890347481, 0.030706996098160744, 0.010624192655086517, -0.01024236436933279, 0.003424727823585272, 0.0010207196464762092, -0.015139139257371426, -0.006035559345036745, -0.016626257449388504, 0.024477174505591393, -0.007676749024540186, 0.03421713411808014, -0.0056269364431500435, 0.005358987022191286, -0.010456724092364311, -0.011454835534095764, 0.0001974033220903948, 0.018555494025349617, 0.0012752715265378356, -0.01996222697198391, 0.01033614668995142, 0.024798713624477386, 0.03349367156624794, 0.0162243340164423, 0.0053053973242640495, -0.024303007870912552, -0.006069052964448929, 0.0032086935825645924, 0.010483519174158573, -0.014134328812360764, -0.008467200212180614, 0.0006171208806335926, 0.03528892993927002, -0.0025471935514360666, -0.02896532602608204, -0.015299908816814423, 0.03333289921283722, 0.044318825006484985, -0.01772485114634037, -0.014335290528833866, -0.01515253633260727, -0.013551538810133934, -0.010470121167600155, -0.0015398714458569884, -0.03486021235585213, 0.006866202224045992, 0.035583674907684326, 0.01234576664865017, 0.011501726694405079, 0.00534893898293376, 0.015219523571431637, -0.006018812768161297, -0.0058982353657484055, 0.01851530186831951, -0.017751645296812057, -0.016894206404685974, 0.027156667783856392, -0.01082515437155962, 0.01898421160876751, 0.03126969188451767, -0.018662672489881516, 0.021931655704975128, -0.0044948505237698555, 0.023860890418291092, -0.01247304305434227, 0.014496060088276863, 0.0020900049712508917, -0.01931914873421192, 0.030519433319568634, -0.004149865824729204, -0.02136896178126335, -0.0040761795826256275, -0.0181803647428751, 0.003660858143121004, 0.023191018030047417, 0.006246569566428661, 0.07266786694526672, 0.006293460726737976, -0.008976303972303867, 0.002600783482193947, -0.007073862943798304, 0.015916192904114723, -0.008849027566611767, 0.009465311653912067, -0.018220556899905205, -0.019546905532479286, -0.0024383391719311476, -0.012513235211372375, -0.0005375734181143343, 0.0006933189579285681, -0.033359695225954056, 0.004588632844388485, -0.002885479712858796, 0.03384200483560562, -0.009197361767292023, -0.010383037850260735, 0.01646548882126808, -0.008520789444446564, 0.0224541574716568, -0.01728273369371891, -0.011474931612610817, 0.016411898657679558, 0.010456724092364311, 0.002980936551466584, -0.023271402344107628, -0.04713229462504387, 0.013256794773042202, 0.0018321038223803043, -0.00944521464407444, -0.012613716535270214, -0.016532475128769875, -0.001773489871993661, -0.019654085859656334, 0.009458612650632858, -0.005134579725563526, 0.029876353219151497, 0.00012518258881755173, 0.023901082575321198, -0.017068374902009964, -0.02583031915128231, -0.018046390265226364, 0.009773452766239643, -0.002801745431497693, -0.006702083628624678, -0.02103402465581894]} +{"id": "test:10000024", "text": "\"What do you do in your free time? Of course some of you ,watch TV or play video games . If it is too boring, would you like to do something more interesting? All of you have wild imagination ,so you can make your own games!\\nLisa Green ,who's 35 years old ,has been making her own games since she was 5 .I asked her How she started that hobby.She told me that she didn't like the games which other children play, so she started to make her own games. Then ,I asked her If it was difficult to make her own games. She told me that it wasn't difficult and,If you had imagination ,you would be a very good player too.\\nLisa yesterday invented a new game which is called table tennis with hand .You paint a court of table tennis , then you take the ball and throw it to the other. If you want to use a net make one! Isn't it a fantastic game ?\\nWith a bit practise, you'll be make your own games in no time !!\\nSay hi to the future blogger of the school, Marinos!\\n2013! Marinos is a \\\"present\\\" blogger now!! It is very thrilling, isn't it!!\\nThat sounds very interesting, Raffaela!! I think that you have really wild imagination!!!!\\nTwo missions before winter courses start!\"", "vector_field": [0.005489333067089319, 0.008824337273836136, 0.0005118956323713064, -0.02533438801765442, -0.009438935667276382, -0.0014143846929073334, -0.036953531205654144, -0.015526693314313889, 0.002799656940624118, -0.04220672696828842, 0.007627488113939762, 0.011780878528952599, -0.010338190011680126, -0.007705121301114559, 0.0057707540690898895, 0.016691194847226143, 0.048365648835897446, 0.027948047965765, -0.0007573306211270392, -0.031726207584142685, -0.0033964640460908413, 0.011664427816867828, 0.007368709892034531, -0.02449335902929306, -0.03120865300297737, -0.0004884438822045922, 0.0013844635104760528, -0.00526290200650692, -0.004091930575668812, -0.0029581584967672825, 0.018981382250785828, -0.011043360456824303, 0.014258679933845997, -0.011095115914940834, -0.02818094752728939, -0.017765125259757042, 0.00981416366994381, 0.00101085240021348, 0.003192676231265068, 0.0009582880884408951, 0.011677366681396961, 0.023704085499048233, -0.005987481214106083, 0.007562793325632811, -0.026317745447158813, 0.025554349645972252, -0.018386192619800568, -0.0006760581163689494, -0.004418638069182634, 0.02568373829126358, 0.019291916862130165, 0.0192660391330719, -0.0012421354185789824, -0.003165181027725339, -0.02402755804359913, -0.0005984246381558478, 0.016626499593257904, 0.015125586651265621, 0.015203219838440418, -0.021439775824546814, -0.0018065954791381955, 0.009206035174429417, 0.009231912903487682, 0.001455627498216927, -0.005395525600761175, 0.00862378440797329, -0.011541508138179779, -0.018347375094890594, 0.01260249875485897, -0.014905625022947788, 0.032735444605350494, 0.005039705894887447, 0.00015253762830980122, 0.013935207389295101, 0.012207861989736557, 0.0013060213532298803, -0.021038668230175972, 0.0051076351664960384, -0.0135599784553051, -0.005133512895554304, -0.010564620606601238, -0.011910267174243927, -0.013417650945484638, 0.019990617409348488, 0.004606252070516348, 0.010875154286623001, -0.017221691086888313, 0.04572610929608345, -0.0052208504639565945, -0.0046677119098603725, -0.0015939121367409825, 0.02485564723610878, 0.024946220219135284, 0.006767050363123417, -0.013948146253824234, 0.019887106493115425, -0.04042115807533264, -0.0010076176840811968, 0.0006360283587127924, -0.012324312701821327, 0.01436219085007906, 0.024273397400975227, -0.017079362645745277, -0.011289199814200401, -0.004247197415679693, -0.01280952151864767, 0.025114426389336586, -0.0067605809308588505, 0.036772385239601135, -0.02854323759675026, -0.018761420622467995, 0.020352907478809357, 0.0224101934581995, -0.014051657170057297, -0.0070905233733356, -0.008280903100967407, 0.008093289099633694, -0.009717121720314026, -0.01689821667969227, 0.003108573378995061, 0.02384641207754612, 0.012395476922392845, 0.00043426218326203525, 0.002715553855523467, 0.011638550087809563, 0.01622539386153221, -0.012356660328805447, 0.01275776606053114, -0.013831695541739464, -0.01271894946694374, 0.0219961479306221, 0.008449108339846134, 0.014491580426692963, -0.004865030292421579, -0.018476765602827072, -0.004606252070516348, 0.011011012829840183, -0.0032185539603233337, -0.025373203679919243, -0.018541458994150162, 0.003068139310926199, -0.004787397105246782, -0.0006040853913873434, -0.004538322798907757, 0.004942663945257664, 0.024648625403642654, 0.027145834639668465, 0.0051076351664960384, 0.004066052846610546, 0.014698603190481663, 0.0026945280842483044, 0.013010075315833092, -0.005191737785935402, 0.01010528951883316, 0.00585485715419054, 0.01498325914144516, -0.01689821667969227, 0.020883401855826378, -0.0219961479306221, -0.006935256067663431, 0.027741024270653725, 0.005392291117459536, 0.006019828375428915, -0.008830806240439415, 0.005890439264476299, 0.033563535660505295, 0.03164857625961304, 0.01701466739177704, 0.00951656885445118, -0.0008507333695888519, -0.0020022965036332607, 0.03524559363722801, -0.05310129001736641, 0.005832213908433914, -0.010784582234919071, 0.019964739680290222, 0.0014620969304814935, 0.004321596119552851, -0.005502271931618452, -0.02123275212943554, 0.006595609709620476, 0.01641947776079178, 0.011638550087809563, 0.04769282415509224, -0.026369500905275345, -0.018800238147377968, 0.007129339966922998, -0.02098691277205944, -0.003706997958943248, -0.026473011821508408, 0.007860388606786728, 0.024105191230773926, 0.011806756258010864, -0.02449335902929306, -0.6492227911949158, -0.002916106954216957, 0.017920391634106636, -0.018839053809642792, 0.004716232884675264, 0.034210480749607086, 0.007439873646944761, -0.004408933687955141, -0.015888983383774757, 0.009995308704674244, 0.008565559051930904, 0.00655355816707015, 0.007692182436585426, -0.003393229329958558, 0.006071583833545446, -0.022798361256718636, 0.014750358648598194, -0.014077534899115562, -0.01933073252439499, 0.012796582654118538, -0.020055312663316727, 0.013340016826987267, -0.012486048974096775, -0.01635478250682354, 0.002455158391967416, 0.0222419872879982, 0.004628895316272974, -0.015112647786736488, 0.0133529556915164, 0.04844328388571739, -0.0006517976289615035, 0.018644969910383224, 0.0009388797334395349, 0.01956363394856453, 0.0494525171816349, -0.009697713889181614, -0.007756877224892378, 0.03643597289919853, 0.02229374274611473, 0.03703116253018379, -0.0317520871758461, -0.006618252955377102, 0.008901970461010933, 0.0008090862538665533, 0.004101634956896305, 0.007925082929432392, 0.001399828470312059, 0.008830806240439415, 0.0037878660950809717, -0.019718900322914124, 0.0015858252299949527, -0.006281841080635786, -0.004557731561362743, -0.016626499593257904, 0.02247488871216774, 0.010053534060716629, -0.007472220808267593, 0.022086720913648605, 0.005447281524538994, 0.01141858845949173, -0.02573549374938011, 0.0314156748354435, -0.030121784657239914, 0.010454639792442322, -0.0176098570227623, 0.009141340851783752, -0.010836337693035603, -0.028827892616391182, -0.01742871291935444, -0.022617215290665627, -0.017286384478211403, 0.005295249167829752, -0.03170033171772957, -0.01784275844693184, -0.0026411551516503096, -0.003512914292514324, 0.013314139097929, 0.0014176194090396166, -0.015099708922207355, 0.00564783439040184, 0.022436071187257767, -0.013792878948152065, -0.025657860562205315, -0.012693071737885475, 0.057293497025966644, -0.009076645597815514, -0.014711542055010796, 0.009917675517499447, -0.007187564857304096, 0.0062883105129003525, 0.022047903388738632, 0.04463924095034599, -0.008313249796628952, -0.004389525391161442, -0.01622539386153221, 0.008584967814385891, -0.016781767830252647, 0.022190231829881668, 0.016070127487182617, -0.02699056826531887, -0.006702356040477753, -0.00960067193955183, 0.02877613715827465, -0.007976838387548923, 0.018632031977176666, 0.015901921316981316, -0.021038668230175972, 0.017506346106529236, 0.03436574712395668, -0.029293693602085114, -0.013663490302860737, -0.02591663785278797, -0.003181354608386755, -0.0030099141877144575, 8.12219877843745e-05, -0.020973974838852882, 0.017144056037068367, -0.015526693314313889, 0.0358925387263298, -0.011735592037439346, 0.005013828165829182, 0.012434293515980244, -0.0004658007819671184, 0.007465751376003027, -0.013818756677210331, 0.030535830184817314, -0.003897846909239888, -0.007834510877728462, -0.015863103792071342, -0.010875154286623001, -0.007258729077875614, 0.010655192658305168, 0.006941725499927998, -0.01764867454767227, -0.006188034079968929, 0.009671836160123348, 0.013327077962458134, -0.0021106598433107138, 0.02389816753566265, -0.011353894136846066, -0.01629008911550045, -0.014827991835772991, 0.002219023182988167, 0.01150269154459238, -0.003794335527345538, -0.021452713757753372, 0.0022287273313850164, -0.01992592215538025, -0.012389007024466991, 0.00787332747131586, -0.01825680397450924, -0.0068640923127532005, 0.016613561660051346, -0.010202331468462944, -0.0037425800692290068, -0.022345498204231262, -0.010169983841478825, -0.0036875896621495485, -0.00867553986608982, -0.006999950855970383, -0.008565559051930904, 0.009225443005561829, -0.0030471135396510363, -0.0025651389732956886, -0.008843745104968548, -0.005227319896221161, -0.02543789893388748, 0.012563682161271572, 0.00948422122746706, -0.03496093675494194, -0.005773989018052816, 0.00414692098274827, -0.014219863340258598, 0.012997135519981384, -0.016147760674357414, 0.006068349350243807, 0.010655192658305168, -0.0018777594668790698, 0.019059015437960625, -0.010797521099448204, -0.02052111178636551, -0.029707739129662514, -0.009451874531805515, -0.0023888463620096445, 0.0133529556915164, -0.010279964655637741, 0.008138574659824371, 0.03578902781009674, -0.0015777384396642447, 0.024570992216467857, 0.014103412628173828, 0.045984890311956406, 0.011185688897967339, -0.009639488533139229, 0.005317892413586378, 0.0035873129963874817, 0.01052580401301384, 0.008345597423613071, 0.01251192670315504, 0.013702306896448135, 0.014077534899115562, -0.010668132454156876, -0.0012946997303515673, 0.0016852931585162878, 0.007711590733379126, -0.03793688490986824, 0.02353587932884693, -0.012537804432213306, 0.022500766441226006, 0.0024955924600362778, 0.0033802904654294252, -0.007815102115273476, 0.014892686158418655, -0.013178280554711819, 0.01537142600864172, 0.021323325112462044, -0.009762408211827278, 0.021866759285330772, -0.009380710311233997, 0.008423230610787868, 0.01891668699681759, 0.005929255858063698, -0.005832213908433914, -0.01465978566557169, -0.0020524347200989723, 0.025373203679919243, 0.0017936564981937408, -0.004583609290421009, 0.009057237766683102, -0.030716974288225174, -0.011179219000041485, 0.0003099273599218577, -0.0005705251242034137, 0.023755840957164764, 0.01956363394856453, 0.021802064031362534, 0.022850116714835167, 0.0027446665335446596, 0.0139740239828825, 0.0070905233733356, 0.02052111178636551, 0.022267865017056465, 0.02609778381884098, -0.029034916311502457, 0.02301832288503647, -0.012938911095261574, 0.03524559363722801, -0.015953676775097847, -0.008371475152671337, 0.04000711068511009, -0.043138328939676285, 0.0017014667391777039, -0.04181855916976929, 0.0037361104041337967, 0.00747869024053216, -0.015009136870503426, -0.014297496527433395, 0.0017742482014000416, 0.009594202041625977, 0.022449010983109474, 0.00556049682199955, 0.01594073884189129, -0.02599427103996277, 0.001688527874648571, -0.02480389177799225, 0.007950960658490658, -0.009464813396334648, -0.02442866377532482, -0.016315966844558716, -0.018593214452266693, -0.02426045760512352, -0.032424911856651306, 0.015009136870503426, -0.0009720356902107596, 0.006327127572149038, -0.017027607187628746, -0.004907081834971905, -0.025489654392004013, 0.010661662556231022, 0.017221691086888313, -0.0233159177005291, -0.03586665913462639, 0.015436120331287384, -0.019667144864797592, 0.006401526276022196, -0.025114426389336586, -0.0267059113830328, -0.01548787672072649, -0.0009841659339144826, -0.0016755890101194382, -0.012291965074837208, -0.0049911849200725555, 0.016924096271395683, 0.027016445994377136, -0.0034320461563766003, -0.009464813396334648, 0.02503679320216179, -0.007840979844331741, -0.006349770352244377, -0.006310953758656979, -0.0018389427568763494, 0.019278977066278458, -0.016445355489850044, -0.034158725291490555, 0.014375129714608192, 0.01486680842936039, -0.023678207769989967, -0.009969430975615978, -0.02510148659348488, -0.02015882357954979, -0.013417650945484638, -0.012925972230732441, 0.005379352252930403, -0.009432465769350529, 0.014556274749338627, 0.008274433203041553, -0.006330362055450678, -0.01896844245493412, -0.0028271519113332033, 0.017622796818614006, -0.01570783741772175, -0.019253099337220192, -0.015229098498821259, 0.008707886561751366, 0.10868684947490692, -0.020715195685625076, -0.027559880167245865, 0.02424751967191696, -0.007472220808267593, -0.03705704212188721, -0.01961538940668106, -0.008992543444037437, 0.03149330988526344, -0.015293792821466923, 0.003301039570942521, 0.023328855633735657, 0.009374241344630718, -0.007562793325632811, -0.015720777213573456, 0.003829917637631297, 0.03330475836992264, -0.0016949973069131374, -0.001223535742610693, 0.007278137374669313, 0.005605783313512802, 0.02473919838666916, -0.016742950305342674, 0.020896341651678085, 0.01992592215538025, -0.009820633567869663, 0.025709616020321846, 0.012887155637145042, 0.0012615438317880034, -0.020585807040333748, 0.001648902427405119, 0.01254427433013916, -0.001255074399523437, -0.005916316993534565, -0.009678305126726627, 0.006268902216106653, 0.013508222997188568, 0.02383347414433956, 0.021595042198896408, -0.005566966254264116, 0.02128450758755207, 0.030302928760647774, 0.010215270332992077, -0.008365006186068058, 0.010144106112420559, -0.007096992805600166, -0.025476714596152306, 0.006320658139884472, -0.017752185463905334, -0.026421256363391876, 0.03138979896903038, 0.008423230610787868, -0.018994322046637535, 0.006495333276689053, 0.005589609500020742, -0.004929725080728531, -0.03379643335938454, -0.013217097148299217, -0.0028692034538835287, 0.0015850166091695428, 0.0032185539603233337, -0.017985086888074875, 0.00014212584937922657, -0.0060942270793020725, -0.0032379624899476767, -0.020534051582217216, -0.0005503080319613218, 0.013999901711940765, -0.0141810467466712, -0.004706528969109058, 0.003132833866402507, -0.036901772022247314, -0.03377055749297142, 0.0004888482508249581, 0.011218035593628883, -4.144494960200973e-05, 0.021310385316610336, 0.02996651828289032, -0.0035355573054403067, 0.019227221608161926, -0.0003440941509325057, -0.02991476096212864, 0.008552620187401772, -0.02611072175204754, 0.009626549668610096, 0.009523038752377033, 0.009924144484102726, -0.0013092560693621635, -0.011677366681396961, 0.00014950506738387048, 0.029112549498677254, 0.02581312693655491, 0.019253099337220192, -0.008345597423613071, -0.0050720530562102795, -0.0017111710039898753, 0.015293792821466923, 0.009982369840145111, 0.008720826357603073, -0.01141211949288845, -0.010124697349965572, -0.03229552134871483, -0.03863558918237686, -0.035349104553461075, -0.027016445994377136, 0.0023370906710624695, -0.011172749102115631, 0.012020247988402843, -0.010538742877542973, -0.011334486305713654, 0.024946220219135284, -0.014168106950819492, 0.03276132047176361, 0.010972196236252785, -0.006149217486381531, 0.030251173302531242, 0.0010925292735919356, -0.002607190515846014, 0.008371475152671337, -0.01011175848543644, -0.005178798921406269, -0.043552372604608536, 0.019046077504754066, 7.619300049555022e-06, -0.020107068121433258, 0.0015397303504869342, 0.01557844877243042, -0.007472220808267593, -0.002683206694200635, -0.014375129714608192, 0.00969124399125576, 0.007957430556416512, -0.012188454158604145, -0.028051558881998062, -0.03597017377614975, -0.020184701308608055, 0.035064447671175, -0.01415516808629036, 0.013844634406268597, 0.0024972097016870975, -0.011638550087809563, 0.009141340851783752, 0.003926959354430437, 0.0024163415655493736, 0.013689368031919003, -0.02730110101401806, -0.012596029788255692, 0.014129290357232094, -0.006812336388975382, 0.008403822779655457, 0.003428811440244317, 0.00832618959248066, -0.016277149319648743, 0.0016917625907808542, -0.0012364747235551476, -0.008093289099633694, -0.015125586651265621, -0.010286434553563595, 0.013042422011494637, 0.026123661547899246, 0.03840268775820732, -0.01255074329674244, 0.0011669279774650931, -0.012072003446519375, 0.017144056037068367, 0.007336362265050411, 0.0011539891129359603, -0.02431221306324005, -0.0222419872879982, 0.02039172314107418, 0.010169983841478825, 0.02235843800008297, 0.006010123994201422, -0.02699056826531887, -0.023393550887703896, 0.013534100726246834, -0.008876092731952667, 0.00029496673960238695, -0.010163514874875546, -0.018062720075249672, 0.007860388606786728, 0.017079362645745277, -0.021905576810240746, 0.0010334955295547843, -0.009037829004228115, -1.7452998690714594e-06, 0.027922170236706734, -0.006453281734138727, 0.018243864178657532, -0.012317842803895473, 0.022280804812908173, -0.03351178020238876, -0.0007775476551614702, 0.007084053475409746, 0.01132154744118452, -0.021543286740779877, -0.011399180628359318, -0.02657652273774147, 0.009322484955191612, -0.00475828442722559, 0.008707886561751366, -0.010868685320019722, -0.015436120331287384, 0.0037425800692290068, -0.010299373418092728, 0.0015858252299949527, -0.021297447383403778, 0.008060941472649574, -0.021724430844187737, -0.001989357639104128, 0.008914909325540066, -0.014504519291222095, 0.0057610501535236835, 0.004001358058303595, -0.0015227480325847864, -0.013074769638478756, -0.02313477173447609, 0.019485998898744583, -0.017286384478211403, -0.04062817990779877, 0.015526693314313889, -0.0016505197854712605, 0.04264665022492409, 0.016665317118167877, 0.00616862578317523, 0.021077485755085945, 0.006019828375428915, -0.0003693654725793749, -0.0029306632932275534, -0.0009041064186021686, -0.00747869024053216, 0.014025779440999031, 0.0309239961206913, -0.010448170825839043, -0.011379771865904331, -0.019524816423654556, 0.012401945888996124, -0.027741024270653725, -0.013197689317166805, 0.002138155046850443, 0.010299373418092728, 0.011108054779469967, -0.0320884995162487, -0.0024972097016870975, -0.020999852567911148, 0.02514030411839485, -0.006492098327726126, 0.007148748263716698, -0.026149539276957512, -0.007187564857304096, -0.01832149736583233, 0.03560788184404373, 0.0046677119098603725, 0.007925082929432392, -0.02116805873811245, -0.005314657464623451, -0.009665366262197495, -0.011036890558898449, -0.003791100811213255, 0.014452763833105564, 0.012938911095261574, 0.01415516808629036, -0.011884389445185661, 0.017506346106529236, 0.02330297790467739, 0.020132945850491524, -0.006572966929525137, -0.01055815163999796, 0.012453701347112656, 0.029397206380963326, 0.010648723691701889, -0.0016497111646458507, -0.026628278195858, 0.008552620187401772, 0.012033186852931976, 0.017325202003121376, -0.0026492420583963394, 0.00393666373565793, 0.00028667773585766554, -0.014297496527433395, 0.02021057903766632, -0.012117289938032627, -0.02675766684114933, -0.009833572432398796, -0.015397303737699986, -0.005835448857396841, 0.0012332398910075426, -0.01694997400045395, 0.003826682921499014, -0.02283717691898346, -0.000991443987004459, -0.0283620934933424, -0.003360882168635726, -0.023742901161313057, 0.013792878948152065, -0.007168156560510397, 0.015669021755456924, 0.02401461824774742, -0.020314089953899384, -0.0014491580659523606, -0.010001777671277523, -0.01251192670315504, 0.0007278137491084635, 0.01861909218132496, -0.009581263177096844, -0.009251321665942669, 0.02526969276368618, -0.009089584462344646, -0.014582152478396893, -0.02437690831720829, 0.019473060965538025, -0.0007908909465186298, -0.010933379642665386, -0.015798410400748253, 0.019460121169686317, -0.002157563343644142, 0.007517507299780846, -0.021659737452864647, -0.031545065343379974, -0.027430491521954536, 0.0031215122435241938, 0.003271927125751972, -0.011108054779469967, 0.006825275253504515, 0.004871499724686146, -0.02409225143492222, 0.015216158702969551, 0.012997135519981384, -0.023510001599788666, -0.00404017511755228, -0.0006036810809746385, 0.002699380274862051, -0.013844634406268597, 0.011088646948337555, -0.01275129709392786, -0.01724756881594658, -0.028931405395269394, 0.010978666134178638, -0.012447232380509377, -0.024985037744045258, 0.006663538981229067, 0.03276132047176361, 0.004214850254356861, 0.00829384196549654, 0.007853918708860874, 0.00951656885445118, -0.025243815034627914, -0.00811916682869196, -0.044380463659763336, -0.006155686918646097, 0.01617363840341568, 0.062831349670887, -0.0009615228045731783, -0.0018793768249452114, -0.0020961035043001175, -0.01706642284989357, -0.03930841013789177, -0.010157044976949692, 0.009497161023318768, 0.007439873646944761, 0.023975802585482597, -0.004386290907859802, 0.007672774139791727, 0.04122336953878403, -0.018399130553007126, 0.00698701199144125, -0.018800238147377968, -0.030173540115356445, 0.0014831227017566562, 0.0063821179792284966, -0.006670008413493633, 0.018166230991482735, -0.01979653351008892, 0.0046677119098603725, 0.017105240374803543, -0.0016254506772384048, -0.024338090792298317, 0.0004512445302680135, -0.010952788405120373, 0.0020427305717021227, -0.015552571043372154, 0.015966616570949554, 0.015720777213573456, 0.00013181514805182815, -0.0019327497575432062, -0.03245078772306442, -0.0018632031278684735, 0.015125586651265621, -0.00536641338840127, -0.006670008413493633, -0.007291076239198446, 0.010208800435066223, -0.014944441616535187, -0.0065406193025410175, -0.003035791916772723, -0.02831033617258072, -0.0022918046452105045, -0.004188972525298595, 0.05475747212767601, -0.019434243440628052, 0.018360314890742302, 0.028387971222400665, 0.005673712585121393, -0.013178280554711819, -0.015888983383774757, 0.009658897295594215, 0.0008879327797330916, -0.005182033870369196, 0.004075756762176752, -0.024351030588150024, 0.024881524965167046, 0.005198207218199968, 0.01537142600864172, -0.008513803593814373, -0.011987901292741299, -0.017221691086888313, 0.008636723272502422, -0.012938911095261574, 0.015436120331287384, 0.01436219085007906, 0.004127512685954571, 0.003917255438864231, -0.011890859343111515, 0.0020071484614163637, 0.003168415743857622, -0.03133804351091385, 0.003978715278208256, -0.014996198005974293, -0.014219863340258598, 0.022073781117796898, 0.009865919128060341, -0.019809473305940628, 0.0017030840972438455, 0.023924045264720917, -0.0034902712795883417, 0.012091412208974361, 0.21012790501117706, 0.006812336388975382, -0.0027770136948674917, 0.025580227375030518, -0.012414884753525257, 0.026550645008683205, 0.018269741907715797, 0.006514741573482752, -0.002967862645164132, 0.025360265746712685, -0.02611072175204754, -0.005757815204560757, -0.012143167667090893, -0.001444305875338614, -0.011877920478582382, -0.022319620475172997, -0.01630302704870701, -0.02497209794819355, 0.023031260818243027, -0.03863558918237686, 0.00930954609066248, -0.0141810467466712, -0.03490918129682541, -0.015449059195816517, 0.02259133756160736, 0.03364116698503494, -0.010221739299595356, 0.013986962847411633, 0.01713111810386181, -0.007213442586362362, -0.009024890139698982, -0.003525853157043457, -0.007200503721833229, 0.0052564325742423534, -0.00876611191779375, 0.00344175030477345, 0.017635734751820564, -0.003946367651224136, 0.013301200233399868, 0.005304953549057245, 0.01933073252439499, 0.03263193368911743, 0.008817867375910282, -0.01477623637765646, 0.014310435391962528, 0.03327887877821922, -0.012854808010160923, -0.004560966044664383, 0.01653592847287655, -0.011392710730433464, -0.03074285201728344, -0.03276132047176361, 0.030587585642933846, 0.005237024277448654, 0.00749162957072258, 0.017169935628771782, 0.014646846801042557, 0.009089584462344646, -0.0113280164077878, 0.01652298867702484, 0.0017111710039898753, 0.0550680048763752, -0.007853918708860874, 0.034986816346645355, -0.020482296124100685, 0.013482345268130302, 0.008850215002894402, 0.005987481214106083, -0.009076645597815514, -0.010694010183215141, -0.020961035043001175, -0.011470344848930836, -0.001348072779364884, -0.012324312701821327, -0.026188354939222336, -0.03920489922165871, 0.03591841831803322, 0.00758220162242651, 0.026175417006015778, 0.02270778827369213, -0.018942564725875854, 0.014711542055010796, -0.012706010602414608, -0.01635478250682354, -0.021297447383403778, -0.03312361240386963, 0.0029419849161058664, 0.006120104808360338, -0.026291867718100548, -0.021957332268357277, 0.004418638069182634, -0.010972196236252785, -0.00373934512026608, 0.011929675936698914, -0.003257370786741376, -0.007899205200374126, 0.02080576866865158, 0.018114475533366203, -0.005625191610306501, -0.001688527874648571, -0.015319670550525188, -0.007148748263716698, 0.03783337399363518, 0.02485564723610878, -0.00747869024053216, 0.014323374256491661, -0.002193145453929901, -0.00025615000049583614, 0.022966567426919937, -0.014685663394629955, 0.005563731770962477, -0.043785274028778076, 0.015009136870503426, -0.013754062354564667, -0.008746704086661339, -0.003512914292514324, -0.004871499724686146, -0.020948097109794617, -4.6549754188163206e-05, 0.01842500828206539, -0.018230926245450974, -0.0077245295979082584, 0.009432465769350529, -0.007051706314086914, -0.0026168946642428637, -0.012110820971429348, -0.032735444605350494, 0.010176453739404678, -0.0014992962824180722, -0.029992396011948586, -0.005596078932285309, -0.0025344090536236763, 0.005013828165829182, -0.014789175242185593, 0.004680650774389505, 0.010461109690368176, -0.018528521060943604, 0.008960195817053318, 0.008397352881729603, 0.007355771027505398, -0.00013646508159581572, -0.01795920915901661, -0.00876611191779375, 0.01574665494263172, 0.039101388305425644, -0.030044151470065117, -0.010952788405120373, 0.01346940640360117, -0.0035970171447843313, -0.007776285521686077, -0.04205146059393883, 0.013314139097929, -0.007750407792627811, 0.008662601001560688, 0.004596548154950142, -0.005599313881248236, -0.02611072175204754, -0.012473110109567642, -0.017855696380138397, 0.007064645178616047, -0.03185559809207916, 0.028983160853385925, 0.017765125259757042, 0.008436169475317001, -0.009108993224799633, -0.03358941152691841, -0.16292676329612732, 0.0072069731540977955, 0.018062720075249672, -0.03519383817911148, 0.023147711530327797, -0.012531335465610027, 0.02561904303729534, -0.007032298017293215, -0.023988740518689156, 0.0036390686873346567, 0.011871450580656528, 0.01623833365738392, -0.024467481300234795, -0.018813176080584526, 0.0038040396757423878, 0.027870414778590202, -0.011140402406454086, 0.02062462456524372, 0.05224732309579849, 0.0017467529978603125, 0.027741024270653725, -0.030070029199123383, 0.007640426978468895, 0.043190084397792816, 0.012207861989736557, 0.0038913774769753218, 0.012376068159937859, -0.01221433188766241, 0.013094177469611168, -0.023264162242412567, -0.0016836758004501462, -0.01230490393936634, 0.007511037867516279, -0.0017192577943205833, 0.01022820919752121, -0.01061637606471777, 0.010338190011680126, -0.00475504994392395, -0.019071955233812332, 0.027275223284959793, 0.014724480919539928, 0.021064545959234238, 0.012867746874690056, -0.0036390686873346567, 0.010519334115087986, 0.01992592215538025, 0.005097930785268545, -0.00384285650216043, -0.014491580426692963, -0.018761420622467995, 0.004822979215532541, -0.03038056194782257, 0.011049829423427582, -0.019537756219506264, 0.01528085395693779, 0.007433404214680195, -0.004331300500780344, 0.024713320657610893, 0.01660062186419964, 0.009503629989922047, -0.0019456887384876609, -0.03586665913462639, 0.009645958431065083, 0.017454590648412704, -0.009206035174429417, -0.008390883915126324, 0.007756877224892378, 0.01724756881594658, -0.0325801782310009, 0.025942515581846237, -0.04225848242640495, -0.018709665164351463, 0.015151464380323887, -0.021452713757753372, -0.007239320781081915, -0.0002221853646915406, -0.012492518872022629, -0.03185559809207916, 0.021802064031362534, 0.0018146822694689035, -0.01570783741772175, 0.02080576866865158, -0.015332609415054321, 0.010370536707341671, -0.0009364536963403225, -0.0014823139645159245, 0.008093289099633694, 0.005450516007840633, -0.021129241213202477, -0.03206261992454529, 0.02294068969786167, -0.0019521581707522273, -0.028879649937152863, -0.013689368031919003, -0.012169045396149158, 0.014853869564831257, 0.026136599481105804, -0.009904736652970314, 0.0002755583554971963, -0.03757459670305252, 0.005625191610306501, -0.009296607226133347, -0.007840979844331741, 0.023031260818243027, 0.03263193368911743, -0.013456467539072037, 0.014543335884809494, 0.001227579195983708, 0.03441750258207321, -0.013061830773949623, -0.006074818782508373, 0.017027607187628746, 0.017920391634106636, 0.02128450758755207, -0.0027042324654757977, 0.016199516132473946, 0.010331720113754272, 0.001319768954999745, -0.003360882168635726, -0.013023014180362225, 0.024997975677251816, -0.012887155637145042, 0.007925082929432392, 0.01563020423054695, 0.013572917319834232, 0.019822411239147186, -0.10754822939634323, -0.03651360794901848, 0.018217986449599266, 0.019240159541368484, 0.00747869024053216, 0.023406488820910454, 0.009497161023318768, 0.008843745104968548, 0.015681959688663483, 0.009717121720314026, -0.01161914225667715, -0.01183263398706913, -0.00617186026647687, 0.0034482197370380163, -0.008099758066236973, -0.02169855311512947, 0.0012720567174255848, -0.013113586232066154, -0.03716055303812027, 0.026473011821508408, 0.008423230610787868, 0.002988888416439295, -0.013029483146965504, -0.01961538940668106, -0.01594073884189129, -0.020857524126768112, -0.03527146950364113, 0.026123661547899246, 0.011146871373057365, -0.0015106177888810635, -0.017079362645745277, -0.012356660328805447, -0.002170502208173275, -0.03358941152691841, -0.0073816487565636635, -0.014879747293889523, -0.0053243618458509445, 0.007297545671463013, 0.02015882357954979, -0.021685615181922913, 0.0027284927200526, 0.007562793325632811, 0.025010915473103523, -0.012796582654118538, -0.0014426885172724724, 0.0007359005394391716, -0.0133529556915164, 0.0179462693631649, 0.006226850673556328, -0.024868587031960487, -0.030872240662574768, -0.015177342109382153, -0.02015882357954979, -0.029267815873026848, 0.01825680397450924, -0.013831695541739464, 0.002568373689427972, 0.014711542055010796, -0.001018939190544188, -0.012440762482583523, -0.005272606387734413, -0.019408365711569786, 0.0029824189841747284, -0.0003386355529073626, 0.023147711530327797, -0.00142004550434649, -0.005431107711046934, -0.006941725499927998, 0.0186061542481184, -0.01705348491668701, -0.005266136489808559, 0.0013221949338912964, 0.0008131297072395682, 0.006644130684435368, -0.0030293224845081568, -0.0036908243782818317, -0.007950960658490658, -0.02093515731394291, 0.028284458443522453, 0.012110820971429348, -0.017752185463905334, -0.030535830184817314, -0.007931551896035671, -0.031001631170511246, 0.040162380784749985, -0.011230974458158016, -0.007795693818479776, 0.018179168924689293, -0.0109139708802104, -0.017791002988815308, -0.0267059113830328, 0.028387971222400665, 0.012984196655452251, -0.010499926283955574, -0.003364116884768009, 0.0016755890101194382, 0.0012858043191954494, -0.01795920915901661, 0.02212553843855858, 0.003697293810546398, -0.007200503721833229, -0.03656536340713501, -0.03749696537852287, 0.019059015437960625, 0.0019311323994770646, 0.011664427816867828, -0.006605314090847969, -0.010357597842812538, -0.0010812077671289444, 0.007737468462437391, -0.011476813815534115, -0.031907353550195694, -0.014944441616535187, 0.01641947776079178, 0.011573855765163898, -0.013417650945484638, -0.022552521899342537, -0.004767988808453083, 0.008759642951190472, 0.017454590648412704, 0.00880492851138115, -0.004939428996294737, -0.022151416167616844, 0.028517359867691994, 0.014892686158418655, 0.015837226063013077, 0.012958318926393986, 0.010687540285289288, -0.015591387636959553, 0.01855439879000187, -0.012887155637145042, -0.03682414069771767, 0.015785470604896545, -0.0031700332183390856, -0.004984715487807989, -0.010169983841478825, -0.01742871291935444, -0.014556274749338627, -0.0033107437193393707, 0.02859499305486679, -0.012059064581990242, 0.0494525171816349, -0.014750358648598194, -0.019693022593855858, 0.008157983422279358, -0.00475828442722559, -0.011256852187216282, 0.013547039590775967, -0.017674552276730537, -0.011761469766497612, 0.011334486305713654, 0.01920134387910366, 0.0175322238355875, 0.0088890315964818, -0.03703116253018379, -0.02217729389667511, -0.029448961839079857, -0.012744827196002007, 0.01658768393099308, -0.0011580325663089752, 0.015086770057678223, -0.016859401017427444, 0.03529734909534454, -0.008869623765349388, 0.006307718809694052, -0.010053534060716629, -0.004641834180802107, -0.010221739299595356, -0.014905625022947788, 0.030225295573472977, -0.007944491691887379, -0.005182033870369196, -0.00799624715000391, 0.017687490209937096, -0.0025182354729622602, 0.029863005504012108, 0.007763346657156944, -0.011101585812866688, -0.005628426093608141, 0.021866759285330772, -0.01447864156216383, 0.02859499305486679, -0.002141389762982726, 0.00846851710230112, -0.03775574266910553, 0.023225344717502594, 0.022668972611427307, 0.01723462902009487, 0.0027576053980737925, 0.02075401321053505, -0.006071583833545446, -0.007627488113939762, 0.0074592819437384605, -0.008028593845665455, -0.022992445155978203, -0.004816509783267975, 0.02391110733151436, 0.0237299632281065, -0.010538742877542973, 0.010344658978283405, 0.014582152478396893, 0.026188354939222336, 0.05214381217956543, -0.012706010602414608, -0.0010399649618193507, -0.03485742583870888, -0.007116401102393866, 0.0061298091895878315, -0.013404712080955505, -0.029578350484371185, 0.025010915473103523, 0.011962022632360458, 0.013378833420574665, 0.005450516007840633, 0.00799624715000391, 0.001131345983594656, -0.022979505360126495, 0.014168106950819492, -0.012971257790923119, -0.02978537231683731, -0.0226560328155756, 0.03312361240386963, 0.014879747293889523, -0.007543385028839111, 0.013158872723579407, -0.017739247530698776, 0.02294068969786167, 0.013741123490035534, 0.025347325950860977, -0.027326980605721474, 0.016393600031733513, -0.03320124372839928, 0.001829238492064178, 0.006298014894127846, -0.03102750889956951, 0.007905674166977406, 0.01979653351008892, -0.011606203392148018, 0.002270778873935342, 0.043552372604608536, -0.014388068579137325, 0.06277959793806076, 0.007730999030172825, 0.002618512138724327, 0.029034916311502457, 0.014892686158418655, 0.0038525606505572796, 0.012084942311048508, -0.0005693120765499771, -0.02771514654159546, -0.0027252580039203167, 0.0006756537477485836, -0.006479159463196993, 0.01604424975812435, -0.03477979078888893, -0.01867084763944149, -0.005243493709713221, -0.018657909706234932, 0.02503679320216179, -0.03545261546969414, 0.021129241213202477, 0.038195665925741196, 0.009749469347298145, 0.02378171868622303, 0.010357597842812538, -0.020197639241814613, 0.03286483511328697, 0.05786281079053879, -0.005780458450317383, -0.018864931538701057, -0.015138525515794754, 0.008196800015866756, 0.01213022880256176, -0.01701466739177704, -0.016742950305342674, 0.007588671054691076, -0.020728135481476784, 0.000243615431827493, 0.008371475152671337, 0.018282681703567505, 0.026136599481105804, 0.0016577979549765587, -0.012020247988402843, -0.03260605409741402, -0.015086770057678223, -0.014258679933845997, 0.0022044668439775705, -0.0022529878187924623, 0.009115463122725487, -0.04673534631729126]} +{"id": "test:10000025", "text": "\"Locksmith Westington \u2013 Putting in place the best of locking and keying services is just one part of the story. However the challenge lies in keeping the locking and keying systems in good condition at all points of time. Without proper maintenance it is quite obvious that the locks and keys would become damaged and replacing the same with new ones could be a big drain on resources and it would also take lot of time. It also could expose the entire home, office and other such places to unwanted risks. Further, you also might need to make some changes to your entire locking and keying systems or even a part of it. All these cannot be done by you on your own because of some obvious reasons.\\nHence you have no other option but to hire a good locksmith. While you could come across many such locksmiths in this city, you may have some good reasons to choose us over others. We are considered an experienced, reliable and well known service provider in Westington Gloucestershire, South West, United Kingdom with years of experience. Further we also would like to point out that we offer our services across the board covering automobile locksmith and commercial locksmiths. We also are considered one of the best when it comes to domestic locksmiths. It would be interesting to look at each one of the above in some detail for the benefits of our customers and readers.\\nAre you stuck with your car, motorbike, SUV or even big sized vehicles in the road, home or office because you have lost your keys? If this is the case all you have to do is to get in touch with us. You can be sure that you will have a replacement key ready within a few minutes. This is because we have some of the finest molds to take care of almost all major brands and makes of automobiles in Westington Gloucestershire, South West, United Kingdom. Further we can make the keys right at the doorstep of our customers and also in the middle of the road if the need arises.\"", "vector_field": [0.017661208286881447, 0.013324083760380745, -0.0015414468944072723, -0.04062486067414284, -0.018218645825982094, 0.02808934450149536, -0.00978232454508543, -0.004561459179967642, -0.013256103731691837, 0.006787804886698723, -0.014466147869825363, 0.00632893992587924, -0.017076581716537476, 0.01408546045422554, -0.0242144837975502, -0.004806187469512224, 0.03744339570403099, -0.006539677735418081, -0.0076069640927016735, -0.03842230513691902, -0.013833933509886265, 0.01492841262370348, 0.01729411631822586, -0.015499444678425789, -0.011862513609230518, -0.013616397976875305, 0.0028279689140617847, -0.016559932380914688, -0.02054356224834919, -0.014126247726380825, -0.0032222529407590628, -0.0027192006818950176, -0.03823196142911911, -0.0020105091389268637, -0.017742784693837166, -0.020788289606571198, 0.013432851992547512, -0.015744172036647797, 0.0066008600406348705, -0.0026716147549450397, 0.02227025479078293, -0.006801400799304247, 0.012338373810052872, -0.02766786701977253, -0.006607657764106989, 0.015662595629692078, -0.035159267485141754, 0.0022790301591157913, -0.011624583043158054, 0.01772918924689293, -0.013534821569919586, 0.014411764219403267, -0.0030098154675215483, 0.006196378730237484, -0.013963095843791962, -0.004612444434314966, 0.014180632308125496, 0.02394256368279457, 0.006322141736745834, -0.029177024960517883, -0.012127635069191456, 0.0022178480867296457, -0.008402330800890923, -0.009836709126830101, 0.0016808059299364686, -0.03173307329416275, -0.024935070425271988, -0.00030314840842038393, -0.020829077810049057, 0.015744172036647797, 0.00980271864682436, 0.029285792261362076, 0.003558753989636898, -0.002700506243854761, 0.003793284995481372, 0.005625346675515175, -0.012807435356080532, -0.006271156948059797, -0.03083573654294014, 0.0005578610580414534, 0.0009721142705529928, -0.015839343890547752, -0.025098223239183426, 0.03600221872329712, 0.016491953283548355, 0.0026087332516908646, 0.0012023965828120708, 0.009014150127768517, 0.015948113054037094, 0.005604952573776245, -0.009986264631152153, 0.014561319723725319, 0.00084125273860991, -0.0010426435619592667, -0.008810210973024368, 0.008252774365246296, -0.020570753142237663, 0.021277746185660362, -0.02629467099905014, -0.01289580948650837, -0.00574091263115406, -0.01373196393251419, -0.011067147366702557, -0.00923168659210205, -0.03959156200289726, -0.03679078817367554, 0.02383379451930523, -0.018612928688526154, 0.018408989533782005, -0.015716981142759323, -0.023330742493271828, -0.0017164954915642738, 0.018721697852015495, 0.01764761283993721, 0.014765259809792042, -0.0006101206527091563, 0.014697279781103134, -0.02429605834186077, 0.0037898861337453127, -0.011916897259652615, 0.021930353716015816, -0.004714414477348328, 0.02817092090845108, 0.010224195197224617, 0.022718923166394234, 0.004231756087392569, -0.006186181679368019, 0.00031292051426135004, -0.0009338755044154823, -0.011386653408408165, -0.009625970385968685, 0.015553828328847885, 0.02629467099905014, 0.022324638441205025, 0.01752524822950363, 0.01616564951837063, -0.015485848300158978, 0.008028440177440643, -0.013738761655986309, -0.036682017147541046, 0.02014927752315998, 0.028116535395383835, -0.004496878478676081, -0.00788568239659071, 0.014289400540292263, 0.030264705419540405, -0.005638942588120699, 0.004439095035195351, -0.021114593371748924, -0.01201206911355257, 0.010577690787613392, -0.01737569272518158, 0.049598220735788345, -0.026865703985095024, -0.011033156886696815, -0.007579772267490625, 0.0051494864746928215, 0.01823224127292633, -0.02175360545516014, -0.020557157695293427, -0.0003507344226818532, -0.00900055468082428, 0.038014426827430725, 0.008844200521707535, 0.03135238587856293, 0.0163967814296484, 0.0056695337407290936, 0.026308268308639526, 0.0122839892283082, 0.01647835597395897, 0.007783712353557348, 0.007056326139718294, -0.03556714579463005, 0.019754992797970772, 0.012623889371752739, 0.012923001311719418, -0.01631520502269268, 0.007301053963601589, -0.006641647778451443, -0.008259572088718414, 0.0004282741283532232, 0.01060488261282444, 0.009326858446002007, 0.03162430599331856, -0.03964594751596451, -0.005747710820287466, 0.0230044387280941, 0.0070495279505848885, 0.0404617078602314, 0.010924388654530048, -0.02414650283753872, 0.034180354326963425, -0.00900055468082428, -0.0062371669337153435, -0.6417313814163208, 0.012351969256997108, 0.03447946533560753, -0.0051630823872983456, 0.00230452255345881, 0.015716981142759323, 0.017348501831293106, -0.0023691037204116583, -0.033554937690496445, 0.016138456761837006, -0.00874223094433546, -0.009877496398985386, 0.013786347582936287, -0.00755257997661829, -0.03048224002122879, -0.023289954289793968, -0.007804106455296278, -0.004575055092573166, 0.02969367243349552, -0.019673418253660202, 0.0017419880023226142, 0.021427301689982414, 0.0002540753339417279, 0.007980854250490665, 0.023371530696749687, -0.008347946219146252, 0.009156908839941025, -0.035431187599897385, -0.005666134413331747, 0.012827829457819462, -0.03687236085534096, 0.007035932037979364, -0.021155381575226784, 0.013779549859464169, 0.04872807860374451, 0.008701442740857601, -0.05090343952178955, 0.027844615280628204, 0.017090177163481712, 0.029557712376117706, 0.004544464405626059, -0.035975027829408646, 0.018612928688526154, 0.005139289423823357, 0.006743617821484804, 0.018300220370292664, 0.02656659111380577, -0.008014844730496407, -0.008232380263507366, 0.003140676999464631, 0.028306880965828896, 0.003545158077031374, -0.012848223559558392, -0.009768729098141193, 0.00038451197906397283, -0.01510515995323658, 0.017973916605114937, -0.02324916608631611, 0.026811320334672928, 0.011162319220602512, -0.00773612642660737, 0.03681797906756401, -0.008613068610429764, -0.011067147366702557, -0.03268479183316231, 0.027368756011128426, -0.003745699068531394, -0.019238345324993134, 0.008762624114751816, -0.019890952855348587, 0.006023029796779156, 0.012209211476147175, -0.022120697423815727, -0.008837402798235416, -0.006362929940223694, 0.013140537776052952, 0.02868756838142872, -0.019904550164937973, 0.021141786128282547, 0.019075194373726845, -0.0026886097621172667, -0.01800110936164856, -0.028741952031850815, -0.007226276211440563, 0.03545837849378586, -0.007817702367901802, 0.005190274678170681, -0.020951442420482635, 0.025492507964372635, 0.012216009199619293, 0.017158156260848045, -0.02171281911432743, -0.02531575970351696, -0.022052718326449394, 0.023072417825460434, 0.031787458807229996, -0.006257560569792986, 0.008871392346918583, 0.03358212858438492, -0.0212913416326046, -0.024663150310516357, -0.010870005004107952, -0.026580188423395157, 0.019075194373726845, 0.0037966840900480747, 0.01968701370060444, -0.03679078817367554, -0.007647752296179533, 0.030373472720384598, -0.016179244965314865, 0.01800110936164856, 0.006485293619334698, 0.013262901455163956, -0.004496878478676081, -0.0064309099689126015, -0.023289954289793968, -0.0003447861527092755, -0.012535515241324902, 0.003956437110900879, -0.00546899251639843, 0.0010044047376140952, -0.00046013976680114865, -0.0095851831138134, -0.0020241050515323877, -0.00815760251134634, 0.007409822195768356, 0.021100997924804688, -0.004245351999998093, -0.005543770734220743, -0.007267063949257135, -0.011855714954435825, -0.022039122879505157, 0.024527190253138542, -0.020951442420482635, 0.02152247354388237, 0.0012491329107433558, 0.021114593371748924, -0.00954439491033554, 0.020122086629271507, -0.0036097390111535788, 0.006560071837157011, -0.03173307329416275, -0.002766786841675639, -0.0163967814296484, -0.019428689032793045, -0.013256103731691837, -0.009789123199880123, -0.0019017410231754184, -0.013840732164680958, 0.006757213734090328, -0.004639636259526014, 0.0006963703199289739, -0.01889844611287117, 0.0011199708096683025, -0.01179453358054161, -0.003517966018989682, -0.01465649250894785, -0.010516509413719177, 0.005893867462873459, 0.0051120975986123085, -0.0024625761434435844, 0.02320837788283825, -0.040135402232408524, -0.0015448459889739752, -0.0025084626395255327, 0.027763040736317635, 0.007722530048340559, -0.020394006744027138, 0.00811681430786848, -0.030074359849095345, 0.00279737776145339, 0.004789192229509354, -0.02163124270737171, 0.019455881789326668, 0.02770865522325039, 0.01322891190648079, -0.018123473972082138, 0.026580188423395157, -0.006573667749762535, 0.02081548236310482, 0.0106456708163023, -0.012814234010875225, -0.028225304558873177, -0.011175915598869324, 0.023031631484627724, -0.01066606491804123, 0.004826581571251154, -0.010475721210241318, 0.0005400162772275507, -0.00574091263115406, 0.003966634161770344, -0.019986126571893692, -0.013256103731691837, 0.006889774929732084, -0.0005710321711376309, -0.014126247726380825, 0.01201206911355257, 0.015404272824525833, 0.011393451131880283, 0.03064539283514023, 0.03216814622282982, -0.018069088459014893, 0.018558545038104057, -0.004598848521709442, 0.016016092151403427, -0.012358767911791801, 0.020434793084859848, -0.028388455510139465, 0.027531906962394714, -0.01772918924689293, -0.004863970447331667, -0.01399028766900301, 0.009428828954696655, -0.015431464649736881, 0.020720310509204865, 0.03273917734622955, -0.04973418265581131, 0.042392339557409286, 0.008171197958290577, -0.01573057658970356, 0.009891092777252197, -0.007035932037979364, 0.030074359849095345, -0.003115184372290969, -0.029612096026539803, 0.02383379451930523, -0.006798001937568188, 0.02343950979411602, 0.008769422769546509, 0.0014811146538704634, -0.0359206423163414, 0.0042181601747870445, 0.023616258054971695, 0.001007803832180798, 0.014670087955892086, 0.012923001311719418, 0.03915649279952049, -0.011447835713624954, 0.026049943640828133, -0.014778856188058853, 0.025111818686127663, -0.012746253982186317, 0.006413914728909731, -0.012630688026547432, 0.02645782381296158, 0.006862583104521036, 0.0197278019040823, -0.020570753142237663, 0.002661417704075575, 0.024676747620105743, -0.00927247479557991, 0.01596170850098133, 0.0026580188423395157, -0.010319367051124573, 0.010822419077157974, -0.011101136915385723, 0.02594117447733879, -0.007287458050996065, 0.031787458807229996, 0.02308601513504982, -0.017470864579081535, 0.015526636503636837, -0.0038000831846147776, -0.012270393781363964, 0.00466682855039835, 0.020013317465782166, 0.01877608150243759, -0.002326616086065769, 0.0061555905267596245, -0.008191592060029507, -0.002129474189132452, 0.026539400219917297, 0.018735293298959732, -0.024513594806194305, 0.021468089893460274, 0.012066453695297241, -0.006937360856682062, 0.0081983907148242, -0.0046294392086565495, 0.027382351458072662, -0.004418701399117708, -0.029258599504828453, 0.0095104044303298, 0.01125069335103035, -0.002987721934914589, -0.013555215671658516, -0.023643450811505318, 0.001606027944944799, -0.008436320349574089, 0.025696447119116783, 0.00561514962464571, 0.0019391300156712532, -0.0003084593336097896, -0.018381796777248383, 0.020448390394449234, -0.009394838474690914, 0.027763040736317635, -0.020489178597927094, 0.026974471285939217, -0.021658433601260185, 0.0038578661624342203, 0.009972669184207916, -0.027423139661550522, -0.005401012487709522, 0.03287513554096222, 0.016301609575748444, -0.0004456939932424575, -0.014738067984580994, 0.0041671753861010075, -0.03616537153720856, -0.0045308684930205345, -0.017049388960003853, -0.011644977144896984, -0.01426220778375864, -0.005547169595956802, -0.0058904686011374, 0.0093336571007967, 0.0021974542178213596, 0.04231076315045357, 0.010496115311980247, -0.0019374305848032236, -0.013337680138647556, 0.0007367334328591824, 0.002751491265371442, 0.09446503221988678, 0.002386098727583885, -0.027246391400694847, 0.01783795654773712, 0.0007222876884043217, -0.012134433723986149, 0.006682435981929302, -0.038993339985609055, 0.026362651959061623, 0.0067810071632266045, 0.027790231630206108, -0.01596170850098133, -0.01426220778375864, 0.00817799661308527, 0.03415316343307495, 0.015322696417570114, -0.004884364549070597, 0.0015304002445191145, -0.03148834407329559, -0.02031243033707142, -0.004816384520381689, -0.029122639447450638, 0.019931741058826447, 0.005394214764237404, -0.007117507979273796, 0.03880299627780914, 0.02547891065478325, 0.014588512480258942, -0.020529964938759804, -0.033201441168785095, -0.02746392786502838, 0.007899277843534946, 0.012494727969169617, -0.0006381624261848629, 0.013140537776052952, 0.006570268888026476, -0.01070005539804697, -0.0013298591366037726, 0.022351831197738647, 0.01123029924929142, 0.022827690467238426, 0.010183406993746758, -0.002782082185149193, -0.013262901455163956, 0.015581020154058933, -0.026471419259905815, 0.02942175231873989, 0.019496669992804527, -0.008001248352229595, -0.009666758589446545, -0.003399000968784094, -0.006655243691056967, 0.006713027134537697, -0.035349611192941666, 0.020475581288337708, -0.009476414881646633, -0.01148182526230812, 0.009524000808596611, -0.018844060599803925, -0.0025186596903949976, -0.01232477743178606, -0.016451165080070496, -0.020135682076215744, -0.009911486878991127, -0.03388124331831932, -0.007634156383574009, -0.011060348711907864, 0.021984737366437912, -0.016777468845248222, 0.01666870154440403, 0.023072417825460434, -0.00839553214609623, -0.0026325262151658535, -0.012787041254341602, 0.024432018399238586, 0.006614455953240395, 0.02246059849858284, -0.01322211418300867, -0.016111264005303383, 0.03317425027489662, -0.0021226759999990463, 0.0017436875496059656, 0.0019476275192573667, -0.017348501831293106, -0.014099055901169777, 0.02362985536456108, 0.026865703985095024, -0.00811681430786848, -0.013881520368158817, 0.007029133848845959, 0.014506936073303223, 0.011386653408408165, 0.02625388279557228, 0.003354813903570175, 0.008042036555707455, -0.009143312461674213, 0.025152606889605522, 0.018286624923348427, 0.00978232454508543, -0.004948945250362158, 0.0038136790972203016, -0.003738901112228632, 0.0041807712987065315, -0.025954771786928177, 0.005207269452512264, -0.009320060722529888, 0.009619172662496567, -0.012277191504836082, 0.00323754851706326, -0.007090316154062748, 0.0007584021077491343, -0.030074359849095345, 0.010652469471096992, -7.074170571286231e-05, -0.031787458807229996, 0.026199499145150185, 0.0030760958325117826, 0.00872863456606865, 0.014316592365503311, -0.004602247383445501, 0.008715038187801838, -0.042908988893032074, 0.02148168534040451, 0.0219439510256052, -0.0049183545634150505, 0.0026291273534297943, 0.0060570198111236095, -0.017674805596470833, -0.012358767911791801, -0.017756380140781403, -0.014235015958547592, 0.014235015958547592, -0.00842952262610197, -0.007103912066668272, -0.028470031917095184, -0.003623334923759103, 0.0009236785117536783, 0.023058822378516197, -0.01658712513744831, -0.007484599947929382, -0.018830465152859688, -0.007518589962273836, 0.018966425210237503, 0.0009687152924016118, -0.000837853760458529, -0.02581881172955036, -0.013534821569919586, 0.008517896756529808, 0.005771503783762455, 0.0029792243149131536, -0.008089622482657433, -0.028007768094539642, -0.035784680396318436, -0.016818257048726082, 0.006825193762779236, -0.04587291553616524, 0.0032137553207576275, 0.0030370072927325964, -0.011644977144896984, 0.00546899251639843, 0.04848334938287735, -0.009891092777252197, 0.005190274678170681, 0.015975303947925568, 0.014602107927203178, -0.011617785319685936, 0.0003214180178474635, 0.020394006744027138, -0.031597111374139786, 0.006903370842337608, -0.0029197419062256813, 0.030237512663006783, 0.0006373127107508481, 0.035594336688518524, -0.011379855684936047, 0.01787874475121498, 0.003521365113556385, -0.030346279963850975, 0.003959835972636938, -0.016818257048726082, -0.01152261346578598, -0.0019476275192573667, -0.022474193945527077, -0.012046059593558311, -0.04260987788438797, 0.021236957982182503, 0.03241287171840668, 0.02010848931968212, 0.004850374534726143, -0.00044441939098760486, 0.035431187599897385, -0.03344617038965225, 0.021005826070904732, -0.020217258483171463, -0.014806048013269901, -0.003592744003981352, -0.00980271864682436, -0.043371252715587616, 0.022977245971560478, 0.004469686187803745, 0.005424805451184511, 0.008905382826924324, -0.003749098163098097, -0.005050915293395519, -0.009979466907680035, 0.01118951104581356, -0.014058267697691917, -0.022664537653326988, -0.009891092777252197, -0.009571586735546589, -0.0023843992967158556, -0.003534961026161909, -0.023181186988949776, -0.01835460588335991, -0.01042813528329134, -0.00715149799361825, -0.009748334996402264, 0.010536902584135532, -0.0207339059561491, -0.009007352404296398, 0.0012202413054183125, -0.012249999679625034, 0.034180354326963425, 0.012644283473491669, 0.03915649279952049, 0.007518589962273836, 0.010305770672857761, -0.029557712376117706, -0.008021642453968525, -0.0050985016860067844, -0.01158379577100277, 0.019360709935426712, 0.004245351999998093, -0.004908157512545586, -0.010842813178896904, 8.146980690071359e-05, -0.019048001617193222, -0.013392063789069653, -0.02614511549472809, 0.016464760527014732, 0.007369033992290497, -0.015676192939281464, -0.005587957799434662, 0.006648445967584848, -0.002506763208657503, -0.005441800691187382, -0.021304938942193985, 0.024744726717472076, -0.0030404063872992992, -0.02664816752076149, -0.015635404735803604, 0.012392757460474968, -0.03888457268476486, 0.027423139661550522, -0.004343923181295395, -0.004493479151278734, -0.013405660167336464, -0.021903162822127342, 0.0072194780223071575, 0.02191675826907158, -0.014520532451570034, 0.026349054649472237, -0.026158710941672325, -0.012943395413458347, 0.002467674668878317, 0.01314733549952507, -0.005142688285559416, -0.031162040308117867, 0.014874028041958809, 0.024935070425271988, -0.035703107714653015, 0.01259669754654169, 0.009116120636463165, -0.0005157983978278935, -0.03263041004538536, 0.005472391378134489, -0.00757297407835722, -0.007335043977946043, -0.0010010057594627142, 0.008857796899974346, 0.012331575155258179, 0.017987513914704323, -0.01658712513744831, -0.012671475298702717, -0.010298972949385643, -0.02571004256606102, 0.0003524339117575437, 0.005479189567267895, 0.008055632002651691, 0.0027990771923214197, -0.018245836719870567, -0.008123612031340599, -0.020407602190971375, 0.006199777591973543, -0.01492841262370348, -0.02097863331437111, 0.006230368744581938, 0.028905104845762253, 0.011141925118863583, -0.001959524117410183, 0.0020105091389268637, 0.004999930504709482, -0.029448945075273514, -0.016559932380914688, -0.0034244933631271124, -0.01310654729604721, 0.03306548297405243, 0.007389428094029427, -0.020394006744027138, 0.006672238931059837, -0.007722530048340559, -0.009768729098141193, -0.018014704808592796, -0.026308268308639526, -0.0015575921861454844, -0.0006704529514536262, 0.02031243033707142, 0.013079355470836163, -0.020869866013526917, 0.006723223719745874, -0.026552995666861534, 0.0010825818171724677, 0.006223570555448532, -0.008694644086062908, -0.009156908839941025, 0.01921115256845951, 0.012766648083925247, -0.002185557736083865, -0.014806048013269901, -0.03399001061916351, -0.021807990968227386, 0.015186736360192299, -0.009326858446002007, -0.03464261814951897, -0.016111264005303383, -0.013643589802086353, -0.021033018827438354, 0.03708989918231964, -0.021495282649993896, -0.0014394769677892327, 0.029448945075273514, -0.03627413883805275, 0.009877496398985386, -0.006872780155390501, -0.035703107714653015, 0.008789816871285439, -0.01937430538237095, -0.020652329549193382, -0.01289580948650837, 0.0011675568530336022, -0.0062371669337153435, 0.027423139661550522, 0.020557157695293427, -0.02320837788283825, -0.02183518186211586, -0.007430216297507286, -0.05617868900299072, -0.01623362861573696, 0.007579772267490625, 0.00422495836392045, 0.012114039622247219, 0.02575083076953888, 0.002540753223001957, 0.002350409282371402, 0.012725859880447388, 0.017321309074759483, 0.0017810765421018004, 0.01040774118155241, 0.019197557121515274, -0.0068183960393071175, 0.0313795767724514, -0.005040718242526054, -0.02844284102320671, 0.01941509358584881, -0.0028602592647075653, -0.006954356096684933, -0.013922307640314102, 0.00072271260432899, -0.021073805168271065, 0.001896642497740686, -0.003431291552260518, 0.02367064356803894, 0.016872640699148178, 0.020516369491815567, -0.011026359163224697, -0.041930075734853745, -0.002319818129763007, 0.004126387182623148, -0.0009840107522904873, -0.012651081196963787, 0.004778995178639889, 0.007301053963601589, -0.0042793420143425465, 0.02492147497832775, -0.012317979708313942, 0.012154827825725079, 0.03583906590938568, -0.0066756377927958965, 0.04758601263165474, -0.007538984064012766, 0.028007768094539642, 0.019551053643226624, -0.005224264692515135, 0.01295019406825304, 0.011400248855352402, -0.003235848853364587, -0.022922862321138382, -0.021223362535238266, -0.014479744248092175, -0.03896614536643028, 0.008606270886957645, 0.004391509108245373, 0.029367368668317795, -0.004503676202148199, -0.012589899823069572, 0.0004830830148421228, 0.018123473972082138, -0.0011174215469509363, 0.02175360545516014, 0.009965870529413223, -3.73624570784159e-05, 0.0014003884280100465, -0.014874028041958809, 0.0016706089954823256, 0.015268312767148018, -0.019224749878048897, -0.022841285914182663, -0.0015830846969038248, -0.024119310081005096, 0.017905937507748604, -0.0060944086872041225, -0.004544464405626059, 0.00042976118857041, 0.01596170850098133, 0.010870005004107952, -0.016886236146092415, 0.20394006371498108, -0.007525388151407242, 0.007695338223129511, 0.036682017147541046, 0.022528577595949173, 0.026947280392050743, 0.028986679390072823, 0.0017079979879781604, -0.004112791270017624, -0.005581159610301256, -0.009952275082468987, 0.006961153820157051, -0.012902608141303062, -0.0010545400436967611, -0.0024115911219269037, -0.02519339509308338, -0.0018388595199212432, -0.02554689161479473, 0.01461570430546999, 0.045954491943120956, 0.00921809021383524, -0.01964622549712658, 0.013562014326453209, -0.004700818564742804, 0.05090343952178955, 0.011277885176241398, -0.019523860886693, -0.00031122102518565953, 0.021549666300415993, 0.02394256368279457, -0.009829910472035408, 0.005560765508562326, 0.028034960851073265, 0.010251387022435665, -0.009211292490363121, -0.0073758321814239025, 0.010271781124174595, -0.017824361100792885, 0.010686459019780159, 0.030455049127340317, 0.017362097278237343, -0.03200499340891838, 0.03333740308880806, -0.009054938331246376, -0.004656631499528885, 0.00502712232992053, 0.022351831197738647, 0.0029418354388326406, -0.022161485627293587, 0.004846975207328796, -0.03529522567987442, -0.0453018844127655, 0.013426054269075394, 0.023018034175038338, -0.008042036555707455, -0.01156340166926384, -0.009109322912991047, -0.00974153634160757, -0.00870824046432972, 0.01013582106679678, -0.011923694983124733, 0.008456714451313019, -0.0031032878905534744, -0.0038306741043925285, -0.024282462894916534, 0.04590011015534401, -0.0011429140577092767, -0.014602107927203178, 0.0033140259329229593, -0.0061182016506791115, -0.0006445355829782784, 0.02516620233654976, -0.0010222495766356587, 0.02339872345328331, -0.020122086629271507, -0.0363285206258297, 0.01240635383874178, 0.018028300255537033, 0.021454494446516037, 0.011420642957091331, -0.009612374939024448, -0.013364871963858604, 0.014629299752414227, -0.012671475298702717, 0.0036403299309313297, -0.011162319220602512, -0.014058267697691917, 0.0024183890782296658, -0.003983628936111927, -0.01764761283993721, 0.005006728228181601, -0.006077413912862539, -0.012148029170930386, -0.030427856370806694, -0.004653232172131538, -0.016342395916581154, 0.035159267485141754, 0.020394006744027138, -0.008497502654790878, -0.012732657603919506, -0.015213928185403347, 0.011148722842335701, 0.018572140485048294, 0.014112652279436588, -0.0041807712987065315, 0.01983656920492649, -0.01877608150243759, 0.014167035929858685, -0.015608212910592556, -0.001228738809004426, 0.01620643585920334, -0.03165149688720703, 0.01123029924929142, -0.0021396710071712732, 0.02348029799759388, 0.004561459179967642, 0.008850998245179653, -0.009809516370296478, -0.004510474391281605, -0.01948307268321514, 0.031243616715073586, -0.03219533711671829, 0.007715732324868441, 0.010557296685874462, 0.009129717014729977, -0.031325191259384155, -0.05098501592874527, 0.026090731844305992, -0.00792647060006857, -0.01461570430546999, 0.0023742022458463907, -0.013963095843791962, 0.003081194357946515, 0.001535498653538525, -0.010944782756268978, 0.00338540505617857, 0.017756380140781403, -0.03681797906756401, 0.000984860584139824, 0.017905937507748604, 0.03366370499134064, -0.01666870154440403, 0.01772918924689293, 0.0013859426835551858, 0.026824915781617165, -0.028660375624895096, -0.019510265439748764, 0.008191592060029507, -0.014370976015925407, -0.0296664796769619, -0.005611750762909651, -0.00032481702510267496, -0.008313956670463085, -0.021971141919493675, 0.012195615097880363, 0.017634017392992973, -0.006284752860665321, -0.015295504592359066, 0.010197003372013569, 0.010244589298963547, -0.019659820944070816, 0.016288012266159058, 0.01310654729604721, -0.008150804787874222, 0.011971281841397285, -0.0047993892803788185, -0.1730499416589737, 0.021468089893460274, 0.007158296182751656, -0.02868756838142872, 0.020869866013526917, -0.00563554372638464, 0.022311042994260788, 0.021495282649993896, -0.04038013145327568, 0.008300360292196274, -0.005445199552923441, -0.0006912717944942415, -0.04133184999227524, -0.01647835597395897, -0.003793284995481372, -0.012997779995203018, -0.0007286608451977372, 0.012780243530869484, -0.011223501525819302, 5.528687688638456e-05, 0.028034960851073265, -0.005567563697695732, 0.01764761283993721, -0.023031631484627724, 0.01877608150243759, -0.001238086144439876, 0.004965940490365028, 0.0010035550221800804, 0.009265677072107792, -0.03091731294989586, -0.0033837053924798965, 0.0008183094905689359, 0.01748446188867092, -0.01377275213599205, 0.0016850547399371862, -0.007810904178768396, 0.0083751380443573, -0.00373210315592587, -0.0032596418168395758, 0.04377913102507591, 0.008925776928663254, 0.02844284102320671, -0.0061419946141541, -0.015118756331503391, 0.02014927752315998, 0.04225638136267662, 0.005560765508562326, -0.0077497223392128944, 0.015023584477603436, 0.0007673244690522552, 0.007878884673118591, 0.011461431160569191, 0.006825193762779236, 0.006726623047143221, 0.02316759154200554, 0.0026291273534297943, -0.005934655666351318, 0.015621808357536793, 0.010006658732891083, 0.01287541538476944, -0.00371170905418694, 0.010169810615479946, 0.012630688026547432, -0.009659960865974426, -0.01009503286331892, -0.010734044946730137, -0.003057401394471526, -0.0006067216745577753, -0.021413706243038177, -0.002343611093237996, 0.009591980837285519, 0.010067841038107872, 0.010312569327652454, -0.0014819643693044782, -0.005723917856812477, -0.0012066453928127885, -0.008986958302557468, 0.010067841038107872, 0.010264982469379902, 0.0003097339649684727, 0.007960460148751736, 0.016152052208781242, -0.03192341700196266, -0.0034091980196535587, -0.022256657481193542, 0.013840732164680958, 0.011672168970108032, -0.004680424463003874, 0.017076581716537476, 0.03991786763072014, 0.02410571463406086, -0.03040066547691822, -0.0070631238631904125, -0.003976831212639809, 0.003922447096556425, 0.004765399266034365, 0.004728010389953852, 0.01772918924689293, 0.012732657603919506, 0.03135238587856293, 0.007498196326196194, -0.005523376632481813, -0.033201441168785095, -0.006767410784959793, -0.00017143711738754064, 0.014506936073303223, 0.025778023526072502, 0.02214789018034935, -0.0008382786181755364, 0.003361611859872937, 0.000679375312756747, 0.01484683621674776, 0.012249999679625034, 0.019591841846704483, -0.04269145056605339, 0.015553828328847885, 0.012685071676969528, 0.005234461277723312, 0.010414538905024529, -0.006886376067996025, 0.03428912162780762, -0.009503606706857681, -0.0030557019636034966, -0.0009568187524564564, -0.006383324041962624, 0.0083751380443573, -0.1060488298535347, -0.0097619304433465, 0.004877566359937191, -0.0004626890004146844, -0.0352136492729187, 0.0033106268383562565, -0.01377275213599205, 0.019007213413715363, 0.0033803065307438374, 0.025016646832227707, -0.04590011015534401, -0.01292979996651411, 0.0095851831138134, 0.009680354967713356, 7.074170571286231e-05, -0.036138176918029785, 0.0029486333951354027, -0.00044017063919454813, -0.015621808357536793, 0.00974153634160757, -0.001949327066540718, -0.022447003051638603, -0.021767202764749527, -0.008762624114751816, -0.05710321664810181, -0.01483323983848095, -0.017389288172125816, 0.016247224062681198, -0.005553967785090208, 0.007491398137062788, 0.005207269452512264, 0.011019561439752579, -0.0013375069247558713, -0.013528023846447468, 0.006923764944076538, 0.01741648092865944, -0.027015259489417076, -0.020965037867426872, 0.023657046258449554, -0.0013528023846447468, -0.01484683621674776, -0.004976137541234493, 0.017661208286881447, -0.014167035929858685, 0.035703107714653015, -0.004547863267362118, -0.011556603014469147, 0.02352108620107174, -0.004017618950456381, -0.0011165718315169215, -0.020162872970104218, 0.006573667749762535, -0.05617868900299072, -0.0273959469050169, 0.013031769543886185, 0.006981547921895981, -0.02942175231873989, 0.011706159450113773, -0.021454494446516037, 0.02152247354388237, 0.0011989976046606898, -0.008334350772202015, -0.011407047510147095, 0.01434378419071436, 0.029911208897829056, -0.0038408711552619934, -0.006322141736745834, -0.0051120975986123085, 0.008313956670463085, 0.001959524117410183, 0.01156340166926384, -0.012861819937825203, -0.02183518186211586, -7.456558523699641e-05, -0.05639622360467911, 0.008531492203474045, -0.008130410686135292, -0.016383184120059013, -0.0053058406338095665, 0.0010621878318488598, -0.003534961026161909, 0.00450027734041214, 0.0018422584980726242, -0.03344617038965225, -0.0023419116623699665, 0.016954217106103897, 0.01379314623773098, -0.005271850619465113, 0.00477559631690383, -0.03399001061916351, 0.003281735349446535, 0.03695393726229668, 0.019238345324993134, -0.02187597006559372, -0.00632893992587924, 0.016913428902626038, -0.0030726969707757235, 0.014629299752414227, 0.002287527546286583, 0.017362097278237343, -0.016913428902626038, -0.023493895307183266, -0.0613451711833477, 0.025778023526072502, 0.013398861512541771, -0.01343964971601963, -0.006570268888026476, -0.004238554276525974, 0.005244658328592777, 0.0027378953527659178, 0.0043745143339037895, 0.02195754647254944, -0.010271781124174595, 0.013154134154319763, -0.00923848431557417, 0.003715108148753643, -0.039346836507320404, -0.0013562013627961278, 0.010598084889352322, 0.0120324632152915, -0.023738622665405273, 0.01987735740840435, -0.002447280799970031, -0.025057435035705566, -0.010985570959746838, 0.01066606491804123, 0.0074506099335849285, 0.010305770672857761, -0.03276636824011803, -0.0004397457523737103, -0.00022093506413511932, -0.023806603625416756, 0.03706270828843117, -0.016220033168792725, -0.0040991953574121, 0.023153994232416153, 0.009184100665152073, -0.015363484621047974, -0.001772579038515687, 0.026281075552105904, 0.009693950414657593, 0.022596558555960655, -0.03388124331831932, -0.002143070101737976, 0.0034329909831285477, 0.005054314620792866, -0.0072806598618626595, 0.015295504592359066, -0.005445199552923441, -0.006682435981929302, -0.003444887464866042, -0.013759155757725239, 0.02590038813650608, 0.014806048013269901, -0.024268867447972298, -0.01670948788523674, -0.01850416138768196, -0.012086847797036171, -0.010822419077157974, 0.0038306741043925285, 0.016872640699148178, -0.0191703662276268, 0.01036695297807455, -0.01537708006799221, 0.009524000808596611, -0.003993825986981392, 0.03156992048025131, -0.007695338223129511, -0.01259669754654169, 0.008286763913929462, -0.00013989013677928597, -0.03994505852460861, -0.017280520871281624, 0.012956991791725159, 0.004850374534726143, 0.03254883363842964, 0.010768035426735878, -0.003589344909414649, 0.012569505721330643, -0.006967952009290457, -0.029258599504828453, 0.014275804162025452, 0.0030285099055618048, -0.005203870590776205, 0.004524070303887129, -0.003721906105056405, 0.019809378311038017, -0.018952829763293266, -0.016423972323536873, 0.03431631252169609, -0.010292175225913525, 0.02833407185971737, 0.0013094651512801647, 0.012426747940480709, -0.026158710941672325, -0.0053568254224956036, -0.0010893797734752297, -0.00847031082957983, -0.003715108148753643, 0.02488068677484989, 0.000665779341943562, -0.0034397889394313097, 0.012766648083925247, 0.008810210973024368, 0.008864594623446465, -0.011644977144896984, -0.0045682573691010475, 0.024228079244494438, 8.609032374806702e-05, -0.01898002065718174, -0.015880132094025612, 0.019279133528470993, 0.011672168970108032, -0.01346004381775856, 0.026797723025083542, -0.002185557736083865, -0.00599583750590682, 0.012059655040502548, 0.017742784693837166, -0.009381243027746677, -0.0363285206258297, 0.012630688026547432, -0.006124999839812517, -0.0006789504550397396, -0.0038272750098258257, -0.006842189002782106, 0.015798555687069893, 0.020475581288337708, 0.017117368057370186, -0.029992785304784775, -0.01520033273845911, -0.013704772107303143, 0.011175915598869324, 0.014806048013269901, -0.040244173258543015, 6.250975275179371e-05, 0.0028262692503631115, -0.010883601382374763, -0.007980854250490665, 0.04326248541474342, -0.01585294120013714, 0.050577133893966675, 0.01760682463645935, -0.035431187599897385, 0.0025152608286589384, -0.01839539408683777, 0.0033446168527007103, -0.003704911097884178, 0.014506936073303223, -0.002647821791470051, -0.01658712513744831, 0.0043609184212982655, -0.009612374939024448, 0.015866536647081375, -0.013065760023891926, -0.014860432595014572, -0.0019408295629546046, -0.0034941730555146933, -0.009088928811252117, 0.01510515995323658, 0.03282075375318527, 0.024486403912305832, 0.006362929940223694, 0.016260821372270584, -0.008273168466985226, 0.002216148655861616, -0.03439788892865181, 0.02085627056658268, -0.012338373810052872, -0.030699776485562325, -0.014547724276781082, -0.0025526497047394514, 0.02104661427438259, -0.04706936702132225, -0.02066592499613762, 0.024948667734861374, 0.00225013867020607, -0.014751664362847805, -0.03725305199623108, 0.014874028041958809, 0.016288012266159058, -0.001779376994818449, 0.016029689460992813, 0.009422030299901962, -0.028306880965828896, 0.00272429920732975, -0.0072058821097016335, 0.0006704529514536262, 0.009598778560757637, 0.003970033023506403]} +{"id": "test:10000026", "text": "\"Movers & Moving Companies in Randallsville, New York for every moving service .: Movers MAX :.\\nMovers MAX Movers Directory is your one-stop source for moving resources. Research your move before making that all-important move. Inside, you will find helpful Randallsville, New York moving guides and moving tips on every type of moving services as well as free Randallsville, New York moving estimates from professional movers in Randallsville, New York .\\nWith information available on all Randallsville, New York move types you can certainly find out what you need to know about moving anywhere in Randallsville, New York . From local Randallsville, New York moving to long distance moving across the country or around the world to an international location, our Randallsville, New York moving services directory offers all the information you need to plan your next Randallsville, New York move.\\nOur nationwide directory of residential movers and commercial moving companies can offer free Randallsville, New York moving and storage quotes to any state or country. From smaller residential and household local moves or long distance moving to office movers and corporate relocation Randallsville, New York Movers can help you with your Randallsville, New York move.\\nRandallsville, New York Movers is a local moving companies directory with local Randallsville, New York movers ready to help you with your local move. Whether you are moving to Randallsville, New York or moving from Randallsville, New York our local Randallsville, New York moving companies can help with full service residential moving services and YOU-Pack they move or self service moves.\\nNeed to move your car? Randallsville, New York Movers offers Randallsville, New York auto shipping company quotes and reliable Randallsville, New York car shipping companies to move cars anywhere in Randallsville, New York or around the world.\\nLet Randallsville, New York Movers help you with your next Randallsville, New York move and get a FREE Randallsville, New York Moving Company Quote now!\"", "vector_field": [-0.0026292293332517147, -0.008707122877240181, 0.02501538209617138, -0.00970586109906435, -0.019022950902581215, 0.010469996370375156, -0.006381201557815075, -0.008526143617928028, -0.006743160542100668, -0.01855374500155449, -0.012165841646492481, 0.02048419415950775, 0.0030146485660225153, -0.0015190544072538614, 0.001587759586982429, 0.007775413803756237, 0.012983600609004498, -0.0070649017579853535, 0.02000158093869686, -0.024747265502810478, -0.015322927385568619, 0.009940464049577713, 0.006558829452842474, -0.010798441246151924, -0.0064817457459867, 0.014692850410938263, 0.029627006500959396, 0.008351867087185383, 0.015296115539968014, -0.007198960520327091, 0.0014562143478542566, -1.819744284148328e-05, -0.003958087880164385, -0.03145020827651024, -0.004232908599078655, -0.013077441602945328, -0.021033834666013718, -0.016274744644761086, -0.011207320727407932, -0.03691980987787247, 0.015738509595394135, -0.024559583514928818, -0.01585916243493557, -0.004085443448275328, -0.007232475560158491, 0.028045112267136574, -0.005483007058501244, -0.041236504912376404, -0.009236655198037624, 0.009015457704663277, 0.001434429781511426, 0.0016505996463820338, -0.008224510587751865, 0.01984071172773838, 0.0007473781006410718, -0.006109732203185558, -0.013633785769343376, 0.012145732529461384, -0.00806364044547081, -0.008914913982152939, 0.009109299629926682, 0.010329235345125198, -0.021637098863720894, 0.014585603959858418, 0.011502250097692013, -0.041370563209056854, -0.005151211749762297, -0.019317882135510445, 0.009591910988092422, 0.002523658098652959, 0.021194705739617348, 0.011904426850378513, 0.015229086391627789, 0.018178381025791168, 0.02043057046830654, -0.011596091091632843, -0.00216672639362514, 0.02010882832109928, -0.003390013240277767, -0.011857505887746811, -0.005228295456618071, -0.025484587997198105, -0.025243282318115234, 0.036437198519706726, 0.048770613968372345, 0.026717931032180786, -0.01671713963150978, 0.012239573523402214, -0.016113875433802605, 0.01286965049803257, 0.027723371982574463, 0.03198644146323204, -0.0022940821945667267, -0.024090375751256943, -0.00026958397938869894, 0.006149949971586466, 0.0035257479175925255, 0.006877219304442406, -0.03362196311354637, -0.035713277757167816, -0.0067565664649009705, 0.010751520283520222, -0.008961834944784641, -0.003381634596735239, -0.029466137290000916, -0.0061600045301020145, -0.019398316740989685, -0.013968932442367077, 0.002848750678822398, -0.024291465058922768, -0.019720057025551796, -0.003669861238449812, 0.004212799482047558, 0.01551060937345028, -0.014464951120316982, -0.02309834025800228, 0.028420478105545044, -0.0077083841897547245, -0.011073261499404907, -0.019049763679504395, 0.006689536850899458, 0.03195963054895401, 0.023996535688638687, 0.0019857469014823437, 0.034185007214546204, 0.004092146642506123, -0.014451544731855392, 0.005164617672562599, -0.001070795115083456, -0.004001657012850046, -0.008619984611868858, 0.028018301352858543, 0.007540810853242874, 0.0064750430174171925, 0.005908644292503595, 0.03262992575764656, -0.014625821262598038, 0.01917041651904583, -0.007567622698843479, -0.030887160450220108, 0.030887160450220108, 0.015215680003166199, -0.01620771549642086, -0.006247142795473337, -0.0041424185037612915, 0.005345596931874752, 0.026986047625541687, 0.01618090458214283, 0.01841968670487404, 0.01160279382020235, 0.008800963871181011, -0.017105910927057266, -0.010671084746718407, 0.013834874145686626, 0.003743593581020832, 0.011984861455857754, -0.024492552503943443, 0.008921616710722446, -0.025055600330233574, -0.0040184142999351025, 0.010027602314949036, 0.034640807658433914, 0.00024381955154240131, 0.007366534322500229, 0.0033296868205070496, 0.005576848518103361, 0.0017260077875107527, -0.011073261499404907, -0.011408409103751183, -0.010657679289579391, 0.008378678932785988, 0.017320403829216957, -0.04595537483692169, 0.0317719504237175, 0.002924158703535795, 0.011891020461916924, 0.00433345278725028, 0.014773285947740078, 0.006273954641073942, -0.0022840278688818216, 0.002327597001567483, -0.0011269322130829096, 0.02614147774875164, 0.04402492940425873, -0.008921616710722446, -0.009826513938605785, 0.006029297132045031, 0.0037000244483351707, 0.0006816054810769856, -0.02344689331948757, -0.004976935219019651, 0.009886840358376503, 0.009732672944664955, -0.011730150319635868, -0.6220331192016602, -0.012635047547519207, 0.011059856042265892, -0.0442662350833416, 0.0055936058051884174, 0.027589313685894012, -0.003212385345250368, 0.035150233656167984, -0.019022950902581215, 0.03689299896359444, -0.028527725487947464, 0.028822654858231544, 0.0007511485600844026, -0.03126252442598343, -0.03370239585638046, -0.0006137382006272674, 0.023406676948070526, -0.013754438608884811, 0.03238862007856369, -0.017883451655507088, -0.00622703367844224, 0.025162847712635994, -0.00819769874215126, 0.027696559205651283, 0.030190054327249527, -0.015027998015284538, 0.008097155019640923, -0.0004511918523348868, -0.010369452647864819, 0.00915621966123581, -0.02678496018052101, 0.018593963235616684, 0.004175933543592691, 0.006950951647013426, 0.041692305356264114, 0.010624164715409279, -0.010798441246151924, 0.027133513242006302, 0.024184217676520348, 0.030806725844740868, -0.023326240479946136, -0.027562500908970833, 0.03193281963467598, 0.02368820086121559, -0.0052785673178732395, -0.027642935514450073, 0.01951896958053112, -0.004360264167189598, -0.009075785055756569, -0.012051891535520554, -0.028313230723142624, -0.003260981757193804, -0.014505168423056602, -0.017695769667625427, -0.009967275895178318, -0.02506900578737259, 0.0035827229730784893, 0.0005504791624844074, 0.002962700789794326, 0.0028923198115080595, -0.0012325035640969872, 0.0055936058051884174, -0.023929506540298462, 0.001105147646740079, -0.03265673667192459, 0.009699158370494843, -7.263266888912767e-05, -0.016516050323843956, 0.009732672944664955, 0.02547118254005909, -0.01276240311563015, 0.03351471573114395, -0.031101655215024948, 0.024090375751256943, -0.002302460838109255, 0.042550280690193176, 0.018567152321338654, -0.006759917829185724, -0.006568884011358023, -0.018433092162013054, 0.016408804804086685, 0.005600308533757925, -0.014867126941680908, -0.04576769471168518, 0.03842126950621605, -0.020805934444069862, -0.017749393358826637, -0.008821072988212109, 0.035686466842889786, -0.0021482931915670633, -0.0014989456394687295, -0.02309834025800228, -0.012883055955171585, -0.03780459612607956, -0.014089586213231087, 0.015296115539968014, -0.019639622420072556, 0.018379468470811844, 0.013452806510031223, -0.03171832486987114, -0.03919880837202072, 0.0010967690031975508, 0.013539944775402546, 0.004547946620732546, -0.013131065294146538, 0.015282710082828999, -0.002010882832109928, 0.012500988319516182, 0.04702784866094589, -0.012058594264090061, 0.01110677607357502, -0.02619510143995285, -0.03415819630026817, -0.02248167060315609, -0.008472519926726818, -0.030806725844740868, 0.024653423577547073, 0.012239573523402214, 0.0017813070444390178, -0.03198644146323204, -0.022280581295490265, -0.005650580860674381, 0.026557059958577156, 0.0036832671612501144, -0.005348948296159506, 0.02573930099606514, 0.01711931638419628, -0.010389561764895916, -0.016757357865571976, 0.01868780516088009, -0.030216867104172707, -0.011167102493345737, 0.015389956533908844, 0.0028822654858231544, -0.0091025959700346, 0.0063175237737596035, 0.021529853343963623, -0.010262205265462399, 0.008144075982272625, -0.02301790565252304, -0.02234761230647564, 0.004340155515819788, -0.02165050618350506, -0.03764372691512108, -0.002999566961079836, -0.024264652281999588, -0.01852693408727646, -0.00951817911118269, -0.029761066660284996, 0.01871461607515812, -0.015430174767971039, 0.010295719839632511, -0.028232796117663383, -0.010349343530833721, -0.014974374324083328, -0.01097271777689457, -0.024103783071041107, -0.007366534322500229, 0.021744346246123314, -0.007829037494957447, 0.008331757970154285, -0.007507295813411474, -0.035900961607694626, 0.01844649948179722, -0.009109299629926682, -0.027696559205651283, 0.017642145976424217, -0.0056204176507890224, 0.022307394072413445, -0.0045848130248487, -0.005526576191186905, -0.023701606318354607, -0.005144508555531502, 0.03769735246896744, 0.01908998191356659, 0.030592231079936028, 0.008083748631179333, 0.021744346246123314, 0.023487111553549767, -0.00010955121979350224, 0.017374027520418167, -0.010221987962722778, -0.027348006144165993, 0.0074804844334721565, 0.01639539748430252, 0.01610046811401844, 0.0018416335806250572, 0.006498503033071756, -0.03437269106507301, -0.03348790109157562, -0.029680630192160606, 0.02026969939470291, -0.009638831950724125, 0.009350605309009552, 0.00044365102075971663, -0.006103029474616051, -0.022039275616407394, 0.03469443321228027, 0.01636858657002449, 0.013238311745226383, 0.010858767665922642, -0.0047155204229056835, -0.003844137769192457, -0.007279396057128906, 0.006381201557815075, -0.01949215866625309, 0.021328764036297798, -0.005241701379418373, 0.010818549431860447, 0.007694978266954422, 0.011716743931174278, -0.04531189426779747, -0.004950123373419046, -0.022629134356975555, 0.017749393358826637, 0.05107642337679863, -0.015456986613571644, 0.042496658861637115, -0.002909077098593116, 0.012500988319516182, 0.010644272901117802, -0.010966014117002487, 0.04311332851648331, 0.009008754976093769, -0.021100863814353943, 0.005422680638730526, -0.0013791305245831609, 0.0110665587708354, -0.009431040845811367, 0.0028236147481948137, -0.0035927772987633944, 0.015202274546027184, -0.0023912747856229544, 0.03370239585638046, 0.029814690351486206, 0.0041692303493618965, 0.018097946420311928, 0.008036828599870205, 0.004001657012850046, -0.029600195586681366, 0.028313230723142624, -0.005962267518043518, 0.026449812576174736, 0.008767449297010899, -9.551693801768124e-05, -0.00562712037935853, -0.01556423306465149, 0.020309917628765106, -0.025323718786239624, 0.025484587997198105, -0.01161620020866394, 0.042496658861637115, -0.02563205361366272, -0.002756585134193301, 0.006977763492614031, 0.006991169415414333, 0.020725499838590622, -0.003780459752306342, 0.0429256446659565, 0.011140290647745132, 0.007574325427412987, 0.023433487862348557, 0.015121839009225368, -0.0020929938182234764, -0.010322531685233116, -0.022146522998809814, 0.01984071172773838, -0.018781645223498344, 0.01825881563127041, 0.003390013240277767, -0.0034520155750215054, -0.017682362347841263, 0.015658074989914894, 0.01010803785175085, 0.012145732529461384, 0.007829037494957447, 0.0036296434700489044, -0.025565024465322495, -0.008512737229466438, -0.00311351683922112, 0.006478394381701946, -0.030484983697533607, 0.0050741275772452354, -0.004444051068276167, -0.0012107189977541566, -0.00819769874215126, -0.015322927385568619, 0.023594358935952187, -0.042845211923122406, 0.015336332842707634, 0.025511400774121284, 0.028849465772509575, -0.03139658272266388, 0.0038541920948773623, 0.023621169850230217, -0.00320400670170784, 0.041960421949625015, 0.002069533569738269, 0.02323240041732788, -0.025940388441085815, 0.012159137986600399, 0.011388299986720085, -0.008814370259642601, -0.005623769015073776, 0.024975165724754333, 0.0003653103776741773, 0.0022203498519957066, 0.005767882335931063, 0.0032911449670791626, -0.03423863276839256, -0.006971060764044523, -0.012534502893686295, -0.029546571895480156, 0.011777070350944996, -0.006890625227242708, -0.0019807196222245693, -0.01655626855790615, 0.016676921397447586, 0.04407855123281479, 0.011388299986720085, -0.01281602680683136, 0.002411383669823408, -0.007942987605929375, 0.011019637808203697, 0.1082659363746643, -0.017977293580770493, -0.02544437162578106, 0.003924572840332985, -0.015497203916311264, -0.025591835379600525, 0.011609497480094433, -0.035927772521972656, 0.01320479717105627, 0.0084121935069561, 0.007178851868957281, 0.0061398958787322044, 0.027750182896852493, 0.014424732886254787, 0.0014134830562397838, 0.0011788801057264209, -0.004440699703991413, -0.01132127083837986, -0.0008923292625695467, -0.022602323442697525, -0.003659806912764907, 0.009162923321127892, 0.0021482931915670633, 0.035364724695682526, -0.006324226502329111, 0.01580553874373436, 0.0012861271388828754, 0.005238350015133619, 0.0006351038464345038, -0.03842126950621605, -0.0006552126724272966, 0.02640959434211254, 0.023353053256869316, 0.014759880490601063, -0.006330929696559906, -0.015027998015284538, -0.011334676295518875, -0.006759917829185724, 0.0010733087547123432, 0.02218674123287201, 0.01331204455345869, 0.001391698489896953, -0.005271864589303732, -0.003696673084050417, 0.047564081847667694, -0.021489635109901428, -0.020296510308980942, 0.010798441246151924, 0.026851989328861237, -0.0030950838699936867, 0.048690177500247955, -0.014009150676429272, 0.022092899307608604, 0.0012802621349692345, 0.01266856212168932, 0.013023817911744118, -0.008526143617928028, -0.00017343864601571113, -0.025645459070801735, -0.03447993844747543, -0.013694112189114094, -0.008673608303070068, 0.02482770010828972, -0.022414641454815865, -0.004521134775131941, -0.008606578223407269, -0.015953004360198975, 0.017977293580770493, -0.031048031523823738, 0.0035458568017929792, 0.028179172426462173, -0.03817996382713318, 0.003669861238449812, -0.013432697393000126, 0.029412513598799706, 0.014907345175743103, -0.010188473388552666, 0.005382463335990906, -0.002206943929195404, 0.02165050618350506, -0.01006782054901123, -0.003371580271050334, -0.0003669861180242151, -0.0058751292526721954, -0.03638357296586037, 0.006780026480555534, 0.042174916714429855, 0.00979970209300518, -0.0049434201791882515, -0.004082092083990574, 0.021181300282478333, 0.0012836134992539883, 0.004269774537533522, -0.0045378925278782845, 0.035766903311014175, 0.004112255293875933, -0.006528666242957115, -0.011683229357004166, 0.002448249841108918, 0.003371580271050334, 0.020283104851841927, -0.020966805517673492, 0.00935730803757906, -0.030806725844740868, 0.013727626763284206, -0.009779593907296658, 0.009792999364435673, -0.013157877139747143, -0.00971926748752594, -0.0016380316810682416, -0.012366930022835732, -0.019773680716753006, -0.014170020818710327, 0.004145770333707333, -0.028125548735260963, 0.02536393515765667, 0.019049763679504395, -0.0016522754449397326, -0.019559187814593315, 0.012588126584887505, 0.015188868157565594, -0.01938491128385067, 0.010711302980780602, 0.015175462700426579, -0.017575116828083992, 0.02560524083673954, -0.00655547808855772, -0.02590017206966877, -0.002123157260939479, -0.0019153659231960773, 0.009196437895298004, 0.02232079952955246, -0.03372920677065849, -0.03686618432402611, -0.010449888184666634, 0.009243357926607132, 0.009900246746838093, 0.04303289204835892, -0.010718005709350109, -0.020578034222126007, -0.011756962165236473, 0.011388299986720085, -0.02186499908566475, -0.0035894259344786406, -0.0011495547369122505, -0.012755700387060642, 0.009692455641925335, -0.005178023129701614, -0.0044071851298213005, 0.024640018120408058, 0.005999133922159672, -0.018218599259853363, -0.012259682640433311, -0.0071453372947871685, -0.0198138989508152, -0.014143208973109722, -0.013157877139747143, 0.008787558414041996, 0.023728417232632637, 0.011756962165236473, 0.02638278342783451, 0.010731411166489124, -0.013392480090260506, 0.0006250494043342769, 0.0018634181469678879, -0.014773285947740078, -0.041263315826654434, 0.013117658905684948, -0.015403362922370434, 0.011803882196545601, 0.03815314918756485, 0.04989670589566231, -0.0017913614865392447, 0.018433092162013054, 0.003361525945365429, 0.009725970216095448, 0.029090771451592445, -0.03724155202507973, -0.013667300343513489, -0.048100318759679794, 0.007896066643297672, 0.007446969393640757, -0.008552955463528633, -0.002738152164965868, -0.03700024634599686, -0.014934157021343708, 0.04029809311032295, 0.005687446799129248, 0.028742218390107155, -0.017052287235856056, -0.0020460733212530613, -0.015818946063518524, 0.0021114270202815533, 0.023004500195384026, -0.00313027435913682, 0.008003314025700092, -0.005958916153758764, -0.03941330313682556, 0.0008780854986980557, -0.008867993019521236, -0.005781288258731365, -0.0038407861720770597, -0.009752782061696053, 0.0030598933808505535, -0.03217412531375885, -0.017950480803847313, -0.04488290473818779, -0.006471691187471151, 0.005298676434904337, -0.019103387370705605, 0.005348948296159506, -0.02226717583835125, -0.02662408910691738, -0.03340746834874153, 0.016381992027163506, -0.006639264989644289, 0.008519439958035946, -0.0003343092685099691, -0.042362600564956665, -0.03126252442598343, -0.015296115539968014, -0.013144470751285553, 0.013245015405118465, 0.007775413803756237, 0.02619510143995285, 0.02250848151743412, -0.016194310039281845, -0.023741822689771652, -0.023768635466694832, -5.2104907808825374e-05, 0.002739827847108245, 0.014625821262598038, 0.0033497957047075033, -0.002310839481651783, -0.008258025161921978, 0.008573063649237156, -0.03973504528403282, -0.011891020461916924, -0.041719116270542145, 0.04257709160447121, 0.00021072376694064587, 0.01868780516088009, -0.028179172426462173, 0.009270169772207737, 0.01166982389986515, 0.028742218390107155, -0.0008361921063624322, 0.00331460521556437, -0.0076815723441541195, -0.0009761160472407937, -0.030163243412971497, -0.002346029970794916, -0.024867918342351913, 0.020712092518806458, 0.03831402212381363, -0.036464009433984756, -0.01551060937345028, -0.02331283502280712, 0.00485963374376297, -0.011073261499404907, 0.013278529979288578, 0.00482611870393157, -0.0022035925649106503, 0.020993616431951523, 0.0002586916962172836, 0.028420478105545044, -0.011327973566949368, -0.010657679289579391, 0.006830298807471991, 0.030109619721770287, -0.025779517367482185, -0.0035726686473935843, 0.0009258439531549811, 0.010208581574261189, -0.009142814204096794, 0.01572510413825512, -0.030350925400853157, 0.007420157548040152, -0.04737640172243118, -0.0008646796341054142, 0.02291065827012062, 0.015537421219050884, -0.007058199029415846, -0.019130198284983635, -0.0005663986667059362, -0.02466682903468609, 0.002843723399564624, 0.009833217598497868, 0.0004604083951562643, -0.02272297628223896, -0.025270095095038414, 0.00931709073483944, 0.00508083077147603, -0.0034955847077071667, 0.0033229838591068983, 0.0005207348731346428, 0.013023817911744118, 0.02683858387172222, -0.015121839009225368, 0.006813541520386934, -0.006997872143983841, -0.004625030793249607, -0.047671329230070114, 0.0009786296868696809, 0.006123138125985861, -0.028634971007704735, 0.012373632751405239, -0.019317882135510445, -0.008921616710722446, -0.006149949971586466, -0.018674399703741074, 0.010577243752777576, -0.008077045902609825, 0.0007528242422267795, 0.01989433355629444, 0.013633785769343376, 0.005509818904101849, 0.004692059941589832, -0.019210634753108025, 0.014692850410938263, -0.022066088393330574, -0.012112217955291271, 0.013157877139747143, 0.008030125871300697, 0.011964753270149231, -0.006351038347929716, 0.01890229806303978, 0.011354785412549973, -0.0031637889333069324, -0.024800889194011688, 0.0028102088253945112, -0.007587731350213289, -0.001105147646740079, -0.007601137273013592, -0.0027146919164806604, 0.008258025161921978, -0.0067766751162707806, 0.026691118255257607, -0.0063443356193602085, -0.010650976561009884, 0.010416373610496521, 0.014183427207171917, 0.022629134356975555, 0.023795446380972862, -0.013781250454485416, 0.0035626140888780355, 0.020939992740750313, -0.008532846346497536, -0.001890229876153171, 0.0077351960353553295, 0.014947562478482723, 0.029921937733888626, 0.0037771083880215883, -0.02514944225549698, -0.028098735958337784, -0.007983204908668995, -0.04919959977269173, -0.00013259258412290365, 0.005201483611017466, 0.02627553604543209, 0.023862477391958237, 0.010268907994031906, -0.0008805991383269429, 0.014719662256538868, 0.029734253883361816, 0.025430964305996895, 0.009786296635866165, 0.005895238369703293, -0.010731411166489124, 0.0028286417946219444, 0.021087458357214928, 0.014545385725796223, -0.010992825962603092, 0.012903165072202682, 0.008418896235525608, 0.014170020818710327, -0.013834874145686626, 0.0070112780667841434, -0.006800135597586632, -0.009276872500777245, -0.019559187814593315, -0.008190996013581753, -0.0004084605898242444, -0.011777070350944996, 0.027589313685894012, -0.02528350055217743, -0.0004989503067918122, 0.016435615718364716, -0.011622902937233448, -0.009270169772207737, 0.0048462278209626675, 0.023835664615035057, 0.001938826171681285, 0.007359831128269434, -0.009806405752897263, -0.0014394569443538785, 0.01286965049803257, -0.004993692506104708, 0.005392517428845167, 0.008385381661355495, -0.007534107659012079, 0.015684885904192924, -0.00775530468672514, -0.010322531685233116, 0.03901112824678421, -0.03182557225227356, -0.011689932085573673, -0.005938807502388954, -0.02170412987470627, -0.017937075346708298, 0.027482066303491592, 0.00014159966667648405, 0.006250494159758091, 0.0077351960353553295, -0.0028554536402225494, -0.013888497836887836, 0.0004951799055561423, -0.017829827964305878, -0.0034888817463070154, -0.0013665624428540468, -0.008164184167981148, 0.0187414288520813, -0.009002052247524261, -0.034640807658433914, 0.01930447481572628, -6.001228393870406e-05, -0.0019773682579398155, -0.02178456448018551, 0.028929902240633965, -0.03166470304131508, -0.0083451634272933, -0.028929902240633965, 0.022173335775732994, -0.014853721484541893, 0.01327182725071907, 0.014330891892313957, 0.17727942764759064, 0.018044322729110718, 0.006089623551815748, 0.03319297358393669, -0.0014469977468252182, 0.009229952469468117, 0.010630867443978786, -0.008244619704782963, 0.007185554597526789, 0.012078703381121159, -0.012581423856317997, 0.03249586746096611, -0.000811056059319526, 0.005791342817246914, 0.012769105844199657, -0.028715407475829124, -0.04330101236701012, -0.028554536402225494, -0.020671876147389412, 0.010228690691292286, 0.0056405263021588326, 0.008894804865121841, -0.007701681461185217, -0.014612415805459023, 0.002848750678822398, 0.006562181282788515, -0.007279396057128906, 0.007292801979929209, -0.012648453004658222, 0.015845756977796555, -0.024157406762242317, -0.015845756977796555, 0.014170020818710327, 0.010490105487406254, -0.023647982627153397, -0.004423942416906357, 0.011301161721348763, -0.008653499186038971, -0.0017176291439682245, 0.009250061586499214, -0.00011740623449441046, -0.023192182183265686, -0.01847331039607525, 0.004919960163533688, -0.006645967718213797, 0.028876278549432755, -0.01933128759264946, 0.0010087928967550397, -0.004350210074335337, 0.0003181803331244737, -0.04504377767443657, -0.02517625316977501, 0.017079098150134087, 0.020872963592410088, 0.013030520640313625, 0.010275611653923988, 0.006826947443187237, 0.0009526557405479252, -0.010449888184666634, -0.006595695856958628, -0.006904031150043011, 0.019264258444309235, -0.022736381739377975, 0.00405863206833601, -0.018030915409326553, 0.021690722554922104, 0.015188868157565594, -0.0023812204599380493, 0.018620776012539864, -0.010650976561009884, -0.010490105487406254, 0.006243791431188583, -0.0014981076819822192, -0.024264652281999588, 0.025216471403837204, 0.006294063292443752, -0.003044811775907874, 0.0662250742316246, 0.012038485147058964, 0.04338144510984421, -0.047671329230070114, 0.007701681461185217, -0.0012777484953403473, -0.025565024465322495, -0.004397130571305752, -0.008767449297010899, 0.005600308533757925, -0.005526576191186905, 0.01989433355629444, -0.0029761067125946283, -0.0040720379911363125, -0.0016103819943964481, -0.006089623551815748, -0.015027998015284538, -0.0020092071499675512, 0.023808853700757027, 0.02603423036634922, 0.0018768239533528686, -0.019612811505794525, 0.011857505887746811, -0.018781645223498344, 0.019425127655267715, 0.0020795881282538176, 0.006434825249016285, -0.0168243870139122, -0.011354785412549973, 0.022012464702129364, 0.03289804235100746, -0.01250769104808569, -0.048287998884916306, 0.0029828096739947796, -0.021958841010928154, -0.005288621876388788, -0.020578034222126007, 0.04490971565246582, -0.0010624164715409279, 0.0020343430805951357, -0.014183427207171917, 0.03128933906555176, -0.010429779067635536, 0.009699158370494843, -0.0344531275331974, 0.0009509800001978874, 0.00391116738319397, -0.015376551076769829, -0.020229481160640717, -0.04340825974941254, 0.00475238636136055, 0.00013112631859257817, -0.029841501265764236, 0.01850012317299843, -0.0019220688845962286, -0.0038575436919927597, 0.015524015761911869, 0.003746944945305586, 0.0014000771334394813, -0.014706256799399853, -0.03265673667192459, -0.004651842173188925, -0.027348006144165993, 0.015430174767971039, 0.016542863100767136, 0.02010882832109928, 0.023661388084292412, -0.006066163070499897, 0.018915705382823944, -0.0031704918947070837, 0.015591044910252094, -0.034640807658433914, -0.013821467757225037, -0.0388234443962574, -0.024680236354470253, 0.005550036672502756, -0.015376551076769829, 0.012239573523402214, -0.0019522320944815874, -0.030887160450220108, -0.034533560276031494, 0.029787877574563026, 0.028447289019823074, -0.0198138989508152, 0.017950480803847313, 0.00790276937186718, -0.004698763135820627, -0.020846152678132057, 0.0027213948778808117, -0.16902141273021698, 0.015792133286595345, 0.01933128759264946, -0.017521493136882782, 0.015483797527849674, 0.016087062656879425, 0.021261734887957573, 0.017105910927057266, -0.011743555776774883, 0.014116398058831692, 0.011395002715289593, -0.007621245924383402, -0.029841501265764236, -0.028554536402225494, -0.0010733087547123432, -0.018004104495048523, -0.023299429565668106, 0.004487620200961828, -0.0010791737586259842, 0.020470786839723587, 0.04649161174893379, 0.013808062300086021, 0.0012358550447970629, -0.029385700821876526, -0.001430240459740162, 0.0017796313622966409, -0.007152040023356676, -0.0009124380885623395, 0.0018500122241675854, -0.0168243870139122, -0.02301790565252304, 0.005043964367359877, -0.011395002715289593, -0.018299033865332603, 0.021114269271492958, -0.0015190544072538614, -0.020551223307847977, -0.013278529979288578, -0.011535764671862125, 0.0012408823240548372, 0.014759880490601063, 0.03128933906555176, -0.006270602811127901, 0.0225352942943573, 0.00821110513061285, 0.01257472112774849, 0.009122705087065697, -0.0031721678096801043, -0.0065990472212433815, -0.042630717158317566, -0.02140919864177704, -0.0074000488966703415, 0.014947562478482723, 0.009491367265582085, 0.036303140223026276, 0.002178456401452422, 0.014867126941680908, -0.013982338830828667, -0.0007582703838124871, -0.027160324156284332, -0.029224831610918045, 0.009008754976093769, 0.007554216776043177, -0.013285232707858086, -0.01132127083837986, -0.024090375751256943, -0.03313934803009033, -0.01335226185619831, -0.010007494129240513, 0.0037502965424209833, 0.011086667887866497, 0.00815748143941164, -0.0009434391977265477, 0.008680311031639576, 0.014263862743973732, -0.006773323751986027, -0.014062774367630482, 0.00025345501489937305, 0.004849579185247421, -0.01852693408727646, -0.0331125371158123, 0.027535689994692802, -0.03166470304131508, -0.02678496018052101, 0.000784663250669837, 0.004862985108047724, -0.0022019166499376297, -0.006910733878612518, 0.002178456401452422, 0.047590892761945724, -0.01136148814111948, -0.03434588015079498, 0.003512341994792223, -0.035579219460487366, 0.020805934444069862, -0.005171320401132107, -0.003810622962191701, -0.0030950838699936867, -0.010523620061576366, -0.024680236354470253, 0.020859558135271072, -0.020551223307847977, -0.021717535331845284, -0.0007494728197343647, 0.034292254596948624, 0.008774152025580406, 0.009565099142491817, 0.021395793184638023, 0.018151570111513138, -0.035525597631931305, 0.01839287579059601, -0.0010473348665982485, 0.016167497262358665, -0.0007628786843270063, -0.007024683989584446, 0.009283576160669327, 0.0023259210865944624, -0.02590017206966877, 0.02232079952955246, 0.026020824909210205, 0.0461430586874485, -0.018044322729110718, -0.02018926478922367, 0.013901903294026852, 0.024438928812742233, 0.009880137629806995, -0.06788740307092667, -0.0010305774630978703, -0.0037938656751066446, 0.016328368335962296, -0.04716190695762634, 0.026315754279494286, -0.011515655554831028, 0.03986910358071327, 0.0017779555637389421, 0.03796546906232834, -0.003343092743307352, 0.016355181112885475, 0.02183818817138672, 0.017923669889569283, 0.03716111555695534, -0.002245485782623291, 0.034774865955114365, -0.027187136933207512, -0.03220093622803688, 0.01564466953277588, -0.003961439244449139, -0.014639227651059628, -0.03702705726027489, -0.011006232351064682, -0.0016447346424683928, 0.04086114093661308, -0.03238862007856369, 0.0020025041885674, 0.013928715139627457, -0.015631262212991714, -0.007386642973870039, -0.006471691187471151, 0.01650264486670494, -0.020671876147389412, -0.000784663250669837, 0.024546176195144653, -0.0523902028799057, -0.017414245754480362, 0.01997477002441883, -0.011086667887866497, 0.0049534747377038, -0.0034989360719919205, -0.010356046259403229, -0.010563838295638561, -0.016676921397447586, -0.027589313685894012, -0.009384119883179665, 0.022361017763614655, 0.0037938656751066446, 0.003971493802964687, 0.007560919504612684, -0.0055165220983326435, -0.009015457704663277, -0.041558247059583664, -0.011569279246032238, 0.005563442595303059, -0.017789609730243683, -0.006297414656728506, -0.021583475172519684, -0.0013741032453253865, -0.019773680716753006, 0.0035827229730784893, -0.03252267837524414, -0.006659373641014099, 0.01169663481414318, 0.010999529622495174, -0.00805023405700922, 0.009095893241465092, -0.002394626382738352, 0.00866690557450056, -0.03112846612930298, 0.013667300343513489, -0.036303140223026276, 0.010201878845691681, -0.021637098863720894, 0.008030125871300697, -0.041558247059583664, -0.0030615690629929304, 0.030190054327249527, 0.003706727409735322, -0.003586074337363243, -0.03319297358393669, 0.008888102136552334, -0.010637570172548294, 0.011542467400431633, 0.008579767309129238, -0.008586470037698746, 0.015095027163624763, 0.006635913625359535, -0.021744346246123314, 0.02215992845594883, 0.024170812219381332, -0.005245052743703127, -0.014880533330142498, 0.029305266216397285, -0.008190996013581753, -0.001066605793312192, 0.024546176195144653, -0.0031051381956785917, -0.00415247306227684, -0.03330022096633911, -0.0071185254491865635, -0.07501933723688126, -0.0016430588439106941, 0.018352657556533813, -0.005178023129701614, -0.012903165072202682, -0.017199750989675522, 0.007259286940097809, -0.012836135923862457, 0.010195176117122173, 0.011140290647745132, -0.024841105565428734, -0.0017176291439682245, 0.009672346524894238, 0.012159137986600399, -0.005932104308158159, 0.005426032468676567, -0.019961364567279816, -0.019049763679504395, 3.6362155242386507e-06, 0.0110665587708354, 0.00584831740707159, -0.009055675938725471, 0.025377342477440834, 0.022173335775732994, 0.008532846346497536, 0.024345088750123978, -0.002273973310366273, 0.004769143648445606, -0.023138558492064476, -0.0014210238587111235, -0.009699158370494843, -0.020980210974812508, 0.001317966147325933, 0.041585057973861694, -0.002972755115479231, -0.015953004360198975, -0.009256764315068722, 0.011743555776774883, 0.010657679289579391, 0.0019254203652963042, -0.027267571538686752, -0.017749393358826637, 0.002813560189679265, -0.024197623133659363, -0.004903202876448631, 0.021074052900075912, 0.018017509952187538, -0.004648490808904171, 0.009538287296891212, 0.016006628051400185, 0.03858213871717453, -0.0004905716050416231, -0.005268513225018978, -0.019451940432190895, -0.008003314025700092, -0.008566360920667648, 0.01690482161939144, -0.022414641454815865, -0.0030682720243930817, 0.025270095095038414, 0.03405094891786575, 0.02568567730486393, 0.02579292468726635, 0.01663670502603054, 0.018593963235616684, 0.001972340978682041, -0.03233499452471733, 0.007889363914728165, 0.018406281247735023, -0.03847489133477211, -0.016087062656879425, 0.017749393358826637, 0.03678575158119202, -0.00019951727881561965, 0.014773285947740078, -0.0012660183710977435, -0.01270877942442894, 0.009846623055636883, -0.027803806588053703, 0.010469996370375156, 0.019693246111273766, 0.008311648853123188, -0.018245410174131393, 0.02175775170326233, -0.028340041637420654, 0.0034218523651361465, -0.0409683883190155, 0.0255248062312603, 0.00657558673992753, -0.0060795689933001995, -0.026476623490452766, -0.0018516879063099623, -0.014625821262598038, 0.003844137769192457, -0.0035190449561923742, -0.0009643859229981899, 0.0076815723441541195, 0.0038709493819624186, 0.05064743757247925, 0.03158426657319069, 0.0061331926845014095, -0.0030565420165657997, -0.007212366443127394, -0.017883451655507088, 0.0022974335588514805, 0.0055165220983326435, -0.01959940418601036, 0.0015601099003106356, -0.013794656842947006, 0.021127676591277122, 0.030029183253645897, -0.007393346168100834, 0.0011981510324403644, 0.014759880490601063, -0.02264254167675972, 0.02317877672612667, 0.012460771016776562, -0.011180508881807327, -0.0063443356193602085, 0.010101335123181343, -0.011381597258150578, 0.012480879202485085, 0.023929506540298462, 0.003974845167249441, 0.0049970438703894615, 0.015041403472423553, -0.013519835658371449, 0.004698763135820627, 0.010845361277461052, -0.004986989311873913, 0.014746474102139473, 0.0006736457580700517, -0.01925085112452507, 0.017642145976424217, -0.011629605665802956, 0.002748206490650773, 0.004581461660563946, -0.0056472294963896275, -0.02242804691195488, 0.05657283589243889, 0.00826472882181406, 0.00786255206912756, 0.011240835301578045, -0.020470786839723587, 0.023795446380972862, 0.041638679802417755, 0.007353128399699926, -0.002674474148079753, -0.021033834666013718, 0.004236259963363409, -0.0015911110676825047, 0.020805934444069862, 0.003101786831393838, -0.01125424075871706, 0.03171832486987114, 0.006485097110271454, 0.024546176195144653, -0.008311648853123188, 0.0027331248857080936, 0.028876278549432755, 0.01825881563127041, 0.026557059958577156, -0.014974374324083328, -0.00935730803757906, -0.030270490795373917, -0.016435615718364716, -0.003222439670935273, -0.01125424075871706, -0.01270877942442894, -0.011864208616316319, -0.0062639000825583935, 0.0005923725548200309, -0.008204402402043343, 0.006786729674786329, -0.0074804844334721565, -0.015550827607512474, -0.0172265637665987, 0.004008359741419554, 0.010061116889119148, -0.011904426850378513, 0.004507728852331638, -0.0028638322837650776, -0.002049424685537815, -0.0017033853800967336, -0.004266423173248768, -0.007574325427412987, -0.005188077688217163, -0.03689299896359444]} +{"id": "test:10000027", "text": "\"Vashikaran Specialist Chandigarh is a black magic study of attraction, by which you can attract your partner, Girlfriend,Wife, Husband Anything that is living! From ages it\u2019s going in India and everywhere in the world! It\u2019s A Science for Welfare of Mankind. I need to recover its roots, it\u2019s As Simple as Singing a Song and your Mate will love you for this! Kindly Don\u2019t Use These Powers in Wrong Way on the other aspect if you done as such then these powers may hurt you!\\nThe usage of Online Vashikaran Specialist Chandigarh to win the individual love you need or treasured. It is advised that Online Vashikaran Specialist Pandit Chandigarhnot to be mishandle for imperfect or hurting other. We All understand that Love is God and shaitaan is for heartedness. operate some individual to do as you wish is want yet to make his/her identity to do likewise is not that much horrible. Use vashikaran mantra in Hindi not to lose. Regularly pick humanity and deal with your problems with the help of them.\\nOnline Vashikaran Specialist Pandit ji Chandigarh similarly oversee disturbed impact of the enemy identity so they may stay a long way from their country, nation, home, work and even from your relatives. Vashikaran Mantra in Hindi is used for well-defined purposes, some use it for in receipt of well off, some use it as love spells to recover their love one. The key thing to be recall is this real strategy and fitting duty are important for vashikaran mantra in Hindi to get fulfilment. One can start included vashikaran mantra in Hindi yet after bundles of put into practice and meeting to get most great benefits.\\nIn present time vashikaran mantra in Hindi is uncommonly overseeing way to deal with light up every one of the worry people refuse in their regular life. Accepting that you are looking the strong strategy to get comes about that serve to totally handle your friendship problems between problems like How to get love back, how to recover your ex, how to get sweetheart yet again with the use of Love Spells then vashikaran mantra in Hindi will be the result of all request that make your presence hellfire.\\nThere are two types of Vashikaran to be specific Pandit Vashikaran you have the full information to get the better results. You can get the information about this study on the web. Various individuals offering these services give data through their sites. On the other aspect that you need to realize this strategy, you must think the experienced person who is knowledgeable with the harmless methods. It can be useful in resolving many problems in your expert and individual life. It is operate to contact individuals who have attempted these strategies.\\nPandit Aman Sharma Love Vashikaran Specialist Chandigarh coordinating is likewise one of the services offered by All Love Vashikaran Specialist Pandit Chandigarh of two people (male and a female) are coordinated for shared compatibility. It incorporates analysis of different factor, for example, Ashta koot and Manglik Dosh and tells what sort of association is suitable.\"", "vector_field": [-0.0023531136102974415, 0.0050523714162409306, 0.029981397092342377, 0.0002577358391135931, -0.024030383676290512, 0.0060808188281953335, -0.003448127070441842, -0.0369841568171978, -0.019530510529875755, -0.026892727240920067, 0.0023547778837382793, 0.013240008614957333, -0.018145935609936714, 0.03219139575958252, -0.004243592731654644, 0.012188263237476349, 0.04191005229949951, -0.02036924473941326, -0.0037676445208489895, -0.017293887212872505, -0.023564420640468597, 0.030221035704016685, 0.026519957929849625, 0.006942850537598133, -0.03283042833209038, 0.024070322513580322, 0.005275368224829435, -0.007914716377854347, -0.006856314372271299, -0.004027252551168203, -0.012940460816025734, -0.005325292702764273, -0.01625545509159565, -0.013080250471830368, -0.0471554659307003, -0.012813985347747803, -0.007588542066514492, -0.016521720215678215, 0.004223622847348452, 0.0029255826957523823, 0.052986662834882736, -0.013253321871161461, -0.008826673030853271, 0.0112763075158, -0.007382187061011791, 0.02288544550538063, -0.02536170557141304, -0.03788945823907852, -0.029928144067525864, 0.024070322513580322, 0.029608627781271935, 0.020169546827673912, -0.013845761306583881, 0.013725942000746727, -0.011775553226470947, -0.019330812618136406, 0.0007297315169125795, 0.011868746019899845, 0.001501898979768157, -0.017506899312138557, 0.006440275814384222, -0.01756015233695507, 0.009558900259435177, 0.0007268192712217569, -0.0074487533420324326, -0.004689585883170366, -0.004493215586990118, 0.01146269217133522, -0.001681627589277923, 0.0069761332124471664, 0.02910272404551506, 0.01718738302588463, -0.029608627781271935, -0.015190397389233112, 0.027585016563534737, -0.013346514664590359, -0.019370751455426216, -0.010717151686549187, 0.003484738292172551, 0.003717720042914152, 0.022099964320659637, -0.03836873546242714, -0.010504139587283134, 0.009425767697393894, 0.000663997430820018, -0.004350098315626383, 0.0146312415599823, 0.037250421941280365, -0.005954342894256115, -0.01817256212234497, 0.0011807172559201717, 0.00556825939565897, -0.00924603920429945, -0.005837852135300636, -0.009292636066675186, 0.02405701018869877, -0.03378898277878761, 0.018265753984451294, -0.011882059276103973, -0.01837226003408432, 0.010424260050058365, -0.010623958893120289, -0.0345611497759819, -0.0032151455525308847, 0.00924603920429945, -0.0011357851326465607, -0.006476887036114931, -0.019850028678774834, 0.02718561887741089, 0.006956163793802261, -0.021021593362092972, 0.037250421941280365, 0.005215458571910858, -0.0455312542617321, 0.01874503120779991, -0.008859955705702305, -0.0031835264526307583, -0.023138396441936493, -0.032164767384529114, -0.02031599171459675, 0.011389469727873802, 0.029688507318496704, 0.003441470442339778, 0.002963858190923929, 0.037782952189445496, 0.007528632413595915, -0.012374648824334145, -0.019077861681580544, -0.014538048766553402, -0.01690780371427536, -0.014950759708881378, -0.0019603734835982323, -0.015882685780525208, 0.04401354491710663, 0.0023714194539934397, 0.0024013742804527283, -0.029608627781271935, -0.021128099411725998, -0.011882059276103973, -0.012833955697715282, 0.0070826392620801926, 0.01959707774221897, -0.009905043989419937, -0.014511422254145145, -0.027744775637984276, 0.011948625557124615, 0.01397889293730259, -0.004729525651782751, 0.008360709995031357, 0.021114785224199295, 0.007521975785493851, -0.00866025686264038, -0.0029455525800585747, 0.006124086678028107, 0.007308964151889086, 0.004293517209589481, -0.006882940884679556, 0.005941029638051987, -0.01589599810540676, -0.024416467174887657, -0.008014565333724022, 0.0008994752424769104, -0.017054250463843346, 0.010004893876612186, 0.022938698530197144, 0.014178591780364513, -0.0021683925297111273, -0.013845761306583881, -0.02779802866280079, 0.0016075727762654424, 0.020595569163560867, 0.02902284450829029, -0.03690427914261818, 0.044492822140455246, -0.0016242142301052809, -0.017866356298327446, 0.02833055704832077, 0.0068629710003733635, -0.008540438488125801, 0.031685490161180496, -0.01544334925711155, -0.010384321212768555, 0.013166786171495914, 0.024296648800373077, -0.04137752577662468, 0.019237620756030083, 0.0019670298788696527, -0.0083340834826231, 0.004756152164191008, -0.022565927356481552, 0.008074475452303886, 0.0046196915209293365, -0.006047535687685013, -0.01821250095963478, -0.6364789605140686, -0.025960801169276237, -0.011515945196151733, -0.0016824596095830202, 0.014191905036568642, -0.0054883798584342, 0.01726726070046425, -0.0011524265864863992, -0.024496346712112427, 0.03096657618880272, -0.008274173364043236, 0.022539300844073296, -0.035626206547021866, -0.0033482778817415237, -0.01641521416604519, -0.023298155516386032, -0.011289620772004128, -0.0176799725741148, -0.0002030267787631601, 0.0008653600816614926, 0.0046496461145579815, 0.004509857390075922, -0.031925130635499954, 0.02645339071750641, -0.01254106406122446, -0.00423693610355258, 0.02211327850818634, 0.006540125235915184, 0.011356187053024769, 0.017932923510670662, -0.0036944218445569277, 0.03011452965438366, -0.02150086872279644, -0.019397377967834473, 0.04933883622288704, -0.012614287436008453, 0.004310158547013998, 0.04055210202932358, 0.028117544949054718, 0.02096834033727646, 0.002477925270795822, -0.0024962308816611767, 0.0378095768392086, 0.012674196623265743, -0.00506235659122467, -0.0020668793004006147, 0.0033649192191660404, -0.016441840678453445, -0.02019617334008217, 0.01861189864575863, 0.005328620783984661, -0.009938327595591545, -0.005777942482382059, -0.0022382871247828007, 0.009578870609402657, -0.03666463866829872, 0.023471226915717125, -0.029076097533106804, 0.02686610072851181, 0.014178591780364513, 0.024030383676290512, 0.01775985024869442, -0.0020502377301454544, -0.013599466532468796, 0.02710573934018612, 0.0065967063419520855, -0.004955850541591644, 0.00021072349045425653, 0.0035579612012952566, 0.002216653199866414, -0.007595198694616556, 0.02015623264014721, -0.00889989547431469, 0.004563109949231148, 0.015603107400238514, 0.018625210970640182, 0.012920491397380829, -0.016814611852169037, -0.009618810378015041, 0.004027252551168203, 0.0020169545896351337, -0.02162068895995617, -0.02251267433166504, -0.026493331417441368, -0.0016441841144114733, -0.023883936926722527, -0.012694166041910648, 0.01420521829277277, 0.0037676445208489895, -0.012234860099852085, 0.012154980562627316, -0.004203652963042259, -0.02490905672311783, -0.02096834033727646, 0.003797599347308278, 0.00731562077999115, -0.020289365202188492, -0.017293887212872505, -0.015177084133028984, -0.00760185532271862, -0.00412377342581749, -0.03096657618880272, 0.008001252077519894, 0.0027641598135232925, 0.0033699118066579103, -0.0002893547643907368, -0.02979501336812973, -0.007515319157391787, 0.05093642324209213, -0.035173557698726654, 0.013433050364255905, -0.020209485664963722, 0.0006115766009315848, -0.00484601641073823, 0.00617733970284462, -0.016428528353571892, -0.025574717670679092, 0.017826417461037636, 0.028117544949054718, -0.035466451197862625, 0.0008745128870941699, -0.02251267433166504, 0.0030304244719445705, 0.0010484169470146298, 0.00688959751278162, 0.006936193909496069, -0.006823031231760979, -0.005837852135300636, -0.01063061598688364, -0.0023747477680444717, 0.009465707466006279, -0.005841180216521025, -0.006373709533363581, -0.006849657744169235, 0.020981652662158012, 0.0013845760840922594, 0.00412377342581749, -0.007941342890262604, 0.021114785224199295, -0.0052653830498456955, 0.009738628752529621, 0.0018638524925336242, 0.02288544550538063, 0.021913578733801842, -0.0027242200449109077, -0.05170859023928642, -0.012008534744381905, -0.013326545245945454, -0.03472090885043144, -0.015629734843969345, 0.012654227204620838, -0.02304520457983017, -0.005142235662788153, 0.03474753350019455, 0.016348648816347122, -0.011755583807826042, 0.008633630350232124, -0.014325037598609924, -0.019863341003656387, -0.01548328809440136, 0.01877165585756302, 0.015017325058579445, -0.024150202050805092, 0.0007813202682882547, -0.012001878581941128, -0.00010203343845205382, -0.03389548882842064, -0.0033815607894212008, 0.007002759724855423, -0.03152573108673096, -0.013872387818992138, 0.003944044932723045, -0.012887208722531796, 0.03514693304896355, -0.02779802866280079, 0.01637527532875538, -0.003524678060784936, -0.01979677565395832, -0.04047222435474396, 0.019290871918201447, -0.0005113112856633961, 0.0026293632108718157, -0.023977130651474, -0.003574602771550417, 0.007695048116147518, 0.015842745080590248, -0.00451318547129631, 0.01034438144415617, 0.0045364839024841785, 0.006749808322638273, -0.01942400448024273, 0.03264404460787773, 0.008347396738827229, 0.004746166989207268, -0.015110517852008343, -0.015190397389233112, 0.006387022789567709, -0.003880806965753436, 0.00810775812715292, 0.030087903141975403, 0.029289109632372856, -0.005927716381847858, 0.042495835572481155, -0.00023402164515573531, 0.0025012234691530466, -0.00760185532271862, 0.009272665716707706, -0.008427275344729424, 0.019171053543686867, -0.002829061821103096, -0.008607004769146442, -0.007375530432909727, 0.0058977617882192135, -0.003947373013943434, 0.018878161907196045, 0.028969591483473778, 0.005185503978282213, 0.017733225598931313, 0.01682792417705059, 0.010211248882114887, 0.027012547478079796, -0.013819134794175625, 0.0017723239725455642, -0.009791881777346134, -0.0030620433390140533, 0.021900266408920288, 0.0076151685789227486, 0.0013396439608186483, -0.01254106406122446, -0.013299918733537197, -0.0034647686406970024, 0.014191905036568642, 0.001938739325851202, 0.03557295352220535, 0.02227303758263588, 0.027252186089754105, 0.003528006374835968, -0.0035080364905297756, 0.029608627781271935, -0.01075043436139822, 0.0005965991877019405, 0.028596822172403336, 0.00616735452786088, -0.0031868547666817904, 0.0005645642522722483, 0.03133934736251831, 0.0008270845282822847, 0.011995221488177776, 0.002432993147522211, 0.03133934736251831, -0.02739863097667694, 0.028437063097953796, 0.004613034892827272, -0.007095952518284321, 0.017626719549298286, -0.014404917135834694, 0.014484795741736889, -0.010564049705862999, 0.015416722744703293, 0.03642500191926956, 0.004839359782636166, 0.014058772474527359, 0.003840867429971695, -0.01164242159575224, 0.018305692821741104, 0.03272392228245735, 0.018598584458231926, -0.010790374130010605, -0.028729954734444618, 0.0048027485609054565, 0.0013138495851308107, -0.0009111242834478617, -0.0014403252862393856, -0.004433306399732828, 0.0378095768392086, -0.016774671152234077, 0.006357068195939064, 0.0022732343059033155, 0.03205826133489609, 0.013313231989741325, -0.001323002390563488, -0.015469974838197231, 0.02110147289931774, 0.004000626038759947, 0.023670926690101624, -0.019197680056095123, -0.02023611217737198, 0.00623724889010191, -0.006516826804727316, 0.017333827912807465, -0.012747419066727161, 0.014271784573793411, 0.03165886551141739, -0.0219402052462101, -0.007022729609161615, -0.008054505102336407, 0.028437063097953796, -0.02072870172560215, 0.020595569163560867, -0.018039429560303688, -0.012707479298114777, 0.02466941811144352, -0.038342107087373734, -0.012700823135674, 0.004746166989207268, 0.0010084772948175669, -0.017613405361771584, -0.0012073436519131064, 0.0009460715227760375, -0.0201162938028574, 0.009718659333884716, -0.01601581834256649, -0.00220999657176435, 9.438873530598357e-05, -0.004682929255068302, -0.002298196544870734, -0.03360259532928467, 0.017586778849363327, 0.031019829213619232, -0.006503513548523188, 0.0292624831199646, -0.0024945668410509825, -0.005468409974128008, -0.009152846410870552, 0.1101270541548729, -0.012361335568130016, -0.02434989996254444, -0.003136930288746953, -0.004310158547013998, 0.00019855436403304338, -0.010597332380712032, -0.030780192464590073, 0.018358945846557617, -0.015177084133028984, 0.033389586955308914, -0.028011038899421692, -0.0016658181557431817, 0.00045181778841651976, 0.015629734843969345, 0.0032384435180574656, 0.004742838907986879, 0.02666640281677246, -0.020262738689780235, 0.0076617649756371975, 0.02247273549437523, 0.001995320664718747, 0.001366270356811583, 0.011629108339548111, -0.014418230392038822, 0.012521094642579556, 0.023804059252142906, 0.008467215113341808, -0.019277559593319893, -0.03312332183122635, -0.03749006241559982, 0.04816727340221405, 0.0102645019069314, 0.02182038687169552, -0.022246411070227623, 0.01613563671708107, -0.008866612799465656, 0.03437476605176926, -0.016974370926618576, 0.012454528361558914, 0.00866025686264038, 0.004806076642125845, 0.017094189301133156, -0.017586778849363327, 0.015350156463682652, -0.01404545921832323, 0.006433619186282158, 0.019157741218805313, -0.028836460784077644, -0.0199165940284729, 0.00573134608566761, -0.010084773413836956, -0.015749553218483925, -0.00033907138276845217, -0.000837901548948139, -0.0003421916626393795, 0.0009236054611392319, -0.012427901849150658, 0.006400336045771837, 0.005388530436903238, -0.00029060288215987384, -0.031605612486600876, -0.006357068195939064, -0.0008994752424769104, -0.011615795083343983, -0.0333363339304924, -0.013319888152182102, -0.016162263229489326, -0.00847387220710516, 0.007608511950820684, 0.023098457604646683, -0.03181862458586693, -0.0004041813954245299, 0.012394619174301624, 0.0191044881939888, -0.011309590190649033, 0.012261486612260342, -0.004902597516775131, -0.004689585883170366, 0.017586778849363327, -0.014844253659248352, -0.033309705555438995, -5.179679283173755e-05, -0.024855803698301315, -0.03365584835410118, 0.006327113602310419, -0.008706853725016117, -0.0023048531729727983, -0.05056365579366684, 0.005588228814303875, 0.012853925116360188, 0.014364977367222309, 0.009525617584586143, 0.008946491405367851, 0.011529258452355862, -0.001719071064144373, 0.029981397092342377, -0.019477257505059242, 0.01845213957130909, 0.005228771828114986, -0.017134130001068115, -0.017892982810735703, -0.0024496347177773714, 0.0030221035704016685, 0.008527125231921673, 0.019490571692585945, 0.01069052517414093, 0.013519586995244026, -0.017786476761102676, 0.004196996334940195, 0.009405798278748989, -0.0184920784085989, 0.020662136375904083, 0.007908059284090996, 0.019783461466431618, 0.02138105034828186, 0.015030638314783573, 0.04869979992508888, 0.005042386706918478, -0.012387962080538273, -0.012234860099852085, -0.036638014018535614, 0.00717583205550909, 0.021966831758618355, -0.004110460169613361, -0.025348393246531487, 0.011842119507491589, -0.02710573934018612, 0.0012381304986774921, 0.02870332822203636, 0.014218531548976898, 0.00932591874152422, -0.027851281687617302, -0.02548152580857277, -0.008081131614744663, -0.00831411313265562, -0.018545331433415413, 0.010623958893120289, 0.00016662341658957303, -0.01385907456278801, -0.0195038840174675, -0.012228203006088734, 0.004995790310204029, -0.030460674315690994, 0.03165886551141739, -0.03203163668513298, -0.0075086625292897224, 0.0116890175268054, -0.01278070267289877, -0.00010073331941384822, -0.014125338755548, -0.006277188658714294, -0.010763747617602348, -0.00902637094259262, -0.00028956279857084155, -0.016388587653636932, 0.010996729135513306, -0.0031036471482366323, 0.031073082238435745, 0.015682987868785858, 0.032936934381723404, -0.01447148248553276, -0.017253948375582695, -0.004230279475450516, -0.003840867429971695, 0.002116803778335452, -0.014684494584798813, -0.007102609146386385, -0.03040742129087448, 0.021687254309654236, 0.0012581003829836845, -0.00185719586443156, 0.009778568521142006, -0.0008728487300686538, -0.003654482075944543, 0.029741760343313217, -0.011875403113663197, -0.021594062447547913, 0.0004226950986776501, -0.026480017229914665, -0.006629989482462406, 0.0229653250426054, -0.004183683078736067, -0.00645358907058835, -0.04521173611283302, 0.01296708732843399, 0.011269650422036648, 0.007994595915079117, 0.013393111526966095, 0.0024629479739814997, 0.02108815871179104, -0.011030012741684914, 0.006982789840549231, 0.0030054620001465082, 0.024003757163882256, -0.009092937223613262, 0.0018771656323224306, -0.006813046522438526, 0.0005038226372562349, -0.003518021432682872, 0.030167782679200172, 0.005475066602230072, 0.018585272133350372, 0.0023214947432279587, -0.05804568901658058, 0.005451768171042204, 0.0036411688197404146, -0.01556316763162613, 0.02231297641992569, -0.019543824717402458, -0.018971355631947517, -0.05373220145702362, -0.03235115483403206, -0.009971610270440578, -0.002810755977407098, -0.023417973890900612, -0.020262738689780235, 0.018931414932012558, -0.028729954734444618, 0.009039684198796749, 0.0048693143762648106, -0.014684494584798813, 0.02474929764866829, -0.011129861697554588, 0.029688507318496704, 0.025188634172081947, -0.0035979009699076414, 0.012527750805020332, 0.0233780350536108, 0.0036811085883527994, -0.01190868578851223, -0.007495349273085594, -0.00415705656632781, 0.0040505509823560715, -0.02512206882238388, 0.013686002232134342, 0.002103490522131324, -0.04028584063053131, -0.030434047803282738, 0.011003386229276657, -0.010237875394523144, -0.0025345063768327236, -0.023790745064616203, -0.004047222435474396, -0.029475495219230652, 0.029076097533106804, -0.00039482052670791745, 0.022326288744807243, -0.007934685796499252, -0.010044833645224571, -0.02739863097667694, -0.002824069233611226, 0.01205513160675764, 0.003354934509843588, 0.04198993369936943, 0.012367992661893368, -0.041963305324316025, -0.019237620756030083, -0.006606691051274538, 0.01658828742802143, 0.015190397389233112, 0.004310158547013998, -0.022179843857884407, -0.013659375719726086, 0.009319262579083443, 0.010810344479978085, -0.00045639422023668885, 0.010810344479978085, 0.004965835250914097, 0.02516200765967369, -0.007894746027886868, 0.0015218687476590276, 0.008207607083022594, -0.013739255256950855, 0.012887208722531796, 0.02044912427663803, -0.0011748926481232047, -0.005491707939654589, 0.005871135275810957, 0.0006989446701481938, 0.023417973890900612, 0.016082383692264557, -0.016188889741897583, -0.001807271153666079, 0.010131369344890118, -0.021474242210388184, -0.004127101972699165, -0.007462066598236561, 0.029076097533106804, -0.008567065000534058, 0.005009103566408157, -0.0032833758741617203, -0.002214988926425576, 0.0058145541697740555, 0.0010700509883463383, -0.0009751942125149071, 0.01103666890412569, 0.025148695334792137, -0.03056718036532402, 0.006580065004527569, -1.341724146186607e-05, 0.013060280121862888, -0.01821250095963478, 0.0034814102109521627, 0.029608627781271935, -0.02508212812244892, 0.0009602168574929237, 0.022046711295843124, -0.02207333780825138, -0.011615795083343983, 0.01979677565395832, 0.008194293826818466, 0.009405798278748989, 0.014098712243139744, -0.00281907687895, 0.005255398340523243, 0.013686002232134342, -0.009652093052864075, -0.0030820132233202457, 0.014258471317589283, -0.0009310941677540541, 6.380990089382976e-05, 0.006646630819886923, -0.0439070388674736, 0.022539300844073296, -0.0015393424546346068, 0.007368873804807663, 0.02423008158802986, -0.011289620772004128, -0.027744775637984276, -0.012394619174301624, -0.01801280304789543, -0.0016991011798381805, -0.02377743273973465, -0.013819134794175625, -0.0018671808065846562, -0.0036877652164548635, 0.014165278524160385, -0.007049356121569872, -0.011236367747187614, 0.010863597504794598, 0.02104821987450123, 0.0012597645400092006, -0.001194862532429397, -0.005911075044423342, 0.03767644613981247, -0.01967695727944374, -0.012820642441511154, -0.02162068895995617, 0.001995320664718747, -0.013026997447013855, 0.02227303758263588, 0.044359687715768814, -0.015736239030957222, -0.01613563671708107, 0.0006839672569185495, -0.03429488465189934, 0.0013321551959961653, 0.0069628204219043255, 0.03410850092768669, 0.013965579681098461, 0.006583393085747957, 0.004972491879016161, 9.527281508781016e-05, -0.000883665750734508, 0.016961056739091873, -0.00010296952677890658, 0.0009369186591356993, -0.010823657736182213, 0.005255398340523243, 0.00520214531570673, 0.005465081427246332, -0.007894746027886868, 0.010237875394523144, 0.019570451229810715, -0.002551147947087884, 0.01119642797857523, 0.023138396441936493, 0.00609746016561985, -0.004819389898329973, -0.0004035573510918766, 0.020702075213193893, 0.007874776609241962, -0.00452317064628005, 0.0049691637977957726, -0.025175321847200394, -0.0030736923217773438, -0.02409694902598858, -0.017586778849363327, -0.007402156945317984, 0.025667911395430565, 0.011549228802323341, -0.005395187065005302, 0.005485051311552525, -0.007708361372351646, -0.006666600704193115, 0.015150457620620728, 0.006120758131146431, 0.031472478061914444, -0.020702075213193893, -0.019903281703591347, 0.034933920949697495, -0.000441000796854496, -0.01914442703127861, 0.005901089869439602, -0.003954029642045498, 0.017174068838357925, 0.005178847350180149, 0.019770149141550064, -0.011529258452355862, 0.01678798533976078, -0.010663898661732674, 0.008913208730518818, 0.007721674628555775, 0.006909567397087812, -0.014125338755548, 0.0032434361055493355, -0.02971513383090496, 0.022046711295843124, 0.011236367747187614, 0.02934236265718937, 0.029395615682005882, -0.005172190722078085, -0.015043951570987701, -0.003531334688887, -0.020622195675969124, -0.013686002232134342, -0.03157898411154747, 0.008753449656069279, -0.023684239014983177, -0.0018022787990048528, -0.018145935609936714, 0.0011940305121243, 0.009006401523947716, -0.00660336297005415, -0.010490826331079006, 0.19682282209396362, 0.0012855589156970382, -0.0026626463513821363, 0.05077666416764259, -0.0037643162067979574, 0.017546840012073517, 0.026732970029115677, 0.003877478651702404, 0.02068876288831234, -0.0056681083515286446, -0.002995477057993412, 0.0076151685789227486, -0.028916338458657265, -0.002386396750807762, -0.011069952510297298, -0.006600034423172474, -0.040658608078956604, -0.029901519417762756, 0.005208801943808794, 0.0003249260480515659, 0.02215321734547615, -0.021807072684168816, 0.0026227065827697515, -0.019197680056095123, 0.019863341003656387, 0.01113651879131794, 0.018385572358965874, -0.013792508281767368, 0.0073955003172159195, 0.005747987888753414, -0.00853378139436245, -0.0014028818113729358, -0.010004893876612186, 0.00961215328425169, 0.004739510361105204, -0.007189145311713219, 0.023111769929528236, -0.009219412691891193, 0.011815492995083332, 0.002343128900974989, -0.0028873072005808353, -0.02678622305393219, 0.01930418610572815, 0.00677643483504653, -0.01156254205852747, 0.019823402166366577, -0.004865986295044422, -0.005505021195858717, 0.020994966849684715, 7.956736226333305e-05, -0.04648980498313904, -0.03189850226044655, 0.040871620178222656, 0.011835463345050812, -0.01385907456278801, 0.01645515486598015, 0.032617416232824326, 0.015110517852008343, 0.014990698546171188, 0.00014654955884907395, -0.02365761250257492, -0.006084146909415722, 0.004692913964390755, -0.0019304186571389437, -0.012407932430505753, 0.02666640281677246, -0.0051522208377718925, -0.036478254944086075, -0.01674804463982582, 0.017932923510670662, 0.01885153539478779, -0.030753565952181816, 0.005801240913569927, 0.01428509782999754, -0.0058811199851334095, -0.012394619174301624, 0.007635138463228941, -0.017932923510670662, 0.028729954734444618, 0.04803413897752762, -0.001193198375403881, -0.01779979094862938, 0.009006401523947716, -0.003528006374835968, -0.016308708116412163, -0.00642030593007803, 0.0007634306093677878, -0.011063295416533947, -0.028916338458657265, 0.0075685721822083, 0.003957358188927174, -0.011835463345050812, -0.013725942000746727, -0.023963816463947296, 0.014151965267956257, 0.002131781307980418, 0.03064705990254879, -0.03203163668513298, -0.0025927519891411066, 0.011336216703057289, 0.004160385113209486, 0.014418230392038822, 0.020781954750418663, 0.006709869019687176, -0.008826673030853271, 0.00945239420980215, 0.01775985024869442, 0.006656615994870663, 0.0011224718764424324, -0.006486872211098671, 0.010963446460664272, 0.004905926063656807, -0.02122129127383232, -0.009971610270440578, 0.0006523483316414058, 0.02747851051390171, -0.014218531548976898, -0.001546831103041768, 0.015802806243300438, 0.01645515486598015, 0.0038075842894613743, -0.023883936926722527, -0.007921372540295124, 0.009019714780151844, 0.00926600955426693, -0.017693284898996353, -0.028969591483473778, 0.0007858967292122543, 0.00031722933636046946, -0.011649077758193016, 0.026440078392624855, -0.04097812622785568, 0.009918357245624065, 0.00023339760082308203, -0.003314994741231203, 0.03072693943977356, 0.04238932952284813, -0.016042444854974747, 0.005415156949311495, 0.013366485014557838, 0.031685490161180496, 0.012947117909789085, 0.01174892671406269, 0.007914716377854347, -0.011049982160329819, 0.012873895466327667, 0.0012697494821622968, -0.0014736083103343844, 0.0014203554019331932, -0.02284550480544567, -0.02645339071750641, 0.000968537584412843, 0.015576480887830257, -0.016002504155039787, 0.005591557361185551, -0.005681421607732773, -0.04044559970498085, -0.027425257489085197, 0.016654852777719498, -0.044040169566869736, -0.02044912427663803, 0.0223529152572155, 0.027984412387013435, 0.006443604361265898, -0.011808836832642555, -0.0166681669652462, -0.16774672269821167, 0.0357593409717083, 0.011262994259595871, -0.030673686414957047, 0.026493331417441368, -0.007362217176705599, 0.04587739706039429, 0.006909567397087812, -0.0007617664523422718, 0.006007595919072628, 0.006443604361265898, -0.005960999522358179, -0.02771814912557602, -0.032697297632694244, -0.02019617334008217, -0.012740762904286385, 0.0015393424546346068, 0.02486911602318287, 0.020049726590514183, 0.0026410124264657497, -0.003098654793575406, -0.002155079273506999, 0.006060848943889141, -0.002852359786629677, 0.01775985024869442, -0.007249054498970509, -0.03520018607378006, -0.020994966849684715, 0.009312605485320091, -0.018971355631947517, -0.010224562138319016, -0.0024629479739814997, 0.011575855314731598, -0.005165534093976021, 0.004100475460290909, -0.0040705204010009766, 0.011243023909628391, -0.004230279475450516, -0.013819134794175625, -0.01471112109720707, 0.03613211214542389, 0.002772480482235551, -0.0025345063768327236, 0.004220294300466776, 0.030087903141975403, 0.026972606778144836, 0.005774614401161671, -0.00024795893114060163, -0.006480215582996607, -0.021434303373098373, 0.026147186756134033, 0.017160756513476372, 0.03706403821706772, 0.011362843215465546, 0.025388332083821297, 0.0036012292839586735, 0.03613211214542389, -0.0036012292839586735, -0.01304696686565876, -0.0016059086192399263, -0.01914442703127861, -0.005045714788138866, -0.010743778198957443, -0.002963858190923929, -0.008041191846132278, -0.004623019602149725, -0.00847387220710516, -0.009871761314570904, -0.02158074826002121, 0.02629363350570202, -0.00876010674983263, 0.04361414536833763, 0.001500234822742641, -0.010337724350392818, 0.01335317175835371, -0.004263562615960836, -0.02454959973692894, 0.026120560243725777, 0.0031901830807328224, -0.012600974179804325, -0.0219402052462101, 0.020635509863495827, -0.012234860099852085, -0.010990072973072529, -0.000748037185985595, 0.013106876984238625, 0.012321395799517632, -0.02187363989651203, -0.03219139575958252, 0.02601405419409275, 0.0010367679642513394, -0.022406168282032013, -0.004333456978201866, -0.02044912427663803, -0.004290189128369093, 0.006337098311632872, 0.0233780350536108, 0.005807897541671991, 0.0008886582218110561, -0.0004676272510550916, 0.025534778833389282, -0.010377664119005203, -0.004429977852851152, 0.0007705033058300614, 0.0064136493019759655, -0.0011391134466975927, 0.004333456978201866, 0.011775553226470947, 0.01589599810540676, -0.024576226249337196, -0.008913208730518818, 0.006956163793802261, 0.030593806877732277, 0.011589168570935726, -0.02247273549437523, 0.0467294417321682, 0.0028506957460194826, 0.006580065004527569, 0.007761613931506872, 0.015682987868785858, 0.05248076096177101, 0.004100475460290909, 0.0034647686406970024, 0.0031968397088348866, -0.016468467190861702, -0.02694598026573658, -0.10757090896368027, -0.005325292702764273, 7.036900910861732e-07, 0.01702762395143509, -0.005588228814303875, 0.006876284256577492, 0.0018904788885265589, 0.004616362974047661, -0.010876910760998726, 0.039566922932863235, -0.01938406564295292, -0.03858174383640289, -0.0024063666351139545, 0.0033749041613191366, 0.00415705656632781, -0.015070578083395958, -0.011782210320234299, -0.003448127070441842, -0.006553438492119312, 0.01837226003408432, 0.005974312778562307, -0.0037776294630020857, -0.03344283998012543, -0.006540125235915184, -0.006773106753826141, -0.0013479647459462285, -0.014085398986935616, 0.02060888335108757, 0.006616676226258278, 0.010284471325576305, -0.03876813128590584, 0.013646062463521957, -0.0053186360746622086, -0.011629108339548111, -0.014444855973124504, 0.016894491389393806, -0.023604359477758408, -0.011282963678240776, 0.01641521416604519, -0.038794755935668945, -0.005701391492038965, -0.010570705868303776, -0.011163144372403622, -0.03746343404054642, 0.006823031231760979, -0.011129861697554588, -0.01097010262310505, 0.029661880806088448, -0.0006373709766194224, -0.023138396441936493, -0.021061532199382782, 0.0018172560958191752, -0.039806563407182693, -0.01018462236970663, 0.05218786746263504, -0.024735985323786736, -0.0023531136102974415, 0.020742015913128853, -0.03727705031633377, -0.010104742832481861, -0.017373766750097275, 0.02979501336812973, -0.04121776670217514, 0.004633004777133465, -0.005471738055348396, 0.017666658386588097, -0.022406168282032013, -0.025188634172081947, 0.024416467174887657, -0.028969591483473778, -0.02211327850818634, -0.003664467018097639, -0.03591910004615784, 0.0054783946834504604, -0.01894472911953926, 0.012294769287109375, 0.0025777744594961405, 0.0057679577730596066, 0.013792508281767368, 0.0012273135362192988, -0.009192786179482937, -0.01369931548833847, 0.02731875143945217, 0.012248173356056213, 0.004273547325283289, -0.019770149141550064, 0.01233470905572176, -0.029049471020698547, 0.0036245272494852543, -0.026879414916038513, 0.006606691051274538, 0.013160130009055138, 0.008460558950901031, -0.005255398340523243, 0.00889989547431469, 0.018518704921007156, -0.010504139587283134, -0.0076617649756371975, 0.01967695727944374, 0.02381737157702446, -0.030167782679200172, -0.015430036000907421, -0.057246897369623184, 0.01298705767840147, 0.02454959973692894, 0.0023165023885667324, -0.034614402800798416, -0.026027368381619453, 0.007222427986562252, -0.020635509863495827, 0.009865105152130127, 0.002965522464364767, -0.008646943606436253, 0.008220920339226723, -0.015004011802375317, 0.011229710653424263, -0.02902284450829029, -0.016282081604003906, 0.0034081873018294573, -0.0302742887288332, -0.007641795091331005, -0.01443154364824295, 0.014511422254145145, -0.01140278298407793, 0.015509914606809616, 0.022858818992972374, -0.0006764786085113883, -0.002820740919560194, -0.006233920808881521, -0.014844253659248352, -0.0017057578079402447, -0.0033699118066579103, 0.006077490281313658, -0.02556140534579754, 0.009439080953598022, 0.02003641426563263, 0.0029322393238544464, -0.007362217176705599, 0.02568122372031212, 0.017853043973445892, -0.01349296048283577, 0.04238932952284813, -0.024443093687295914, -0.022739000618457794, 0.021647315472364426, 0.0006215614848770201, 0.006290501914918423, 0.003977328073233366, -0.01018462236970663, 0.009632122702896595, 0.023484541103243828, 0.009399141184985638, 0.030620433390140533, 0.008646943606436253, -0.03405524790287018, 0.00017213592946063727, -0.01248115487396717, -0.006673257332295179, 0.004646318033337593, 0.0009485677583143115, -0.0028906355146318674, 0.0010692189680412412, 0.028969591483473778, 0.001983671449124813, 0.008973117917776108, 0.003045401768758893, -0.017653346061706543, 0.010623958893120289, -0.016814611852169037, 0.007036042865365744, 0.006203966215252876, -0.03352271765470505, -0.01752021349966526, 0.0032817116007208824, 0.02755839005112648, 0.014538048766553402, -0.0016208859160542488, -0.0018888147315010428, -0.0013438042951747775, 0.017293887212872505, -0.031685490161180496, 0.002594416029751301, -0.02182038687169552, 0.025375019758939743, -0.02971513383090496, 0.01645515486598015, 0.017853043973445892, 0.0007326437626034021, -0.01670810580253601, 0.006823031231760979, 0.014990698546171188, -0.00995829701423645, 0.008200950920581818, -0.006882940884679556, 0.009139533154666424, -0.017786476761102676, -0.004147071857005358, 0.00617733970284462, 0.00048718106700107455, -0.0213411096483469, 0.039487045258283615, 0.004746166989207268, 0.0005645642522722483, -0.004210309591144323, -0.0036411688197404146, -0.01326663512736559, -0.01235467940568924, 0.0006789748440496624, 0.005907746497541666, 0.011262994259595871, -0.008021222427487373, 0.006227264180779457, 0.03376235440373421, -0.0007875608862377703, 0.038714878261089325, 0.0095522440969944, -0.004842687863856554, -0.005045714788138866, -0.003930731676518917, -0.007548602297902107, -0.017506899312138557, 0.019171053543686867, -0.010044833645224571, 0.0088399862870574, 0.017986176535487175, 0.00642030593007803, -0.0022665776778012514, 0.01581611856818199, 0.006699883844703436, -0.020555630326271057, 0.02706580050289631, -0.007402156945317984, 0.002076864242553711, -0.00567143689841032, -0.028197424486279488, 0.009099594317376614, -0.02324490249156952, 0.010271158069372177, 0.002286547562107444, 0.026679717004299164, -0.010896880179643631, 0.04731522500514984, 0.00527203967794776, -0.018838223069906235, -0.004350098315626383, 0.005564930848777294, -0.0033865533769130707, -0.0070293862372636795, -0.0040871622040867805, -0.019370751455426216, -0.04321474954485893, 0.0034780818969011307, -0.01775985024869442, 0.04792763292789459, 0.01012471318244934, -0.0075086625292897224, -0.005801240913569927, -0.040179334580898285, 0.047741249203681946, 0.0038142409175634384, -0.0013537892373278737, 0.025534778833389282, 0.01576286554336548, 0.022499362006783485, 0.031605612486600876, -0.016614913940429688, -0.02865007519721985, -0.003990641329437494, 0.022086651995778084, -0.012208233587443829, -0.03466765582561493, -0.009578870609402657, 0.005118937697261572, -0.02504218928515911, -0.020622195675969124, 0.01906454749405384, -0.044972095638513565, -0.014325037598609924, -0.011649077758193016, 0.005804568994790316, 0.04345439001917839, -0.005854493472725153, 0.03714391589164734, -0.0014261798933148384, -0.0146312415599823, -0.007548602297902107, -0.02963525429368019, -0.00326673430390656, -0.002211660612374544, -0.0184920784085989]} +{"id": "test:10000028", "text": "\"membership fees and with many users renewing their accounts, although the overall share of Internet traffic using online dating services in the.S. Enhanced Performance Those who hv performance issues shall not be consuming soya products as it reduces performance Enhanced Performance Agree lucky. One report in China Daily suggests that dating for Chinese university women is \\\"difficult\\\" and \\\"takes work\\\" and steals time away from academic advancement, and places women in a precarious position of having to balance personal success against traditional Chinese relationships. Having said so, if ur objective is muscle building den u shall not hv * more dan once a week as it will reduce test levels in body and hence achieving goals will be difficult. Dating in North Africa is predominantly done under family supervision, usually in a public place. \\\"Parents explore dating scene for choosy children\\\". Dating in Korea is also considered a necessary activity supported by society. Moreover th3 Hon'ble Supreme Court of India has also done a great job by discussing another important point about legalizing prostitution. Somebody is giving tip to them. I have to do it for sure as my * is little thick.\\n21 While some of what happens on a date is guided by an understanding of basic, unspoken rules, there is considerable room to experiment, and there are numerous sources of advice available. In fact, the smarter you are, the more clueless you will be, and the more problems you're going to have in your dating life. According to a 2007 Centers of Disease Control (CDC) report, approximately 72 percent of 8th and 9th graders report that they are \\\"dating.\\\" a b c Sharon Jayson. \\\"Should I follow any rules?\\\". \\\"Can you be beautiful but not superficial?\\\". 138 Iran has a large population of young people with sixty percent of the 70-million population being under the age of thirty.\"", "vector_field": [-0.011748201213777065, -0.01504208892583847, 0.020573075860738754, -0.03464072570204735, -0.0051947361789643764, -0.010135568678379059, -0.023880688473582268, -0.0011254118289798498, -0.03126448765397072, -0.03615042194724083, -0.009469928219914436, 0.02511589787900448, -0.003504902822896838, 0.010931591503322124, -0.0068039377219974995, 0.0050163171254098415, 0.038154203444719315, -0.03719348832964897, 0.008516073226928711, -0.0023348864633589983, -0.004937401041388512, 0.002341748680919409, 0.006477979943156242, -0.021849459037184715, -0.00903074350208044, 0.019749604165554047, 0.027078505605459213, -0.013642187230288982, 0.01881633698940277, -0.007967092096805573, 0.010821795091032982, -0.019667256623506546, -0.007685739081352949, -0.0101630175486207, -0.02265920490026474, 0.014246066100895405, -0.011061974801123142, -0.007033823523670435, 0.02396303601562977, 0.01047868188470602, 0.019077101722359657, -0.004920245613902807, -0.00874939002096653, -0.009435617364943027, -0.0008401988307014108, 0.03730328381061554, 0.014726424589753151, -0.000119875228847377, -0.012235421687364578, 0.010464956983923912, 0.030660608783364296, 0.01571459136903286, -0.0064127882942557335, -0.007630840875208378, -0.02475905977189541, -0.023125840350985527, -0.018473222851753235, 0.03035866841673851, 0.007009805645793676, 0.01900848001241684, -0.003474022727459669, 0.017087044194340706, -0.03027632273733616, 0.0017584558809176087, -0.018912408500909805, -0.006742177531123161, -0.017512505874037743, 0.015344029292464256, 0.003530636429786682, -0.004069324117153883, 0.01607142947614193, 0.029782239347696304, 0.00013467199460137635, -0.009641485288739204, 0.014204892329871655, -0.015934184193611145, 0.010046359151601791, 0.0007505604298785329, -0.014019611291587353, -0.008241582661867142, 0.011775650084018707, -0.02865682728588581, -0.0077543617226183414, 0.027078505605459213, 0.005078077781945467, 0.02265920490026474, -0.0033058971166610718, 0.0256374292075634, 0.00338309770449996, -0.00633044121786952, -0.0011348474072292447, 0.004807018209248781, -0.0018236474134027958, 0.01869281567633152, -0.006786782294511795, 0.026968710124492645, -0.008234720677137375, 0.030221423134207726, -0.020737770944833755, 0.022988595068454742, -0.0049030897207558155, 0.02893131785094738, 0.0013432888081297278, -0.007328900974243879, 0.001682971022091806, 0.0063853394240140915, -0.006532878149300814, -0.03071550652384758, 0.017732098698616028, -0.014438210055232048, -0.00620005838572979, 0.027394169941544533, 0.013635324314236641, -0.038511041551828384, 0.01224228460341692, -0.004793293308466673, -0.004398713354021311, -0.005757442209869623, -0.02091618999838829, -0.0024206647649407387, 0.009758143685758114, 0.0020243688486516476, 0.026570698246359825, 0.00793964322656393, 0.026570698246359825, -0.00890722218900919, 0.0011194073595106602, 0.0023966466542333364, -0.006628950126469135, 0.00036005457513965666, 0.019269246608018875, 0.001129700685851276, 0.018034037202596664, 0.045620352029800415, -0.009236611425876617, 0.0007282581063918769, -0.008516073226928711, -0.015412651933729649, -0.016208674758672714, -0.0438910610973835, 0.006025070324540138, 0.01399216242134571, -0.019351592287421227, -0.012605984695255756, -0.004608012270182371, 0.021561242640018463, 0.024580640718340874, -0.0018905545584857464, -0.01027967594563961, -0.010265951044857502, 0.008454312570393085, 0.00903074350208044, 0.023921862244606018, 0.0017404424725100398, 0.02131420187652111, 0.013745120726525784, -9.510673407930881e-05, 0.018583018332719803, 0.019612358883023262, 0.004096773453056812, 0.00580547796562314, 0.008605282753705978, -0.00797395408153534, -0.014589179307222366, -0.003051993204280734, 0.024264976382255554, 0.005150131415575743, 0.006848542485386133, 0.0183771513402462, -0.019337868317961693, 0.006755901966243982, 0.022755278274416924, -0.026364829391241074, 0.010622789151966572, 0.011432536877691746, 0.003091451246291399, -0.0017979139229282737, 0.001364733325317502, -0.017828170210123062, -0.013086343184113503, -0.02364737167954445, -0.0012961106840521097, 0.023029768839478493, 0.034860316663980484, -0.0202436875551939, 0.020010370761156082, 0.012921649031341076, 0.003640432609245181, 0.0020655423868447542, -0.00992283783853054, 0.012626571580767632, 0.0039011987391859293, -0.016428267583251, -0.012194248847663403, -0.6710748076438904, -0.013642187230288982, 0.020820118486881256, -0.0016975533217191696, -0.00354779209010303, -0.0026059458032250404, 0.007397523615509272, 0.01059534028172493, 0.0020552489440888166, 0.026996158063411713, -0.012756953947246075, 0.02444339543581009, -0.011137459427118301, -0.023112114518880844, 0.00023524709104094654, -0.005908411927521229, 0.017032146453857422, -0.023029768839478493, -0.005410897545516491, -0.004999161697924137, -0.02110833302140236, 0.0037296421360224485, -0.00010105760156875476, 0.012022691778838634, -0.0202436875551939, 0.0163459200412035, 0.00032788768294267356, 0.00934640783816576, 0.0030245441012084484, 0.020984811708331108, -0.012605984695255756, 0.030413568019866943, 0.002537323161959648, 0.020463280379772186, 0.050780776888132095, -0.02722947485744953, -0.01395098865032196, 0.025843298062682152, 0.025774674490094185, 0.009627760387957096, -0.019502563402056694, -0.007946505211293697, 0.018445773050189018, 0.008296480402350426, 0.005771166644990444, -0.002175338566303253, 0.015069538727402687, -0.01266088243573904, 0.01781444437801838, 0.004642323590815067, 0.007960230112075806, 0.028958767652511597, -0.011871721595525742, -0.014836221002042294, 0.0053216880187392235, -0.010087532922625542, -0.01865164190530777, -0.04218921810388565, 0.01623612456023693, -0.0001611560583114624, 0.0013261331478133798, 0.013072618283331394, 0.004666341468691826, -0.01758112758398056, -0.027641210705041885, 0.0035855344031006098, -0.030605711042881012, 0.03562888875603676, -0.0026128082536160946, 0.015330304391682148, 0.0026848618872463703, 0.0013853202108293772, -0.025102172046899796, -0.011425674892961979, 0.03535439819097519, 0.017004698514938354, 0.02492375299334526, -0.0035718099679797888, -0.032993778586387634, 0.001993488520383835, -0.010780621320009232, -0.012310907244682312, 0.014973466284573078, -0.0011794521706178784, 0.001852812129072845, -0.014081371948122978, -0.011892308481037617, 0.012276595458388329, -0.0030365530401468277, -0.017430158331990242, 0.011350189335644245, 0.032691840082407, 0.008440588600933552, -0.010087532922625542, -0.008927809074521065, 0.0071298955008387566, -0.005709405988454819, -0.02028486132621765, 0.02246706187725067, -0.016359644010663033, -0.008721941150724888, -0.0011013938346877694, 0.027970600873231888, 0.0031189003493636847, 0.008886636234819889, 0.02464926242828369, -0.023880688473582268, -0.004659479483962059, 0.04183237999677658, -0.018596744164824486, 0.01544010080397129, -0.0035992590710520744, -0.02230236865580082, -0.01364904921501875, 0.0048035867512226105, -0.00781612191349268, -0.003739935578778386, 0.01255794893950224, 0.032527144998311996, 0.0016280728159472346, -0.010814933106303215, 0.004783000331372023, 0.013827468268573284, -0.03175857290625572, 0.005760873202234507, 0.03337806835770607, 0.0018287941347807646, 0.010567891411483288, 0.017649751156568527, -0.007685739081352949, -0.0006390486378222704, -0.00793964322656393, 0.006083399523049593, -0.021739661693572998, 0.02658442221581936, 0.012928511016070843, -0.00012234135647304356, -0.004817311652004719, 0.013806881383061409, -0.021163230761885643, -0.0014788185944780707, -0.015261681750416756, 0.015028364956378937, -0.018637917935848236, 0.009531688876450062, -0.0031446339562535286, 0.006659829989075661, -0.008763114921748638, -0.01454800646752119, -0.008378827944397926, -0.0007312603411264718, -0.012324631214141846, -0.012715781107544899, 0.03189581632614136, -0.010959040373563766, 0.012139350175857544, -0.0002864996495191008, -0.010567891411483288, -0.0060662440955638885, 0.00443645566701889, 0.004161965101957321, 0.026323655620217323, -0.02071032114326954, 0.0032372744753956795, 0.02286507375538349, -0.011206082068383694, -0.00189227017108351, 0.0182948037981987, -0.01349121704697609, -0.02253568544983864, 0.0020432400051504374, -0.006457393057644367, 0.0005138122360222042, 0.0239355880767107, 0.0006767910672351718, 0.0007818695739842951, -0.013772569596767426, 0.005812339950352907, -0.02504727430641651, 0.002106715925037861, 0.014644077979028225, -0.004278623498976231, -0.030029280111193657, -0.014795047231018543, 0.015110711567103863, 0.017718372866511345, 0.002031231066212058, 0.003474022727459669, -0.004340383689850569, 0.009943424724042416, 0.005349137354642153, -0.0019608926959335804, -0.014836221002042294, -0.006340734660625458, 0.004086480010300875, -0.02270037867128849, 0.024388497695326805, 0.010334573686122894, -0.007822984829545021, 0.013278486207127571, 0.027051055803894997, -0.007047548424452543, 0.0201613400131464, -0.025857022032141685, 0.004487922415137291, -0.007424972951412201, 0.00030065307510085404, -0.019255520775914192, 0.0011983233271166682, 0.015330304391682148, 0.01176878809928894, -0.01638709381222725, -0.0005455502541735768, -0.012036416679620743, 0.0029010234866291285, 0.03420154005289078, -0.00809747539460659, 0.014451934024691582, -0.008008265867829323, 0.013937264680862427, -0.0010362023022025824, -0.0019866263028234243, 0.011631542816758156, 0.01888495869934559, 0.0017207134515047073, 0.022453337907791138, 0.012729505077004433, 0.016524339094758034, 0.012853026390075684, -0.02838233672082424, -0.014012749306857586, 0.01204327866435051, -0.014932293444871902, 0.04691045731306076, 0.004083049017935991, -0.0033419239334762096, 0.010506130754947662, -0.021698487922549248, 0.0091062281280756, 0.0016409396193921566, 0.019543737173080444, 0.02099853754043579, 0.0365072600543499, -0.023825790733098984, 0.012784403748810291, -0.004467335995286703, 0.02114950679242611, 0.01598908193409443, -0.004223725292831659, 0.0329388827085495, 0.001626357319764793, 0.021259302273392677, -0.02293369546532631, -0.025335490703582764, 0.006718159187585115, -0.0042443121783435345, 0.01446565892547369, 0.01932414434850216, 0.014410761184990406, 0.01714194379746914, 0.024059107527136803, 0.004391850903630257, 0.014246066100895405, -0.01741643436253071, -0.00463203014805913, -0.002219943329691887, 0.012159937061369419, -0.020216237753629684, 0.0025012963451445103, -0.009716969914734364, -0.01741643436253071, -0.02389441430568695, -0.00938758160918951, -0.02618641033768654, 0.026598146185278893, -0.012276595458388329, -0.0075622182339429855, -0.010149292647838593, 0.022041602060198784, 0.021094609051942825, -0.013024582527577877, -0.010183604434132576, 0.004549683071672916, 0.015385203063488007, -0.019516287371516228, -0.04460473731160164, -0.026460900902748108, 4.704405728261918e-05, -0.009840491227805614, -0.008968982845544815, -0.009373856708407402, 0.010245365090668201, -0.011295291595160961, -0.01003949623554945, -0.006800506729632616, 0.02475905977189541, 0.028794072568416595, -0.007013237103819847, 0.01101393811404705, 0.011377639137208462, 0.011604093946516514, -0.0007291158544830978, -0.0399932935833931, -0.008653318509459496, 0.03820910304784775, 0.0019282969878986478, -0.01953001134097576, -0.004933970049023628, -0.0007750072982162237, -0.011302153579890728, 0.013923539780080318, -0.009984598495066166, -0.02253568544983864, 0.021643590182065964, -0.003300750395283103, 0.012722643092274666, 0.014630353078246117, 0.006097124423831701, -0.005695681553333998, -0.013010858558118343, 0.002693439833819866, -0.05473344027996063, -0.027709834277629852, -0.009126815013587475, 0.10375747829675674, -0.016579236835241318, 0.005572160705924034, 0.01972215622663498, -0.012393253855407238, -0.0016117750201374292, -0.0018425186863169074, -0.027668660506606102, 0.008516073226928711, 0.019516287371516228, 0.01705959625542164, -0.0073083145543932915, 0.0006896578124724329, 0.0001335997658316046, 0.018541844561696053, 0.003453435841947794, 0.008660181425511837, -0.01968098245561123, 0.008776839822530746, 0.010986489243805408, 0.02687263675034046, -0.009078779257833958, 0.009867940098047256, 0.048118215054273605, 0.014575455337762833, 0.005225616507232189, 0.005716268438845873, 0.035656340420246124, 0.009449341334402561, -0.026598146185278893, -0.014987191185355186, 0.0272980984300375, 0.01745760813355446, 0.017196841537952423, -0.01877516321837902, 0.029315603896975517, 0.02718830108642578, 0.010458094999194145, 0.012743229977786541, 0.010073808021843433, 0.029260706156492233, 0.019392766058444977, 0.009092504158616066, -0.00705441040918231, -0.012489326298236847, -0.028039222583174706, -0.006742177531123161, 0.014136269688606262, -0.011830548755824566, 0.0013175553176552057, 0.01630474627017975, -0.005225616507232189, -0.029233258217573166, -0.019475113600492477, -0.00903074350208044, -0.009943424724042416, 0.017512505874037743, -0.006848542485386133, -0.012715781107544899, -0.03282908722758293, -0.0010696558747440577, -0.019022203981876373, 0.02658442221581936, -0.0012437858385965228, -0.021602416411042213, -0.021561242640018463, -0.007864157669246197, -0.019104551523923874, 0.0003004386380780488, 0.00290273898281157, 0.00841313973069191, -0.033872149884700775, -0.01035516057163477, -0.004350677132606506, -0.00028349741478450596, -0.005205029621720314, 0.007582805119454861, -0.00048121646977961063, -0.009497377090156078, 0.011974656023085117, 0.000946135085541755, -0.0062995608896017075, -0.023070942610502243, -0.03497011214494705, -0.017128217965364456, 0.027792181819677353, -0.015810662880539894, 0.008550385013222694, -0.012331494130194187, 0.00785729568451643, 0.0018270786385983229, -0.012317769229412079, -0.008207271806895733, -0.013765707612037659, 0.010300262831151485, -0.002846125280484557, -0.014026474207639694, 0.017375260591506958, 0.0008504922152496874, -0.006635812111198902, -0.0032166875898838043, -0.0021135781425982714, -0.027476517483592033, -0.009991460479795933, 0.0015851836651563644, -0.0013655911898240447, -0.033268269151449203, -0.02322191186249256, -0.0035169117618352175, -0.01869281567633152, 0.01957118511199951, -0.035189706832170486, -0.0046114432625472546, 0.0035340674221515656, 0.011864859610795975, 0.013902952894568443, 0.016977248713374138, 0.020861292257905006, 0.02020251378417015, -0.016894901171326637, -0.01579693891108036, -0.04004818946123123, 0.026968710124492645, 0.03140173479914665, -0.004340383689850569, -0.0019077102188020945, -0.021959254518151283, -0.037797365337610245, -0.0184594988822937, -0.00045376739581115544, 0.005304532591253519, 0.01594790816307068, -0.031182141974568367, -0.022988595068454742, -0.02746279165148735, -0.011521746404469013, -0.006062813103199005, 0.0036678817123174667, -0.013806881383061409, -0.009833628311753273, -0.007836708799004555, 0.018308527767658234, 0.015426375903189182, -0.03156642988324165, 0.031017446890473366, -0.029754789546132088, -0.024415945634245872, -0.0010370600502938032, 0.0019591771997511387, 0.020614249631762505, -0.021163230761885643, -0.004800155758857727, 0.0007205380243249238, 0.006927458569407463, 0.003973252605646849, -0.035381849855184555, -0.009332682937383652, -0.01478132326155901, 0.0361778698861599, 0.012214835733175278, 0.025266867130994797, -0.01774582266807556, -0.007836708799004555, 0.004275192506611347, 0.009806179441511631, 0.0017121356213465333, -0.002868427662178874, 0.01797913946211338, -0.022549409419298172, 0.016812553629279137, 0.010780621320009232, 0.02091618999838829, -0.013758845627307892, -0.0036781751550734043, 0.0029936640057712793, 0.032993778586387634, -0.003074295585975051, -0.028437234461307526, -0.03826400265097618, -0.03134683519601822, 0.0073220389895141125, -0.021204404532909393, -0.02620013616979122, 0.005808908957988024, -0.029754789546132088, -0.0035992590710520744, 0.022919971495866776, -0.005163855850696564, 0.0028100982308387756, -0.0256374292075634, 0.01670275814831257, -0.02095736376941204, 0.015522448346018791, -0.003918354399502277, -0.0221102237701416, -0.020614249631762505, -0.016126327216625214, -0.009854215197265148, 0.007733774837106466, 0.018747713416814804, 0.0011399941286072135, 0.011981518007814884, 0.009888526983559132, -0.008927809074521065, -0.0014642361784353852, -0.001993488520383835, -0.016812553629279137, -0.039581555873155594, 0.004299210384488106, -0.015302855521440506, 0.002964499406516552, -0.01865164190530777, -0.0019729018677026033, -0.01563224382698536, -0.01877516321837902, 0.0037879713345319033, -0.03211541101336479, 0.03063316084444523, -0.015508723445236683, -0.0030262598302215338, 0.005026610568165779, 0.007500458043068647, 0.0063578905537724495, -0.028327438980340958, 0.04106380417943001, 0.024937478825449944, 0.02341405488550663, -0.009531688876450062, -0.013113792054355145, 0.017032146453857422, -0.0026882931124418974, 0.019612358883023262, 0.026323655620217323, -0.006097124423831701, -0.008488624356687069, -0.005270221270620823, 0.0002053318894468248, -0.017196841537952423, -0.01976332813501358, 0.022055326029658318, 0.009751281701028347, 0.012791265733540058, -0.011755063198506832, 0.003962959162890911, -0.020092718303203583, 0.005465795751661062, 0.011535471305251122, 0.005911842919886112, -0.012605984695255756, -0.011864859610795975, 0.0011725898366421461, 0.043479323387145996, 0.001796198426745832, 0.006501997821033001, -0.0043575395829975605, -0.014877394773066044, -0.015742041170597076, -0.028162743896245956, 0.011281566694378853, 0.01023850217461586, 0.0025493321008980274, 0.02326308563351631, -0.0073494878597557545, -0.012015829794108868, 0.003969821613281965, -0.009435617364943027, -0.015453825704753399, -0.018239906057715416, -0.03266439214348793, 0.03615042194724083, -0.0008161808946169913, -0.000850063341204077, 0.019900573417544365, -0.00531825702637434, 0.02194553054869175, 0.0015062675811350346, -0.006872560363262892, -3.6643432395067066e-05, -0.021808285266160965, -0.008138648234307766, 0.01148743461817503, 0.00809747539460659, -0.016057705506682396, 0.006429944187402725, -0.012057003565132618, -0.023043492808938026, 0.01043750811368227, -0.010231640189886093, 0.027449067682027817, -0.013511803932487965, 0.0032372744753956795, 0.009874802082777023, -0.004783000331372023, 0.005891256034374237, 0.003685037372633815, -0.0015234232414513826, 0.019077101722359657, 0.014836221002042294, -0.018555570393800735, 0.02047700434923172, 0.0026677062269300222, 0.014561730436980724, -0.018857508897781372, 0.020614249631762505, -0.020545627921819687, -0.026735391467809677, 0.009737556800246239, -0.01130901649594307, -0.032005611807107925, -0.022439613938331604, 0.01075317244976759, 0.003422555746510625, -0.009799317456781864, 0.007150482386350632, 0.006604931782931089, -0.01329221110790968, 0.008955258876085281, -0.026598146185278893, -0.01896730624139309, 0.00972383189946413, -0.021959254518151283, -0.007081859745085239, -0.005911842919886112, -0.025253143161535263, 0.02762748673558235, -0.005126113537698984, 0.0200789924710989, -0.027490241453051567, -0.010986489243805408, -0.006152022164314985, -0.025623705238103867, 0.02706478163599968, -0.03197816386818886, -0.00017616726108826697, -0.012050140649080276, -0.0031463494524359703, -0.006965200882405043, 0.023194462060928345, -0.01583811268210411, 0.01699097268283367, 0.019077101722359657, 0.010540442541241646, 0.006556896027177572, 0.018322253599762917, 0.0010773759568110108, -0.009435617364943027, -0.023908138275146484, -0.0007578516378998756, -0.006100555416196585, 0.00817296002060175, 0.005455502308905125, 0.027037331834435463, 0.04169513285160065, -0.010965902358293533, -0.020051544532179832, -0.001517418771982193, -0.03971880301833153, -0.01575576514005661, -0.011741339229047298, 0.018267353996634483, 0.005716268438845873, -0.0037090552505105734, -0.0066255186684429646, 0.02142399735748768, -0.003935510292649269, 0.031099794432520866, -0.030386118218302727, -0.030248872935771942, 0.01399216242134571, -0.017759546637535095, 0.01638709381222725, 0.01603025570511818, -0.05075332522392273, -0.0022919971961528063, 0.03266439214348793, -0.03266439214348793, -0.012077589519321918, 0.011947207152843475, 0.0010576469358056784, 0.01789679192006588, 0.007630840875208378, -0.008584695868194103, -0.004892796277999878, -0.012832439504563808, 0.01948883756995201, -0.035299502313137054, -0.0048756408505141735, 0.0003973681596107781, -0.005544711835682392, -0.009716969914734364, -0.00789846945554018, -0.008927809074521065, -0.005174149293452501, -0.012482463382184505, 0.013443181291222572, -0.017526229843497276, 0.005482951179146767, -0.024855131283402443, 0.012406978756189346, -0.004378126468509436, 0.0006055950652807951, 0.031099794432520866, 0.01924179680645466, -0.013408869504928589, 0.003832576097920537, -1.9715615053428337e-05, -0.0004541962989605963, -0.009161126799881458, 0.005771166644990444, 0.0007149624289013445, 0.01718311570584774, -0.010622789151966572, 0.009778730571269989, -0.004837898071855307, 0.013758845627307892, 0.0029404815286397934, -0.010808070190250874, -0.0008792279404588044, 0.027215750887989998, -0.00017992006905842572, -0.027476517483592033, 0.014534281566739082, 0.003230412257835269, 0.007967092096805573, -0.005747148767113686, -0.02821764163672924, -0.003561516525223851, -0.0063990638591349125, -0.017690924927592278, -0.00423744972795248, 0.0030811578035354614, 0.0025973680894821882, -0.006731884088367224, 0.0029542059637606144, -0.017841894179582596, -0.005126113537698984, 0.21366354823112488, 0.0034946093801409006, 0.006666692439466715, 0.04070696607232094, -0.007733774837106466, 0.005458933301270008, 0.022727828472852707, -0.014369587413966656, 0.009943424724042416, -0.011974656023085117, -0.035299502313137054, 0.014849945902824402, -0.03329572081565857, -0.0025887901429086924, 0.004666341468691826, -0.0019608926959335804, -0.02937050350010395, -0.022686654701828957, 0.022000428289175034, -0.0031806607730686665, -0.0045977188274264336, -0.024319874122738838, -0.01888495869934559, -0.01619495078921318, 0.02769611030817032, 0.01630474627017975, 0.013587288558483124, 0.0018819767283275723, 0.012132488191127777, -0.005112389102578163, 0.008317067287862301, -0.01063651405274868, -0.001731006894260645, 0.00149082753341645, -0.010286537930369377, 0.0018373719649389386, 0.007246553897857666, -0.004364402033388615, 0.004433024674654007, 0.009243473410606384, 0.004014426376670599, -0.009202299639582634, 0.01785561814904213, -0.0218220092356205, 0.01141194999217987, 0.016922350972890854, 0.009936562739312649, 0.0017241445602849126, -0.03217030689120293, 0.005911842919886112, -0.04221666604280472, -0.01841832511126995, 0.02983713708817959, 0.005386879667639732, -0.013498079031705856, 0.00798081699758768, 0.012214835733175278, 0.012654020451009274, -0.0402403324842453, 0.0034036843571811914, -0.008269031532108784, 0.001090242643840611, -0.008406276814639568, 0.024813957512378693, -0.02110833302140236, 0.0030331220477819443, -0.019022203981876373, -0.0025338921695947647, 0.013614737428724766, 0.008344517089426517, 0.0013338531134650111, -0.010828657075762749, -0.019653532654047012, 0.00306057115085423, -0.018871234729886055, -0.022919971495866776, 0.016455715522170067, 0.010924729518592358, 0.010430646128952503, 0.005304532591253519, -0.02876662276685238, -0.02079266868531704, 0.0032561456318944693, 0.014698975719511509, -0.0018150695832446218, -0.017649751156568527, 0.016977248713374138, -0.016291022300720215, -0.008021989837288857, -0.007925918325781822, -0.005078077781945467, -0.005956447683274746, -0.008248444646596909, 0.000954712915699929, 0.007877882570028305, -0.0010662247659638524, 0.027037331834435463, 0.0001710205542622134, -0.006519153714179993, -0.018473222851753235, -0.013710809871554375, 0.040267784148454666, 0.032444797456264496, 0.03241734951734543, -0.010231640189886093, -0.0024772784672677517, -0.014987191185355186, 0.0018957012798637152, 0.005362861789762974, -0.012070727534592152, 0.0043438151478767395, -0.02607661485671997, 0.01718311570584774, -0.012098176404833794, -0.015193059109151363, 0.01646944135427475, -0.007081859745085239, -0.03184092044830322, -0.009517963975667953, -0.0057540107518434525, -0.012489326298236847, -0.021657314151525497, 0.003849731758236885, 0.013889227993786335, -0.028629377484321594, -0.003980115056037903, -0.006193195935338736, -0.0075896671041846275, -0.004412437789142132, -0.01781444437801838, 0.014904843643307686, -0.014328413642942905, -0.01212562620639801, -0.005167286843061447, -0.0035340674221515656, -0.006073106080293655, 0.023551300168037415, -0.03225265443325043, -0.0017704649362713099, 0.010828657075762749, 0.01357356458902359, -0.011542333289980888, 0.02068287320435047, -0.0006557753658853471, 0.014005887322127819, -0.0034414269030094147, 0.0035958278458565474, 0.011164908297359943, -0.017841894179582596, 0.02012016624212265, 0.0016400818713009357, 0.009037605486810207, 0.011995242908596992, -0.005476089194417, 0.003811989212408662, -0.002672852948307991, -0.0363425649702549, -0.00672502163797617, -0.020449556410312653, -0.007040685974061489, -0.019269246608018875, 0.010917866602540016, 0.04320483282208443, -0.0007359781302511692, -0.006083399523049593, 0.0016512329457327724, -0.17786996066570282, 0.02039465680718422, 0.01774582266807556, -0.02972734160721302, 0.008701354265213013, 0.016277296468615532, 0.01297654677182436, -0.01877516321837902, -0.010101256892085075, -0.008145511150360107, 0.028437234461307526, 0.0010001753689721227, -0.014314688742160797, 0.002206218894571066, -0.006522584706544876, 0.015536172315478325, 0.007164206821471453, 0.02389441430568695, 0.0474594384431839, -0.00025862170150503516, 0.02127302810549736, -0.009174850769340992, 0.006302992347627878, 0.002132449531927705, -0.016977248713374138, 0.03184092044830322, 0.006165747065097094, -0.014355862513184547, -0.01575576514005661, -0.003688468597829342, -0.02108088508248329, 0.0110551118850708, 0.010362023487687111, 0.00872880406677723, -0.003897767746821046, 0.009270922280848026, -0.004827604629099369, 0.0005498391692526639, -0.01180996187031269, 0.03776991739869118, 0.010142430663108826, 0.02313956432044506, -0.012990271672606468, 0.009682659059762955, 0.02956264652311802, 0.033432964235544205, 0.007404386065900326, -0.018198732286691666, -0.010423783212900162, -0.0025716344825923443, 0.004731533117592335, -0.005846651270985603, 0.04114615172147751, 0.01474014949053526, -0.007452421821653843, 0.019228072836995125, -0.011995242908596992, -0.004045306239277124, 0.016085153445601463, 0.004192844964563847, 8.341943612322211e-05, -0.00560647202655673, 0.020051544532179832, -0.011576644144952297, -0.03507990762591362, -0.010066946037113667, -0.016675308346748352, 0.02167103998363018, -0.03873063623905182, 0.01544010080397129, 0.0009427039185538888, -0.005146700423210859, 0.009634623304009438, -0.02614523656666279, -0.0017807582626119256, -0.006580913905054331, 0.019063377752900124, 0.015975357964634895, -0.001861389959231019, 0.014095096848905087, 0.004402144346386194, 0.011206082068383694, -0.009751281701028347, 0.017402708530426025, -0.02333170734345913, 0.025857022032141685, 0.03631511703133583, -0.0012926795752719045, 0.003962959162890911, -0.011494297534227371, 0.022755278274416924, -0.024512017145752907, -0.0034774537198245525, 0.004587425384670496, -0.014218617230653763, 0.0011442829854786396, 0.0060525196604430676, -0.0007372648105956614, -0.021341649815440178, -0.013841192238032818, 0.013800019398331642, -0.006213782820850611, 0.00511925108730793, 0.022563133388757706, 0.01630474627017975, -0.013978437520563602, 0.011823685839772224, -0.0004207427555229515, 0.024196352809667587, -0.0027929425705224276, -0.016853727400302887, 0.0021170093677937984, 0.03420154005289078, 0.01442448515444994, -0.01598908193409443, 0.05294925346970558, 0.004354108590632677, -0.0014101958367973566, 0.0012523637851700187, 0.00471094623208046, 0.043396975845098495, -0.016085153445601463, 0.00898956973105669, 0.013868642039597034, -0.006759332958608866, -0.018761437386274338, -0.12033671140670776, -0.015700867399573326, -0.010938453488051891, 0.03996584191918373, -0.006697572767734528, 0.028025498613715172, -0.0027020175475627184, 0.024470843374729156, -0.03167622536420822, 0.019296694546937943, -0.018185008317232132, -0.013237313367426395, -0.035217154771089554, -0.010663962922990322, 0.014616629108786583, -0.024031659588217735, 0.004275192506611347, -0.02798432484269142, -0.014493107795715332, 0.041091255843639374, 0.012647158466279507, -0.0038463007658720016, 0.0010267667239531875, 0.008612144738435745, -0.011508021503686905, -0.02234354056417942, -0.022961145266890526, 0.010787483304738998, 0.008564108982682228, 0.01571459136903286, -0.019502563402056694, -0.006165747065097094, 0.004525665193796158, -0.013710809871554375, -0.018830060958862305, -0.015659693628549576, -0.03571123629808426, -0.00756908068433404, 0.015193059109151363, -0.022151397541165352, 0.004240881185978651, 0.01888495869934559, 0.006049088202416897, 0.005709405988454819, -0.003190954215824604, 0.004460473544895649, -0.01270205620676279, 0.021492620930075645, -0.00847489945590496, -0.0010499268537387252, -0.029892034828662872, -0.018912408500909805, -0.03656215965747833, -0.024951202794909477, 0.019516287371516228, -0.021135782822966576, 0.025102172046899796, 0.003685037372633815, 0.004179120529443026, -0.013031445443630219, 0.007184793706983328, -0.0007106735138222575, -0.028958767652511597, 0.03167622536420822, 0.01619495078921318, -0.0021358805242925882, -0.019584910944104195, 0.010444370098412037, 0.011247255839407444, -0.023084666579961777, -0.0064127882942557335, -0.010663962922990322, -0.021574966609477997, -0.012496188282966614, -0.028272539377212524, 0.013038307428359985, -0.016689034178853035, 0.0024017933756113052, 0.006031932774931192, -0.012839301489293575, -0.004659479483962059, -0.007850433699786663, -0.004714377224445343, -0.01850067265331745, -0.005565298721194267, -0.0002588361385278404, 0.0036781751550734043, -0.01067768782377243, -0.002044955501332879, -0.009284647181630135, -0.00797395408153534, 0.02020251378417015, -0.0014719562605023384, -0.019269246608018875, 0.0002832829486578703, 0.025966817513108253, -0.01478132326155901, -0.012214835733175278, 0.013209863565862179, 0.009360131807625294, -0.017773272469639778, 0.0011485719587653875, -0.042765647172927856, 0.005829495843499899, 0.034585826098918915, -0.009051330387592316, -0.02226119488477707, -0.003815420437604189, 0.007795535493642092, -0.0007994541083462536, -0.009977736510336399, -0.008186684921383858, -0.01310006808489561, 0.016483165323734283, -0.01003949623554945, -0.022480785846710205, -0.03107234463095665, -0.020188789814710617, 0.026049165055155754, -0.017828170210123062, 0.015371478162705898, 0.01161095593124628, 0.012640295550227165, 0.009579724632203579, 0.018185008317232132, 0.025500183925032616, 0.00354779209010303, 0.0028718586545437574, -0.009216024540364742, 0.012633433565497398, 0.0071161710657179356, 0.00281696068122983, 0.03142918273806572, -0.026049165055155754, -0.003094882471486926, -0.020861292257905006, -0.010416921228170395, -0.04150298982858658, 0.00189227017108351, 0.039471760392189026, 0.019735880196094513, 0.040542274713516235, -0.012990271672606468, -0.03749542683362961, 0.01309320516884327, -0.00797395408153534, -0.01446565892547369, -0.00038235692773014307, -0.018239906057715416, 0.00013327809574548155, 0.02051817812025547, -0.00954541377723217, 0.05330609157681465, 0.0033419239334762096, -0.03845614567399025, -0.01710077002644539, -0.012413840740919113, -0.006862266920506954, -0.0021701918449252844, -0.0021924942266196012, -0.017992863431572914, -0.019035927951335907, 0.027311822399497032, 0.0200789924710989, 0.01944766379892826, -0.028245091438293457, -0.015028364956378937, -0.009435617364943027, -0.03606807440519333, 0.007445559836924076, 0.006014776881784201, -0.020367208868265152, -0.0008157519623637199, 0.028437234461307526, 0.024745333939790726, 0.015934184193611145, -0.005314825568348169, 0.010581615380942822, -0.011466847732663155, 0.016140051186084747, -0.00914740189909935, 0.010924729518592358, 0.009140539914369583, -0.006224076263606548, -0.01246187649667263, 0.020737770944833755, 0.011315878480672836, 0.0023297397419810295, -0.0062858364544808865, 0.011432536877691746, -0.0031034601852297783, 0.003798264777287841, 0.010169879533350468, 0.023661097511649132, 0.008021989837288857, -0.01513816136866808, 0.0027088799979537725, 0.02012016624212265, -0.010128705762326717, -0.007651427760720253, -0.0091885756701231, 0.011466847732663155, 0.03873063623905182, -0.008282756432890892, -0.001111687277443707, -0.0023949311580508947, -0.00992283783853054, 0.0043849884532392025, -0.010924729518592358, -0.008227857761085033, 0.003640432609245181, 0.01690862514078617, -0.006855404935777187, -0.010410059243440628, 0.011556057259440422, 0.017292913049459457, 0.008145511150360107, 0.0008427721913903952, -0.0007625694270245731, -0.021808285266160965, -0.014849945902824402, 0.024745333939790726, -0.0019385904306545854, 0.022123949602246284, 0.017869343981146812, -0.007473008707165718, 0.022165123373270035, 2.96471371257212e-05, 0.020984811708331108, -0.027449067682027817, -0.012599121779203415, -0.016441991552710533, 0.008927809074521065, -0.0077543617226183414, -0.01957118511199951, -0.0049030897207558155, -0.01482249703258276, -0.022796450182795525, -0.001194892218336463, 0.005414328537881374, -0.00992283783853054, 0.03755032643675804, 0.01544010080397129, -0.005733423866331577, -0.0017301490297541022, 0.03447603061795235, 0.028821522369980812, 0.0010276244720444083, -0.005524124950170517, -0.008296480402350426, -0.0044707669876515865, 0.007006374653428793, -0.02821764163672924, 0.0236061979085207, -0.036095526069402695, -0.008042576722800732, 0.006203489378094673, -0.013621600344777107, 0.019708430394530296, -0.024306150153279305, -0.012798127718269825, 0.016455715522170067, -0.029535196721553802, 0.025253143161535263, 0.008502349257469177, -0.009243473410606384, 0.008207271806895733, 0.008131786249577999, 0.019118275493383408, 0.0006523442571051419, -0.0200378205627203, 0.007527906913310289, 0.0006373330252245069, -0.023798342794179916, -0.015412651933729649, -0.001821931917220354, 0.009668934158980846, -0.03167622536420822, 0.0004477629263419658, -0.016332196071743965, 0.03189581632614136, 0.0019248658791184425, -0.0069308895617723465, -0.030935099348425865, -0.019749604165554047, -0.014836221002042294, -0.00324413669295609, -0.007994540967047215, -0.007507320027798414, -0.018034037202596664]} +{"id": "test:10000029", "text": "\"This topic contains 1 reply, has 2 voices. Last updated by Raam Dev 4 years, 11 months ago.\\nI need a solution for a site I\u2019m developing with S2 Members Pro, the issue is the clients run a chain of shops across the UK and a magazine that is subscribed from all over the world. The client wants its clients to be able to not only sign up online but also in store, the problem is my client will not accept the 0.01p charge on the checkout with a 100% discount code, this is because they feel there customers will not like having to pay online.\\nI know you suggest 2 options on the plugin: 1 being a free registration form but this is open to abuse if the link gets leaked and will also require a lot of moderation on the admin side of things to make sure each sign up actually did join in store. 2 being a free trial, we can set the trial up to last a year but then the checkout will still require a form of payment to start running once the trial period has ended.\\nIs there any solution to this problem? we need it to be as secure as it can without being open to abuse and easy to administrate.\\nWhat is the best solution for this?\\nLet us know if that answers your question.\"", "vector_field": [0.005962589755654335, -0.0077254134230315685, -0.01112297736108303, -0.03508119657635689, -0.021113432943820953, 0.008419756777584553, -0.004334590397775173, -0.024659303948283195, -0.01408910471946001, -0.03651032969355583, 0.0022515603341162205, 0.012767829932272434, 0.004341331776231527, -0.0030183715280145407, 0.022272920235991478, 0.012882430106401443, 0.018174270167946815, 0.00369417667388916, -0.001877423725090921, 0.00405820133164525, -0.007907425984740257, 0.019145002588629723, -0.004715468268841505, -0.007745637092739344, -0.006838945671916008, -0.008992758579552174, 0.016435042023658752, -0.016313700005412102, 0.003754847450181842, 0.00026206407346762717, -0.0016490656416863203, -0.0024875022936612368, -0.015221625566482544, -0.009107358753681183, -0.03920681029558182, 0.010900517925620079, -0.0039469716139137745, 0.003593058791011572, 0.028474822640419006, -0.008055731654167175, 0.018956249579787254, 0.005028933752328157, 0.0050356751307845116, -0.03319365903735161, -0.027099616825580597, -0.0006467337370850146, -0.01355655025690794, -0.023877324536442757, -0.026695145294070244, 0.0051367930136621, -0.0024875022936612368, 0.0016768730711191893, -0.028313033282756805, 0.008756816387176514, 0.0025178375653922558, -0.01080614048987627, -0.005126681178808212, 0.007031070068478584, 0.0321420319378376, -0.011446555145084858, 0.014035174623131752, 0.025562623515725136, -0.025279494002461433, -0.015936192125082016, -0.014304823242127895, -0.004954780917614698, -0.015936192125082016, 0.019509028643369675, -0.012309428304433823, 0.005399699788540602, -0.011372401379048824, 0.013637444004416466, -0.0009707325370982289, -0.016596829518675804, 0.005092975217849016, -0.011109494604170322, 0.008824228309094906, -0.026263708248734474, -0.0008670866372995079, 0.009808443486690521, 0.007246788125485182, 0.0015403637662529945, -0.021032538264989853, 0.02785463072359562, 0.01807989366352558, 0.007260270416736603, 0.02263694442808628, 0.02672211080789566, -0.023054897785186768, -0.009653395973145962, 0.009053429588675499, 0.02801642008125782, -0.0029391625430434942, 0.023742500692605972, -0.014102586545050144, 0.014507059007883072, -0.03130612522363663, 0.02947251871228218, 0.006046854890882969, 0.025522176176309586, -0.019158486276865005, 0.006316503044217825, -0.03696873039007187, -0.02149094082415104, -0.011210612952709198, 0.012417287565767765, 0.004152578301727772, -0.0055817123502492905, 0.001021291594952345, 0.014237411320209503, -0.0016440097242593765, 0.04786250740289688, 0.006559186149388552, 0.024942433461546898, 0.008177073672413826, 0.002342566382139921, 0.0047053564339876175, -0.017769798636436462, -0.01015898585319519, -0.011756650172173977, 0.015814851969480515, 0.004749174229800701, 0.03464975953102112, -0.0034464376512914896, 0.01945509761571884, -0.002613899763673544, -0.03179148957133293, 0.00866243988275528, 0.014412682503461838, -0.011520707979798317, 0.019145002588629723, 0.004048089496791363, 0.0039469716139137745, 0.0181877538561821, 0.0004975846968591213, 0.021935859695076942, -0.01023313868790865, 0.018147306516766548, -0.006633338984102011, -0.02251560240983963, 0.026897381991147995, 0.007988319732248783, 0.014628400094807148, 0.011824062094092369, 0.016596829518675804, 0.04082469642162323, -0.012754347175359726, 0.00047946773702278733, 0.0027183883357793093, -0.022461673244833946, -0.004543567541986704, 0.006282797083258629, 0.01070502307265997, -0.0036301352083683014, 0.023149274289608, 0.017891140654683113, 0.00388967152684927, 0.02149094082415104, -0.0045873853377997875, -0.023472853004932404, 0.008554580621421337, 0.02651987411081791, -0.001771249808371067, 0.0036368765868246555, 0.022974003106355667, 0.022030236199498177, 0.030443251132965088, -0.008871416561305523, 0.006121008191257715, -0.0022481896448880434, -0.02161228097975254, 0.009612948633730412, -0.02637156844139099, 0.0054907058365643024, 0.01396776270121336, 0.0016633906634524465, -0.007846754975616932, 0.012363357469439507, -0.005639012437313795, -0.02290659211575985, -0.001267345272935927, 0.004145836923271418, -0.01685299538075924, 0.05250045284628868, -0.026492908596992493, -0.0002506883174646646, 0.029364660382270813, -0.012956582941114902, 0.017661940306425095, -0.015019389800727367, 0.030470216646790504, 0.007941131480038166, 0.011062306351959705, 0.006286167539656162, -0.63507479429245, -0.020816819742321968, 0.027436677366495132, -0.01764845661818981, -0.0018993326229974627, 0.02126174047589302, 0.004557050298899412, -0.004122242797166109, -0.035620491951704025, 0.028933223336935043, -0.015450826846063137, 0.005207575857639313, -0.025994060561060905, -0.022218989208340645, -0.022016754373908043, -0.011109494604170322, 0.03224989399313927, 0.0026678291615098715, -0.012558852322399616, -0.0005553062073886395, -0.030955582857131958, 0.00048283833893947303, -0.01512724906206131, 0.024578409269452095, 0.0042166197672486305, -0.0026425498072057962, -0.006121008191257715, -0.0016473802970722318, 0.00702432869002223, 0.00405820133164525, -0.015329484827816486, -0.0018572001717984676, 0.0050356751307845116, 0.0031531956046819687, 0.03869447857141495, -0.018241683021187782, 0.004455931950360537, 0.026897381991147995, 0.012349875643849373, 0.026142366230487823, -0.012410545721650124, -0.012039780616760254, 0.028744470328092575, -0.01377226784825325, 0.010678058490157127, 0.010091573931276798, 0.017634974792599678, 0.006498515140265226, -0.0012024613097310066, 0.02896018885076046, -0.014345269650220871, -0.001052469597198069, -0.02455144375562668, -0.00762429554015398, 0.04408743605017662, -0.0032559987157583237, 0.016556384041905403, -0.016799066215753555, 0.005804171785712242, 0.015895744785666466, -0.006882763467729092, -0.009565760381519794, -0.013846421614289284, -0.03858662024140358, -0.027288371697068214, -0.021288704127073288, -0.017621492967009544, 0.002152127679437399, 0.009242182597517967, 0.00029450611327774823, 0.028825365006923676, 0.0030183715280145407, -0.02153138816356659, -0.015450826846063137, -0.03041628748178482, 0.026978276669979095, 0.017190055921673775, -0.0026594027876853943, -0.03464975953102112, 0.011210612952709198, 0.011075789108872414, 0.004722209647297859, -0.023189721629023552, -0.0007571208407171071, 0.019360721111297607, 0.014345269650220871, -0.00505926925688982, -0.01166901458054781, 0.004493008833378553, 0.013610479421913624, 0.004442449659109116, 0.02801642008125782, 0.012208309955894947, -0.005392958875745535, -0.021854965016245842, 0.01929331012070179, 0.012228533625602722, 0.007475988939404488, 0.025212081149220467, -0.020331453531980515, -0.022380778566002846, 0.000507696473505348, 0.006282797083258629, 0.015059837140142918, 0.029418589547276497, 0.0016237861709669232, -0.02604798972606659, -0.00838605035096407, 0.04640641063451767, -0.025239046663045883, 0.019198933616280556, -0.03375992178916931, 0.00030377524672076106, -0.018403472378849983, -0.014331787824630737, -0.026776039972901344, 0.012107192538678646, -0.001956632826477289, -0.014277857728302479, -0.004246954806149006, 0.010482563637197018, 0.009114100597798824, -0.008635475300252438, -0.017149608582258224, 0.00807595532387495, 0.0296612735837698, 0.013583514839410782, 0.0019094444578513503, 0.014250893145799637, -0.004155948758125305, 0.011702720075845718, 0.0009580928017385304, 0.030470216646790504, -0.034757617861032486, 0.012188086286187172, 0.003677323693409562, 0.03276222571730614, 0.010374704375863075, 0.021544869989156723, -0.022340331226587296, 0.006050225347280502, -0.001572384499013424, 0.026614250615239143, -0.0502084456384182, -0.013886868953704834, -0.04495031014084816, 0.006313132122159004, 0.004115501418709755, -0.017311397939920425, 0.014183481223881245, -0.002221224829554558, -0.01925286278128624, 0.016367629170417786, -0.014803671278059483, -0.03454190120100975, 0.0009016352705657482, -0.007078258320689201, -0.0364564023911953, -0.010132021270692348, -0.021908894181251526, 0.022407744079828262, 0.03171059861779213, -0.04354814067482948, 0.004533455707132816, -0.020061805844306946, 0.0032273486722260714, 0.010132021270692348, 0.005339029245078564, -0.009296112693846226, -0.030470216646790504, 0.004958151374012232, -0.01658334769308567, 0.008183814585208893, 0.010738728567957878, 0.009680360555648804, 0.01443964708596468, -0.01580136828124523, 0.004860403947532177, -0.01771586947143078, -0.00369417667388916, -0.008534356951713562, 0.018632672727108, -0.01001742109656334, 0.008628733456134796, 0.01563958078622818, 0.00561878876760602, -0.007408577017486095, 0.03009270876646042, -0.007489471230655909, 0.03443404287099838, 0.00043733525671996176, 0.014237411320209503, 0.001637268578633666, 0.03642943501472473, -0.025522176176309586, -0.005949107464402914, 0.01568002812564373, 0.007826531305909157, 0.01404865738004446, 0.028582680970430374, 0.007091740611940622, 0.013630703091621399, 0.006205272860825062, -0.021747106686234474, 0.02825910411775112, -0.02495591714978218, -0.0068625397980213165, -0.028690539300441742, 0.04511209949851036, 0.020183147862553596, -0.013536326587200165, 0.0051671285182237625, 0.015100284479558468, -0.013873386196792126, -0.010415151715278625, 0.024659303948283195, -0.009403971955180168, 0.03327455371618271, -0.003970565740019083, 0.018686601892113686, -0.0024639079347252846, -0.012963324785232544, 0.01748666912317276, -0.009606207720935345, -0.016259770840406418, -0.0006783331045880914, -0.008089438080787659, 0.00702432869002223, 0.0033335224725306034, -0.03359813243150711, -0.012619523331522942, -0.006474921014159918, -0.001065952004864812, 0.029445555061101913, 0.02235381305217743, 0.02165272831916809, 0.04012361168861389, -0.00895905215293169, 0.024888504296541214, -0.008642216213047504, 0.008217521011829376, 0.03834393620491028, 0.015666544437408447, -0.011345436796545982, 0.04761982336640358, 0.0009951693937182426, 0.006279426161199808, 0.007873719558119774, -0.01508680172264576, 0.010961188934743404, 0.006990622729063034, 0.017068713903427124, -0.01400821004062891, -0.00850739236921072, 0.03909895196557045, 0.021544869989156723, -0.010543234646320343, 0.008251226507127285, -0.0015883948653936386, 0.019482063129544258, -0.003690805984660983, 0.012498181313276291, 0.008675922639667988, -0.026654697954654694, 0.010603904724121094, 0.023203205317258835, -0.011837543919682503, -0.026573803275823593, 0.005456999875605106, -0.027962490916252136, 0.01166901458054781, -0.002779059112071991, 0.005901919212192297, -0.0016018772730603814, 0.012808277271687984, 0.007293976377695799, -0.002549858298152685, 0.008669180795550346, 0.017149608582258224, 0.02089771442115307, -0.019279826432466507, -0.01968429982662201, 0.0080355079844594, 0.02192237786948681, 0.0036166529171168804, -0.0204662773758173, -0.009983714669942856, -0.011129718273878098, 0.007988319732248783, 0.019738228991627693, -0.016529418528079987, 0.029796097427606583, -0.0055817123502492905, -0.014871083199977875, -0.007341165095567703, -0.0034009346272796392, 0.029068047180771828, -0.001368463272228837, 0.005625530146062374, -0.01108253002166748, 0.018322577700018883, 0.020479761064052582, -0.02023707702755928, -0.02955341339111328, 0.050828635692596436, -0.009734290651977062, -0.010280327871441841, -0.025050293654203415, 0.007192858494818211, -0.02676255814731121, 0.013832938857376575, -0.008568063378334045, -0.01104208268225193, -0.011244318448007107, 0.023836877197027206, 0.006656933575868607, 0.0025026698131114244, -0.033652063459157944, 0.003545870305970311, -0.0014645253540948033, -0.016691207885742188, -0.008372568525373936, -0.016799066215753555, 0.006397397257387638, 0.0647694319486618, 0.014116069301962852, 0.018713567405939102, 0.011547672562301159, -0.015032872557640076, 0.020371900871396065, -0.015693509951233864, -0.011985850520431995, 0.03615978732705116, 0.008911863900721073, -0.01941465213894844, 0.0006989780231378973, -0.01551823876798153, 0.0022127984557300806, 0.045759253203868866, -0.00473569193854928, 0.021949341520667076, 0.004519973415881395, -0.010462339967489243, -0.0026931087486445904, 0.02228640206158161, 0.0009243868407793343, -0.0048839980736374855, 0.035539597272872925, 0.02349981665611267, 0.016313700005412102, 0.04392564669251442, 0.020344937220215797, 0.0012580761685967445, -0.03335544839501381, 0.005028933752328157, 0.018713567405939102, -0.00521431677043438, -0.0029560155235230923, 0.0023156015668064356, 0.039503421634435654, 0.008332121185958385, 0.015841815620660782, 0.011547672562301159, -0.0043952614068984985, 0.013846421614289284, 0.0230279341340065, -0.016246287152171135, -0.0010406725341454148, 0.007084999233484268, -0.030578074976801872, -0.00895905215293169, 0.02196282334625721, 0.004519973415881395, -0.0068018692545592785, 0.031144335865974426, 0.0039200070314109325, -0.006731086410582066, 0.014143033884465694, -0.007058034650981426, -0.009902819991111755, -0.009397230111062527, 0.0020055065397173166, -0.02212461270391941, 0.0010533122112974524, -0.03775070980191231, -0.02643897943198681, 0.006077190395444632, 0.0030267981346696615, -0.027800701558589935, -0.0371035560965538, -0.0010364592308178544, -0.009976973757147789, -0.008682663552463055, -0.018376506865024567, 0.014790189452469349, -0.024510996416211128, -0.014412682503461838, 0.009525313042104244, 0.001813382375985384, 0.004159319214522839, 0.012012815102934837, 0.017500150948762894, 0.01791810430586338, -0.003690805984660983, -0.02604798972606659, -0.02836696244776249, -0.0071321879513561726, -0.013448690995573997, -0.020951643586158752, 0.02368857152760029, -0.01524859108030796, 0.015423862263560295, -0.014062140136957169, 0.004584014881402254, -0.00027891708305105567, -0.004267178475856781, 0.01658334769308567, -0.032088104635477066, 0.02963430806994438, 0.01972474716603756, 0.008736592717468739, 0.011540931649506092, 0.01968429982662201, 0.010523010976612568, -0.008460204117000103, -0.023526782169938087, 0.0068288338370621204, -0.022111130878329277, -0.019616886973381042, -0.02409304305911064, 0.0025869349483400583, -0.005453629419207573, -0.010860070586204529, -0.025117704644799232, 0.0025279494002461433, 0.0059524779208004475, -0.010152244940400124, -3.0598719604313374e-05, 0.017729351297020912, 0.009174770675599575, 0.005453629419207573, 0.00988259632140398, 0.025050293654203415, -0.017378808930516243, 0.019495544955134392, -0.03696873039007187, 0.033220626413822174, 0.006367061752825975, -0.010442116297781467, -0.004061571788042784, 0.012262240052223206, -0.03370599076151848, 0.014547506347298622, -0.004307625815272331, 0.014318305067718029, 0.009855631738901138, -0.014493576250970364, -0.005349141079932451, -0.02828606776893139, -0.024537961930036545, -0.027261406183242798, 0.02184148319065571, -0.018174270167946815, -0.002893659286201, -0.0018420324195176363, 0.0037683299742639065, -0.0023038045037537813, -0.01685299538075924, 0.03292401134967804, -0.028474822640419006, -0.0009361839038319886, -0.007496212609112263, -0.0033149842638522387, 0.01619235798716545, -0.0015403637662529945, -0.008190556429326534, -0.022933557629585266, -0.025279494002461433, -0.0016322126612067223, -0.04500424116849899, -0.02255604974925518, -0.018767496570944786, 0.012579075992107391, 0.02271783910691738, 0.034595828503370285, 0.00465479725971818, -0.007078258320689201, 0.0006699065561406314, 0.02471323311328888, 0.007037810981273651, 0.0057333894073963165, 0.001983597641810775, 0.0013819456798955798, -0.00036802730755880475, 0.004108760505914688, 0.011345436796545982, 0.039476457983255386, 0.00072341482155025, 0.013900350779294968, 0.02409304305911064, 0.0018959620501846075, -0.012336392886936665, -0.017392292618751526, -0.027059169486165047, -0.005574970971792936, -0.012383581139147282, -0.012909394688904285, -0.010037644766271114, -0.04635247960686684, -0.004782880190759897, 0.003643617732450366, -0.016097981482744217, 0.011230836622416973, -0.01074547041207552, 0.026668179780244827, 0.00838605035096407, 0.032357752323150635, -0.0012496496783569455, -0.014250893145799637, -0.006751310080289841, -0.01713612675666809, -0.027059169486165047, 0.0033453197684139013, 0.0032088104635477066, 0.037508025765419006, 0.027167029678821564, 0.004337961319833994, 0.0035795762669295073, -0.014021692797541618, 0.029256800189614296, -0.029930921271443367, -0.022448189556598663, 0.007084999233484268, 0.01196562685072422, -0.003411046462133527, -0.017783280462026596, 0.0002549015625845641, 0.0029812948778271675, -0.032627400010824203, 0.01736532710492611, 0.0060704490169882774, 0.002499299356713891, -0.006404138635843992, -0.025751378387212753, 0.01752711646258831, -0.04956129193305969, 0.01094096526503563, 0.03772374615073204, 0.009552277624607086, 0.0230279341340065, 0.02416045404970646, -0.029930921271443367, 0.0036132822278887033, 0.008675922639667988, 0.003984048031270504, 0.01882142573595047, 0.047134459018707275, -0.008844451978802681, -0.02039886638522148, 0.009410712867975235, -0.01108253002166748, -0.035027265548706055, -0.03354420140385628, 0.021046021953225136, 0.027207477018237114, 0.005069381091743708, -0.025360388681292534, -0.015491274185478687, -0.01576092094182968, 0.0007920907810330391, -0.0008915234939195216, -0.004284031689167023, -0.022529084235429764, -0.0045739030465483665, -0.023958219215273857, 0.03203417360782623, -2.0789750124095008e-05, 0.026587286964058876, 0.01972474716603756, -0.02145049348473549, -0.011379143223166466, -0.024982880800962448, 0.015396896749734879, 0.012107192538678646, 0.0009800017578527331, 0.026034507900476456, -0.018403472378849983, -0.002979609649628401, 0.029499484226107597, 0.017351845279335976, -0.012794794514775276, -0.01658334769308567, -0.008359085768461227, 0.026425497606396675, 0.012012815102934837, -0.0015875522512942553, 0.00020971446065232158, -0.003788553411141038, -0.036995697766542435, -0.003384081646800041, -0.0077254134230315685, -0.04079773277044296, -0.008163590915501118, 0.009646655060350895, 0.02825910411775112, -0.0020105624571442604, -0.019616886973381042, 0.0045401970855891705, 0.013307125307619572, -0.010010679252445698, -0.008493909612298012, 0.010967929847538471, 0.0296612735837698, -0.009552277624607086, -0.028474822640419006, -0.014075621962547302, -0.02372901886701584, 0.005847989581525326, -0.020412348210811615, 0.022812215611338615, 0.008251226507127285, 0.02651987411081791, -0.04942646622657776, 0.0158687811344862, 0.01771586947143078, 0.01007809117436409, -0.013799233362078667, 0.005433405749499798, -0.02329758182168007, -0.019495544955134392, 0.012612782418727875, 0.00685579888522625, -0.01658334769308567, -0.024349208921194077, 0.004685132764279842, -0.015963157638907433, 0.004978375043720007, -0.0016583347460255027, -0.01874053105711937, 0.023796429857611656, 0.009572501294314861, 0.01039492804557085, -0.019792158156633377, 0.010219656862318516, -0.0028363591991364956, -0.015275555662810802, -0.013805974274873734, -0.013859903439879417, 0.009127582423388958, -0.0184708833694458, 0.011648790910840034, 0.02216506004333496, -0.013246454298496246, -0.01046908088028431, -0.005494076758623123, -0.0051401639357209206, -0.028124280273914337, -0.021935859695076942, -0.010610646568238735, -0.002775688422843814, -0.01697433739900589, 0.007233305834233761, -0.0004432337882462889, -0.01886187307536602, -0.00232739862985909, -0.012167862616479397, -0.020695479586720467, -0.007563624531030655, 0.0006298806983977556, -0.011891474016010761, -0.016259770840406418, -0.01822820119559765, -0.023782948032021523, -0.005527782719582319, -0.04106738045811653, 0.04527388885617256, 0.028151243925094604, -0.015073319897055626, -0.011015118099749088, -0.009491607546806335, -0.04459976777434349, -0.011689238250255585, -0.011689238250255585, 0.008419756777584553, 0.01921241544187069, 0.026924345642328262, 0.00846694502979517, 0.01396776270121336, -0.011797097511589527, 0.0408516600728035, -0.019738228991627693, 0.00408516637980938, -0.006936693098396063, -0.022111130878329277, -0.0051671285182237625, 0.002922309562563896, -0.0550081767141819, 0.016205841675400734, 0.029769131913781166, 0.022529084235429764, 0.030551111325621605, 0.023351510986685753, -0.03473065420985222, 0.019751710817217827, 0.04176846519112587, -0.0036335058975964785, 0.01304421853274107, -0.010691540315747261, 0.01212067436426878, -0.028339996933937073, 0.0045132325030863285, 0.006515368353575468, 0.0013861588668078184, -0.008844451978802681, -0.014655365608632565, -0.019670816138386726, -0.0034177876077592373, -0.003309928346425295, -0.017540598288178444, 0.0006235608598217368, -0.0018015853129327297, -0.004179542884230614, 0.03138701990246773, 0.01662379503250122, -0.001349082333035767, 0.010772434994578362, -0.002994777401909232, -0.012916135601699352, -0.010334257036447525, 0.024254832416772842, -0.015302520245313644, -0.004610979463905096, 0.0033705991227179766, -0.023203205317258835, -0.002920624101534486, -0.020304489880800247, -0.0014485149877145886, -0.02161228097975254, -0.00958598405122757, -0.01803944632411003, 0.003923377487808466, -0.0028447858057916164, 0.009990455582737923, 0.015922710299491882, -0.032465610653162, 0.006879393011331558, -0.015828333795070648, 0.0004929501446895301, 0.031009512022137642, -0.026344602927565575, -0.012147638946771622, -0.005662606563419104, -0.02397170104086399, 0.003842483041808009, 0.0018572001717984676, -0.017257466912269592, 0.015841815620660782, -0.0017763057257980108, 0.006785016041249037, 0.0052850996144115925, 0.1958722621202469, 0.007105222903192043, -0.016691207885742188, 0.025090740993618965, 0.038559652864933014, 0.008918605744838715, 0.024457067251205444, -0.003122860100120306, -0.015626097097992897, -0.0025363757740706205, -0.007138928864151239, 0.016826031729578972, -0.02149094082415104, -0.013799233362078667, 0.00312791601754725, -0.025333423167467117, -0.025630036368966103, -0.036051928997039795, -0.02259649708867073, 0.013273419812321663, 0.004496379289776087, 0.027692843228578568, -0.011918438598513603, -0.005150275304913521, 0.022259436547756195, -0.010307292453944683, -0.008918605744838715, 0.0061176372691988945, 0.008325380273163319, 0.01563958078622818, -0.01192517951130867, -0.00021698232740163803, 0.022367296740412712, -0.009970231913030148, -0.01316556055098772, -0.01131847221404314, 0.024618856608867645, -0.027328817173838615, 0.02947251871228218, 0.027692843228578568, -0.015747439116239548, 0.011642049066722393, -0.015302520245313644, 0.0043817791156470776, -0.02321668714284897, -0.003321725409477949, 0.0030588186345994473, -0.011210612952709198, -0.019980911165475845, 0.008952311240136623, -0.006791757419705391, -0.007718672044575214, 0.02695131115615368, 0.00581428362056613, 0.005810913164168596, -0.020102253183722496, 0.0031852161046117544, 0.003508793655782938, -0.02074940875172615, 0.011561155319213867, -0.029769131913781166, 0.009397230111062527, -0.020695479586720467, 0.011426331475377083, -0.041121311485767365, 0.02325713448226452, -0.017850693315267563, 0.00019749604689422995, -0.011803838424384594, -0.014156516641378403, 0.0077388957142829895, 0.0013971133157610893, 0.004169431049376726, -0.004894109908491373, -0.01151396706700325, -0.03041628748178482, -0.006950175389647484, 0.01658334769308567, 0.014143033884465694, 0.018416954204440117, -0.025683965533971786, 0.0026324379723519087, 0.011540931649506092, -0.010185950435698032, 0.01524859108030796, -0.03564745560288429, 0.02401214838027954, 0.013913833536207676, 0.002275154460221529, -0.006232237908989191, 7.515383185818791e-05, 0.008332121185958385, -0.02801642008125782, -0.016408076509833336, -0.0059524779208004475, -0.012417287565767765, 0.022138094529509544, 0.03171059861779213, -0.01343520823866129, -0.005288470070809126, -0.015032872557640076, 0.07873719930648804, -0.007610812783241272, -0.00874333456158638, -0.012531887739896774, -0.014587952755391598, -0.0047323210164904594, 0.042577408254146576, -0.012309428304433823, -0.010826364159584045, -0.00972754880785942, -0.014749742113053799, 0.011871250346302986, -0.011251060292124748, 0.010132021270692348, -0.0007436384330503643, 0.008102920837700367, 0.0005569914937950671, -0.014507059007883072, -0.005382847040891647, 0.02645246312022209, -0.026843450963497162, -0.014345269650220871, -0.02664121612906456, -0.003707659197971225, -0.01752711646258831, -0.03049718216061592, -0.009599466808140278, -0.018969731405377388, -0.05921468511223793, 0.01913152076303959, 0.010421892628073692, 0.009491607546806335, -0.017190055921673775, -0.015976639464497566, -0.01988653466105461, 0.02058761939406395, -0.022542567923665047, 0.023553747683763504, 0.007334423717111349, 0.01328016072511673, 0.02011573500931263, -0.01654290035367012, -0.02011573500931263, 0.012936359271407127, -0.025764860212802887, 0.01666424237191677, -0.008338862098753452, -0.021167362108826637, -0.01078591775149107, 0.008028767071664333, 0.0251851174980402, 0.011716202832758427, -0.018767496570944786, 0.014911530539393425, -0.004624462220817804, -0.027638914063572884, -0.04160667583346367, 0.014925013296306133, -0.019994394853711128, -0.024349208921194077, 0.0028565828688442707, 0.04365599900484085, -0.007401835639029741, -0.045678358525037766, 0.003148139687255025, -0.17138822376728058, 0.010091573931276798, 0.007125446572899818, -0.015437344089150429, 0.011601602658629417, 0.024066077545285225, 0.015221625566482544, 0.01964385248720646, -0.004398631863296032, -0.01619235798716545, 0.012019556947052479, -0.016313700005412102, -0.010873553343117237, -0.004493008833378553, 0.013644185848534107, -0.0014948607422411442, -0.024079561233520508, -0.00405820133164525, 0.027773737907409668, 0.0058075422421097755, 0.048240017145872116, -0.004398631863296032, 0.015329484827816486, -0.013792491517961025, 0.030901653692126274, 0.010138762183487415, -0.01288917101919651, 0.014951977878808975, 0.0023577341344207525, -0.009336560033261776, -0.006319873500615358, 0.012403804808855057, 0.008183814585208893, 0.002664458705112338, -0.0028700651600956917, -0.0013760471483692527, 0.021153880283236504, 0.01273412350565195, 0.019657334312796593, 0.03300490602850914, -0.01108253002166748, 0.0021217921748757362, -0.016839513555169106, -0.020641548559069633, 0.003067245241254568, 0.013050960376858711, 0.036294613033533096, -0.0027453529182821512, 0.003943601157516241, -0.007840014062821865, 0.0018504589097574353, 0.0020981980487704277, 0.018012482672929764, 0.002308860421180725, -0.0001457784092053771, 0.019778676331043243, -0.013441949151456356, -0.010866811498999596, -0.015855299308896065, 0.0019549475982785225, -0.001894276705570519, -0.033732958137989044, -0.008359085768461227, -0.01070502307265997, -0.013408243656158447, -0.02687041647732258, 0.009120841510593891, -0.007914166897535324, -0.017473185434937477, 0.004412114154547453, 0.0032559987157583237, -0.014763223938643932, 0.01713612675666809, -0.019980911165475845, 0.03394867479801178, -0.00870288722217083, -0.036645155400037766, -0.009174770675599575, 0.004304255358874798, 0.005039045587182045, 0.0018925914773717523, 0.014911530539393425, -0.017459703609347343, 0.05123310908675194, -0.009646655060350895, -0.002991406712681055, 0.002806023694574833, -0.00493118679150939, 0.004846921656280756, -0.012221792712807655, 0.012255498208105564, -0.02263694442808628, -0.003805406391620636, 0.012336392886936665, 0.02091119810938835, -0.003515535034239292, 0.0054907058365643024, -0.00529521144926548, -0.011190389283001423, 0.008359085768461227, 0.009478124789893627, -0.0014502003323286772, -0.0323038212954998, 0.012289204634726048, 0.026654697954654694, -0.0067108627408742905, 0.024430103600025177, 0.022151578217744827, 0.022542567923665047, -0.0025077257305383682, -0.004223360680043697, 0.030982548370957375, 0.01870008371770382, 0.023324545472860336, -0.018969731405377388, 0.024268314242362976, 0.0015353079652413726, -0.023944737389683723, 0.00016252607747446746, 0.013489138334989548, 0.03378688544034958, -0.009242182597517967, -0.004701985977590084, 0.012815018184483051, -0.013940798118710518, 0.0012420658022165298, -0.10354480892419815, -0.010287068784236908, 0.014722777530550957, 0.0030605040956288576, -0.031090406700968742, 0.024767162278294563, -0.01611146330833435, 0.04802429676055908, -0.01803944632411003, 0.021989788860082626, -0.02828606776893139, -0.027800701558589935, -0.017931587994098663, 0.024039113894104958, 0.0023290840908885, -0.0037615885958075523, -0.027962490916252136, -0.024079561233520508, -0.020735926926136017, -0.0006593734724447131, -0.01933375746011734, 0.004189654719084501, 0.004287402145564556, -0.0026728850789368153, -0.04055504873394966, -0.0028211914468556643, -0.01799899898469448, 0.013462172821164131, -0.0012344819260761142, 0.001757767517119646, 0.0016591774765402079, -0.021544869989156723, 0.005396329332143068, -0.01611146330833435, -0.00020497456716839224, -0.001480535720475018, -0.01882142573595047, -0.005018822383135557, 0.021639246493577957, -0.024821091443300247, -0.013792491517961025, -0.00525476410984993, 0.00024394711363129318, -0.018794460222125053, -0.01481715403497219, -0.016718171536922455, -0.003148139687255025, 0.023917771875858307, 0.013529584743082523, -0.01720353774726391, -0.008884899318218231, 4.8715690354583785e-05, 0.011116236448287964, -0.027315335348248482, 0.031656667590141296, -0.005827765911817551, -0.0062086437828838825, 0.016556384041905403, -0.007887202315032482, -0.005824395455420017, -0.00889838207513094, -0.007678224705159664, -0.0371844507753849, 0.011803838424384594, 0.021598799154162407, 0.02192237786948681, 0.001334757311269641, -0.00461772084236145, 0.012329651974141598, 0.022879626601934433, -0.012457734905183315, 0.004712097812443972, -0.0008822543313726783, 0.019994394853711128, -0.020102253183722496, -0.009538795799016953, -0.03758892044425011, -0.042065076529979706, 0.013570032082498074, 0.002593676093965769, -0.006751310080289841, -0.01728443242609501, -0.005790689494460821, -0.026492908596992493, -0.012936359271407127, 0.00573675986379385, 0.016987819224596024, 0.021383080631494522, 0.015181178227066994, -0.05228473246097565, 0.02641201578080654, 0.029957884922623634, 0.014264375902712345, 0.0019262974383309484, 0.0004065785324200988, 0.019145002588629723, -0.02715354599058628, -0.00353238801471889, 0.016394594684243202, -0.01728443242609501, -0.010334257036447525, -0.02455144375562668, -0.05339029058814049, 0.0252660121768713, 0.026816487312316895, -0.012167862616479397, 0.00992304366081953, -0.00854783970862627, 0.019859571009874344, -0.002994777401909232, -0.0009471383527852595, -0.00019149215950164944, -0.005996295716613531, 0.024025630205869675, -0.019751710817217827, -0.01658334769308567, -0.021437009796500206, -7.989373261807486e-05, 0.026182813569903374, 0.007462506648153067, 0.00850739236921072, -0.007988319732248783, 0.008648957125842571, -0.016610313206911087, -0.003217236837372184, 0.03405653312802315, -0.01054997555911541, 0.020735926926136017, -0.010570199228823185, 0.011938662268221378, 0.006559186149388552, 0.021383080631494522, 0.011048824526369572, -0.03346330672502518, -0.011325213126838207, 0.0004925287794321775, -0.007374871056526899, -0.02169317565858364, -0.0034262139815837145, 0.022030236199498177, 0.013347572647035122, 0.012855465523898602, -0.0027858002576977015, -0.004155948758125305, -0.004789621569216251, -0.010543234646320343, -0.009619690477848053, -0.0077658602967858315, -0.019118038937449455, 0.004910963121801615, 0.020142700523138046, -0.032519541680812836, 0.025926649570465088, 0.010327516123652458, -0.03076682984828949, -0.03891019523143768, -0.00734790600836277, -0.03516209125518799, -0.012248757295310497, 0.0018116970313712955, 0.035189054906368256, -0.02727488800883293, 0.04354814067482948, -0.0204662773758173, 0.010758952237665653, -0.014264375902712345, -0.002664458705112338, -0.016435042023658752, -0.020533690229058266, 0.017351845279335976, 0.006333355791866779, -0.024025630205869675, 0.012410545721650124, -0.009990455582737923, 0.006471550557762384, 0.003623394062742591, 0.021113432943820953, -0.005756983533501625, -0.0062693143263459206, 0.00025553355226293206, -0.018484365195035934, 0.02035841904580593, 0.008102920837700367, -0.009808443486690521, -0.019279826432466507, 0.01204652152955532, 0.032357752323150635, 0.010671316646039486, 0.016906924545764923, 0.00044955365592613816, -0.014331787824630737, 0.007240047212690115, -0.015747439116239548, 0.0014316620072349906, -0.01059042289853096, 0.004159319214522839, 0.030362356454133987, 0.00738835334777832, 0.0022566162515431643, 0.009896079078316689, 0.009505089372396469, -0.0027251294814050198, 0.04759285971522331, 0.00781979039311409, 0.00437166728079319, -0.004543567541986704, -0.0027453529182821512, -0.0004904221859760582, -0.022178541868925095, -0.025508694350719452, -0.005362623371183872, -0.0007057192269712687, -0.002881862223148346, -0.00621875561773777, 0.021787552163004875, 0.024497514590620995, -0.013219489715993404, 0.004914333578199148, 0.018956249579787254, 0.006471550557762384, -0.02939162403345108, 0.011783614754676819, -0.02204371802508831, -0.004830068442970514, 0.02165272831916809, -0.02887929417192936, 0.02432224340736866, 0.0038458537310361862, -0.00232739862985909, 0.0042570666410028934, 0.020641548559069633, -0.0013077924959361553, 0.0018201235216110945, 0.0005641540628857911, -0.012424028478562832, -0.012956582941114902, 0.0007322626770474017, -0.006067078560590744, -0.011271283961832523, 0.03980003669857979, -0.008345603942871094, 0.08801308274269104, 0.028636610135436058, -0.01843043603003025, 0.02368857152760029, -0.0032003838568925858, 0.021126914769411087, 0.00666030403226614, -0.000864558678586036, 0.0159496758133173, -0.030308427289128304, 0.018133822828531265, 0.0009117470472119749, 0.004139096010476351, -0.015787886455655098, -0.01945509761571884, -0.009538795799016953, -0.01878097839653492, 0.006569297984242439, -0.015653062611818314, 0.01192517951130867, 0.004455931950360537, -0.013145336881279945, 0.0029037711210548878, 0.0012134157586842775, -0.011722943745553493, 0.012511664070189, 0.009943267330527306, -0.011540931649506092, -0.015019389800727367, -0.022583015263080597, -0.006629968527704477, 0.0026846821419894695, -0.002344251610338688, -0.014237411320209503, 0.0031683631241321564, -0.009700584225356579, 0.0026341232005506754, 0.027423195540905, -0.007017587311565876, 0.013940798118710518, 0.010280327871441841, 0.018538296222686768, -0.024618856608867645, -0.02974216639995575, -0.02471323311328888, 0.012916135601699352, -0.005979442968964577, -0.014169998466968536, -0.02354026399552822]} +{"id": "test:10000030", "text": "\"The ebook makes a speciality of geological historical past because the severe think about making a choice on the current biodiversity and landscapes of Amazonia. the various riding mechanisms for panorama evolution are explored through reviewing the historical past of the Amazonian Craton, the linked sedimentary basins, and the position of mountain uplift and weather swap.\\nThis e-book provdes an perception into the Meso- and Cenozoic list of Amazonia that used to be characterised by way of fluvial and long-lived lake platforms and a hugely varied wildlife. This fauna comprises giants akin to the ca. 12 m lengthy caiman Purussaurus, but additionally a diversified fish fauna and fragile molluscs, while fossil pollen and spores shape relics of ancestral swamps and rainforests.\\nWith exceptional wit, readability, and intelligence, Richard Dawkins, one of many world's most famed evolutionary biologists, has brought numerous readers to the wonders of technological know-how in works equivalent to The egocentric Gene. Now, within the Ancestor's story, Dawkins deals a masterwork: an exciting opposite journey via evolution, from present-day people again to the microbial beginnings of lifestyles 4 billion years in the past.\\nColonial dwelling is characterised through department of work and finely coordinated association; besides the fact that, many occasions inside of a colony are the results of conflicts among participants trying to maximize their very own pursuits. This interaction of cooperation and clash has raised many questions in evolutionary biology, in particular how cooperative habit is maintained within the absence of direct replica through staff.\\nTo this point, the dominant paradigms by which glossy scientists have seen nature were established essentially round Newtonian and Darwinian methods. As theoretical ecologist Robert E. Ulanowicz observes in his new paintings, a 3rd Window, neither of those types is enough for explaining how actual change\u2014in the shape of inventive increase or emergence\u2014takes position in nature.\\nEcology is in a hard kingdom as a systematic self-discipline. whereas a few theoretical ecologists try to construct a definition of ecology from first rules, many others are wondering even the feasibility of a normal and common thought. even as, it really is more and more vital that ecology is safely and functionally outlined for a iteration of researchers tackling escalating environmental difficulties within the face of doubt and war of words.\\nSee Chapter 15). , Chapter 16. (9) Middle Miocene Honda Group (La Venta Fauna), Magdalena Valley. , Chapter 16). Map made by D. Riff and J. van Arkel. et al. argue that in the past 30 Ma well-documented episodes of marine influence in Amazonia are limited to the Miocene. However, there is no evidence for fully established marine corridors (\u2018seaways\u2019) throughout the South American continent in the Cenozoic. ). Megafans are low-gradient river systems choked by sediments, which force them to continuously change their courses.\\n2004): 11 30 20 10 0 0 500 1000 1500 2000 2500 3000 Age (Ma) 3500 Fig. 2 Histograms for U-Pb ages for 369 grains of detrital zircon collected from the mouth of the Amazon River outlet show peaks in the Archean, Trans-Amazonian and Grenvillian intervals. Pb-Pb ages give similar results. The error bars correspond to 1\u03c3. (After Rino et al. B. F. de Roever are based on Rb-Sr isochrons, which are now no longer thought to reflect the age of crystallization. Santos et al. (2000), on the basis of new U-Pb zircon ages, make a different subdivision.\\nThe Itacai\u00fanas Supergroup in the Caraj\u00e1s Basin itself consists of a lower-grade metamorphic greenstone sequence, the Gr\u00e3o Par\u00e1 Group, and a higher-grade Salobo Group (Tallarico et al. 2005). 74 Ga, according to SHRIMP zircon U-Pb datings (Trendall et al. 1998; Tallarico et al. 50 Ga. 7 Ga. The Gr\u00e3o Par\u00e1 Group shows the classic greenstone succession of a mainly metabasaltic unit, locally with conspicuous pillow structures, minor meta-andesites and metarhyolites, followed by the BIF and topped by intermediate to acid metavolcanics and metasediments.\"", "vector_field": [0.002158969873562455, -0.0041159712709486485, 0.011498255655169487, 0.006017257925122976, -0.006978347431868315, 0.03334563598036766, -0.03242633491754532, -0.028860552236437798, -2.5259620088036172e-05, -0.03510067239403725, -0.002310445997864008, 0.006306281313300133, 0.0023208926431834698, -0.0020614678505808115, 0.008475697599351406, 0.004808931145817041, 0.03635426610708237, 0.008928384631872177, -0.015461009927093983, -0.0032837234903126955, -0.011881298385560513, 0.013740798458456993, 0.025434058159589767, -0.014513848349452019, -0.007382283918559551, 0.022383641451597214, -0.001247501466423273, -0.03167417645454407, 0.0006938301958143711, 0.010718240402638912, -0.0022320961579680443, -0.001941331778652966, -0.016923537477850914, -0.01213898230344057, -0.0420929454267025, -0.014708852395415306, 0.0018368654418736696, 0.008545341901481152, 0.0037120352499186993, -0.004380619619041681, -0.007807113230228424, 0.006177439354360104, -0.00041525348206050694, -0.01761997863650322, -0.02135290764272213, -0.006842541508376598, 0.001266653649508953, -0.009952154010534286, -0.02819196693599224, 0.0023261159658432007, 0.020921114832162857, 0.025336556136608124, -0.020767897367477417, 0.005272064823657274, -0.009610897861421108, -0.026241930201649666, 0.005258135963231325, -0.0006041633314453065, 0.01605994999408722, -0.010920208878815174, 0.003281982382759452, 0.02367902360856533, -0.009896439500153065, 0.0024427699390798807, -0.016742462292313576, -0.0072012091986835, 0.001159575767815113, 0.008009081706404686, 0.016631031408905983, -0.013183644972741604, 0.03214775770902634, -0.000601986947003752, -0.004558212123811245, -0.007639967370778322, 0.043262969702482224, -0.01310007181018591, -0.00396971870213747, -0.005877969320863485, 0.003304616780951619, 0.0068320948630571365, 0.01495957188308239, -0.01417259220033884, -0.003952307626605034, 0.02306615561246872, 0.01930536888539791, 0.01199969369918108, 0.004704464692622423, 0.01068341825157404, -0.0024253588635474443, -0.004439816810190678, -0.0039348965510725975, -0.0024845562875270844, 0.001130847493186593, 0.012737922370433807, -0.0005797878839075565, -0.0015112789114937186, -0.017341403290629387, 0.012320056557655334, 0.01945858635008335, -0.027328379452228546, -0.0075424653477966785, 0.006919150240719318, -0.008552306331694126, -0.005411353427916765, -0.04086720943450928, -0.009764115326106548, 0.0023121871054172516, 0.008601057343184948, 0.02537834271788597, -0.014834212139248848, -0.003980165347456932, 0.020308244973421097, 0.0032401958014816046, -0.03992004692554474, 0.00514670554548502, -0.0226065032184124, 0.022690076380968094, 0.008761239238083363, 0.014820283278822899, -0.030281292274594307, 0.009666613303124905, 0.0010168051812797785, 0.007291746325790882, -0.007925508543848991, 0.012027551420032978, 0.019430728629231453, -0.02553156018257141, -0.00795336626470089, -0.006567446980625391, -0.01759212277829647, -0.014569563791155815, 0.013037391938269138, 0.007779255975037813, -0.000645514577627182, -0.02665979601442814, 0.018316421657800674, -0.0327327698469162, -0.03036486543715, -0.03159060329198837, -0.030448438599705696, 0.018121417611837387, 0.01870642974972725, -0.016617102548480034, -0.02709158882498741, -0.0018943218747153878, 0.028637690469622612, -0.008844811469316483, 0.02646479196846485, 0.010251624509692192, 0.005592428147792816, 0.024250106886029243, -0.010195909067988396, -0.003501361468806863, 0.007215138059109449, 0.008023010566830635, -0.0051188478246331215, -0.008503555320203304, 0.01621316745877266, -0.010502343066036701, -0.020642537623643875, -0.00926964171230793, 0.01043269969522953, -0.005258135963231325, 6.801190465921536e-05, 0.008670701645314693, 0.02791338972747326, 0.01667281799018383, -0.0120763024315238, 0.014061161316931248, -0.01310007181018591, -0.02522512525320053, 0.014277058653533459, -0.016296740621328354, 0.016241025179624557, -0.005094472318887711, 0.03145131468772888, -0.005268582608550787, 0.031395599246025085, -0.021631484851241112, -0.031089166179299355, 0.0217986311763525, 0.005052685737609863, -0.006859952583909035, -0.003987129777669907, -0.01355275884270668, -0.010558058507740498, 0.01976502127945423, -0.009004993364214897, 0.02899983897805214, -0.005292958114296198, -0.014583492651581764, 0.005292958114296198, -0.00849659088999033, -0.00911642424762249, -0.6476352214813232, -0.0030312633607536554, 0.001519113895483315, -0.028609832748770714, 0.0042308843694627285, -0.0028240717947483063, -0.00013308569032233208, 0.016296740621328354, -0.007765327114611864, -0.002277364954352379, -0.024542611092329025, 0.021102188155055046, 0.006250565871596336, -0.00802997499704361, -0.014625279232859612, -0.02213292382657528, -0.01144254021346569, -0.03691142052412033, 0.028400899842381477, -0.001461657346226275, -0.032342761754989624, -0.0036075690295547247, 0.01271006464958191, 0.022188637405633926, -0.002141558798030019, 0.026436934247612953, 0.011616650968790054, -0.0358528271317482, 0.007639967370778322, 0.01884571649134159, -0.043847981840372086, 0.02477940171957016, 0.010523236356675625, 0.015628155320882797, 0.046856608241796494, -0.036827847361564636, -0.015614226460456848, 0.04446084797382355, 0.018163204193115234, 0.04097864031791687, -0.014005445875227451, -0.02039181813597679, 0.001757645164616406, 0.0007708740886300802, 0.016087807714939117, 0.016617102548480034, 0.034682806581258774, -0.006922632455825806, 0.03239847719669342, -0.023497948423027992, -0.0008487885352224112, 0.01113610528409481, 0.006745039485394955, 0.007779255975037813, 0.005191974341869354, -0.010411806404590607, 0.024876903742551804, -0.0327049121260643, 0.012354878708720207, 0.006435122806578875, -0.022216495126485825, 0.0014102947898209095, -0.007030580658465624, -0.03206418454647064, -0.028721263632178307, 0.029696281999349594, -0.022397570312023163, 0.0028414828702807426, 0.016324598342180252, -0.012368807569146156, 0.024041173979640007, 0.013225431554019451, -0.010662524960935116, -0.024180462583899498, 0.019528230652213097, -0.0015017028199508786, 0.014089019037783146, 0.0036911419592797756, -0.013225431554019451, 0.021784702315926552, 0.018316421657800674, -0.002103254431858659, -0.00958304014056921, -0.0066196802072227, 0.028804836794734, -0.033262062817811966, -0.03036486543715, -0.001558288699015975, -0.009603933431208134, -0.004725357983261347, -0.0068703992292284966, 0.005331262480467558, -0.0342370830476284, -0.006762450560927391, -0.010161086916923523, 0.010878422297537327, 0.014256165362894535, 0.0060068112798035145, -0.004105525091290474, -0.034376371651887894, -0.0039732009172439575, 0.0058118076995015144, -0.00329068792052567, -0.018302492797374725, 0.009234819561243057, 0.012459345161914825, -0.02246721461415291, 0.0032732768449932337, 0.05006024241447449, -0.00689129251986742, 0.004300528671592474, -0.004638303071260452, 0.011728080920875072, -0.0041124895215034485, -0.020503249019384384, -0.02352580614387989, -0.009395000524818897, 0.000756945286411792, 0.010655560530722141, -0.016798177734017372, 0.022091137245297432, 0.0008170133805833757, 0.0002844529808498919, -0.03877788409590721, -0.0001944596297107637, 0.025016192346811295, -0.008148370310664177, -0.030977735295891762, -0.0124663095921278, -0.02069825306534767, 0.006515213754028082, -0.012083266861736774, 0.030922019854187965, -0.009151246398687363, 0.019096435979008675, 0.015266005881130695, 0.042789388447999954, 0.0071141538210213184, -0.007159422617405653, -0.033122774213552475, -0.022829364985227585, -0.015864945948123932, -0.005198938772082329, -0.029556993395090103, -0.003819983685389161, -0.03822072967886925, -0.013469185680150986, 0.0005941519630141556, -0.03543496131896973, -0.003945343196392059, 0.007709611672908068, -0.017508549615740776, -0.02927841618657112, 0.019291440024971962, -0.026910513639450073, 0.017564265057444572, -0.008663737215101719, -0.00016562257951591164, -0.010920208878815174, -0.012912032194435596, 0.013573652133345604, 0.012563811615109444, -0.02040574699640274, -0.0012422781437635422, -0.014541706070303917, -0.009353214874863625, -0.020168956369161606, 0.029807712882757187, 0.009137317538261414, -0.03206418454647064, -0.0020353514701128006, -0.012354878708720207, -0.007841935381293297, 0.005850111600011587, -0.00857319962233305, 0.001412906451150775, -0.01151914894580841, -0.0058118076995015144, -0.005623768083751202, -0.006059044506400824, -0.003363814204931259, 0.006407265085726976, 0.016728533431887627, -0.024723686277866364, 0.008155334740877151, -0.014061161316931248, 0.03504495695233345, 0.01504314411431551, -0.0072708530351519585, 0.043402258306741714, 0.012361843138933182, 0.039669331163167953, -0.016310669481754303, -0.011358967050909996, 0.013197573833167553, -0.009952154010534286, -0.006825130432844162, 0.022049350664019585, 0.008893562480807304, 0.020308244973421097, 0.02291293814778328, -0.012508096173405647, 0.01604602113366127, -0.004182133357971907, 0.01410991232842207, -0.01618530973792076, 0.011393789201974869, -0.020976830273866653, 0.02774624526500702, 0.016087807714939117, 0.01930536888539791, -0.025824064388871193, -0.02479333057999611, -0.03008628822863102, -5.855824838363333e-06, 0.029696281999349594, -0.005801361054182053, 0.00981286633759737, -0.01713247038424015, 0.013671154156327248, 0.00596154248341918, 0.011017710901796818, 0.0171185415238142, -0.01651960052549839, -0.0420650877058506, 0.013378648087382317, 0.0008474827045574784, 0.020823612809181213, 0.008043903857469559, -0.01823284849524498, -0.0038025726098567247, 0.00747282151132822, -0.02649264968931675, 0.016282811760902405, 0.02759302780032158, 0.027328379452228546, -0.011031639762222767, -0.00030121111194603145, 0.002552459482103586, -0.011811654083430767, 0.006501284893602133, 0.0210325438529253, 0.027941247448325157, -0.015558511950075626, 0.029724139720201492, -0.0042308843694627285, 0.012243447825312614, 0.022383641451597214, -0.0072012091986835, -0.010857529006898403, 0.003920967690646648, -0.012640420347452164, -0.018971076235175133, -0.014061161316931248, -0.0072708530351519585, -0.022509001195430756, 0.009471609257161617, -0.0051571521908044815, 0.01777319610118866, 0.005181527696549892, 0.004634820856153965, 0.02303829789161682, 0.0038095370400696993, -0.0027387577574700117, 0.024709757417440414, -0.021074330434203148, -0.0005053556524217129, 0.008280693553388119, -0.014123841188848019, -0.006654502358287573, -0.003694624174386263, -0.022313997149467468, 0.008315515704452991, -0.031869180500507355, -0.006330656819045544, -0.011860405094921589, 0.008580164052546024, -0.005696894600987434, 0.001960483845323324, -0.010397877544164658, -0.009819830767810345, -0.0390007458627224, 0.02101861499249935, 0.010948066599667072, -0.006292352452874184, -0.005578499287366867, -0.03479423746466637, -0.009172139689326286, -0.012424523010849953, 0.030587727203965187, 0.010321268811821938, -0.00011970094783464447, -0.02681301161646843, 0.003102648537606001, -0.009422858245670795, 0.0025071906857192516, 0.020962901413440704, -0.01533565018326044, 0.012584704905748367, -0.0296127088367939, -0.019667519256472588, 0.0035692646633833647, 0.015140646137297153, 0.02197970636188984, 0.017411047592759132, -0.017216043546795845, -0.003146176226437092, 0.018915360793471336, -0.0030451922211796045, 0.0061252061277627945, -0.005801361054182053, -0.019709305837750435, 0.0028171073645353317, 0.0014007186982780695, -0.00902588665485382, 0.010857529006898403, -0.0015426187310367823, -0.019138222560286522, 0.029194843024015427, -0.018762143328785896, -0.019570017233490944, -0.033568497747182846, -0.0233586598187685, -0.021589698269963264, 0.06719271093606949, 0.03769143298268318, -0.027523383498191833, 0.03836001828312874, 0.007298710756003857, -0.01794034242630005, -0.008315515704452991, -0.016895679756999016, 0.02525298297405243, -0.0008226719801314175, -0.012884174473583698, 0.016282811760902405, 0.004620891995728016, 0.00026921831886284053, 0.00259250495582819, 0.00942982267588377, 0.015084930695593357, -0.031729891896247864, -0.007437999360263348, -0.024835117161273956, -0.0074240704998373985, -0.0023121871054172516, 0.025030121207237244, 0.03908431902527809, 0.013392576947808266, 0.0071141538210213184, 0.010864493437111378, -0.009471609257161617, 0.01683996431529522, -0.009722328744828701, -0.015266005881130695, 0.005066614598035812, 0.0035797113087028265, 0.022996511310338974, 0.028010891750454903, 0.004694018047302961, 0.02041967585682869, -0.005815289914608002, 0.029501277953386307, -0.007340497337281704, -0.0026395146269351244, 0.025308698415756226, 0.02273186296224594, -0.021227547898888588, -0.002299999352544546, -0.021227547898888588, 0.0065221781842410564, 0.02522512525320053, -0.023790454491972923, -0.0031775161623954773, 0.024194391444325447, -0.024681899696588516, 0.00795336626470089, -0.02710551768541336, 0.009318392723798752, -0.013364719226956367, 0.023762596771121025, 0.009785008616745472, 0.006219225935637951, 0.0019535194151103497, 0.006476909387856722, -0.009875546209514141, -0.006741557270288467, -0.0024619221221655607, -0.007403177209198475, -0.02930627390742302, -0.02277364954352379, 0.010509307496249676, -0.02369295246899128, -0.00013330332876648754, -0.01089931558817625, -0.030476296320557594, -0.0027892496436834335, -0.01533565018326044, 0.03484995290637016, 0.013824371621012688, 0.015600298531353474, -0.005630732513964176, 0.023637237027287483, 0.02369295246899128, 0.008601057343184948, -0.006323692388832569, 0.004916879814118147, -0.015628155320882797, 0.019570017233490944, -0.007737469393759966, -0.012069338001310825, -0.019597874954342842, -0.00011023369006579742, 0.01051627192646265, 0.02464011311531067, -0.006059044506400824, 0.01837213709950447, 0.03161846101284027, 0.0009680542279966176, 0.005310369189828634, 0.021687200292944908, 0.03211989998817444, 0.016798177734017372, -0.018776072189211845, 0.01654745824635029, -0.030615584924817085, -0.012320056557655334, -0.014304916374385357, 0.0136293675750494, 0.006915668025612831, -0.008844811469316483, -0.007834970951080322, -0.021269334480166435, -0.006595304701477289, -0.018023915588855743, -0.030448438599705696, 0.005028310231864452, 0.007159422617405653, -0.009151246398687363, 0.0008026492432691157, -0.008740345947444439, 0.004039363004267216, -0.0018560176249593496, -0.01916608028113842, 0.02043360471725464, -0.023414375260472298, 0.03025343455374241, 0.018608927726745605, 0.004645267501473427, 0.033874932676553726, -0.019834665581583977, -0.004300528671592474, -0.004300528671592474, 0.007981223985552788, -0.00408463180065155, 0.048082347959280014, 0.005226796492934227, -0.005435728933662176, -0.018483567982912064, 0.015516725368797779, -0.013935801573097706, 0.0007660860428586602, -0.011748974211513996, 0.0030121111776679754, -0.01681210659444332, -0.0023731256369501352, -0.0023539734538644552, -0.007173351477831602, -0.016171380877494812, -0.022662218660116196, -0.0097501864656806, -0.008851775899529457, -0.006539589259773493, -0.003205373650416732, -0.03462709113955498, -0.03705070912837982, 0.009367142803966999, -0.009165175259113312, -0.018288563936948776, -0.020670395344495773, -0.004398030694574118, -0.016171380877494812, 0.023915814235806465, 0.010982888750731945, 0.034989241510629654, -0.002717864466831088, 0.013190609402954578, 0.019361084327101707, -0.009374107234179974, -0.010711275972425938, -0.0039732009172439575, 0.008621950633823872, -0.025030121207237244, 0.018790001049637794, 0.02167327143251896, 0.005421800073236227, -0.010467521846294403, -0.020057525485754013, -0.008378195576369762, 0.03150703012943268, -0.025057978928089142, -0.019737163558602333, -0.030114145949482918, -0.05421103537082672, -0.005320815835148096, 0.004669643007218838, -0.012417558580636978, 0.006577893625944853, -0.024082960560917854, -0.020921114832162857, 0.06641269475221634, 0.0007155940402299166, 0.004697500262409449, -0.02026645839214325, 0.03571353852748871, -0.024264035746455193, 0.03721785545349121, -0.005501891020685434, 0.00658137584105134, -0.013364719226956367, -0.0024932618252933025, -0.017034968361258507, -0.009478573687374592, 0.009645720012485981, -0.03284420073032379, 0.015962447971105576, 0.023316873237490654, -0.009165175259113312, 0.023302944377064705, -0.0036423909477889538, 0.008823918178677559, 0.004373655188828707, 0.005136258900165558, -0.010411806404590607, 0.010634667240083218, -0.0029006805270910263, -0.018260706216096878, -0.01757819391787052, 0.028776979073882103, -0.012974712066352367, 0.01199969369918108, 0.020990759134292603, -0.016477813944220543, -0.018748214468359947, 0.03914003446698189, 0.02292686700820923, 0.05526962876319885, -0.003073049709200859, 0.031702034175395966, -0.0020614678505808115, -0.01887357421219349, -0.011358967050909996, 0.012250412255525589, -0.008816953748464584, 0.0028031785041093826, 0.026088712736964226, -0.0014329291880130768, 0.01151914894580841, -0.02306615561246872, 0.00323323137126863, -0.00572823453694582, -0.01853928342461586, -0.013594545423984528, 0.03755214437842369, 0.008796061389148235, 0.010244660079479218, -0.013517936691641808, 0.003160105086863041, 0.011553971096873283, -0.0012614303268492222, -0.019890379160642624, 0.00903981551527977, 1.953301762114279e-05, -0.014221343211829662, -0.01347615011036396, 0.022174708545207977, -0.0026778189931064844, 0.03551853448152542, -0.011407718062400818, -0.012737922370433807, 0.007389248348772526, 0.0005732587305828929, 0.0013598027871921659, 0.015809230506420135, -0.01098985318094492, 0.01837213709950447, -0.0012414077064022422, 0.01215291116386652, -0.014889927580952644, 0.0030852374620735645, -0.02632550336420536, -0.015224219299852848, -0.011881298385560513, 0.03198061138391495, -0.007723540533334017, 0.015447081066668034, 0.0054566222243011, 0.033902790397405624, -0.011526113376021385, -0.00818319246172905, -0.0001331945095444098, -0.02494654804468155, -0.004373655188828707, -0.008336408995091915, 0.00959696900099516, 0.007215138059109449, 0.0027840263210237026, -0.02211899496614933, -0.001781150116585195, 0.006261012516915798, -0.020628608763217926, 0.0060834200121462345, -0.006609233561903238, -0.025350484997034073, 0.0018020434072241187, 0.000737793103326112, -0.0011508702300488949, 0.0342649407684803, 0.016728533431887627, -0.005776985548436642, -0.019138222560286522, 0.00825283583253622, 0.0028745639137923717, 0.04008719325065613, 0.04373655095696449, 0.003746857400983572, -0.01918000914156437, -0.0034195296466350555, -0.00251589622348547, 0.006180921569466591, 0.004297046456485987, -0.012097195722162724, -0.052260998636484146, -0.0066196802072227, 0.012682206928730011, 0.01619923859834671, -0.03195275366306305, -0.029111269861459732, 0.003247160231694579, -0.0023957600351423025, 0.008900526911020279, -0.011839511804282665, -0.03317848965525627, 0.02245328575372696, 0.01550279650837183, 0.012967747636139393, 0.01993216574192047, -0.0039975764229893684, 0.014736710116267204, -0.0071524581871926785, -0.02277364954352379, 0.011233607307076454, -0.017104612663388252, 0.02397152967751026, -0.010711275972425938, 0.027774102985858917, -0.014931714162230492, -0.002712641144171357, -0.030309149995446205, -0.016784248873591423, -0.026144428178668022, 0.006128688342869282, 0.014736710116267204, -0.009213926270604134, 0.011964871548116207, 0.04797091707587242, 0.010634667240083218, 0.0073474617674946785, 0.015015287324786186, -0.004791520070284605, 0.007312639616429806, -0.017968200147151947, -0.05955971032381058, -0.012278269976377487, 7.541159720858559e-05, 0.02991914376616478, 0.015823159366846085, -0.013692047446966171, -0.024403324350714684, -0.005853593815118074, -0.03473852202296257, -0.01961180381476879, -0.003266312414780259, -0.0004309234209358692, 0.025364413857460022, -0.00669977068901062, 0.009130353108048439, 0.012271305546164513, -0.010885386727750301, 0.021088259294629097, 0.006661466788500547, -0.007249959744513035, -0.019110364839434624, 0.009464644826948643, 0.03613140434026718, 0.008921420201659203, -0.03340135142207146, 0.0005519301630556583, 0.042009372264146805, -0.016491742804646492, 0.005557605996727943, -0.004895986523479223, -0.01665888912975788, 0.02542012929916382, 0.0022634360939264297, 0.0195003729313612, 0.014722781255841255, 0.009652684442698956, 0.012570776045322418, -0.03646569699048996, -0.006529142614454031, -0.010697347111999989, -0.00267085456289351, 0.0011839511571452022, 0.00015191138663794845, -0.03067130036652088, -0.0019239207031205297, 0.020308244973421097, 0.016742462292313576, -0.020447533577680588, 0.024988334625959396, -0.014569563791155815, 0.022495072335004807, 0.0006202685763128102, -0.0038478414062410593, 0.04164722561836243, 0.014555634930729866, -0.018803929910063744, -0.022035421803593636, 0.005543677136301994, -0.007068885024636984, 0.010662524960935116, 0.0068703992292284966, -0.01760604977607727, 0.027996962890028954, -0.012918996624648571, 0.00416472228243947, -0.01794034242630005, 0.00420302664861083, -0.016477813944220543, 0.001835124334320426, -0.023637237027287483, 0.01667281799018383, 0.0452130064368248, 0.003732928540557623, -0.013141858391463757, 0.0021572287660092115, -0.005857076030224562, -0.0014538224786520004, -0.00902588665485382, -0.00408463180065155, 0.003393413033336401, 0.008399088867008686, 0.005136258900165558, -0.022411499172449112, 0.011658436618745327, 0.025364413857460022, -0.0037642682436853647, -0.0343206562101841, 0.0007791443495079875, 0.177620530128479, 0.006745039485394955, 0.006772897206246853, 0.022815436124801636, -0.006274941377341747, 0.013455256819725037, 0.02741195261478424, 0.008991064503788948, -0.0035657824482768774, 0.017090683802962303, -0.013538829982280731, 0.009847688488662243, -0.0052755470387637615, 0.006236637011170387, 0.0016035573789849877, -0.025991210713982582, -0.00878909695893526, -0.015168503858149052, 0.0067171817645430565, -0.009778044186532497, 0.023581521585583687, -0.018274635076522827, -0.009478573687374592, -0.018887503072619438, 0.02913912758231163, -0.016714604571461678, -0.01575351506471634, -0.014068125747144222, 0.011045568622648716, 0.0028763050213456154, -0.005780467763543129, 0.008740345947444439, -0.003069567494094372, -0.009645720012485981, -0.012257376685738564, -0.0031339884735643864, 0.01316275168210268, -0.0014050714671611786, 0.0022059795446693897, 0.01183254737406969, -0.003553594695404172, 0.017954271286725998, 0.014388489536941051, 0.0021520054433494806, 0.0005815289914608002, 0.03117273934185505, -0.0014546929160133004, 0.014221343211829662, -0.008197121322154999, 0.010007869452238083, -0.017104612663388252, 0.031869180500507355, 0.004248295444995165, 0.034069936722517014, -0.0011839511571452022, -0.005327780265361071, 0.017494620755314827, -0.012250412255525589, -0.02023860067129135, -0.012891138903796673, -0.05003238469362259, 0.012278269976377487, -0.0016540494980290532, -0.00902588665485382, -0.013824371621012688, 0.0053382269106805325, -0.016631031408905983, -0.015126717276871204, 0.01137986034154892, -0.01441634725779295, 0.0005475774523802102, -0.02601906843483448, -0.024041173979640007, 0.002141558798030019, -0.018831787630915642, -0.03930718079209328, 0.023887956514954567, -0.00042548245983198285, 0.03309491649270058, 0.006107795052230358, -0.0026969711761921644, 0.010955031029880047, -0.01654745824635029, -0.003994094207882881, 0.0012144205393269658, -0.04733018949627876, 0.03930718079209328, 0.0061252061277627945, -0.028233753517270088, 0.0040115052834153175, -0.021910062059760094, -0.006887810304760933, -0.005561088211834431, -0.025907637551426888, 0.007121118251234293, -0.00766782509163022, -0.0004387583758216351, 0.024375466629862785, -0.028637690469622612, 0.016004234552383423, -0.01976502127945423, 0.030922019854187965, 0.02476547285914421, -0.011247536167502403, -0.0065221781842410564, 0.0018473120871931314, -0.02604692615568638, 0.014346702955663204, 0.016477813944220543, 0.005620285868644714, -0.002277364954352379, 0.000320363265927881, 0.006135652773082256, -0.02245328575372696, -0.002820589579641819, 0.0004635691293515265, 0.01870642974972725, -0.03136774152517319, 0.026214072480797768, -0.006344585679471493, -0.008719452656805515, -0.038109298795461655, 0.015948519110679626, 0.005279029253870249, -0.011310216039419174, -0.023804383352398872, -0.017243901267647743, 0.009764115326106548, 0.011707187630236149, -0.027384094893932343, 0.010544129647314548, 0.005460104439407587, 0.013427399098873138, -0.02693837136030197, -0.012528989464044571, 0.009199997410178185, -0.016867822036147118, -0.026910513639450073, -0.0010272518265992403, 0.03114488162100315, 0.005996364634484053, 0.029083412140607834, 0.027175161987543106, 0.009172139689326286, 0.011630579829216003, -0.006758968345820904, 0.000917562167160213, 0.005773503333330154, -0.028581975027918816, -0.0014895150670781732, -0.012097195722162724, 0.008364266715943813, -0.009576075710356236, 0.01837213709950447, 0.023720810189843178, 0.0037747148890048265, -0.03242633491754532, -0.02444510906934738, -0.011003782041370869, -0.002595987170934677, -0.01074609812349081, 0.009805901907384396, 0.021297192201018333, -0.037775006145238876, 0.006267976947128773, -0.010871457867324352, -0.17973771691322327, -0.0006799013935960829, 0.013998481445014477, -0.024500824511051178, 0.029807712882757187, 0.008399088867008686, 0.025364413857460022, 0.0054566222243011, -0.00747282151132822, -0.011268429458141327, 0.025977281853556633, 0.024654041975736618, -0.037635717540979385, -0.01341347023844719, -0.006550035905092955, 0.018330350518226624, -0.02243935689330101, 0.01794034242630005, 0.012800601311028004, 0.0034682806581258774, -0.01206237357109785, -0.025517631322145462, 0.010007869452238083, -0.01696532405912876, -0.0021868275944143534, 0.03131202608346939, 0.01945858635008335, 0.02277364954352379, 0.007291746325790882, -8.172310481313616e-05, -0.007375319488346577, -0.013559723272919655, 0.014541706070303917, -0.020210742950439453, 0.0052337609231472015, 0.017257830128073692, -0.015669941902160645, 0.003083496354520321, -0.027969105169177055, 0.015976376831531525, -0.004234366584569216, -0.016296740621328354, 0.021854346618056297, -0.014102947898209095, 0.009980011731386185, 0.015656013041734695, 0.0013180162059143186, 0.0067171817645430565, 0.0026813012082129717, -0.014277058653533459, -0.002897198311984539, -0.0005358249763958156, 0.001883875229395926, 0.018664643168449402, 0.01653352938592434, 0.03783072158694267, 0.0056342147290706635, 0.01931929774582386, 0.020962901413440704, -0.014667065814137459, 0.0018699464853852987, -0.015656013041734695, 0.007514608092606068, -0.018023915588855743, 0.00731960404664278, -0.017327474430203438, -0.01504314411431551, 0.010223766788840294, -0.02663193829357624, 0.0037642682436853647, -0.006417711731046438, -0.004091596230864525, 0.01238970085978508, -0.009555182419717312, 8.645673369755968e-05, -0.0024584399070590734, -0.0072012091986835, 0.0011439057998359203, 0.007256924174726009, -0.020753968507051468, -0.011825582943856716, 0.024500824511051178, -0.03930718079209328, -0.0002045798028120771, -0.002460181014612317, -0.010056620463728905, -0.005703859031200409, -0.004753215704113245, -0.01448599062860012, 0.02572656236588955, 0.0046417852863669395, -0.018971076235175133, -0.0020579856354743242, -0.02821982465684414, 0.0097501864656806, 0.015057072974741459, -0.005205903202295303, 0.010056620463728905, -0.009959118440747261, -0.003927932120859623, -0.004704464692622423, -0.009248748421669006, -0.0327606275677681, -0.010704311542212963, 0.02603299729526043, 0.017411047592759132, 0.022550787776708603, 0.005373049061745405, 0.07053562998771667, 0.0041925800032913685, 0.010968959890305996, 0.011191820725798607, 0.009576075710356236, 0.02930627390742302, -0.004098560661077499, 0.007800149265676737, -0.007041027303785086, -0.01887357421219349, 0.012320056557655334, -0.007828006520867348, 0.05153669789433479, -0.02710551768541336, 0.0038095370400696993, -0.011811654083430767, 0.016324598342180252, -0.035462819039821625, -0.11711366474628448, 0.003440422937273979, -0.004648749716579914, 0.015057072974741459, -0.005272064823657274, -0.007960330694913864, -0.01697925291955471, 0.008851775899529457, -0.015154574997723103, 0.025837993249297142, -0.02604692615568638, 0.00046052219113335013, 0.0006738075171597302, -0.000200227033928968, 0.025294769555330276, -0.008468733169138432, -0.015447081066668034, 0.003088719677180052, -0.025057978928089142, 0.018594998866319656, -0.017369261011481285, -0.01575351506471634, 0.00334988534450531, -0.0010028763208538294, -0.0035518535878509283, -0.009283570572733879, -0.028637690469622612, 0.015057072974741459, 0.017703551799058914, -0.011163963004946709, 0.014973500743508339, -0.014973500743508339, 0.011170927435159683, 0.004321421962231398, -0.0006472556851804256, -0.02323330193758011, -0.01128932274878025, -0.004056774079799652, 0.03354064002633095, -0.010335197672247887, -0.005703859031200409, 0.003896592417731881, 0.026715509593486786, -0.012577740475535393, 0.0008683759951964021, -0.013183644972741604, -0.019556088373064995, 0.015516725368797779, -0.009353214874863625, -0.020684324204921722, -0.007382283918559551, -0.013197573833167553, -0.020210742950439453, -0.0024114300031214952, -0.0018995451973751187, 0.0028380006551742554, -0.008566235192120075, 0.01855321228504181, -0.016436029225587845, 0.00982679519802332, -0.005853593815118074, -0.005320815835148096, -0.00872641708701849, 0.01994609460234642, 0.0019535194151103497, 0.003917485475540161, -0.026116570457816124, 0.003320286748930812, -0.003558818018063903, -0.02306615561246872, -0.008740345947444439, 0.014047232456505299, -0.005310369189828634, -0.026297645643353462, -0.02196577750146389, 0.003767750458791852, -0.03613140434026718, -0.005818771664053202, 0.004829824436455965, -0.018803929910063744, -0.01317668054252863, -0.020029667764902115, -0.009381071664392948, -0.027356237173080444, 0.017647836357355118, 0.0025071906857192516, 0.012292198836803436, -0.011581828817725182, 0.014116876758635044, -0.022007564082741737, 0.006553518120199442, 0.013058285228908062, 0.018260706216096878, -0.015419223345816135, -0.004140346776694059, 0.005783949978649616, -0.013155787251889706, -0.0057874321937561035, 0.0005279899924062192, -0.023497948423027992, -0.021311121061444283, 0.013120965100824833, -0.06574411690235138, 0.02589370869100094, 0.0047218757681548595, -0.010766991414129734, 0.003819983685389161, -0.001854276517406106, 0.00630976352840662, 0.0013319450663402677, -0.01417259220033884, 0.019361084327101707, -0.02054503560066223, 0.03936289623379707, 0.00024440756533294916, -0.0265065785497427, -0.027008015662431717, -0.007507643662393093, 0.043374400585889816, 0.02101861499249935, 0.03334563598036766, 0.011414682492613792, 0.00303822779096663, 0.010154122486710548, 0.012340949848294258, 0.008907491341233253, -0.031089166179299355, 0.004102042876183987, -0.012048444710671902, 0.016951395198702812, -0.012271305546164513, 0.009471609257161617, 0.008614986203610897, -0.0038373947609215975, -0.006361996755003929, 0.03791429474949837, 0.0005179786239750683, -0.005724752321839333, -0.00786282867193222, 0.02429189346730709, 0.0075285364873707294, 0.07109278440475464, -0.009708399884402752, -0.018720358610153198, 2.9734803320025094e-05, -0.01533565018326044, -0.0022495072335004807, 0.01572565734386444, -0.011992729268968105, -0.00014070302131585777, -0.0013023463543504477, -0.0002722652570810169, 0.015669941902160645, 0.007410141639411449, -0.0022408016957342625, -0.022244352847337723, -0.0002404900878900662, 0.004624374210834503, 0.002655184594914317, 0.028804836794734, -0.004593034274876118, -0.0005662943003699183, 0.03287205845117569, 0.0046835714019834995, 0.02367902360856533, -0.0013911426067352295, -0.0002929408510681242, -0.02615835703909397, -0.036214977502822876, 0.01668674685060978, -0.0009628309053368866, -0.01495957188308239, -0.026381218805909157, -0.031702034175395966, 0.013120965100824833, 0.027147304266691208, -0.01044662855565548, -0.01441634725779295, -0.01899893395602703, 0.007828006520867348, 0.006400300655514002, 0.014235272072255611, 0.006292352452874184, 0.019249653443694115, -0.01805177330970764, 0.030894162133336067, 0.021603627130389214, 0.0023800900671631098, -0.0005092731444165111, 0.014332774095237255, -0.013134893961250782, 0.015182432718575, -0.0072847818955779076, 0.018009986728429794, 0.018762143328785896, -0.001748069073073566, -0.02147826738655567, -0.00538001349195838, -0.003868734696879983, -0.0034978792537003756, 0.037496428936719894, 0.024431180208921432, -0.008294622413814068, 0.019862523302435875, 0.010014833882451057, 0.00245669879950583, -0.01082270685583353, 0.00592323811724782, -0.0036040868144482374, -0.04253866896033287, 0.0056342147290706635, 0.0015260782092809677, -0.0056481435894966125, 0.0046278564259409904, -0.005007416941225529, 0.0068947747349739075, -0.014848140999674797, 0.01838606595993042, -0.010871457867324352, -0.0049238442443311214, -0.028860552236437798, 0.027788031846284866, 0.01993216574192047, 0.010035727173089981, 0.019124293699860573, 0.006762450560927391, 0.023943671956658363, 0.018762143328785896, 0.03225918859243393, -0.0075424653477966785, 0.017411047592759132, 0.006010293494910002, 0.03211989998817444, -0.00988947506994009, -0.018636785447597504, -0.0024340644013136625, -0.030727015808224678, 0.021380765363574028, 0.01622709631919861, 0.01696532405912876, 0.0029006805270910263, 0.07833578437566757, 0.008928384631872177, -0.007577287498861551, 0.026896584779024124, -0.02646479196846485, 0.03345706686377525, 0.013531865552067757, 0.001810748828575015, 0.006577893625944853, -0.02913912758231163, 0.005620285868644714, 0.002345267916098237, 0.004652231931686401, -0.016296740621328354, -0.022160779684782028, 0.022717934101819992, -0.003903556615114212, 0.019444657489657402, -0.01145646907389164, -0.015488867647945881, 0.017090683802962303, -0.003917485475540161, 0.02540620043873787, 0.018887503072619438, -0.025810135528445244, 0.016491742804646492, 0.0343206562101841, -0.012027551420032978, -0.013894015923142433, -0.015307792462408543, 0.003955789841711521, -0.004283117596060038, -0.041229359805583954, -0.01592066138982773, -0.004544283263385296, -0.011240571737289429, 0.008566235192120075, -0.021102188155055046, 0.004767144564539194, 0.012717029079794884, -0.00919303297996521, 0.006626644637435675, 0.005035274662077427, -0.015586369670927525, 0.026436934247612953, -0.008621950633823872, -0.0007417105953209102, -0.038276445120573044, -0.0303927231580019]} +{"id": "test:10000031", "text": "\"I , , , , , 1 \u201d , , , :0 4 r , , Franka BrunsPost-Gazette CUPBOARD NEARLY BARE Jim Marton strolls through. and includes installation of turnaround lanes for large vehicles. WESTMORELAND COUNTY Man.\\nHe upholds his commitment to the health insurance industry through his understanding and fostering of relationships to support the business endeavors of top brokers across the country. ones you.\\nRestaurants in Westmoreland TN (Tennessee), Places To Eat in Westmoreland TN (Tennessee) Menus, Specials & Envents are available for almost all the Restaurants and Places To Eat.\\nPittsburgh Post-Gazette from Pittsburgh, Pennsylvania \u00b7 Page 23 Publication: Pittsburgh Post-Gazette i Location: Pittsburgh, Pennsylvania Issue Date: Sunday, March 22, 2015 Page: Page 23 Start Free.\\nHe was an avid golfer and lifelong member and past president of Highland Country Club in West View. David (Brenda) Moll, and Cathy Landry; sister of Theresa (Donald) Sheffler, and the late Jennie.\\nShe becomes one of four women on the 19-member board and one of seven women business deans of AACSB-accredited business schools in the country. Wayne Rollins. Hara-lenne, Cathy L. Hillyard, Lisa T.\\nSep FITTS, Mrs. Mildred Louise (Gran Carthage, Tn. Graveltown Community 2 p.m. tomorrow. Mrs. Creighton was a Member of the First Presbyterian Church, Belle Meade Country Club and the James K. Polk.\\nCathy\u2019s Country Cupboard, Westmoreland TN Opening hours. Cathy\u2019s Country Cupboard is listed as a local restaurant for the following areas. Westmoreland. Near places. More restaurants close to Westmoreland TN. Corner Cafe \u2013 Westmoreland. 5234 New Highway 31, Westmoreland, 37186.\\nDishwashers, they say, allow you to spend more time with the family andor guests; encourage even teenage boys tn stack away glasses instead of littering the kitchen with them; save space (you don\u2019t.\\nHillsboro, Ohio, and Joseph Shaw of Wilmington, Ohio, have been duly appointed Executors of tn* Estate of James Shaw. evening with Mr. and Mrs. Joe Stroup and son. Jean Duke and Cathy spent Friday.\\nWestmoreland Little League, Westmoreland, Tennessee. 1,192 likes \u00b7 19 talking about this. Little League of Westmoreland, Tennessee Offering. Tball.\\nCall (931) 296-4865 or hcchamberbellsouth.net (Neighboring Counties Entries Welcome) TN-0000856166 mm Preston. outstanding hardware), country walnut dovetailed child\u2019s chest of drawers, primitive.\\nBest Restaurants in Westmoreland, TN. Reviews, photos, and menus for places like: Cathy\u2019s Country Cupboard Inc, Laquesadilla Mexican Restaurant, Sonic Drive-In, Hardee\u2019s, Kate\u2019s Garage Bar & Grill, Motown Pizza, Subway Sandwiches, Taco Bell.\\nCompanies for \\\"country cupboard\\\" Town And Country Bancshares Inc. Salem, MO US. 522110 \u2013 Commercial Banking. Canoyer Country Greenhouse And Gift Shop. Griswold, IA US. 444220 \u2013 Nursery, Garden Center, and Farm Supply Stores.\\nROSEN High Ridge Country Club acknowledges with sorrow the passing of our. 2010. daughters Elissa and Cathy. Donations should be sent in memory of Milred Blitz to Boys Town Jerusalem Foundation of.\\nThe Tennessean from Nashville, Tennessee \u00b7 Page 6 Publication: The Tennessean i Location: Nashville, Tennessee Issue Date: Wednesday, November 26, 1997 Page: Page 6 Start Free Trial \u00d7 Cancel Show Hide.\\nHonor attendants were Cathy Sounder, sister of the bride. is a programmer analyst at the Westmoreland Intermediate Unit. Following a honeymoon in Myrtle Beach, S.C., the couple are residing : in.\\nShown are Nicole Harris, Jalyn Westmoreland, Hamby, Ellie Campbell and, in front, Georgie Creswell. (Photo: Carol Z. Shane/Shopper News) Patrons can choose one-time-only services or annual memberships.\\n\u201cThat is his job. He\u2019s the head law enforcement officer in our country, and he wanted to make sure that our laws are being obeyed.\u201d She added, \u201cHe needs to make sure that our citizens are protected in.\\nNearby schools include Westmoreland Elementary School, Westmoreland High School and Westmoreland Middle School. The closest grocery stores are ATM (BP Family Market), Mike\u2019s Foodland and Houchens IGA. Nearby restaurants include Pep A Roni\u2019s, Cathy\u2019s Country Cupboard.\\nArrangements by WOODLAWN FUNERAL HOME, 383-4754 Gallatin, TN MARTIN, Allen Palmer House- Age 95. a former member of Brentwood Country Club and Cedar Creek Boat Club. He served in WWII 1943-1945 in.\\nFast Food near Westmoreland, TN 37186. Fast Food \u00b7 $ $ $ \u00b7 open \u00b7 2 on Yelp. 5236 Highway 31 E \u00b7 (615) 644-2355. Cathy\u2019s Country Cupboard.\\nHe was proud of his service to his country and always enjoyed sharing his memories of that time. Upon an Honorable Discharge from the Navy, Donnie settled in Indiana and married Cathy Kielman.\\nOnita\u2019s Country Kitchen. Sitoam Missionary Baptist Church; Westmoreland. Alexander Funeral Home in charge. Survivors: sons, Billy, Murfreesboro, and Jerry Thompson, Nashville; daughters, Elaine.\\nResults for Restaurant in Westmoreland, TN. Get free custom quotes, customer reviews, prices, contact details, opening hours from Westmoreland, TN based businesses with Restaurant keyword.\"", "vector_field": [0.010736041702330112, 0.006669979076832533, -0.0022102613002061844, -0.009549965150654316, -0.0082071078941226, 0.020913127809762955, 0.010333866812288761, -0.002736838534474373, -0.0066597540862858295, -0.020354172214865685, 0.0010719000129029155, 0.03738187626004219, -0.01919536106288433, -0.027484267950057983, -0.003551414469256997, 0.02106309123337269, 0.018104715272784233, 0.022344600409269333, -0.004052429925650358, -0.04681595787405968, -0.0036263964138925076, 0.0008610135409981012, -0.0031799133867025375, 0.003156055463477969, -0.0007140319794416428, 0.023966936394572258, 0.017354896292090416, -0.00982262659817934, -0.010074838064610958, 0.027593331411480904, -0.010027122683823109, -0.008547934703528881, 0.005514576565474272, -0.01674140803515911, -0.0035582310520112514, -0.013530821539461613, 0.010108920745551586, -0.022671794518828392, 0.038554318249225616, -0.02320348285138607, 0.010402032174170017, -0.0016231873305514455, -0.0023039886727929115, 0.00089296605437994, -0.02498941496014595, 0.016455113887786865, 0.014805513434112072, 0.0033043150324374437, -0.0010804206831380725, 0.019672518596053123, 0.028738509863615036, 0.022344600409269333, -0.00688810832798481, 0.006410950794816017, -0.0008277829037979245, -0.0031884340569376945, 0.022126471623778343, 0.045343589037656784, -0.022685427218675613, -0.006571139208972454, -0.002847607247531414, 0.005207832437008619, -0.016441481187939644, 0.010865556076169014, -0.004972661845386028, -0.03849978744983673, -0.019727051258087158, 0.006247353740036488, -0.023912403732538223, 0.01905903033912182, -0.0007591915200464427, 0.03659115731716156, -0.00802987813949585, -0.0036263964138925076, 0.024812186136841774, -0.009386368095874786, 0.0012295324122533202, 0.019236261025071144, -0.02497578226029873, -0.010524729266762733, 0.013087746687233448, -0.030047284439206123, -0.005797462537884712, 0.019808849319815636, 0.034082673490047455, 0.00250848475843668, -0.0015754716005176306, 0.022112838923931122, -0.015555331483483315, -0.013796665705740452, 0.005221465602517128, 0.02500304952263832, -0.021567516028881073, -0.018704570829868317, -0.002842494985088706, 0.004192168824374676, -0.026720816269516945, 0.04831559583544731, 0.004778390750288963, -0.03735461086034775, -0.02107672579586506, 0.019222628325223923, 0.01784568838775158, -0.014573751017451286, -0.005163524765521288, -0.009515882469713688, -0.005715664010494947, -0.013292242772877216, 0.02284902334213257, -0.009938507340848446, -0.01807744987308979, 0.001084681018255651, 0.011179116554558277, 0.011424511671066284, -0.011615375056862831, -0.009400000795722008, 0.021267587319016457, -0.02329891547560692, -0.02096766047179699, -0.03558231145143509, -0.004590936005115509, 0.018704570829868317, 0.020136043429374695, 0.006867658346891403, 0.024035101756453514, 0.023489778861403465, -0.02054503560066223, -0.006867658346891403, 0.0029055478516966105, -0.0010863852221518755, -0.010633793659508228, 0.0068778833374381065, 0.021731112152338028, 0.004297825042158365, -0.007280058693140745, 0.014682815410196781, -0.028738509863615036, 0.011076868511736393, -0.026502685621380806, -0.026693549007177353, 0.015500799752771854, 0.03435533493757248, 0.017327630892395973, -0.016100654378533363, 0.001395685481838882, 0.03370094671845436, 0.008554751053452492, 0.02309441938996315, -0.012631038203835487, 0.005558883771300316, -0.004345540888607502, -0.028929373249411583, 0.0064382171258330345, -0.01416475884616375, 0.013210443779826164, 0.0118198711425066, -0.004427339415997267, 0.0027249096892774105, -0.011383612640202045, -0.00023900474479887635, -0.005742930341511965, 0.008173025213181973, 0.004246701020747423, 0.0032583035062998533, 0.006966498214751482, 0.03176505118608475, 0.009229587391018867, 0.031846851110458374, 0.004584119655191898, 0.006796084810048342, 0.027484267950057983, 0.01076330803334713, -0.03784539923071861, 0.026039162650704384, -0.012835534289479256, 0.01220841333270073, -0.0009108594385907054, 0.017382163554430008, -0.02028600685298443, -0.0066836122423410416, -0.015705294907093048, 0.02051776833832264, 0.014764613471925259, 0.03640029579401016, -0.0026806022506207228, -0.03359188139438629, 0.03473706170916557, 0.021376652643084526, 0.00719144381582737, -0.007941262796521187, -0.006339377257972956, 0.026243658736348152, -0.004052429925650358, -0.008043510839343071, -0.6439171433448792, 0.011417695321142673, -0.012733286246657372, -0.019822482019662857, -0.005862219724804163, 0.0211039911955595, 0.014205657877027988, 0.009229587391018867, -0.032746631652116776, 0.000686765881255269, -0.0048261065967381, 0.004066063091158867, -0.00196657027117908, -0.011008703149855137, -0.00881377886980772, -0.028629444539546967, -0.0001125793278333731, -0.0049079046584665775, 0.016823207959532738, 0.012358376756310463, -0.005105584394186735, 0.03724554553627968, -0.01629151776432991, -0.004393256735056639, 0.028165921568870544, -0.001132396748289466, 0.021553881466388702, 0.00010938407649518922, -0.004618202336132526, -0.005787238012999296, -0.03266483545303345, 0.0221946369856596, 0.004843147937208414, -0.0034934738650918007, 0.049488041549921036, 0.014028428122401237, -0.03091980144381523, 0.030510809272527695, 0.03672748804092407, 0.02922930009663105, -0.006864250171929598, -0.03146512433886528, 0.0038581585977226496, -0.011724439449608326, -0.011540393345057964, 0.014069327153265476, -0.00043071978143416345, -0.010851923376321793, 0.006840392481535673, -0.02598462998867035, 0.015119073912501335, 0.0026499277446419, -0.0096522131934762, -0.028056856244802475, -0.006145105697214603, -0.00339122605510056, 0.013033214025199413, -0.011731255799531937, 0.0005777013138867915, 0.0016973171150311828, 0.005245323292911053, 0.029829155653715134, 0.014969109557569027, -0.00802987813949585, -0.034409865736961365, 0.0017808196134865284, -0.0028288618195801973, -0.014710081741213799, 0.0189499668776989, 0.018241045996546745, -0.01674140803515911, 0.02564380317926407, -0.019372591748833656, -0.0026635609101504087, 0.02966555766761303, -0.00016263825818896294, 0.027470633387565613, -0.021240321919322014, -0.006410950794816017, 0.008213924244046211, 0.02006787806749344, -0.005391879007220268, -0.02174474485218525, -0.0026379988994449377, 0.02966555766761303, 0.0015422409633174539, 0.004976070020347834, 0.011853953823447227, 0.022126471623778343, 0.004475054796785116, 0.009338652715086937, -0.023367080837488174, 0.01638694852590561, -0.01796838454902172, -0.010579261928796768, 0.01221522968262434, -0.012999131344258785, 0.005245323292911053, 0.010436114855110645, -0.029720090329647064, -0.006574547383934259, -0.024062367156147957, 0.004972661845386028, 0.018159247934818268, 0.011519943363964558, 0.0017671865643933415, -0.00510217621922493, -0.012958232313394547, 0.039890360087156296, -0.04283510148525238, -7.689902849961072e-05, -0.015923425555229187, -0.010074838064610958, -0.005552067421376705, -0.023721540346741676, -0.027266139164566994, 0.04188079014420509, -0.011799421161413193, -0.001116207567974925, -0.007389123551547527, 0.0023602251894772053, -0.006796084810048342, 0.003365664044395089, -0.012971865013241768, 0.0030844819266349077, 0.01582799293100834, 0.0211994219571352, -0.004601160995662212, 0.0014246556675061584, 0.006577956024557352, -0.010511096566915512, 0.0012184556107968092, 0.045343589037656784, -0.01839100942015648, 0.002825453644618392, 0.009938507340848446, 0.022480931133031845, -0.005684989970177412, 0.020681366324424744, -0.02944742888212204, -1.937120578077156e-05, -0.010347499512135983, 0.012003917247056961, -0.03021088056266308, -0.011567658744752407, -0.023789705708622932, -0.005010152701288462, -0.00544300302863121, -0.04820653423666954, 0.014982743188738823, -0.002511892933398485, -0.008513852022588253, 0.00437962356954813, -0.0037252360489219427, -0.003813851159065962, -0.006366643123328686, -0.0021932199597358704, 0.01630515046417713, -0.0012218637857586145, -0.009577231481671333, -0.019113563001155853, 0.016359683126211166, -0.025493839755654335, -0.010006672702729702, 0.00671769492328167, 0.009781727567315102, 0.02655721828341484, -0.015419000759720802, 0.022467298433184624, -0.018131982535123825, 0.003663887269794941, -0.016209719702601433, -0.0015695070615038276, 0.007586802821606398, 0.008370704017579556, 0.010681509971618652, 0.00011194028047611937, 0.008479769341647625, -0.002000652952119708, 0.0038683833554387093, 0.013196811079978943, 0.004809065256267786, -0.013414939865469933, -0.010395214892923832, 0.014587383717298508, -0.014737348072230816, -0.00485337246209383, 0.00571907265111804, -0.03672748804092407, 0.022903556004166603, -0.008554751053452492, -0.01344902254641056, -0.0023738581221550703, 0.0321195125579834, -0.006618855055421591, -0.0012499820441007614, 0.003418492153286934, 0.020885862410068512, 0.0017569618066772819, -0.002220486057922244, -0.00046650657895952463, 0.007804932072758675, -0.004171719308942556, -0.020340539515018463, 0.01885453425347805, -0.009038724936544895, 0.019440757110714912, -0.03351008519530296, 0.035555046051740646, -0.019140828400850296, 0.013374040834605694, -0.01627788506448269, -0.006956273689866066, 0.007211893796920776, -0.007443655747920275, 0.025684703141450882, -0.00875924713909626, 0.01510544028133154, 0.0028288618195801973, 0.01556896511465311, 0.014587383717298508, 0.010395214892923832, 0.006247353740036488, 0.01730036549270153, -0.012842351570725441, -0.005466860719025135, 0.020245106890797615, 0.0007885878439992666, -0.01472371444106102, 0.002779441885650158, -0.0007003989303484559, 0.022589996457099915, 0.01649601384997368, 0.04196258634328842, 0.01638694852590561, -0.025521105155348778, 0.01685047335922718, -0.009638579562306404, 0.02286265604197979, 2.2819414880359545e-05, 0.01238564308732748, 0.02955649420619011, 0.021363019943237305, 0.002953263698145747, -0.017354896292090416, -0.01798201911151409, -0.003803626401349902, 0.001644489006139338, -0.02567106857895851, -0.026966210454702377, -0.0063973176293075085, 0.020463237538933754, -0.010006672702729702, 0.011853953823447227, 0.01544626709073782, 0.014178391546010971, -0.02174474485218525, -0.03138332441449165, 0.041335467249155045, 0.026379989460110664, 0.005674764979630709, -0.016141554340720177, 0.03138332441449165, -0.00635641859844327, -0.018786368891596794, 0.0009798768442124128, 0.0038547501899302006, -0.026952577754855156, -0.008288905955851078, -0.013040030375123024, 0.001367567223496735, -0.013721683993935585, 0.011697173118591309, -0.018827268853783607, -0.006881291512399912, -0.004536403808742762, -0.022262802347540855, 0.0002204722841270268, 0.01020435243844986, 0.012685570865869522, -0.034191738814115524, -0.01761392503976822, 0.015132706612348557, 0.027416102588176727, -0.012985498644411564, 0.0029396305326372385, -0.012753736227750778, 0.008847861550748348, -0.004008122254163027, 0.008050327189266682, -0.004693184047937393, 0.011908485554158688, -0.006076940800994635, 0.008397970348596573, -0.005402103532105684, -0.006721103098243475, 0.01661871187388897, -0.013987529091536999, -0.004584119655191898, 0.022003773599863052, 0.010858739726245403, 0.003195250639691949, 0.0015294599579647183, 0.001076160348020494, 0.025848299264907837, 0.0019614577759057283, -0.0068438006564974785, -0.011765338480472565, 0.0009474983089603484, -0.02767512947320938, 0.008513852022588253, 0.0025630169548094273, -0.0028697610832750797, 0.02020420879125595, 0.000518482644110918, -0.0066665709018707275, 0.0003348622703924775, -0.01361261960119009, 0.011090502142906189, 0.019127195701003075, -0.010374765843153, -0.006366643123328686, -0.04141726344823837, 0.013251342810690403, 0.12531517446041107, 0.0032395580783486366, -0.01582799293100834, 0.015214505605399609, -0.006669979076832533, -0.0035105152055621147, -0.021799277514219284, -0.026025528088212013, 0.005569108761847019, -0.0016649385215714574, -0.0055486592464149, -0.002932813949882984, 0.0009279007790610194, 3.024837133125402e-05, 0.03315562382340431, -0.013476288877427578, -0.017382163554430008, -0.019767949357628822, 0.024130532518029213, -0.013387673534452915, -0.004035388585180044, -0.018800001591444016, -0.01674140803515911, 0.015269037336111069, 0.001373531762510538, 0.011540393345057964, 0.02455315738916397, 0.026461787521839142, 0.009372735396027565, -0.04081740975379944, -0.006925599183887243, -0.003121972782537341, 0.008841045200824738, 0.0005180566222406924, 0.0032923861872404814, -0.005695214495062828, 0.014314722269773483, 0.006816534791141748, 0.022794490680098534, -0.010354315862059593, 0.03132879361510277, 0.013571720570325851, -0.0007302212761715055, -0.032528504729270935, 0.02755243331193924, -0.025166645646095276, 0.021690212190151215, 6.406477768905461e-05, 0.0014280639588832855, -0.022112838923931122, 0.04474373161792755, 0.012569690123200417, 0.022589996457099915, -0.01663234457373619, -0.0031969547271728516, -0.014273823238909245, -0.002039847895503044, -0.009924874641001225, -0.020054245367646217, 0.007014214061200619, 0.011185932904481888, -0.00841842032968998, 0.00021578592713922262, -0.012256128713488579, -0.0037627271376550198, -0.00474089989438653, -0.015487166121602058, 0.013367224484682083, -0.014355622231960297, -0.0010062908986583352, -0.0020551851484924555, -0.022126471623778343, -0.014110226184129715, -0.007709500379860401, 0.027947790920734406, 0.005729297176003456, 0.00384793384000659, 0.010790574364364147, -0.0070619299076497555, 0.011465410701930523, 0.001316443202085793, 0.012133431620895863, -0.0035105152055621147, -0.023912403732538223, -0.010122553445398808, 0.019249893724918365, 0.019127195701003075, 0.005119217559695244, -0.024621322751045227, 0.0034627995919436216, 0.008213924244046211, -0.008057143539190292, 0.005265772808343172, -0.009761277586221695, 0.004175127483904362, 0.006533648353070021, 0.006243945565074682, 0.006083757150918245, 0.012187963351607323, 0.0055622924119234085, 0.0368092879652977, 0.012099348939955235, -0.013960262760519981, -0.009693112224340439, 0.010858739726245403, 0.0013087746920064092, -0.006005366798490286, -0.017382163554430008, -0.007791298907250166, -0.002619253471493721, 0.0007574874325655401, -0.012358376756310463, 0.008507034741342068, -0.006124656181782484, -0.03547324612736702, 0.02642088755965233, -0.003389521734789014, 0.01025888416916132, -0.007082379423081875, -0.02207193896174431, 0.00202280655503273, -0.013905730098485947, 0.02432139590382576, 0.0002920459082815796, -0.02432139590382576, 0.03939956799149513, -0.0058008707128465176, -0.024703120812773705, -0.012351560406386852, 0.0051498920656740665, -0.013483105227351189, 0.025589270517230034, -0.008493402041494846, -0.01873183622956276, 0.007048296742141247, 0.0003823649894911796, 0.014042060822248459, 0.02511211298406124, -0.006247353740036488, -0.0017705948557704687, -0.027279771864414215, 0.00017073289200197905, -0.004481871612370014, 0.015078174881637096, -0.014764613471925259, -0.03413720428943634, -0.009461349807679653, 0.0008252267143689096, 0.03255577012896538, 0.03550051152706146, 0.02020420879125595, 0.004454605281352997, -0.021785644814372063, 0.025834666565060616, -0.011424511671066284, -0.019263526424765587, -0.023585209622979164, 0.010504280216991901, 0.008452503010630608, 0.004556853324174881, 0.03713648021221161, 0.013864831067621708, 0.00792081281542778, -0.013871647417545319, -0.003578680567443371, 0.0010054388549178839, -0.019249893724918365, -0.012399276718497276, -0.009202321991324425, -0.010511096566915512, 0.03326468914747238, 0.026393622159957886, 0.0012644671369343996, 0.013012764044106007, 0.0009083032491616905, 0.01905903033912182, -0.0037559105549007654, -0.012828717939555645, -0.006479116156697273, -0.01170398946851492, 0.0011477340012788773, -0.001576323644258082, -0.019713418558239937, 0.0016649385215714574, -0.017477594316005707, -0.016127921640872955, 0.01215388160198927, 0.0011605150066316128, 0.012481074780225754, -0.004372806753963232, 0.02722523920238018, -0.03828165680170059, 0.02310805208981037, 0.004590936005115509, -0.0002020463434746489, -0.010306600481271744, -0.01863640546798706, -0.03091980144381523, -0.008582017384469509, -0.004607977345585823, -0.02073589898645878, -0.002237527398392558, -0.007430022582411766, -0.016155187040567398, -0.013973895460367203, -0.00029886243282817304, 0.002876577666029334, -0.03220130875706673, 0.007436839397996664, -0.001454478013329208, -0.022453665733337402, -0.018118347972631454, -0.02107672579586506, -0.005541842430830002, 0.004113778471946716, -0.013892097398638725, -0.018813636153936386, 0.01905903033912182, -0.010790574364364147, -0.02644815482199192, -0.002087563741952181, -0.010361132211983204, 0.024839451536536217, 0.0019222627161070704, 0.03323742374777794, -0.015814360231161118, -0.010177086107432842, -0.009577231481671333, 0.023939669132232666, 0.003929732367396355, -0.003355439053848386, 0.0009372734930366278, 0.004444380756467581, 0.007879913784563541, -0.005368020851165056, -0.003810442751273513, 0.0014118747785687447, 0.0023636333644390106, -0.03580043837428093, 0.00468295905739069, 0.017368530854582787, 0.0006986947846598923, -0.03310109302401543, 0.009038724936544895, 0.003810442751273513, -0.010783758014440536, -0.0003418918058741838, -0.028302252292633057, -0.01366033498197794, -0.029038436710834503, -0.02578013390302658, 0.023067152127623558, -0.03454619646072388, 0.01762755773961544, -0.002757288282737136, -0.03847252205014229, -0.018486442044377327, -0.024934884160757065, 0.0113222636282444, 0.0010131074814125896, 0.007239159662276506, 0.031301528215408325, -0.011506310664117336, 0.023694274947047234, 0.005221465602517128, -0.014996375888586044, -0.016864106059074402, -0.0034968822728842497, 0.006295069586485624, 0.022903556004166603, 0.009802176617085934, -0.004011530429124832, -9.98515824903734e-05, 0.00357186421751976, 0.031846851110458374, -0.0032583035062998533, 0.004246701020747423, -0.010333866812288761, -0.016577811911702156, 0.00020130079064983875, 0.008881944231688976, 0.010279334150254726, -0.011172300204634666, 0.0009091552929021418, -0.0024317987263202667, -0.016986804082989693, -0.027307037264108658, -0.008288905955851078, 0.03803626447916031, -0.02443045936524868, -0.014860045164823532, -0.021567516028881073, -0.0023482961114495993, -0.033537350594997406, 0.0006096538272686303, -0.017232200130820274, -0.0053271218203008175, 0.03405540809035301, -0.02764786407351494, 0.01905903033912182, -0.012297028675675392, -0.010081654414534569, -0.021840177476406097, 0.0011656273854896426, 0.005834953393787146, -0.02053140290081501, 0.03601856902241707, -0.01322407741099596, -0.009400000795722008, -0.040435682982206345, 0.01697317138314247, -0.0014536259695887566, 0.0179547518491745, 0.006680204067379236, 0.03858158737421036, 0.01574619486927986, -0.014028428122401237, -0.0008554750820621848, -0.017695723101496696, -0.010824657045304775, -0.021008560433983803, -0.013885281048715115, -0.004723858553916216, 0.01941348984837532, 0.011042785830795765, 0.00580427935346961, -0.015596231445670128, 0.0226445272564888, -0.0025544962845742702, -0.0027777377981692553, 0.009079623967409134, 0.022344600409269333, -0.0051158093847334385, -0.007245976477861404, -0.01775025576353073, -0.007157361134886742, -0.004328499548137188, 0.03416446968913078, -0.011860770173370838, -0.014314722269773483, 0.004158086143434048, -0.0030640321783721447, 0.02188107557594776, 0.029501961544156075, 0.0032872736919671297, -8.110611088341102e-05, 0.005170341581106186, -0.006274620071053505, -0.00034316990058869123, -0.009372735396027565, 0.01127454824745655, 0.008411603979766369, 0.013033214025199413, -0.01907266303896904, -0.0015507616335526109, 0.0175048615783453, -0.029283832758665085, -0.032174043357372284, -0.008718348108232021, 0.026039162650704384, 0.020994925871491432, 0.008841045200824738, 0.011015519499778748, 0.013646702282130718, -0.009263670071959496, 0.02999275177717209, -0.01400116179138422, -0.013803482055664062, 0.019372591748833656, -0.004393256735056639, 0.017259465530514717, 0.018813636153936386, -0.026148226112127304, 0.00585199473425746, 0.0049147214740514755, 0.020572301000356674, 0.0002926849410869181, -0.0022613853216171265, -0.02411689981818199, -0.03089253418147564, -0.00858883373439312, -0.02318985015153885, -0.0012184556107968092, -0.005429369863122702, 0.018827268853783607, -0.01327179279178381, -0.007293691858649254, 0.0050919512286782265, 0.014819146133959293, 0.003895649453625083, -0.02199014090001583, 0.005811095703393221, 0.007423206232488155, 0.016550546512007713, 0.015691662207245827, -0.02297172136604786, 0.03948136791586876, -0.002868056995794177, 0.002694235183298588, 0.027620598673820496, -0.00053680210839957, 0.02020420879125595, 0.023953301832079887, -0.020885862410068512, -0.002285243244841695, -0.029365630820393562, -0.02933836542069912, -0.011001886799931526, -0.01806381717324257, -0.012515157461166382, 0.02051776833832264, -0.001256798510439694, 0.005865627899765968, -0.002491443417966366, -0.010388398543000221, -0.010845106095075607, 0.019563455134630203, -0.015459900721907616, 0.006755185779184103, -0.012535607442259789, -0.03310109302401543, -0.0026005080435425043, -0.018990864977240562, -0.005313488654792309, 0.009372735396027565, -0.03244670480489731, -0.019140828400850296, 0.012337927706539631, -0.014519218355417252, 0.0012610588455572724, -0.01183350384235382, -0.031192462891340256, 0.001305366400629282, -0.0023397754412144423, -0.0034679118543863297, 0.006383684463799, 0.18704570829868317, -0.008943293243646622, 0.0018046775367110968, -0.0005137962871231139, -0.013810299336910248, 0.031628720462322235, 0.027484267950057983, 0.025943730026483536, 0.007654968183487654, 0.016223352402448654, -0.027307037264108658, 0.00987034197896719, 0.011983468197286129, 0.001703281537629664, -0.0031969547271728516, -0.02542567439377308, -0.04106280580163002, -0.021826542913913727, -0.014396521262824535, -0.007021030411124229, -0.0009134156280197203, -0.012781002558767796, 0.004219434689730406, 0.0006854877574369311, 0.04327136278152466, -0.012283395044505596, -0.0042330678552389145, 0.007280058693140745, 0.013339958153665066, 0.013932996429502964, -0.020476870238780975, 0.0028407908976078033, 0.024185065180063248, -0.015487166121602058, 0.00948179978877306, 0.018131982535123825, 0.0037081947084516287, 0.012815085239708424, 0.014737348072230816, 0.004584119655191898, -0.00026222356245853007, -0.015487166121602058, 8.659128798171878e-05, -0.0055554755963385105, -0.000733203487470746, 0.029856421053409576, 0.0065915887244045734, -0.0016513054724782705, -0.012106165289878845, 0.001028444617986679, -0.033755477517843246, -0.00755272014066577, 0.01618245244026184, 0.00947498343884945, -0.0006079496815800667, -0.011451778002083302, -0.002341479528695345, -0.013769399374723434, -0.013810299336910248, 0.003946773707866669, -0.016591444611549377, 0.02621639147400856, -0.0023312547709792852, 0.015378101728856564, -0.04231704771518707, 0.03146512433886528, 0.0178593210875988, -0.00114432570990175, 0.011458594352006912, -0.023380713537335396, 0.02017694152891636, -0.0027606964576989412, 0.0066256714053452015, -0.007280058693140745, -0.0036434377543628216, -0.013987529091536999, 0.02253546379506588, 0.027034375816583633, 0.017872953787446022, 0.03468252718448639, -0.023694274947047234, -0.003800217993557453, 0.0026806022506207228, -0.011124584823846817, -0.0044682384468615055, -0.03421900421380997, 0.01004075538367033, -0.00672451127320528, -0.0040558381006121635, -0.03190138190984726, 0.00432168273255229, 0.01539173536002636, -0.004120595287531614, -0.004822698421776295, 0.020054245367646217, 0.003091298509389162, 0.0033605515491217375, 0.012678754515945911, -0.011731255799531937, -0.0014135788660496473, -0.013742133975028992, 0.018881801515817642, -0.010061205364763737, -0.0009952140972018242, -0.024907616898417473, -0.026693549007177353, -0.008704714477062225, 0.036209430545568466, 0.006346193607896566, -0.011785788461565971, -0.019781583920121193, -0.025166645646095276, -0.021935608237981796, -0.0014451052993535995, 0.015282670967280865, 0.018909066915512085, 0.011472227983176708, -0.009897608309984207, -0.002358520869165659, 0.00875924713909626, 0.014437420293688774, -0.007634518668055534, 0.0014331763377413154, 0.015691662207245827, -0.004822698421776295, -0.024048734456300735, -0.05458680912852287, -0.008963743224740028, 0.01432835590094328, -0.03839072212576866, 0.006571139208972454, -0.017109502106904984, -0.00619282154366374, 0.004042204935103655, -0.01293096598237753, 0.004774982575327158, -0.0007464105146937072, -0.009747644886374474, -0.029365630820393562, -0.009543148800730705, 0.025357509031891823, -0.0035752723924815655, 0.012337927706539631, -0.030183615162968636, -0.0001257863623322919, -0.018690938130021095, -0.008745613507926464, -0.00755272014066577, -0.03132879361510277, -0.0208449624478817, 0.002240935806185007, 0.017600292339920998, 0.002723205601796508, -0.006417767144739628, 0.003517331788316369, -0.025575637817382812, -0.022726327180862427, -0.04027208685874939, -0.008534301072359085, 0.022221902385354042, -0.040790144354104996, -0.0074845547787845135, 0.046461500227451324, -0.0007131799357011914, -0.013973895460367203, -0.0057327053509652615, -0.17450328171253204, 0.024457726627588272, -0.008329804986715317, -0.022044673562049866, 0.030974332243204117, 0.008970559574663639, 0.009420450776815414, 0.023176217451691628, -0.014955476857721806, 0.0017194708343595266, 0.03356461599469185, -0.0034474623389542103, 0.0004417966410983354, -0.01685047335922718, 0.007948079146444798, 0.0006556654116138816, -0.03154692053794861, 0.015119073912501335, 0.0096522131934762, 0.005742930341511965, 0.033864542841911316, -0.028029590845108032, 0.026066428050398827, -0.004839739762246609, -0.01772299036383629, 0.027729662135243416, -0.016087021678686142, 0.013796665705740452, -0.0026056203059852123, 0.0006437365082092583, -0.012290211394429207, -0.0240896325558424, -0.004890863783657551, -0.008173025213181973, -0.007232343312352896, -0.008275273256003857, -0.0197543166577816, -0.028874840587377548, -0.027947790920734406, 0.014764613471925259, -0.015773460268974304, 0.01873183622956276, -0.0178593210875988, 0.0061621470376849174, -0.022494563832879066, 0.019699783995747566, 0.024784918874502182, -0.006359826773405075, -0.008350254967808723, -0.0013777920976281166, 0.0032702323514968157, -0.020572301000356674, 0.033646415919065475, 0.032174043357372284, 0.011192750185728073, -0.004042204935103655, -0.007252792827785015, 0.023571576923131943, 0.0006522571784444153, 0.028193186968564987, -0.0221946369856596, -0.009202321991324425, 0.003715011291205883, 0.025970997288823128, 0.009911241009831429, -0.04395301640033722, -0.01630515046417713, -0.028138654306530952, -0.015146340243518353, 0.0027760337106883526, -0.0073073250241577625, 0.010354315862059593, -0.004846556112170219, -0.019263526424765587, -0.0025544962845742702, 0.0055622924119234085, -0.026598118245601654, 0.0032361496705561876, 0.020572301000356674, -0.00034253086778335273, -0.016359683126211166, 0.014042060822248459, -0.0506877526640892, -0.020681366324424744, -0.010054388083517551, 0.008213924244046211, 0.0008188362116925418, -0.00309641077183187, 0.029829155653715134, 0.0026482236571609974, 0.012017550878226757, -0.02689804509282112, -0.017763888463377953, -0.029392896220088005, 0.00038939452497288585, -0.009856709279119968, -0.0009969181846827269, 0.02609369345009327, -0.005524801090359688, -0.025916464626789093, 0.017777523025870323, -0.0018711386946961284, -0.03345555067062378, 0.006775635294616222, 0.022589996457099915, 0.009509066119790077, 0.008070777170360088, 0.047143153846263885, 0.013387673534452915, -0.026584485545754433, 0.018445542082190514, 0.016441481187939644, 0.011479044333100319, 0.0003830040222965181, -0.0023636333644390106, 0.009802176617085934, 0.01817288063466549, -0.034982454031705856, 0.03165598586201668, 0.017545759677886963, 0.04158085957169533, -0.005020377691835165, 0.006762002129107714, -0.007102828938513994, -0.00787309743463993, 0.008466135710477829, -0.08381611108779907, -0.003297498682513833, 0.02988368831574917, 0.022617261856794357, -0.0268844123929739, 0.004403481259942055, -0.022944455966353416, 0.0074845547787845135, -0.017123134806752205, 0.036318495869636536, -0.00527258962392807, -0.01863640546798706, -0.0165096465498209, 0.011349529959261417, 0.017545759677886963, -0.017709357663989067, 0.00521464878693223, -0.00512944208458066, -0.020681366324424744, 0.01840464398264885, -0.0002609454677440226, -0.02263089455664158, -0.02031327225267887, -0.02343524619936943, 0.00841842032968998, -0.004918129649013281, -0.009638579562306404, 0.006216679699718952, 0.019018132239580154, 0.0015405367594212294, -0.01490094419568777, 0.004856781102716923, 0.008125308901071548, -0.01997244544327259, 0.01042248122394085, -0.0066495295614004135, -0.03506425395607948, -0.0025374549441039562, 0.024580422788858414, -0.0033861135598272085, -0.02742973528802395, -0.011356346309185028, 0.028356783092021942, -0.030047284439206123, 0.0105997109785676, 0.006980131380259991, -0.04744308069348335, 0.04834286496043205, 0.007620885502547026, -0.013155912049114704, 0.020340539515018463, -0.01222204603254795, -0.021444818004965782, -0.01607338897883892, 0.011131401173770428, -0.0030640321783721447, 0.00786628108471632, -0.004539811983704567, -0.02040870487689972, -0.024825818836688995, -0.009911241009831429, 0.0098771583288908, 0.006520015187561512, 0.03582770749926567, 0.026066428050398827, 0.04005395621061325, -0.019904280081391335, -0.008173025213181973, 0.010804207064211369, -0.0013888688990846276, 0.011117767542600632, -0.013312691822648048, -0.02177201211452484, 0.02028600685298443, -0.03266483545303345, -0.0015430930070579052, -0.03209224343299866, 0.0003663887328002602, 0.021117623895406723, -0.020599568262696266, -0.00719144381582737, -0.01998608000576496, 0.003234445583075285, -0.007457288913428783, 0.0004008974356111139, -0.0031543513759970665, -0.0018404643051326275, 0.0074913715943694115, 0.003684337018057704, -0.050715018063783646, 0.02376244030892849, 0.05352342873811722, 0.017654825001955032, -0.020435970276594162, -0.004822698421776295, 0.03247397020459175, 0.017941119149327278, 0.007034663576632738, -0.011240465566515923, -0.03187411651015282, -0.03427353501319885, -0.01289006695151329, -0.082998126745224, 0.01764119230210781, -0.0008022208930924535, -0.006410950794816017, -0.005695214495062828, -0.028493113815784454, -0.01695953868329525, -0.003602538490667939, -0.0044511971063911915, -0.00047119296505115926, -0.039099641144275665, 0.034191738814115524, 0.010292966850101948, 0.009795360267162323, -0.02712980844080448, 0.013810299336910248, 0.0006433104281313717, -0.010068021714687347, 0.0013692714273929596, 0.03634576126933098, 0.010279334150254726, 0.026570850983262062, -0.003766135312616825, 0.01272646989673376, 0.024498624727129936, 0.02433502860367298, -0.016891373321413994, 0.02042233757674694, -0.013653518632054329, -0.011431328020989895, 0.05278724431991577, -0.0193998571485281, -0.008868311531841755, 0.04029935225844383, 0.012106165289878845, -0.01038158219307661, -0.027920525521039963, 0.00256472104229033, 0.013851198367774487, 0.01708223484456539, -0.05145120248198509, -0.01166309043765068, -0.010729225352406502, 0.016455113887786865, -0.006901741027832031, 0.007293691858649254, 0.005204424262046814, 0.006629080045968294, 0.016155187040567398, 0.021785644814372063, 0.020026978105306625, 0.024375926703214645, 0.0090250913053751, -0.032064978033304214, -0.005855403374880552, -0.002060297643765807, 0.028302252292633057, -0.011206382885575294, 0.015241771005094051, -0.020790429785847664, 0.0301290825009346, 0.029092969372868538, 0.02263089455664158, 0.020149676129221916, 0.024048734456300735, -0.005422553047537804, 0.008282089605927467, -0.01817288063466549, 0.008820596151053905, -0.03503698855638504, 0.016018856316804886, -0.018936332315206528, 0.009018274955451488, 0.03506425395607948, -0.003006091807037592, -0.01897723227739334, -0.04196258634328842, 0.004144452977925539, 0.011131401173770428, 0.03157418966293335, 0.006584772374480963, 0.010879188776016235, -0.0008972263894975185, 0.015459900721907616, 0.005398695357143879, 0.025698335841298103, -0.010511096566915512, -0.007341407705098391, 0.007729950360953808, 0.023067152127623558, 0.006758593954145908, 0.01717766746878624, 0.007286875508725643, 0.014860045164823532, 0.012617405503988266, -0.0034968822728842497, -0.014178391546010971, -0.006755185779184103, 0.039317771792411804, 0.02665265090763569, 0.021840177476406097, 0.0029754173010587692, -0.019331691786646843, 0.015173605643212795, -0.024730388075113297, 0.003178209299221635, -0.029202034696936607, -0.02789326012134552, -0.03100159950554371, 0.01854097470641136, 0.0042330678552389145, -0.0030844819266349077, 0.03190138190984726, 0.011342713609337807, -0.03179231658577919, 0.011629007756710052, -0.009277303703129292, -0.007102828938513994, -0.027947790920734406, 0.022808125242590904, 0.012610589154064655, -0.014096593484282494, 0.03091980144381523, 0.029611026868224144, -0.002378970617428422, 0.02152661606669426, 0.00753227062523365, -0.011008703149855137, -0.007811748422682285, -0.013728500343859196, 0.0051158093847334385, 0.021567516028881073, -0.04872458800673485, -0.005654315464198589, 0.004147861152887344, 0.015868892893195152, 0.00015624775551259518, 0.011056419461965561, -0.013339958153665066, 0.03558231145143509, 0.0038547501899302006, -0.017014069482684135, 0.02498941496014595, 0.02811138890683651, -0.006935823708772659, 0.007900363765656948, -0.03342828527092934, 0.002315917517989874, -0.006141697522252798, 0.007641335483640432, -0.009522698819637299, 0.02009514346718788, -0.03179231658577919, -0.00820029154419899, 0.002782850293442607, -0.01038158219307661, 0.003520740196108818, 0.024348661303520203, 0.00671769492328167, -0.006792676635086536, -0.02432139590382576, 0.005374837666749954, -0.019113563001155853, -0.04662509635090828, -0.017232200130820274, 0.01042248122394085, -0.014805513434112072, -0.012269762344658375, -0.043216828256845474, 0.021949240937829018, -0.0033094275277107954, -0.019795216619968414, -0.014982743188738823, 0.016455113887786865, -0.026707181707024574, -0.015173605643212795, -0.0031287893652915955, 0.015255404636263847, 0.005381654016673565, 0.0010548586724326015, -0.00535779632627964, -0.004440972115844488, -0.02744336798787117, 0.022017406299710274, -0.002805003896355629, 0.01918172836303711, -0.001584844314493239, -0.03915417566895485]} +{"id": "test:10000032", "text": "\"Check out these two awesome ladies!!\\nThey competed all weekend at the Heart of America CrossFit Competition. Three days of workouts and fun. They did GREAT!!! Very cool to see them compete. More to come on this, once I\u2019ve rested a little.\"", "vector_field": [-0.026786958798766136, -0.0019171058665961027, 0.006693457253277302, -0.0102486377581954, 0.004405404441058636, 0.042333900928497314, 0.010681956075131893, -0.0230709258466959, -0.00260975887067616, -0.03794819116592407, 0.017989281564950943, 0.014365163631737232, 0.012172309681773186, -0.03337865322828293, -0.005639705341309309, 0.006210898049175739, 0.026327379047870636, 0.0032219854183495045, -0.009014337323606014, -0.03999660536646843, -0.005793992895632982, 0.007878517732024193, 0.006227311212569475, 0.004011478275060654, -0.031409021466970444, 0.004500602837651968, 0.011509200558066368, -0.03117266669869423, 0.008817373774945736, -0.014785351231694221, -0.01030116155743599, 0.0030693390872329473, -0.017700402066111565, -0.022952746599912643, -0.018895311281085014, -0.02103564143180847, -0.021482091397047043, -0.005144015420228243, 0.005147297866642475, 0.0052950200624763966, 0.009257257916033268, -0.01411567721515894, 0.009309781715273857, 0.0010233866050839424, -0.02922930009663105, -0.012303617782890797, -0.015258061699569225, -0.018724609166383743, 0.011141536757349968, 0.030463600531220436, 0.010635998100042343, 0.003614269895479083, -0.026616258546710014, 0.011049620807170868, -0.01609843783080578, -0.020733632147312164, 0.007642162032425404, 0.029203036800026894, 0.015034837648272514, -0.007661858107894659, -0.01773979514837265, 0.0016077099135145545, -0.012822287157177925, 0.005304868333041668, 0.010379946790635586, -0.03613613173365593, -0.0024800915271043777, 0.01466717291623354, -0.0037619920913130045, 0.006703305058181286, 0.008344663307070732, 0.0041001117788255215, -0.002785384189337492, 0.005206387024372816, 0.0028444731142371893, -0.028992943465709686, -0.00831840094178915, -0.01411567721515894, 0.015927735716104507, 0.00795730296522379, 0.000403774029109627, -0.008942116983234882, -0.00737954443320632, 0.037921931594610214, 0.02610415406525135, -0.002110786037519574, 0.00019368021457921714, -0.0097890580072999, 0.009769361466169357, -0.024659758433699608, 0.01810745894908905, 0.021849755197763443, -0.0020960138645023108, 0.008837070316076279, -0.04377829656004906, 0.0209568552672863, -0.03006967529654503, 0.019459938630461693, -0.03421902656555176, -0.017056990414857864, -0.02242751233279705, 0.023858776316046715, -0.01080669928342104, -0.013071773573756218, -0.039681464433670044, -0.006220745854079723, -0.012441491708159447, -0.0038604733999818563, 0.027968736365437508, 0.0018317552749067545, -0.033877626061439514, -0.00864667259156704, 0.0031087317038327456, -0.03025350719690323, -0.010839526541531086, -0.019079143181443214, 0.012658150866627693, -0.021350782364606857, -0.005577333737164736, -0.01511362288147211, 0.02381938509643078, 0.006788655649870634, 0.026786958798766136, 0.0021075033582746983, -0.00018403724243398756, -0.0007267111795954406, -0.007747208699584007, -0.017595356330275536, 0.03647753596305847, 0.0014731186674907804, -0.005885908845812082, -0.007668423466384411, 0.019788209348917007, -0.028152568265795708, -0.0007652830681763589, 0.015940867364406586, -0.01699133589863777, 0.004851853474974632, -0.01424698531627655, -0.005196538753807545, -0.001527283457107842, 0.009631487540900707, -0.01429950911551714, -0.015494417399168015, -0.016216615214943886, 0.03574220836162567, 0.015244930982589722, -0.014772220514714718, 0.015061099082231522, 0.004428383428603411, 0.004933921620249748, -0.004359446465969086, 0.0013590442249551415, 0.014627780765295029, 0.026708174496889114, 0.01919732056558132, 0.017595356330275536, 0.01171929482370615, -0.009664314799010754, -0.024594105780124664, 0.011056185699999332, 0.01181121077388525, 0.007609334774315357, -0.0066474988125264645, -0.0015174353029578924, 0.004645042587071657, 0.03261706233024597, -0.006801786832511425, -0.01351165771484375, 0.0014542429707944393, 0.015402501448988914, -0.007983564399182796, -0.03789566829800606, 0.006227311212569475, 0.025132469832897186, 0.008607280440628529, -0.0002921616833191365, 0.0003524815838318318, -0.0239900853484869, -0.018173113465309143, 0.017043858766555786, 0.0021698749624192715, 0.029281822964549065, 0.00615837424993515, -0.011043054983019829, -0.003169461851939559, 0.0021173516288399696, -0.019893256947398186, 0.01264502014964819, -0.008974944241344929, -0.014496471732854843, 0.018816525116562843, -0.001682391739450395, 0.00336478347890079, -0.6533917188644409, 0.006860875524580479, 0.006887137424200773, -0.022217418998479843, 0.024974899366497993, 0.024475926533341408, 0.00955926813185215, -0.020891202613711357, -0.014654042199254036, 0.04151978716254234, -0.005669249687343836, 0.0193942841142416, -0.0018301139352843165, -0.006887137424200773, 0.0031973649747669697, -0.023241626098752022, 0.0024406989105045795, -0.003647096920758486, -0.016702456399798393, 0.014496471732854843, -0.016846897080540657, -0.00404758844524622, 0.009263823740184307, -0.001101351110264659, -0.017687272280454636, 0.011226886883378029, 0.005029120482504368, 0.0018153417622670531, -0.0038539080414921045, 0.017871104180812836, -0.036451272666454315, 0.035243235528469086, -0.0037160341162234545, -0.03151407092809677, 0.05977168679237366, -0.01737213134765625, 0.00012628197146113962, 0.012572800740599632, 0.008292139507830143, 0.027469763532280922, -0.02988584339618683, 0.014076284132897854, 0.009427959099411964, -0.012887941673398018, 0.009729969315230846, 0.02416078746318817, -0.012855114415287971, 0.014141938649117947, -0.002227322431281209, 0.005222800653427839, -0.0025441045872867107, -0.0196700319647789, -0.0004448079562280327, 0.004582670982927084, -0.004832157399505377, 0.010425904765725136, 0.023976953700184822, 0.005938432645052671, -0.016689326614141464, 0.0123167484998703, -0.005370522849261761, 0.019328629598021507, -0.04327932372689247, -0.013334390707314014, -0.01415506936609745, 0.00946078635752201, -0.010005717165768147, -0.0023684792686253786, 0.02500116266310215, 0.007077534683048725, 0.016361054033041, 0.007418937049806118, -0.006581844761967659, 0.025894060730934143, 0.025395087897777557, 0.028073783963918686, 0.001392692094668746, 0.015875212848186493, -0.003660227870568633, 0.02224368043243885, -0.0019121818477287889, -0.011089012958109379, 0.009316346608102322, -0.014470210298895836, 0.013577311299741268, -0.008128004148602486, -0.02546074241399765, 0.006384881678968668, -0.00950674433261156, -0.014417686499655247, 0.022453773766756058, 0.025710228830575943, 0.025802144780755043, -0.01875087060034275, 0.010590040124952793, 0.025342565029859543, -0.015218669548630714, 0.021468959748744965, 0.029807057231664658, -0.0334574356675148, 0.02941313199698925, -0.009447655640542507, 0.02656373381614685, -0.0162691380828619, -0.02198106423020363, -0.010642563924193382, -0.0012178875040262938, 0.004717262461781502, 0.02234872803092003, -0.011509200558066368, -0.03306351229548454, -0.019171059131622314, -0.012704108841717243, 0.017726663500070572, -0.02115381881594658, -0.02978079579770565, 0.011758686974644661, 0.015061099082231522, 0.010990532115101814, -0.03224939852952957, 0.046378206461668015, -0.005537941120564938, 0.02308405563235283, -0.011049620807170868, 0.004861701745539904, -0.006352054420858622, 0.011043054983019829, -0.021350782364606857, -0.025815274566411972, -0.0009060294833034277, 0.010452166199684143, 0.018238767981529236, 0.02501429244875908, -0.011417284607887268, -0.0034074587747454643, 0.00016290476196445525, 0.009592094458639622, -0.013143992982804775, 0.0397077277302742, -0.04679839313030243, -0.0022125502582639456, 0.0063881645910441875, 0.01209352444857359, -0.0155338104814291, -0.010537517257034779, 0.006686891429126263, -0.02665564976632595, 0.012697543948888779, -0.016295399516820908, -0.0018678652122616768, 0.0004891246207989752, -0.004884680733084679, -0.019998302683234215, 0.013117731548845768, 0.0019302366999909282, -0.005813689436763525, -0.01313742808997631, -0.02363555133342743, -0.019354891031980515, -0.030384816229343414, -0.010517820715904236, -0.00498972786590457, -0.002970857545733452, -0.01756909303367138, 0.010268334299325943, -0.0027706120163202286, -0.015336846932768822, 0.01717516779899597, -0.016111567616462708, -0.021166950464248657, 0.009336043149232864, -0.021180080249905586, -0.01360357366502285, 0.005551071837544441, 0.01764787919819355, 0.010832961648702621, 0.008666369132697582, 0.00804265309125185, -0.013249039649963379, 0.012008173391222954, 0.01217887457460165, -0.0073073250241577625, -0.01892157271504402, 0.002429209416732192, -0.004428383428603411, 0.027601072564721107, 0.007871951907873154, 0.013386914506554604, -0.028520232066512108, -0.014535864815115929, 0.009119383990764618, -0.0022355292458087206, 0.010373380966484547, -0.007438633590936661, 0.003039794508367777, -0.005570768378674984, -0.006276552099734545, 0.011863733641803265, 0.02509307861328125, -0.0014993803342804313, 0.031566593796014786, -0.0066474988125264645, 0.028283877298235893, 0.017450915649533272, 0.00841031689196825, -0.027811165899038315, 0.018356945365667343, -0.01710951328277588, 0.043725769966840744, -0.025408217683434486, -0.007674988824874163, -0.015586333349347115, 0.007793166674673557, -0.008265878073871136, -0.005150580778717995, 0.025040553882718086, -0.005662684328854084, 0.010465296916663647, -0.029097991064190865, 0.02232246659696102, -0.00668360898271203, -0.003962237853556871, 0.010557212866842747, -0.0011284335050731897, -0.0048485710285604, -0.013294997625052929, 0.003821081016212702, 0.002650792710483074, 0.01199504267424345, 0.003702903166413307, 0.0012539973249658942, 0.004832157399505377, 0.025710228830575943, 0.01600652188062668, 0.036004822701215744, -0.02519812434911728, 0.011640509590506554, -0.004638477228581905, 0.029491916298866272, -0.004109960049390793, 0.01411567721515894, 0.007162885274738073, 0.010649129748344421, -0.002018870087340474, 0.009362304583191872, -0.016334792599081993, 0.043725769966840744, 0.008344663307070732, 0.012901072390377522, -0.020812416449189186, -0.025224385783076286, 0.019709425047039986, -0.005432894453406334, 0.005682380869984627, -0.003169461851939559, -0.03377257660031319, -0.012743501923978329, -0.012572800740599632, 0.026314247399568558, 0.02857275679707527, 0.0277586430311203, -0.00909312255680561, 0.0159933902323246, -0.0013500167988240719, -0.0009462427697144449, 0.008154265582561493, -0.006086154840886593, -0.007123492658138275, 0.0027968736831098795, -0.03411398082971573, -0.01708325184881687, -0.003624117933213711, 0.018409468233585358, 0.0060303485952317715, 0.01885591819882393, 0.008843636140227318, -0.007911344058811665, -0.029912104830145836, 0.012815721333026886, 0.019525591284036636, -0.008797678165137768, -0.04162483289837837, 0.010859223082661629, 0.015927735716104507, -0.01643984019756317, 0.010688521899282932, -0.018094327300786972, -0.0031152970623224974, -0.007077534683048725, 0.046929702162742615, 0.003844060003757477, 0.004950335249304771, -0.015363109298050404, -0.005084926262497902, -0.005590464454144239, 0.043436892330646515, 0.0275748111307621, -0.013143992982804775, 0.02859901823103428, -0.014050022698938847, -0.000966759747825563, -0.0052621932700276375, -0.0020845243707299232, -0.02112755738198757, 0.01111527532339096, 0.011174364015460014, 0.025395087897777557, 8.87872010935098e-05, 0.005163711495697498, -0.005866212770342827, 0.01699133589863777, -0.021626530215144157, -0.022545689716935158, 0.017621617764234543, 0.017989281564950943, -0.013984368182718754, -0.00634548906236887, -0.009618356823921204, 0.020169004797935486, -0.011410719715058804, -0.013130862265825272, -0.02132452093064785, -0.015704510733485222, -0.020379098132252693, 0.10998409986495972, -0.005912170745432377, -0.005452590528875589, 0.03117266669869423, -0.0005182587192393839, 0.0031070902477949858, 0.005370522849261761, -0.006224028766155243, 0.014877267181873322, -0.0006799324764870107, -0.0035617463290691376, -0.0199457798153162, -0.0024456230457872152, -0.00364381424151361, 0.03014845959842205, 0.0026360205374658108, 0.0025244082789868116, -0.005419763270765543, -0.006880571600049734, 0.0069724880158901215, -0.00011930619075428694, -0.013813666999340057, 0.016032783314585686, 0.009040598757565022, 0.027811165899038315, 0.0004013119905721396, 0.04469745606184006, 0.016663063317537308, 0.0021271996665745974, 0.0008276546723209321, 0.004336467478424311, 0.021587137132883072, -0.0008292960119433701, -0.0058104065246880054, 0.015612594783306122, 0.017253952100872993, 0.024147655814886093, -0.0018153417622670531, 0.03800071403384209, -0.03765931352972984, 0.01643984019756317, 0.03238070756196976, 0.008699196390807629, -0.004697565920650959, 0.004254399333149195, -0.0013377065770328045, 0.021455828100442886, 0.01827816106379032, -0.018895311281085014, -0.012441491708159447, 0.04517016559839249, 0.02271639183163643, 0.01475908886641264, -0.022860830649733543, -0.003610987216234207, 0.007865386083722115, -0.024331487715244293, -0.0201296117156744, -0.0038506253622472286, -0.010878919623792171, 0.000381410529371351, -0.018173113465309143, 0.004773068241775036, 0.020365966483950615, -0.006985618732869625, -0.027732381597161293, -0.021166950464248657, -0.004530147649347782, 0.0015880136052146554, -0.01818624511361122, -0.03513818606734276, -0.01664993353188038, 0.0014509602915495634, 0.014654042199254036, 0.005954846274107695, 0.025683967396616936, 0.01351165771484375, 0.007517418824136257, 0.006933095399290323, 0.003230192232877016, -0.009703706949949265, 0.0008666369249112904, 0.003975368570536375, -0.007123492658138275, 0.023753730580210686, -0.0062141804955899715, -0.010675391182303429, -0.0034665476996451616, -0.004257682245224714, 0.031960517168045044, 0.0273909792304039, 0.004346315283328295, 0.01738526113331318, 0.010215810500085354, -0.011837472207844257, 0.027180884033441544, 0.020379098132252693, 0.026091022416949272, -0.014194462448358536, -0.0028904310893267393, 0.0343765988945961, -0.03077874146401882, 0.020169004797935486, -0.0032416817266494036, 0.00531143369153142, 0.007563376799225807, -0.001839962089434266, 0.028625279664993286, -0.014916659332811832, -0.016505494713783264, 0.010918311774730682, 0.007418937049806118, 0.020011434331536293, 0.020825548097491264, 0.010373380966484547, 0.006933095399290323, -0.00016916243475861847, 0.018343813717365265, -0.01940741389989853, -0.012218267656862736, -0.015021705999970436, 0.0020681107416749, 0.03151407092809677, 0.003238398814573884, -0.011240018531680107, 0.005859647411853075, -0.017424654215574265, -0.037002768367528915, 0.010662260465323925, 0.02694452926516533, -0.0035125056747347116, -0.00045465611037798226, -0.03190799430012703, -0.0157176423817873, -0.002222398528829217, -0.018882179632782936, -0.004372577182948589, 0.04041679576039314, 0.003218702506273985, 0.013761143200099468, -0.02261134423315525, -0.0024981466121971607, -0.0119490846991539, -0.010307726450264454, -0.019998302683234215, -0.029649486765265465, -0.004247833974659443, -0.00831840094178915, 0.002983988495543599, 0.027706120163202286, -0.005633139982819557, 0.003394327824935317, -0.02749602496623993, 0.00813456904143095, -0.03227565810084343, -0.03335238993167877, -0.024423403665423393, 0.005281889345496893, 0.017726663500070572, 0.03821080923080444, 0.02391130104660988, 0.005117753520607948, 0.007169450633227825, 0.010681956075131893, 0.00728762848302722, 0.009099687449634075, 0.024935508146882057, -0.009618356823921204, 0.0032039303332567215, 0.004733675625175238, 0.0033615007996559143, 0.018632693216204643, -0.027968736365437508, -0.012100089341402054, 0.007523984182626009, 0.022480037063360214, -0.025434480980038643, -0.028520232066512108, -0.014969183132052422, -0.0070184459909796715, 0.014181330800056458, -0.01025520358234644, 0.005025837570428848, 0.0029971192125231028, -0.012776329182088375, -0.017713533714413643, 0.02223055064678192, -0.014076284132897854, 0.010813265107572079, -0.006210898049175739, 0.026589995250105858, -0.010846092365682125, -0.005866212770342827, -0.009198169223964214, -0.009250692091882229, -0.011916257441043854, -0.004556409083306789, -0.026314247399568558, 0.0013237550156190991, 0.00015315919881686568, 0.005166994407773018, 0.02804752252995968, 0.00314976554363966, -0.012776329182088375, -0.01628226973116398, 0.008022956550121307, -0.02610415406525135, -0.011568289250135422, -0.017621617764234543, -0.028940420597791672, 0.00714975455775857, 0.01011732965707779, -0.009099687449634075, -0.012855114415287971, -0.0016717228572815657, -0.003656945191323757, -0.02922930009663105, 0.004779634065926075, 0.0027525569312274456, -0.0238062534481287, 0.0007595383212901652, -0.003968803212046623, 0.01919732056558132, -0.0020796002354472876, 0.03899865970015526, 0.0018153417622670531, 0.00941482838243246, 0.0011505917645990849, 0.008686065673828125, -0.013025815598666668, 0.022388119250535965, 0.017687272280454636, 0.03306351229548454, -0.044356051832437515, 0.0008797677583061159, 0.01011076383292675, 0.018501384183764458, -0.04015417769551277, -0.024791067466139793, 0.019341759383678436, 0.0070644039660692215, 0.007215408608317375, -0.017122644931077957, 0.003190799616277218, -0.01680750399827957, -0.014050022698938847, -0.020707370713353157, -0.007208843249827623, 0.004464493133127689, -0.0039720856584608555, -0.014837874099612236, 0.013432872481644154, -0.016610540449619293, 0.006000804249197245, 0.010045109316706657, -0.030279768630862236, 0.0014739392790943384, -0.005524810403585434, -0.013189950957894325, 0.0029068447183817625, -0.011771817691624165, 0.007760339416563511, -0.0035387673415243626, -0.005813689436763525, 0.004487472120672464, 0.00701188063248992, -0.039681464433670044, -0.0043299016542732716, 0.0101567218080163, 0.02804752252995968, 0.009723403491079807, -0.005334412679076195, -0.024699151515960693, -0.005324564874172211, 0.021770969033241272, -0.008036087267100811, -0.01520553883165121, -0.01802867464721203, 0.0032072130125015974, 0.00152892479673028, 0.01448334101587534, -0.002852679928764701, -0.02682635188102722, -0.029465654864907265, -0.017779188230633736, -0.014693435281515121, -0.006046762224286795, -0.017240822315216064, 0.0005978645640425384, -0.022296203300356865, -0.020155873149633408, -0.004008195828646421, 0.0007480488275177777, 0.0200376957654953, 0.02123260498046875, -0.013485395349562168, 0.021849755197763443, 0.01020267978310585, -0.02271639183163643, 0.030647432431578636, -0.044539883732795715, 0.0015789861790835857, -0.04635194316506386, 0.003258095122873783, -0.0022191156167536974, -0.012474318966269493, 0.0007558452780358493, 0.0029954779893159866, -0.009060295298695564, -0.021193211898207664, 0.00860071461647749, 0.00023840721405576915, -0.01259906217455864, -0.005577333737164736, 0.0358472540974617, 0.004543278366327286, -0.017949888482689857, -0.026051631197333336, -0.028073783963918686, 0.01148293912410736, 0.0007513315067626536, -0.002759122522547841, 0.01020267978310585, -0.010721349157392979, 0.020379098132252693, -0.010773872956633568, 0.0050882091745734215, 0.03784314543008804, -0.008443144150078297, 0.012539973482489586, -0.006549017503857613, -0.0033779144287109375, 0.0025260497350245714, -0.0200376957654953, -0.02096998691558838, -0.029675748199224472, -0.019761947914958, 0.03813202306628227, -0.01327530201524496, -0.011194059625267982, -0.0020352837163954973, 0.029439393430948257, 0.00864667259156704, 0.0008009825833141804, -0.0019663465209305286, -0.012901072390377522, 0.01793675869703293, -0.006900268141180277, -0.02427896484732628, -0.01351165771484375, 0.003176027210429311, 0.018685217946767807, 0.0010299519635736942, -0.04165109619498253, 0.00012063979374943301, -0.001033234759233892, -0.020024564117193222, -0.00860071461647749, -0.01810745894908905, 0.0334574356675148, 0.012021304108202457, 0.005514962133020163, 0.014942921698093414, 0.02152148261666298, -0.010097633115947247, 0.016059044748544693, -0.039024922996759415, 0.0021961366292089224, 0.011968781240284443, -0.0006930633098818362, -0.004901094362139702, -0.023583028465509415, -0.029859580099582672, 0.0200376957654953, 0.014128807932138443, 0.025250649079680443, -0.01746404729783535, 0.017989281564950943, -0.013748012483119965, -0.006782090291380882, -0.024318356066942215, -0.010767307132482529, 0.019551854580640793, 0.014522734098136425, 0.030279768630862236, -0.035059403628110886, 0.009736534208059311, 0.015179276466369629, -0.003254812443628907, 0.028100045397877693, 0.020838677883148193, 0.009099687449634075, 0.017621617764234543, -0.01351165771484375, -0.002860886510461569, -0.022269941866397858, 0.000630691705737263, -0.01466717291623354, 0.013235908932983875, -0.002363555133342743, -0.00705783860757947, 0.035794731229543686, 0.04193997383117676, -0.012868245132267475, 0.013734881766140461, -0.004346315283328295, 0.005547789391130209, 0.0023914582561701536, 0.024121394380927086, 0.0059023224748671055, -0.014220723882317543, -0.0207730233669281, -0.006585127208381891, -0.017963020130991936, -0.018133720383048058, -0.0009634770103730261, -0.007090665400028229, -0.0273909792304039, -0.006762394215911627, -0.015336846932768822, 0.005052099470049143, 0.01609843783080578, 0.009329477325081825, -0.000495690037496388, 0.004671304486691952, -0.02489611506462097, -0.014811612665653229, -0.02142956666648388, -0.023753730580210686, 0.021744707599282265, 0.0041920277290046215, -0.00026425858959555626, -0.020654845982789993, 0.016951942816376686, -0.013708620332181454, 0.014076284132897854, 0.22732149064540863, -0.023136578500270844, -0.027889952063560486, 0.021954800933599472, -0.005370522849261761, 0.010445601306855679, -0.0015125111676752567, -0.002155102789402008, -0.0056134434416890144, 0.025802144780755043, -0.023136578500270844, -0.0035584636498242617, -0.0245415810495615, -0.01278289407491684, 0.00831840094178915, -0.006788655649870634, -0.01313742808997631, -0.006906833499670029, -0.0155338104814291, -0.02875658869743347, 0.005876061040908098, 0.012027869932353497, 0.006709870416671038, -0.005075078457593918, 0.032301921397447586, -0.00918503850698471, 0.003417306812480092, 0.026498079299926758, 0.003525636624544859, -0.00946078635752201, -0.007346717640757561, -0.0038276463747024536, 0.013524788431823254, -0.0058793434873223305, -0.007648727390915155, -0.0064341225661337376, 0.004129656124860048, -0.0010956063633784652, 0.0236224215477705, -0.009145645424723625, -0.005728338845074177, 0.01582268811762333, -0.0071563199162483215, 0.009007771499454975, 0.010924877598881721, 0.013774274848401546, 0.003093959530815482, -0.008068914525210857, -0.0010389795061200857, 0.0026885441038757563, -0.025027424097061157, 0.008430013433098793, 0.022033587098121643, 0.038946136832237244, 0.002063186839222908, 0.0019039750332012773, 0.03852595016360283, -0.0006852668593637645, -0.02959696389734745, 0.019906386733055115, -0.018159981817007065, 0.017332738265395164, -0.0056594014167785645, 0.01424698531627655, -0.015927735716104507, 0.0067755249328911304, 0.003725882153958082, 0.0009027468040585518, -0.009907235391438007, 0.012375838123261929, 0.0017775904852896929, -0.009861277416348457, 0.001685674418695271, -0.0358472540974617, -0.005672532599419355, 0.0006475156405940652, 0.031120143830776215, 0.009086556732654572, 0.0119490846991539, 0.008758285082876682, 0.01766100898385048, -0.025999106466770172, 0.02223055064678192, 0.00804265309125185, 0.017949888482689857, -0.042517732828855515, 0.01717516779899597, 0.007990129292011261, -0.008022956550121307, -0.001326217083260417, 0.005967976991087198, -0.001417312421835959, -0.0012884658062830567, -0.01848825439810753, 0.008620411157608032, -0.01223139837384224, 0.004188745282590389, -0.016334792599081993, -0.0064341225661337376, 0.0159014742821455, -0.007839124649763107, -0.0024226440582424402, 0.0009347532759420574, 0.01457525696605444, -0.02196793258190155, -0.016781242564320564, 0.025867799296975136, 0.032118089497089386, -0.0004780454619321972, 0.0014419328654184937, -0.0038933006580919027, -0.025789013132452965, 0.0020960138645023108, -0.0035125056747347116, -0.017240822315216064, 0.02250629849731922, 0.015520678833127022, -0.0281263068318367, 0.006565431132912636, -0.017516570165753365, -0.019801340997219086, -0.0063192276284098625, 0.01043903548270464, -0.006355337332934141, 0.005551071837544441, -0.039839036762714386, -0.025079946964979172, -0.005797275807708502, 0.001212963368743658, -0.012907637283205986, 0.020365966483950615, -0.01829129084944725, 0.006480080541223288, -0.004720544908195734, 0.0006044299807399511, -0.02381938509643078, 0.01643984019756317, 0.0005354929598979652, 0.0014698358718305826, 0.028651541098952293, -0.0014435742050409317, -0.0008247822988778353, -0.010327422991394997, 0.0039523895829916, 0.008022956550121307, -0.03642501309514046, -0.01457525696605444, 0.011587985791265965, -0.006880571600049734, -0.001665978110395372, 0.0024915810208767653, 0.008935552090406418, -0.005626574624329805, 0.012060697190463543, 0.025539526715874672, 0.01217887457460165, -0.0316191166639328, -0.01756909303367138, -0.01619035378098488, -0.009782492183148861, -0.014929790049791336, 0.018711479380726814, 0.015428762882947922, -0.01674184948205948, -0.014089414849877357, -0.01792362704873085, -0.1664993315935135, 0.0329059399664402, 0.011194059625267982, -0.007116927299648523, 0.0026146830059587955, 0.0025079946499317884, 0.021928539499640465, 0.0048485710285604, -0.02124573476612568, 0.00993349775671959, 0.005623291712254286, 0.01466717291623354, -0.010793568566441536, -0.0049404869787395, -0.001529745408333838, 0.008863331750035286, -0.012940464541316032, 0.007228539790958166, 0.03188173472881317, 0.013472264632582664, 0.027732381597161293, -0.005600312724709511, 0.021350782364606857, -0.008843636140227318, 6.27691133558983e-06, 0.027233408764004707, 0.019801340997219086, 0.010780437849462032, -0.009099687449634075, 0.0031366348266601562, 0.00432333629578352, 0.011062751524150372, 0.01792362704873085, 0.01680750399827957, -0.009999151341617107, -0.006224028766155243, -0.024489058181643486, -0.004316770937293768, -0.013839928433299065, 0.026537472382187843, 0.0034271550830453634, -0.0008214995614252985, -0.018882179632782936, -0.017148906365036964, -0.01209352444857359, -0.0008740230114199221, -0.021574007347226143, -0.015796426683664322, 0.01485100481659174, -0.0096052261069417, -0.001169467461295426, -0.003153048222884536, 0.043541938066482544, -0.014509602449834347, 0.018619563430547714, 0.01829129084944725, -0.014365163631737232, 0.011614247225224972, 0.004871550016105175, -0.015074229799211025, 0.013590442016720772, -0.011423850432038307, -0.018251897767186165, -0.00418546237051487, 0.02060232311487198, -0.016242876648902893, -0.01710951328277588, -0.009053729474544525, -0.0420975461602211, 0.0042150067165493965, -0.019735686480998993, -0.00672956695780158, -0.005367239937186241, -0.02270326018333435, 0.03521697223186493, 0.001236763084307313, -0.034796785563230515, 0.03721286356449127, 0.017713533714413643, -0.0036766414996236563, -0.01439142506569624, 0.039471372961997986, -0.01674184948205948, 0.010681956075131893, 0.01511362288147211, 0.003459982108324766, 0.005189973395317793, 0.01810745894908905, -0.00831840094178915, -0.011082448065280914, 0.029859580099582672, -0.009762795642018318, -0.030936311930418015, -0.01089861523360014, 0.003350011305883527, 0.019512461498379707, -0.005298302974551916, -0.02085180953145027, -0.00560359563678503, -0.02326788753271103, 0.012756632640957832, -0.0061452435329556465, -0.03398267179727554, 0.0031973649747669697, 0.02682635188102722, 0.011699598282575607, 0.0040541538037359715, 0.02875658869743347, 0.02409513294696808, -0.012152613140642643, -0.007130058016628027, 0.010997097007930279, 0.005514962133020163, 0.025631442666053772, 0.029912104830145836, 0.010498124174773693, 0.011830907315015793, -0.0277586430311203, 0.01662367209792137, -0.022493166849017143, 0.059141404926776886, 0.0005679097957909107, -0.05672532320022583, 0.0080886110663414, -0.0020746763329952955, -0.013813666999340057, -0.10578222572803497, -0.02232246659696102, 0.007753774058073759, 0.04162483289837837, 0.004086981061846018, 0.012769763357937336, -0.0115814208984375, 0.01995891146361828, -0.005803841166198254, 0.03429781273007393, 0.016400447115302086, -0.005616726353764534, 0.01195565052330494, 0.011043054983019829, 0.02409513294696808, -0.01448334101587534, -0.0032006476540118456, -0.0023011835291981697, 0.013839928433299065, 0.02582840621471405, -0.00918503850698471, 0.0024127960205078125, -0.01195565052330494, -0.035164449363946915, 0.009401697665452957, -0.00837092474102974, -0.025395087897777557, 0.015376240015029907, 0.0083643589168787, 0.0006885495968163013, -0.009395131841301918, -0.002105862135067582, 0.0323544442653656, -0.010813265107572079, -0.0037225994747132063, -0.024607235565781593, -0.03329986706376076, -0.023281019181013107, 0.016124699264764786, -0.039129968732595444, 0.0005416480707935989, 0.004566257353872061, 0.030358552932739258, -0.00909312255680561, -0.01664993353188038, 0.01655801758170128, -0.016702456399798393, 0.027653595432639122, 0.0097890580072999, -0.010905181057751179, -0.019473068416118622, -0.00959866028279066, 0.003663510549813509, -0.006398012861609459, 0.03576846793293953, 0.005015989299863577, 0.021915409713983536, 0.015087360516190529, -0.011200625449419022, -0.007891648449003696, -0.011889996007084846, 0.010012282058596611, -0.02051040716469288, 0.018173113465309143, 0.01511362288147211, -0.0017365565290674567, -0.009624921716749668, 0.01025520358234644, -0.0004181358963251114, -0.004392273258417845, 0.00350594031624496, 0.01718829944729805, -0.016137829050421715, 0.019433675333857536, 0.0005970438942313194, 0.021862884983420372, -0.0167287178337574, -0.0351119264960289, 0.017884233966469765, -0.006617954466491938, -0.018041804432868958, -0.006463666912168264, -0.0002630275848787278, -0.005495266057550907, 0.020536668598651886, -0.032958462834358215, -0.002265073824673891, 0.011121840216219425, 0.0005145656759850681, -0.010084502398967743, 0.0010471863206475973, 0.022939616814255714, 0.01322934404015541, -0.03776435926556587, -0.016886288300156593, 0.005153863225132227, 0.019446806982159615, -0.039208754897117615, -0.031041357666254044, -0.01047842763364315, -0.02112755738198757, -0.013078338466584682, -0.05940401926636696, 0.02455471269786358, -0.004254399333149195, 0.0017119362019002438, 0.01609843783080578, -0.014745958149433136, -0.003350011305883527, 0.009638053365051746, 0.013104600831866264, -0.00043660117080435157, -0.00814113486558199, 0.00018998715677298605, 0.016571147367358208, 0.0005338516202755272, -0.013400045223534107, -0.008108307607471943, 0.007215408608317375, 0.014089414849877357, 0.03025350719690323, 0.007084100041538477, -0.006013934966176748, 0.021101295948028564, 0.019157927483320236, 0.01726708374917507, 0.0008276546723209321, 0.01360357366502285, -0.041362214833498, 0.004917507991194725, -0.006874006241559982, -0.0201296117156744, 0.015455025248229504, -0.021652791649103165, -0.016321660950779915, 0.015468155965209007, -0.006660629995167255, -0.022637605667114258, -0.03006967529654503, 0.02473854459822178, -0.019052881747484207, -0.008869897574186325, -0.02150835283100605, -0.009867843240499496, 0.005025837570428848, 0.005839950870722532, -0.028546495363116264, 0.00042182893957942724, -0.003660227870568633, -0.014036891981959343, -0.003821081016212702, 0.01756909303367138, 0.003912996966391802, 0.025946583598852158, -0.0017381978686898947, -0.04341062903404236, -0.006118981633335352, 0.0019745533354580402, -0.011666771024465561, -0.01680750399827957, 0.004674586933106184, -0.020746761932969093, 0.037370435893535614, 0.0039523895829916, 0.023320412263274193, -0.0013089828426018357, 0.0455903522670269, -0.013642965815961361, -0.00020978604152332991, 0.00023245729971677065, -0.023740598931908607, -0.013419740833342075, -0.02244064398109913, 0.006496494170278311, -0.016610540449619293, 0.008594149723649025, 0.013485395349562168, 0.0028067217208445072, -0.00359785626642406, 0.0006380778504535556, 0.010517820715904236, 0.011601116508245468, 0.0064341225661337376, 0.0007513315067626536, -0.024200178682804108, 0.03453416749835014, 0.0024390576872974634, 0.01264502014964819, -0.02296587824821472, 0.020379098132252693, -0.013255605474114418, 0.0003793588257394731, -0.004559691995382309, 0.0013976161135360599, 0.007944171316921711, -0.00047148001613095403, 0.021311389282345772, 0.024843592196702957, 0.0015592898707836866, -0.010603170841932297, 0.02434461936354637, 0.009073426015675068, 0.0030447186436504126, 0.0010406208457425237, -0.007491156924515963, -0.0027492742519825697, -0.03723912686109543, 0.008797678165137768, -0.012126351706683636, -0.007898213341832161, -0.006631085649132729, 0.004119807854294777, 0.019525591284036636, 0.02106190286576748, 0.006696739699691534, 0.019079143181443214, -0.0016536680050194263, -0.008889594115316868, 0.0017250670352950692, 0.0034895266871899366, -0.026156676933169365, 0.032774630934000015, 0.008147699758410454, 0.004786199424415827, -0.0005761166103184223, -0.01332125999033451, 0.012559670023620129, -0.013334390707314014, 0.014772220514714718, -0.0070184459909796715, -0.003013532841578126, -0.0010028695687651634, 0.008620411157608032, -0.01563885621726513, -0.042438946664333344, 0.005889191757887602, -0.012205136939883232, 0.0009618356707505882, -0.005892474669963121, 0.0363987497985363, -0.04312175139784813, 0.051236625760793686, 0.000741893716622144, -0.009467351250350475, 0.008351228199899197, -0.006571996491402388, 0.010557212866842747, 0.0018514515832066536, 0.020917464047670364, -0.02030031383037567, -0.015599464066326618, -0.009296650066971779, -0.025894060730934143, -0.0332736037671566, -0.02602536976337433, -0.03624118119478226, -0.003955672029405832, 0.0009569115936756134, -0.01387932151556015, 0.005176842678338289, -0.016518624499440193, 0.0079047791659832, -0.007136623840779066, 0.011666771024465561, -0.011141536757349968, -0.0021304823458194733, -0.012664716690778732, 0.015034837648272514, -0.016965074464678764, 0.008128004148602486, -0.00305784959346056, 0.02123260498046875, -0.013301563449203968, -0.013426306657493114, -0.00039433620986528695, 0.014050022698938847, -0.0026425861287862062, -0.004467776045203209, -0.001954857027158141, -0.004684435203671455, 0.017043858766555786, 0.005278606433421373, -0.010471862740814686, -0.0351119264960289, -0.03358874469995499, 0.035637158900499344, 0.01039307750761509, -0.0071563199162483215, -0.004497320391237736, -0.04377829656004906]} +{"id": "test:10000033", "text": "\"* 2. How was the session?\\n* 4. What would you like to hear?\\n* 5. What's your overall feedback of the sessions/conference?\"", "vector_field": [-0.003492793533951044, 0.015043867751955986, 0.01665290631353855, -0.03252091631293297, -0.002447898965328932, 0.012453706003725529, -0.019216906279325485, -0.007750862278044224, -0.015501724556088448, -0.010164421983063221, 0.01274804212152958, 0.018405845388770103, -0.003016949398443103, -0.0101251769810915, -0.0006524460040964186, 0.0025296590756624937, 0.02060355804860592, -0.0113156046718359, 0.012944267131388187, -0.023272208869457245, -0.012211696244776249, 0.0032687706407159567, -0.0074630663730204105, 0.008045198395848274, -0.02047274075448513, -0.012022011913359165, 0.009601911529898643, -0.0024380877148360014, -0.015488642267882824, 0.013434827327728271, 0.005857297219336033, -0.00019326055189594626, 0.0010072850855067372, 0.009693482890725136, -0.01632586680352688, 0.0067893629893660545, -0.011040890589356422, -0.004944853950291872, 0.027471410110592842, 0.0033488955814391375, -0.0037773188669234514, 0.015030785463750362, 0.00702483206987381, -0.013238603249192238, -0.007149107288569212, 0.018484333530068398, 0.004591649863868952, -0.0297476127743721, -0.010040146298706532, 0.010072850622236729, 0.010445676743984222, -0.0014054569182917476, -0.032677896320819855, 0.009327197447419167, -0.0009116256842389703, -0.0020129347685724497, -0.013323633931577206, 0.032337773591279984, -0.005801700055599213, -0.012852695770561695, -0.0008282302878797054, 0.004156685899943113, -0.007894759997725487, 0.023049822077155113, 0.0070640770718455315, -0.010654982179403305, -0.0028714165091514587, 0.020145701244473457, -0.0009377889218740165, -0.00012396882812026888, -0.010968941263854504, 0.01715000905096531, 5.421719833975658e-05, -0.0013792937388643622, 0.03354128450155258, 0.005301327910274267, -0.0032507835421711206, -0.01822270266711712, -0.002642488107085228, 0.024698106572031975, -0.0029024851974099874, -0.0040585738606750965, -0.021362291648983955, 0.01609039679169655, -0.0035614720545709133, 0.04617813229560852, 0.019478537142276764, 0.020538149401545525, -0.02605859376490116, -0.023023657500743866, -0.009909329935908318, 0.0020996006205677986, 0.01963551715016365, 0.03273022174835205, -0.019910231232643127, 0.024096351116895676, 0.026974307373166084, 0.025208288803696632, -0.004683221224695444, 0.013958092778921127, -0.021859392523765564, 0.012067797593772411, -0.014926132746040821, -0.024802759289741516, -0.015161601826548576, -0.010223289020359516, -0.0006189243285916746, -0.01107359491288662, 0.02570539154112339, -0.0021813607309013605, -0.009471096098423004, 0.02857026644051075, 0.0029940565582364798, -0.03500642627477646, -0.0232198815792799, -0.007685454096645117, 0.012865777127444744, 0.004670139867812395, -0.00993549358099699, -0.009667320176959038, 0.019844822585582733, -0.014926132746040821, 0.025875451043248177, -0.0452624186873436, 0.009268330410122871, -0.0059881131164729595, -0.013107786886394024, 0.003427385352551937, 0.011531451717019081, -0.020394250750541687, 0.02057739347219467, 0.017660191282629967, 0.019360803067684174, 0.005334032233804464, -0.015318581834435463, -0.016221214085817337, -0.01666598953306675, 0.023834718391299248, -0.017764845862984657, -0.017777927219867706, 0.013879602774977684, 0.02297133207321167, 0.004997180309146643, 0.0001979617663891986, -0.008254504762589931, 0.05405326932668686, 0.0345354862511158, 0.0036726659163832664, -0.016024990007281303, -0.010020524263381958, -0.00809098407626152, -0.00502661382779479, -0.011426798067986965, 0.020171863958239555, -0.012309808284044266, 0.011832328513264656, -0.0022941897623240948, 0.033750589936971664, -0.0009034496615640819, -0.00527189439162612, -0.010785798542201519, 0.03390756994485855, 0.004493537824600935, 0.0023089065216481686, -0.009209463372826576, 0.040422216057777405, 0.05619865655899048, 0.0004472280270420015, 0.007240678649395704, -0.00499390996992588, -0.02023727260529995, -0.005268624052405357, -0.021401535719633102, 0.01641743816435337, 0.0026948144659399986, 0.01847125217318535, 0.004758440889418125, -0.014128154143691063, -0.010700767859816551, -0.020145701244473457, -0.030140060931444168, -0.024004779756069183, -0.0062857200391590595, 0.032547079026699066, -0.021493107080459595, -0.014298214577138424, 0.011466043069958687, -0.028360961005091667, 0.012519113719463348, -0.005644720513373613, 0.020302679389715195, 0.013984255492687225, 0.008522678166627884, 0.005278435070067644, -0.6568021774291992, -0.018261946737766266, 0.012048175558447838, -0.007469607051461935, 0.0036988290958106518, 0.0042122830636799335, 0.011112838983535767, 0.0020734374411404133, 0.0038459973875433207, 0.031212754547595978, 0.01641743816435337, 0.01926923170685768, 0.01196314487606287, -0.020067211240530014, -0.027235940098762512, -0.010013982653617859, -0.01404966413974762, 0.0056022051721811295, -0.032782550901174545, -0.0075284745544195175, -0.02786385826766491, 0.01966167986392975, -0.0069332607090473175, 0.0007006844971328974, -0.00012028961646137759, -0.009955115616321564, -0.020328843966126442, 0.0012174085713922977, -0.011341767385601997, 0.017359314486384392, -0.03058483637869358, -0.014612173661589622, 0.015737192705273628, -0.0016580958617851138, 0.02667343057692051, 0.029093531891703606, -0.00910480972379446, 0.014311296865344048, 0.000956593721639365, 0.03979429975152016, -0.017045356333255768, 0.00887588132172823, 0.030689489096403122, -0.0020783429499715567, 0.005576041992753744, 0.010955859906971455, 0.009327197447419167, 0.031108101829886436, 0.005003720987588167, 0.01166880875825882, -0.002168279141187668, 0.008378780446946621, -0.013814195059239864, 0.014010419137775898, -0.007639668416231871, -0.0160511527210474, 0.028962714597582817, 0.009359901770949364, -0.025875451043248177, 0.007306086830794811, 0.003172293771058321, 0.03770124167203903, -0.025299860164523125, -0.03356744721531868, -0.011433339677751064, -0.0015150156104937196, -0.004964476451277733, -0.007502311374992132, 0.0231937188655138, -0.0345354862511158, 0.013356338255107403, 0.024109432473778725, -0.009117892012000084, 0.0006916908314451575, 0.0002097147807944566, 0.017045356333255768, -0.0006630747811868787, -0.014232806861400604, 0.020289598032832146, 0.004104359541088343, 0.008293749764561653, -0.019007598981261253, -0.007881678640842438, 0.009889707900583744, -0.008437647484242916, -0.029459817335009575, -0.034587811678647995, -0.0047846040688455105, 0.009065565653145313, 0.0003301065880805254, 0.027942348271608353, 0.01928231306374073, 0.010563410818576813, -0.01022982969880104, 0.009287952445447445, 0.04105013608932495, -0.0016229389002546668, 0.007665831595659256, -0.00037487028748728335, -0.028047000989317894, 0.00532422075048089, -0.004706114064902067, 0.010445676743984222, -0.007103321608155966, 0.023612331598997116, 0.017424723133444786, -0.008660035207867622, 0.00828066747635603, 0.010537248104810715, -0.008064821362495422, 0.001723503926768899, -0.023285290226340294, -0.014429030939936638, 0.004961205646395683, 0.01404966413974762, -0.025914696976542473, 0.007947086356580257, 0.00910480972379446, -0.00408146670088172, -0.02464577928185463, 0.021375373005867004, 0.00982429925352335, 0.024907412007451057, -0.01820961944758892, -0.0010980387451127172, 0.046596743166446686, 0.0018772130133584142, -0.019243068993091583, 0.0009942033793777227, 0.004843471106141806, -0.014716826379299164, 0.00278475065715611, 0.011237114667892456, -0.02796851098537445, -0.006854770705103874, -0.012126665562391281, 0.012872317805886269, -0.00401278818026185, 0.014913051389157772, -0.020381169393658638, -0.006665087305009365, -0.02511671744287014, 0.013245143927633762, -0.01725466176867485, 0.0002067509776679799, -0.019596273079514503, -0.001158541301265359, -0.02857026644051075, 0.004091277718544006, 0.006913638208061457, -0.012865777127444744, -0.013670297339558601, -0.0026294062845408916, 0.0059750317595899105, 0.01819653809070587, -0.016260458156466484, 0.004451022483408451, -0.02906736731529236, -0.017777927219867706, -0.029930755496025085, -0.002962987869977951, -0.006380562204867601, -0.023664657026529312, -0.0007734509999863803, -0.012165910564363003, -0.006907097529619932, 0.013199358247220516, 0.00690055638551712, -0.007175270467996597, -0.02726210467517376, 0.010589574463665485, -0.03118659183382988, -0.0148868877440691, -0.0025034956634044647, -0.003388140583410859, 0.006400184240192175, -0.015122356824576855, -0.030872631818056107, 0.01928231306374073, -0.01155761443078518, -0.0034404671750962734, 0.03518956899642944, 0.004326747264713049, 0.0026065134443342686, 0.023285290226340294, 0.003878701478242874, 0.029956918209791183, 0.016849132254719734, -0.024580370634794235, 0.012564899399876595, 0.012663011439144611, -0.019609354436397552, -0.017777927219867706, 0.004820578265935183, -0.009961656294763088, -0.0004603096458595246, -0.003813293296843767, 0.002045638859272003, 0.016862213611602783, 0.015109275467693806, 0.033044181764125824, 0.011564155109226704, 0.02833479642868042, 0.02010645531117916, -0.004287502262741327, 0.021466944366693497, 0.014389785937964916, -0.016927620396018028, 0.023520760238170624, 0.022801270708441734, 0.011040890589356422, -0.04034372791647911, 0.013120868243277073, -0.011662268079817295, -0.011701512150466442, -0.006413266062736511, 0.011590318754315376, 0.029198184609413147, -0.022068699821829796, 0.03793670982122421, 0.0008359975181519985, 0.005592394154518843, 0.012022011913359165, -0.012041634880006313, -0.01810496672987938, 0.0023056361824274063, 0.01529241818934679, 0.01760786585509777, 0.027890022844076157, 0.013140491209924221, -0.0007407469674944878, 0.010511084459722042, 0.028308633714914322, -0.0010980387451127172, 0.011210951954126358, -0.007050995249301195, 0.045288581401109695, -0.006609490606933832, 0.036000628024339676, -0.015083111822605133, 0.01929539442062378, 0.019020680338144302, 0.027576062828302383, -0.02930283732712269, 0.03393373265862465, -0.01172113511711359, 0.014952296391129494, -0.0030741814989596605, -0.010393350385129452, 0.0014635067200288177, -0.00217154948040843, 0.008136769756674767, -0.0214146189391613, -0.0014128154143691063, -0.001659731031395495, 0.008332994766533375, -0.015266254544258118, 0.0006520372116938233, -0.015266254544258118, 0.021833229809999466, -0.001916457898914814, 0.019910231232643127, 0.04651825502514839, -0.015985744073987007, -0.007358413189649582, 0.0058867307379841805, -0.010851207189261913, -0.02582312561571598, 0.001978595508262515, -0.014899969100952148, -0.0021061415318399668, -0.008424566127359867, 0.006887475028634071, 0.01440286822617054, -0.009477636776864529, 0.012401379644870758, 0.00879739224910736, -0.021584678441286087, -0.025260616093873978, 0.016352029517292976, -0.018183456733822823, -0.002539470326155424, -0.013513317331671715, 0.005255542229861021, 0.014559847302734852, 0.0001958155626198277, 0.008450728841125965, -0.005353654734790325, -0.018732884898781776, 0.025182126089930534, -0.016953784972429276, 0.009464554488658905, -0.0226050466299057, -0.018628232181072235, 0.0044346703216433525, 0.013356338255107403, 0.02523445151746273, -0.009209463372826576, 0.0047846040688455105, -0.0033080156426876783, -0.002837077248841524, -0.021715495735406876, 0.00198023091070354, -0.03817217797040939, 0.029093531891703606, -0.003662854665890336, -0.009693482890725136, -0.034090712666511536, -0.0017300447216257453, -0.017660191282629967, 0.016365110874176025, -0.005572771653532982, 0.0006299619562923908, -0.0101251769810915, 0.048271190375089645, 0.002791291568428278, -0.0033717884216457605, -0.01529241818934679, 0.02272278070449829, 0.0068678525276482105, -0.02284051477909088, -0.014324378222227097, -0.020184945315122604, 0.00497428746894002, 0.07330942153930664, 0.03819834068417549, 0.0166790708899498, 0.01997563987970352, -0.004673410207033157, 0.005507363472133875, -0.0273667573928833, -0.0017414911417290568, 0.028387123718857765, -0.01644360087811947, -0.0024102891329675913, -0.01047838106751442, 0.0007742685847915709, -0.024449555203318596, 0.013709541410207748, -0.004572027362883091, -0.0022712969221174717, 0.006658546626567841, 0.02069512940943241, -0.00031804697937332094, 0.01530549954622984, -0.028177818283438683, 0.004650517366826534, 0.010589574463665485, -0.006079684477299452, -0.0011242020409554243, 0.020629720762372017, 0.007855514995753765, 0.014429030939936638, -0.040160585194826126, 0.017764845862984657, 0.027314430102705956, 0.013892684131860733, 0.003327638143673539, -0.03555585443973541, 0.046832215040922165, 0.023847799748182297, -0.0020783429499715567, 0.007986331358551979, 0.005307869054377079, -0.020682048052549362, 0.016103478148579597, 0.01231634896248579, -0.0030414776410907507, 0.019216906279325485, 0.007600423414260149, 0.005654531996697187, 0.014834561385214329, 0.004261339083313942, 0.0033341788221150637, -0.003062735078856349, 0.01022982969880104, -0.0196485985070467, -0.0027765745762735605, -0.018955273553729057, -0.010059768334031105, -0.010197125375270844, -0.006972505711019039, 0.02010645531117916, -0.005154159851372242, -0.008601167239248753, -0.028360961005091667, 0.0004276055842638016, 0.005530256312340498, -0.03299185633659363, -0.020773619413375854, -0.02010645531117916, -0.008208719082176685, -0.020629720762372017, 0.022212596610188484, -0.02870108187198639, -0.0017496672226116061, -0.03359360992908478, 0.01915149763226509, 0.02880573645234108, 0.013840357773005962, 0.01609039679169655, -0.0014839466894045472, -0.0011830693110823631, -0.00556950131431222, 0.0021453863009810448, -0.01666598953306675, 0.005680695176124573, -0.0006643012166023254, -0.023010576143860817, 0.014023500494658947, -0.030898794531822205, 0.002876322017982602, -0.007469607051461935, 0.014389785937964916, -0.020891353487968445, -0.025378350168466568, 0.028491776436567307, -0.03443083539605141, -0.00401278818026185, 0.021689333021640778, -0.011773461475968361, 0.009065565653145313, 0.012551818042993546, -0.008973994292318821, 0.002387396292760968, -0.04952702671289444, 0.006017547100782394, -0.026333307847380638, -0.0017414911417290568, 0.012421001680195332, -0.001418538624420762, -5.2019895520061255e-05, -0.0220032911747694, -0.017764845862984657, 0.026738839223980904, -0.023756228387355804, 0.013604888692498207, -0.0038198342081159353, 0.02857026644051075, 0.015868009999394417, 0.0023154474329203367, 0.02430565655231476, -0.002936824457719922, 0.009202922694385052, 0.008960912004113197, -0.029224347323179245, 0.030270878225564957, 0.00613201130181551, -0.01535782590508461, 0.024828922003507614, -0.0035909058060497046, -0.01196314487606287, -0.020642802119255066, 0.002192807151004672, -0.014311296865344048, 0.01089699286967516, -0.018301190808415413, -0.013879602774977684, 0.004320206120610237, -0.01701919175684452, -0.008274126797914505, 0.00945801381021738, -0.007999412715435028, 0.001277093542739749, -0.031134264543652534, -0.029826102778315544, -0.00749577023088932, -0.02059047669172287, 0.016705233603715897, -0.029355164617300034, -0.034718628972768784, 0.022591963410377502, -0.015684867277741432, 0.04235829785466194, -0.007031372748315334, 0.03764891251921654, -0.03592213988304138, 0.0067239548079669476, -0.021637005731463432, -0.052091024816036224, -0.008450728841125965, 0.010269074700772762, 0.01030177902430296, 0.015684867277741432, -0.00784897431731224, 0.020982924848794937, 0.001351495273411274, 0.01725466176867485, -0.007783566135913134, -0.011642645113170147, -0.010700767859816551, -0.02807316556572914, -0.009523422457277775, 0.0064819445833563805, -0.0001550376764498651, -0.007600423414260149, -0.005281705409288406, 0.0022320521529763937, 0.005932516418397427, 0.005337302573025227, -0.00684823002666235, -0.032677896320819855, -0.015501724556088448, -0.025025146082043648, 0.005719940178096294, -0.0014070922043174505, -0.023232964798808098, 0.007175270467996597, -0.01976633444428444, -0.01011209562420845, -0.004352910444140434, -0.02571847289800644, -0.009006697684526443, -0.037544261664152145, 0.02059047669172287, -0.015410153195261955, 0.01499154046177864, -0.009588830173015594, 0.008247964084148407, 0.0008216894930228591, -0.0184320081025362, -0.03623609617352486, 0.00026592487120069563, 0.017529375851154327, 0.009608453139662743, 0.024017861112952232, 0.008947830647230148, 0.019583191722631454, -0.014415949583053589, -0.014952296391129494, -0.027314430102705956, -0.004251527599990368, 0.0021012357901781797, -0.003535308875143528, 0.012421001680195332, 0.005000450648367405, 0.005059318151324987, 0.00749577023088932, -0.007469607051461935, 0.008365698158740997, -0.019073007628321648, 0.03084646910429001, -0.006531000602990389, -0.03288720175623894, 0.006442699581384659, -0.028413286432623863, 0.038381483405828476, 0.034326180815696716, 0.008640412241220474, -0.0064394292421638966, -0.011197869665920734, -0.011341767385601997, -0.011387553066015244, -0.0006569427787326276, -0.0038590789772570133, 0.028047000989317894, 0.01878521218895912, 0.005644720513373613, -0.007822810672223568, 0.009098269045352936, 0.017333151772618294, -0.0042907726019620895, -0.01807880401611328, 0.019086088985204697, 0.029826102778315544, -0.025731554254889488, -0.009902789257466793, 0.0005927610909566283, -0.032076142728328705, 0.03118659183382988, -0.00720143411308527, -0.004539323505014181, 0.0003442102170083672, 0.0002516168460715562, -0.009032861329615116, 0.03395989537239075, 0.007986331358551979, 0.020407333970069885, 0.003669395577162504, -0.014703745022416115, -0.003554931376129389, -0.008313371799886227, 0.002125763799995184, -0.0078031886368989944, -0.014311296865344048, 0.019700925797224045, -0.01677064225077629, 0.017411641776561737, 0.00975889153778553, -0.02297133207321167, -0.008156392723321915, -0.005880190059542656, 0.018902946263551712, 0.023847799748182297, -0.00416976772248745, -0.018902946263551712, -0.009994360618293285, 0.0016891646664589643, 0.01951778307557106, -0.006710872985422611, -0.010955859906971455, -0.006086225621402264, -0.011080135591328144, -0.012237858958542347, 0.01572411134839058, 0.011969685554504395, -0.004581838846206665, -0.0009933857945725322, -0.0002469156461302191, -0.0267911646515131, 0.009444932453334332, 0.006400184240192175, 0.01363105233758688, -0.02809932827949524, -0.026921981945633888, -0.01878521218895912, 0.007502311374992132, 0.006704332306981087, 0.016731396317481995, 0.008764687925577164, 0.021257638931274414, 0.01963551715016365, -0.00591943459585309, 0.00681552616879344, -0.0048859864473342896, 0.019452374428510666, -0.03856462612748146, 0.0013801113236695528, -0.018876783549785614, -0.012689175084233284, -0.0002189127990277484, 0.01641743816435337, -0.02034192532300949, -0.025744635611772537, 0.03325348719954491, 0.014742990024387836, -0.008561923168599606, 0.010373727418482304, 0.026359472423791885, 0.007750862278044224, -0.009785054251551628, 0.018772130832076073, -0.014272051863372326, 0.007646209094673395, -0.02357308566570282, -0.022421903908252716, 0.03751809895038605, -0.012819991447031498, 0.014847642742097378, 0.007822810672223568, 0.024698106572031975, 0.025757716968655586, -0.048637475818395615, 0.020538149401545525, -0.006687980145215988, 0.007417280692607164, -0.019321558997035027, -0.008319912478327751, -0.018994517624378204, -0.008548840880393982, -0.02596702240407467, 0.023403024300932884, -0.018065722659230232, -0.008542300201952457, 0.005916164256632328, 0.0107203908264637, 0.017359314486384392, 0.013840357773005962, -0.018732884898781776, -0.015632539987564087, -0.035294219851493835, -0.02142770029604435, -0.04568102955818176, -0.015161601826548576, -0.0060404399409890175, 0.02786385826766491, 0.0065702456049621105, 0.0013236968079581857, -0.048742130398750305, -0.007822810672223568, -0.024318737909197807, 0.010654982179403305, -0.0033129211515188217, 0.028256308287382126, 0.003597446484491229, -0.023285290226340294, -0.014062745496630669, 0.028831899166107178, 0.0053798179142177105, 0.017934905365109444, -0.02345535159111023, -0.003875431139022112, 0.003878701478242874, -0.022474229335784912, 0.0002882045228034258, 0.0104064317420125, -0.037308789789676666, 0.005415792111307383, 0.015135439112782478, -0.012571440078318119, -0.0172677431255579, -0.005275164730846882, -0.003940839320421219, -0.018366599455475807, 0.01334325596690178, 0.02035500667989254, 0.009902789257466793, 0.02464577928185463, -0.01268263440579176, -0.004477185662835836, 0.014258969575166702, 0.005925975739955902, 0.0014177209232002497, -0.012061256915330887, -0.003708640346303582, 0.025025146082043648, -0.004948124289512634, 0.010628819465637207, -0.02238265797495842, -0.026163248345255852, 0.03131740912795067, -0.013500235974788666, 0.024828922003507614, 0.004176308400928974, -0.024070188403129578, 0.031003449112176895, 0.0014128154143691063, -0.001954067498445511, 0.00970656517893076, 0.029459817335009575, -0.023272208869457245, 0.023232964798808098, 0.027994675561785698, -0.03320116177201271, -0.029198184609413147, 0.0036497730761766434, -0.0008633871912024915, -0.029538307338953018, -0.023036738857626915, -0.01997563987970352, 0.00579842971637845, -0.028360961005091667, 0.0032426074612885714, 0.001981866080313921, -0.025613820180296898, 0.0023595979437232018, -0.02344227023422718, -0.01810496672987938, 0.012290185317397118, -0.006187607999891043, 0.010929696261882782, -0.03163136541843414, -0.004679950885474682, 0.018510498106479645, -0.01749013178050518, 0.005850756075233221, -0.005376547574996948, 0.010262534022331238, -0.02891038917005062, 0.006540812086313963, 0.214120015501976, -0.008542300201952457, -0.011250196024775505, 0.02596702240407467, -0.0003233613970223814, 0.012963889166712761, 0.045183926820755005, 0.010347564704716206, 0.00786205567419529, 0.0007407469674944878, -0.027183614671230316, -0.023664657026529312, -0.020773619413375854, -0.005719940178096294, 0.009680401533842087, -0.024828922003507614, -0.040893156081438065, -0.020289598032832146, -0.008476892486214638, -0.028727246448397636, 0.01679680496454239, -0.006066603120416403, 0.016482846811413765, 0.0009982914198189974, 0.03550352528691292, 0.02522137016057968, -0.015619458630681038, -0.0058867307379841805, 0.02332453615963459, -0.0017791008576750755, -0.00015933008398860693, -0.000968857784755528, -0.0019851364195346832, 0.01904684491455555, -0.01266955304890871, -0.009209463372826576, 0.003002232639119029, -0.008614249527454376, -0.006158174481242895, 0.0076069640927016735, -0.006730495486408472, 0.01763402856886387, -0.024371065199375153, -0.02308906614780426, 0.016129642724990845, 0.014690663665533066, -0.0013048920081928372, -0.003685747506096959, 0.0004983281251043081, 0.01656133495271206, -0.017699437215924263, 0.00749577023088932, 0.019949477165937424, 0.03388140723109245, -0.027209777384996414, 0.004536053165793419, -0.0003129369579255581, -0.005654531996697187, 0.00323116104118526, -0.004964476451277733, -0.015122356824576855, 0.0369425043463707, 0.005847485736012459, 0.009536503814160824, 0.0004549952282104641, 0.013225521892309189, 0.0075938827358186245, -0.006907097529619932, 0.016639824956655502, -0.01677064225077629, 0.012983511202037334, -0.005683965515345335, -0.019740169867873192, -0.009170218370854855, -0.018157294020056725, -0.01273496076464653, 0.0202241912484169, 0.036026790738105774, 0.015070030465722084, 0.036026790738105774, -0.02295825071632862, -0.004578568506985903, -0.021166067570447922, -0.01644360087811947, -0.01011209562420845, -0.021009087562561035, 0.02917202189564705, -0.00257544475607574, -0.011034349910914898, -0.013644133694469929, -0.01493921410292387, 0.01238175667822361, -0.005661072675138712, -0.008012495003640652, -0.010798880830407143, -0.00982429925352335, 0.006920178886502981, 0.023389942944049835, -0.010360646061599255, -0.012597603723406792, -0.02165008708834648, 0.06237318366765976, -0.004846741445362568, 0.0034633600153028965, -0.011439880356192589, -0.008365698158740997, -0.001500298734754324, 0.033645935356616974, 0.021597761660814285, -0.026254819706082344, 0.008437647484242916, -0.026634186506271362, 0.016744477674365044, 0.013460990972816944, -0.021336128935217857, -0.029616795480251312, -0.009091728366911411, -0.027576062828302383, -0.0005874466733075678, 0.00020276516443118453, -0.010099013336002827, -0.028256308287382126, -0.014193561859428883, -0.004392154980450869, 0.003976813517510891, -0.008954371325671673, 0.010805421508848667, -0.0036366914864629507, 0.010269074700772762, -0.0321023054420948, 0.038407646119594574, -0.0005122273578308523, 0.015501724556088448, -0.013003134168684483, -0.00021359839593060315, -0.02809932827949524, 0.016757560893893242, -0.0004697120457421988, -0.017791008576750755, -0.006920178886502981, 0.019125334918498993, 0.025286778807640076, -0.01738547720015049, -0.006442699581384659, 0.015423234552145004, -0.0178694985806942, -0.0020914245396852493, 0.004820578265935183, -0.014442112296819687, -0.014978459104895592, -0.03199765086174011, 0.009359901770949364, -0.0025917966850101948, -0.02820398099720478, 0.037334952503442764, 0.001181434141471982, -0.014088909141719341, -0.006226853001862764, -0.005340572912245989, -0.01107359491288662, -0.011106298305094242, -0.020682048052549362, 0.009889707900583744, -0.0046668690629303455, -0.02689581923186779, 0.00803865771740675, -0.16629360616207123, 0.011335226707160473, 0.021388454362750053, -0.004804226569831371, -0.009942034259438515, -0.018745966255664825, 0.032416265457868576, 0.017450885847210884, -0.03592213988304138, 0.007247219793498516, -0.004820578265935183, -0.006017547100782394, -0.03223312273621559, -0.006017547100782394, -0.014965377748012543, -0.009222544729709625, 0.01244062464684248, 0.012571440078318119, 0.02783769555389881, 0.0033717884216457605, 0.022670453414320946, -0.01320589892566204, 0.00028248131275177, -0.01142025738954544, 0.011642645113170147, 0.018602069467306137, -0.02132304757833481, 0.013735705055296421, -0.013513317331671715, -0.02298441343009472, -0.0019099171040579677, 0.008110607042908669, 0.016038071364164352, 0.02167624980211258, 0.008888963609933853, 0.010615738108754158, -0.006992127746343613, -0.009673860855400562, -0.012519113719463348, 0.019308477640151978, 0.0256007369607687, 0.008051739074289799, 0.007312627509236336, -0.014494439586997032, -0.028125490993261337, 0.0031019800808280706, 0.0020750726107507944, 0.01713692769408226, -0.0050822109915316105, -0.024541126564145088, 0.0131012462079525, -0.020786700770258904, 0.016482846811413765, -0.010628819465637207, 0.012466787360608578, 0.004529512021690607, 0.007188352290540934, 0.023756228387355804, 0.003989895340055227, -0.0047846040688455105, -0.017189253121614456, -0.03900940343737602, -0.001916457898914814, -0.006275909021496773, -0.008398402482271194, 0.005157430190593004, 0.013774950057268143, 0.006606219802051783, -0.01113900262862444, 0.009248708374798298, -0.012407920323312283, -0.02833479642868042, 0.007247219793498516, -0.0028648755978792906, 0.02226492390036583, 0.010380268096923828, -0.008980534970760345, 0.003083992749452591, 0.0214407816529274, -0.008431106805801392, 0.007018291391432285, 0.0243841465562582, -0.018353518098592758, 0.017895661294460297, -0.008293749764561653, 0.01238175667822361, -0.002318717772141099, 0.007927464321255684, 0.01160994078963995, -0.019831741228699684, 0.03346279263496399, -0.010452217422425747, -0.01653517223894596, 0.009614993818104267, 0.03252091631293297, -0.009961656294763088, 0.0016842590412124991, -0.0058442153967916965, -0.001895200228318572, -0.015540968626737595, 0.009098269045352936, 0.021977128461003304, -0.016979947686195374, 0.024004779756069183, 0.01166880875825882, 0.0012599239125847816, -0.0078031886368989944, 0.019099170342087746, 0.02094367891550064, -0.012630308046936989, -0.019844822585582733, 0.026765001937747, 0.008869340643286705, 0.008738524280488491, 0.01797415129840374, 0.029826102778315544, -0.001617215690203011, -0.019557027146220207, 0.01988406851887703, 0.008424566127359867, 0.04091931879520416, -0.01785641722381115, -0.014507520943880081, 0.008346076123416424, 0.008483433164656162, -0.022107943892478943, -0.09434466809034348, -0.020511986687779427, 0.003067640820518136, 0.002390666864812374, 0.03202381357550621, 0.026608021929860115, 0.02010645531117916, 0.0208390261977911, 0.010341024026274681, 0.043169356882572174, -0.021597761660814285, -0.006511378102004528, -0.008228341117501259, 0.026608021929860115, 0.0035614720545709133, -0.03249475359916687, 0.0062628271989524364, -0.01054378878325224, -0.0095299631357193, 0.021597761660814285, 0.0032197146210819483, -0.03764891251921654, 0.019112251698970795, -0.015449397265911102, -0.012041634880006313, -0.014742990024387836, -0.01691453903913498, 0.007691994775086641, 0.01926923170685768, 0.005893271416425705, 0.0036497730761766434, -0.007279923651367426, 0.006112388800829649, -0.009955115616321564, 0.004019328858703375, 0.0069986688904464245, -0.025286778807640076, -0.008228341117501259, 0.020904434844851494, -0.011969685554504395, -0.020067211240530014, 0.020865190774202347, 0.011793083511292934, -0.024344902485609055, 0.012362134642899036, -0.001976960338652134, -0.022696617990732193, 0.018183456733822823, -0.004830389749258757, -0.007325709331780672, -0.016692152246832848, -0.017306989058852196, -0.01606423407793045, -0.026751920580863953, 0.02867491915822029, -0.013722623698413372, -0.009006697684526443, -0.014245888218283653, -0.009333738125860691, -0.005265353713184595, 0.0048729050904512405, -0.00046112723066471517, 0.0038296454586088657, 0.008032117038965225, 0.020420415326952934, -0.0005584218306466937, -0.012591063044965267, -0.002384125953540206, 0.008241423405706882, -0.021571597084403038, 0.0061418223194777966, 0.001726774382404983, -0.012067797593772411, 0.017424723133444786, 0.0026457584463059902, -0.0238216370344162, -0.024855084717273712, -0.01524009183049202, -0.006534270942211151, 0.008718902245163918, -0.004892527125775814, -0.00054983701556921, 0.005494282115250826, -0.013657215051352978, 0.0012975335121154785, 0.016469763591885567, -0.003934298176318407, -0.03275638446211815, -0.011551073752343655, -0.011498747393488884, -0.006913638208061457, 0.02927667461335659, 0.008404943160712719, -0.018510498106479645, -0.02713128738105297, -0.002176455222070217, 0.0022909194231033325, 0.0022843785118311644, 0.017777927219867706, -0.0023579627741128206, 0.00047052965965121984, -0.0028419827576726675, -0.05057355761528015, 0.012937725521624088, -0.001445519388653338, -0.02664726786315441, 0.026817329227924347, 0.023520760238170624, 0.03343662992119789, -0.011472583748400211, -0.008450728841125965, -0.010020524263381958, -0.030270878225564957, 0.020433496683835983, -0.016260458156466484, -0.010151339694857597, -0.006432888563722372, 0.00951688177883625, 0.027654552832245827, 0.006109118461608887, 0.010013982653617859, 0.011583778075873852, -0.0018886594334617257, -0.002063626190647483, 0.01701919175684452, 0.015619458630681038, -0.007037913426756859, 0.021990209817886353, -0.022317249327898026, 0.014258969575166702, -0.02489433065056801, -0.023900127038359642, 0.014193561859428883, 0.009189840406179428, -0.0014438842190429568, 0.02477659471333027, -0.02178090438246727, -0.020184945315122604, 0.006459051743149757, 0.039584994316101074, -0.0027733042370527983, 0.012041634880006313, -0.025208288803696632, -0.012774205766618252, 0.020394250750541687, -0.014128154143691063, -0.007332250010222197, -0.00562182767316699, -0.0381198525428772, 0.021846311166882515, 0.005566230975091457, -0.009902789257466793, 0.010556870140135288, 0.008411483839154243, -0.029695285484194756, 3.4492561098886654e-05, -0.0006916908314451575, -0.009693482890725136, -0.010936237871646881, 0.005873648915439844, 0.013310552574694157, -0.04227980971336365, 0.03950650244951248, -0.017463967204093933, 0.004882716108113527, -0.012571440078318119, 0.008823554962873459, -0.0035451201256364584, -0.012996593490242958, -0.0013253319775685668, -0.0027814803179353476, -0.022068699821829796, -0.01363105233758688, -0.0010350834345445037, 0.0208390261977911, 0.04753861948847771, 0.006027358118444681, -0.007188352290540934, -0.009510340169072151, 0.007554637733846903, -0.008947830647230148, 0.02786385826766491, 0.03937568888068199, -5.585240069194697e-05, 0.0003029213403351605, -0.0015068395296111703, 0.04330017417669296, 0.02356000430881977, -0.016103478148579597, -0.014285133220255375, 0.004516430664807558, 0.005438684951514006, 0.011564155109226704, -0.012172451242804527, -0.01106705330312252, -0.009144054725766182, -0.0011569061316549778, 0.026385635137557983, 0.003087263321503997, -0.017699437215924263, 0.026097839698195457, 0.010609196498990059, 0.008575004525482655, 0.006030628457665443, 0.00028840891900472343, -0.020394250750541687, -0.028727246448397636, -0.010641900822520256, -0.011439880356192589, -0.038512300699949265, 0.02940749004483223, 0.02261812798678875, -0.02272278070449829, 0.03511107712984085, -0.0032524187117815018, 0.012571440078318119, -0.01160994078963995, -0.004006247501820326, -0.0031379545107483864, 0.0010775987757369876, -0.03270405903458595, 0.03655005618929863, -0.013212439604103565, 0.0075938827358186245, 0.021859392523765564, -0.0013523128582164645, 0.0036137986462563276, 0.011524911038577557, 0.004997180309146643, 0.008424566127359867, -0.002472426975145936, 0.0032017275225371122, 0.0061189294792711735, 0.0002800285001285374, -0.005680695176124573, -0.012473328039050102, 0.007600423414260149, -0.012623767368495464, -0.012702256441116333, 0.01208742056041956, -0.003007138380780816, 0.06096036732196808, 0.012571440078318119, 0.00532422075048089, 0.027157451957464218, 0.01333017461001873, 0.024933574721217155, 0.025875451043248177, 0.0006336411461234093, -0.01775176264345646, -0.01844508945941925, 0.028491776436567307, -0.012211696244776249, 0.004542593844234943, -0.051855556666851044, -0.0014438842190429568, -0.004804226569831371, 0.007162189111113548, 0.019583191722631454, -0.0006193331209942698, 0.004316935781389475, 0.007449984550476074, -0.024187922477722168, 0.021362291648983955, 0.008555381558835506, -0.018235784024000168, 0.010504543781280518, 0.025443758815526962, 0.003066005650907755, -0.013892684131860733, -0.0369163416326046, 0.006939801387488842, 0.013997337780892849, -0.03537271171808243, 0.004179578740149736, 0.005232649389654398, -0.022291086614131927, -0.004287502262741327, -0.0030055029783397913, 0.008823554962873459, 0.007901300676167011, 0.01535782590508461, 0.008418024517595768, -0.03259940817952156, -0.015567132271826267, -0.012839613482356071, 0.010151339694857597, -0.027759205549955368, 0.0033979518339037895, -0.03484944626688957]} +{"id": "test:10000034", "text": "\"The Champion work \u201cUncle Ian III (Into the Unknown\u2019, with artist Louise Jenkins, and her cousin David whose father sat for the portrait.\\nTenterfield Show 2015 was judged by Tanya Robertson-Cuninghame.\\nI graduated from the National Art School in 1981, and have been a practicing artist since then, updating my qualifications regularly and teaching Visual Arts for 27 years in various establishments. These include: TAFE, New England Girls School, regional High Schools, and G.I Correctional Centre. I also have a Graduate Diploma in Education from UNE.\\nMy partner and I had an exhibition of our combined work of paintings & leadlights at Maitland Regional Art Gallery in October 2013.\\nThe following show results are those of Borderline members only. Congratulations!!!\"", "vector_field": [-0.016316724941134453, -0.005927074700593948, -0.005436605773866177, -0.01526670716702938, 0.0021846590097993612, 0.04285730794072151, -0.013180487789213657, -0.018098993226885796, -0.023805011063814163, -0.02214708738029003, 0.005906350445002317, -0.0027822020929306746, -0.004269151482731104, -0.004210433457046747, 0.0020205937325954437, 0.008455406874418259, 0.014631170779466629, 0.0033469321206212044, -0.013746945187449455, -0.03252292424440384, -0.02343197911977768, 0.027728762477636337, -0.0015793443890288472, -0.01885887421667576, -0.014009449630975723, -0.005826908629387617, 0.008559027686715126, -0.030450519174337387, 2.1830399418831803e-05, -0.011757437139749527, -0.014769330620765686, 0.010935383848845959, 0.00884916353970766, -0.012779823504388332, -0.03943093493580818, -0.023031312972307205, -0.00034129898995161057, -0.006389911752194166, 0.029814980924129486, 0.02538003772497177, 0.005830362439155579, -0.00259223161265254, -0.011515656486153603, -0.016496334224939346, -0.015004202723503113, 0.014617354609072208, 0.013215027749538422, -0.012379158288240433, -0.01142585277557373, 0.025905046612024307, 0.03699931502342224, 0.01312522403895855, -0.004193163476884365, -0.027770210057497025, -0.002029228722676635, -0.022768808528780937, 0.009560689330101013, 0.03622561693191528, 0.005775098223239183, -0.003231222741305828, 0.026139918714761734, -0.012386065907776356, -0.01291798334568739, -0.014520642347633839, -0.01735292747616768, -0.01648251712322235, -0.014396297745406628, 0.0006005653413012624, -0.0012451692018657923, 0.02503463812172413, 0.024412916973233223, 0.02316947467625141, 0.010362018831074238, 0.008282707072794437, 0.0348992794752121, -0.013049235567450523, -0.028433378785848618, 0.007294861134141684, 0.0015966143691912293, -0.014410113915801048, 0.011999217793345451, -0.03492690995335579, 0.0005876127979718149, 0.021953662857413292, 0.010817947797477245, 0.00414480734616518, 0.013360096141695976, 0.019701652228832245, -0.006407181732356548, -0.01565355621278286, 0.0052880835719406605, 0.025587277486920357, 0.005087750963866711, 0.004390041809529066, -0.009899182245135307, -0.006341555621474981, -0.01978454738855362, 0.0115225650370121, -0.00217602401971817, -0.028433378785848618, -0.009767930023372173, -0.005543679930269718, 0.021815503016114235, -0.02124904654920101, -0.018389130011200905, -0.027134673669934273, -0.01177816092967987, 0.001361741917207837, 0.01776740886271, -0.0029393592849373817, -0.010168594308197498, 0.014838410541415215, 0.014879859052598476, -0.018637817353010178, 0.0006579881883226335, -0.02669255994260311, 0.017048975452780724, -0.0018150802934542298, -0.011999217793345451, -0.004987584892660379, 0.013746945187449455, 0.014838410541415215, 0.008393235504627228, -0.01434103399515152, 0.03224660083651543, 0.04277441278100014, -0.03920987993478775, -0.012116653844714165, 0.00036072777584195137, 0.013111407868564129, 0.01565355621278286, -0.002645768690854311, 0.024993188679218292, 0.013788392767310143, 0.007070350926369429, 0.008586659096181393, -0.022989865392446518, -0.00978174526244402, -0.002692397916689515, -0.027010329067707062, -0.004320961888879538, 0.010154778137803078, -0.003462641267105937, -0.011640001088380814, 0.012766007333993912, 0.022478671744465828, -0.016302909702062607, 0.0038235848769545555, 0.015225259587168694, 0.015004202723503113, -0.005709472112357616, 0.002452344400808215, 0.013643324375152588, 0.008883704431355, 0.03473348543047905, -0.009622861631214619, -0.0010802404722198844, 0.00902877189218998, 0.0022727360483258963, -0.011287692002952099, 0.009740297682583332, 0.020185211673378944, -0.008441591635346413, 0.0011467300355434418, 0.02307276241481304, 0.01604040525853634, 0.01862400211393833, 0.0017390921711921692, -0.021843135356903076, -0.009305092506110668, 0.01648251712322235, 0.011494932696223259, -0.04244282841682434, 0.021884582936763763, 0.008489947766065598, 0.016054220497608185, -0.004075727425515652, 0.0015378962270915508, -0.010534718632698059, -0.0158469807356596, -0.010555443353950977, 0.027839289978146553, 0.02474450133740902, 0.03451243042945862, -0.03630851209163666, -0.0026164096780121326, -0.004614552482962608, 0.012869627214968204, 0.0026958519592881203, -0.0079234903678298, -0.016109485179185867, 0.036059826612472534, 0.012102837674319744, -0.009926813654601574, -0.6295686364173889, -0.005187917500734329, 0.004289875738322735, 0.011550196446478367, 0.010728143155574799, 0.013636416755616665, 0.008614291436970234, -0.008006386458873749, -0.014493010006844997, -0.0015568933449685574, -0.002291733166202903, 0.029953142628073692, -0.0018997034057974815, 0.010513994842767715, 0.002483430551365018, -0.039099350571632385, -0.012952523306012154, -0.011854149401187897, 0.024288572371006012, -0.013691680505871773, 0.006528071593493223, 0.05330222472548485, 0.028626803308725357, -0.016924630850553513, -0.001923881471157074, 0.006486623547971249, 0.0018599823815748096, -0.011356772854924202, -0.005395157728344202, 0.0030585224740207195, -0.02973208576440811, 0.01939769834280014, 0.006051418837159872, -0.0020050506573170424, 0.04901925474405289, -0.009912997484207153, -0.006973638664931059, 0.0048943269066512585, -0.007509009446948767, 0.02109706960618496, -0.04219413921236992, -0.011757437139749527, 0.01609566994011402, -0.0028530091512948275, 0.0064900778234004974, 0.04733370244503021, 0.01820952072739601, 0.0005396884516812861, 0.013816025108098984, -0.0308926310390234, -0.015487764030694962, -0.005985792726278305, -0.013546612113714218, -0.003972107078880072, 0.0038892109878361225, -0.010893935337662697, 0.02522806078195572, -0.021856950595974922, 0.019259538501501083, 0.01246896293014288, -0.035451918840408325, 0.009097852744162083, -0.0034695493523031473, -0.006783668417483568, 0.010831763967871666, -0.008586659096181393, -0.012648571282625198, 0.012517319060862064, 0.005205187480896711, -0.006224119104444981, 0.002576688537374139, 0.03379399701952934, -0.01966020278632641, 0.009312001056969166, 0.017587799578905106, 0.016758838668465614, 0.015031835064291954, -0.013173580169677734, 0.003816676791757345, 0.029759718105196953, 0.00034108312684111297, -0.014506826177239418, 0.007916582748293877, 0.028115611523389816, 0.0224095918238163, 0.006476261653006077, -0.008994231931865215, 0.011405128985643387, 0.008724819868803024, -0.009574505500495434, 0.01961875520646572, 0.0018306232523173094, 0.005405519623309374, -0.004182801581919193, -0.007557365577667952, 0.025407670065760612, -0.015612108632922173, 0.008759359829127789, -0.001689872588030994, -0.04103359580039978, 0.013615692965686321, -0.021981295198202133, 0.007115252781659365, -0.01927335560321808, 0.01830623298883438, -0.0018150802934542298, -0.0008850891026668251, 0.02435765229165554, 0.027839289978146553, -0.03288213908672333, -0.01575026847422123, 0.009415620937943459, -0.015156179666519165, 0.013332463800907135, 0.009415620937943459, -0.02122141420841217, 0.015128547325730324, 0.025518197566270828, 0.028378115966916084, -0.012351525947451591, 0.031058423221111298, 0.007743882015347481, 0.009546873159706593, -0.012061390094459057, -0.01367095671594143, 0.015695003792643547, 0.0015802078414708376, -0.02423330768942833, 0.00032014321186579764, 0.008904428221285343, 0.011729804798960686, -0.007764605805277824, 0.05617595836520195, -0.01227553840726614, 0.03299266844987869, 0.018927954137325287, 0.028239954262971878, 0.0025386945344507694, -0.007336309179663658, -0.03194265067577362, -0.0013738309498876333, 0.0026561308186501265, -0.011018279939889908, -0.004490207880735397, -0.012911075726151466, -0.03597692772746086, 0.006058326922357082, 0.004877056926488876, -0.03111368790268898, -0.011501840315759182, -0.010486362501978874, 0.018098993226885796, -0.019646387547254562, 0.012786731123924255, 0.024509627372026443, -0.013387728482484818, -0.009761021472513676, 0.0011087360326200724, -0.023280002176761627, -0.012013033963739872, -0.007916582748293877, 0.01725621521472931, -0.038491446524858475, -0.009629769250750542, 0.0066213300451636314, -0.008683371357619762, 0.006866564508527517, 0.013304832391440868, -0.013083775527775288, -0.0284886434674263, -0.005146469455212355, -0.0010241128038614988, 0.013235751539468765, 0.02148391865193844, 0.024081330746412277, -0.0011173710227012634, 0.008897519670426846, -0.0060755969025194645, 0.002051679650321603, 0.004621460568159819, -0.017587799578905106, 0.005080843344330788, -0.015045651234686375, -0.016026588156819344, 0.008206718601286411, 0.0019825994968414307, -0.007481377571821213, 0.0026699467562139034, -0.004683632403612137, -0.0008086691959761083, -0.022865520790219307, 0.02323855459690094, 0.0016087034018710256, 0.024951741099357605, 0.009837009944021702, 0.0273695457726717, 0.01869308203458786, 0.0025456026196479797, 0.022395776584744453, 0.01750490441918373, 0.005975430831313133, -0.0020085046999156475, 0.017435822635889053, 0.008061650209128857, 0.030809735879302025, -0.016634494066238403, 0.018997034057974815, -0.014990387484431267, 0.04219413921236992, 0.0036888786125928164, 0.0009576232405379415, -0.016634494066238403, -0.036032192409038544, 0.010804131627082825, 0.0014178695855662227, 0.008441591635346413, -0.005923620890825987, 0.005467691924422979, 7.38833550713025e-05, -0.006790576037019491, -0.01930098608136177, -0.000793557905126363, -5.623553806799464e-05, 0.02012994885444641, -0.0024903384037315845, -0.0025248785968869925, 0.012247906066477299, -0.00942943710833788, -0.01171598955988884, -0.013843657448887825, -0.012980155646800995, -0.009070220403373241, 0.022036559879779816, 0.0009023591410368681, 0.00120458472520113, 0.00942943710833788, -0.004003193229436874, -0.020889829844236374, 0.040121737867593765, -0.007964937947690487, 0.02269972860813141, 0.016855550929903984, 0.016717391088604927, -0.0020171396899968386, 0.03976251929998398, 0.011833425611257553, 0.0187207143753767, -0.015584476292133331, -0.009740297682583332, 0.013069959357380867, 0.015432500280439854, 0.012669295072555542, -0.024799764156341553, -0.0005211231764405966, 0.019604939967393875, -0.00405845744535327, -0.008241258561611176, 0.0011579556157812476, 0.012565674260258675, 0.008310339413583279, 0.0046974485740065575, -0.0007987389690242708, 0.00838632695376873, -0.010859395377337933, 0.009947538375854492, 0.019770732149481773, -0.025849781930446625, -0.02707940898835659, -0.017104238271713257, -0.018195705488324165, -0.004538564011454582, -0.0105623509734869, 0.03415321186184883, 0.0018444393062964082, 0.031058423221111298, -0.011943953111767769, 0.014866042882204056, 0.006524617783725262, 0.01740819215774536, 0.023805011063814163, -0.03570060804486275, -0.04382443055510521, 0.03279924392700195, 0.021746423095464706, -0.006959822494536638, -0.025656357407569885, 0.00024026930623222142, 0.01715950295329094, -0.02605702355504036, 0.019066113978624344, 0.020696405321359634, 0.010997556149959564, 0.007419205270707607, 0.01054853480309248, -0.010023525916039944, 0.02007468417286873, 0.006631691940128803, -0.02851627580821514, -0.006324285641312599, -0.014935122802853584, -0.0033952880185097456, 0.005395157728344202, 0.0019998697098344564, -0.013788392767310143, 0.018195705488324165, -0.005526409950107336, -0.016910813748836517, 0.009740297682583332, -0.013926553539931774, -0.009491609409451485, -0.020862197503447533, -0.004915050696581602, -0.00414480734616518, 0.018610185012221336, 0.010728143155574799, 0.006023786962032318, 0.004365863744169474, -0.0070599885657429695, 0.02778402529656887, -0.002163934987038374, -0.027521520853042603, -0.020654957741498947, -0.03517559915781021, 0.0017520446563139558, 0.1453445851802826, 0.0176016166806221, -0.00804783497005701, 0.014285770244896412, 0.00037260091630741954, -0.01898321881890297, -0.0017166411271318793, -0.04559288173913956, 0.014700250700116158, 0.01074195932596922, -0.004179347772151232, 0.006465899758040905, 0.01414760947227478, 0.004231157712638378, -0.009657401591539383, 0.006894196383655071, 0.004877056926488876, 0.0068631102330982685, 0.009450160898268223, 0.0027908370830118656, -7.857863965909928e-05, -0.011253152042627335, 0.001729593612253666, 0.0182509683072567, -0.005630030296742916, -0.031003160402178764, 0.025849781930446625, 0.011819609440863132, 0.001932516461238265, -0.020654957741498947, -0.011764345690608025, 0.013228843919932842, 0.010507087223231792, 0.010513994842767715, -0.01910756342113018, 0.0006325148860923946, 0.014099253341555595, 0.009256736375391483, 0.020572060719132423, -0.03570060804486275, 0.021870767697691917, 0.03009130246937275, 0.007833685725927353, -0.017836488783359528, -0.00077931018313393, 0.002067222725600004, -0.0035645344760268927, 0.01130150817334652, -0.02022666111588478, -0.015639740973711014, 0.02448199689388275, 0.006369187496602535, 0.01085248775780201, -0.005892534740269184, -0.00867646373808384, 0.03752432391047478, -0.0212904941290617, -0.003654338652268052, 0.006932190619409084, 0.029234709218144417, -0.011128808371722698, -0.036087457090616226, 0.00311724073253572, -0.015059467405080795, -0.008324154652655125, -0.025532014667987823, 0.0019653295166790485, -0.00500830914825201, -0.004034279379993677, 0.010244582779705524, 0.006134314928203821, -0.01454827468842268, -0.00813073106110096, 0.003972107078880072, 0.006324285641312599, -0.012634755112230778, 0.02906891703605652, -0.011867965571582317, 0.0017909022280946374, 0.020917462185025215, -0.005446967668831348, 0.00025883459602482617, 0.00020572924404405057, -0.02061350829899311, 0.006569519639015198, 0.010569259524345398, 0.024730684235692024, 0.012786731123924255, -0.0200056042522192, 0.015142363496124744, -0.014976571314036846, -0.002645768690854311, 0.009954445995390415, -0.004089543595910072, 0.0030757924541831017, 0.01368477288633585, 0.001994688529521227, 0.010569259524345398, 0.00614467728883028, -0.001641516457311809, 0.02973208576440811, -0.025628726929426193, -0.025780702009797096, 0.00043822702718898654, 0.028543908149003983, -0.0033072109799832106, 0.006984000559896231, -0.012282446026802063, -0.030975528061389923, -0.011626184917986393, 0.013615692965686321, -0.0040860893204808235, 0.013608784414827824, 0.002562872599810362, -0.009968262165784836, 0.013857472687959671, -0.014589722268283367, 0.0010215223301202059, 0.00141614256426692, -0.0032329498790204525, -0.016220012679696083, -0.00784059427678585, 0.03630851209163666, 0.002583596622571349, -0.03279924392700195, 0.03815986216068268, 0.012240998446941376, -0.013705496676266193, 0.008648831397294998, -0.00804783497005701, 0.012026850134134293, -0.0020119587425142527, -0.00555404182523489, -0.01818188838660717, -0.006227573379874229, -0.020599693059921265, -0.012095930054783821, 0.0022744631860405207, 0.007764605805277824, 0.0002543012087699026, -0.03191501647233963, -0.011660724878311157, -0.009595229290425777, -0.024247122928500175, 0.012648571282625198, -0.018485842272639275, 0.021953662857413292, 0.002238196088001132, 0.004759620409458876, 0.013518980704247952, -0.021304311230778694, -0.013726220466196537, 0.004155169706791639, 0.015335788019001484, -0.006984000559896231, -0.01866544969379902, -0.01534960325807333, -0.008724819868803024, -0.0014886766439303756, 0.01862400211393833, 0.016703573986887932, -0.02231287956237793, 0.01019622664898634, -0.01074195932596922, -0.00023897405480965972, 0.027825474739074707, 0.005232819356024265, 0.002115578856319189, -0.0345676951110363, 0.0032277689315378666, -0.0053537096828222275, 0.023777378723025322, -0.0010197953088209033, -0.0029894423205405474, 0.013802208937704563, 0.019508227705955505, -0.016883183270692825, -0.005215549375861883, -0.005035941023379564, -0.03033999167382717, -0.030809735879302025, -0.00781296193599701, -0.008324154652655125, 0.003764866851270199, 0.0031034245621412992, 0.00894587580114603, 0.04061911255121231, 0.02170497551560402, -0.002322819083929062, -0.011916321702301502, 0.024302387610077858, -0.005478053819388151, 0.03854671120643616, -0.017076605930924416, -0.011557104997336864, 0.021069437265396118, -0.010279122740030289, -0.030008405447006226, 0.02275499328970909, 0.015487764030694962, -0.01074195932596922, 0.0020965817384421825, 0.006186125334352255, -0.0009697122732177377, -0.017297662794589996, 0.006096321158111095, -0.008973508141934872, -0.0069943624548614025, -0.015142363496124744, -0.0064693535678088665, -0.006541887763887644, -0.016302909702062607, -0.006151584908366203, -0.010258398950099945, 0.019895076751708984, -0.002511062426492572, -0.015612108632922173, 0.007232688833028078, 0.011370588093996048, -0.007412297185510397, -0.004510932136327028, -0.007688617799431086, 0.02061350829899311, 0.014410113915801048, 0.023128025233745575, 0.014838410541415215, 0.00955378171056509, -0.013767668977379799, 0.025200430303812027, 0.02275499328970909, -0.006393365561962128, 0.032661084085702896, 0.03429137542843819, 0.001313385902903974, 0.004030825104564428, 0.02973208576440811, 0.008103098720312119, -0.03500980883836746, -0.04653237387537956, 0.012772914953529835, -0.00840014312416315, -0.005319169722497463, -0.02362540178000927, -0.018582552671432495, -0.010327478870749474, 0.003989377059042454, -0.018554922193288803, -0.01881742663681507, -0.02596031129360199, -0.014368666335940361, -0.03293740376830101, 0.0020706767681986094, -0.019439147785305977, 0.011695264838635921, -0.017822671681642532, -0.027811657637357712, 0.007135977037250996, -0.009408713318407536, 0.0034816383849829435, 0.016054220497608185, 0.010603799484670162, 0.026457687839865685, 0.0038132229819893837, -0.005250089336186647, 0.029345236718654633, 0.0023193652741611004, 0.004196617752313614, -0.01339463610202074, -0.011791977100074291, 0.03246765956282616, -0.013111407868564129, 0.024026067927479744, -0.0029928963631391525, -0.009229104965925217, 0.0035179052501916885, 0.004877056926488876, -0.003326208097860217, -0.008545211516320705, -0.0022071099374443293, 0.00971266534179449, 0.0029169083572924137, -0.0017313206335529685, -0.03337951749563217, 0.007888950407505035, -0.01186105702072382, -0.023777378723025322, -0.0012840267736464739, -0.010665970854461193, -0.002574961632490158, -0.04349284619092941, -0.0029393592849373817, -0.012828178703784943, -0.007509009446948767, -0.0016501514473930001, 0.0062068491242825985, -0.0037614128086715937, 0.008351786993443966, 0.041337545961141586, -0.018168073147535324, 0.004832154605537653, -0.0030757924541831017, 0.0014031900791451335, -0.03774537891149521, -0.008455406874418259, -0.01738055981695652, -0.0222437996417284, 0.037054579704999924, -0.00659369770437479, 0.002440255368128419, -0.016717391088604927, 0.015695003792643547, -0.00047794810961931944, -0.02522806078195572, -0.01283508725464344, -0.002583596622571349, 0.014866042882204056, 0.00613776920363307, -0.020309556275606155, -0.02234051190316677, 0.024122780188918114, 0.010417282581329346, 0.0039099352434277534, 0.0041137211956083775, -0.018734529614448547, 0.008441591635346413, -0.03354530781507492, 0.009912997484207153, 0.01434103399515152, -0.049378473311662674, -0.00680093839764595, 0.003702694782987237, 0.01554302778095007, 0.0027649318799376488, -0.03907172009348869, -0.020654957741498947, -0.008040926419198513, -0.0005370979779399931, 0.012420606799423695, -0.0007266365573741496, -0.03699931502342224, 0.0048079765401780605, 0.017325295135378838, 0.007695525884628296, 0.013422268442809582, -0.00613776920363307, -0.012966339476406574, -0.0007693798979744315, -0.01628909260034561, -0.03639141097664833, 0.013118315488100052, -0.013809116557240486, 0.025145165622234344, -0.016634494066238403, -0.005073935259133577, -0.026609664782881737, -0.0051291994750499725, -0.04744422808289528, 0.005557496100664139, 0.0019480595365166664, 0.030450519174337387, -0.007467561401426792, -0.004604190122336149, 0.008745543658733368, 0.022188536822795868, -0.018168073147535324, 0.005557496100664139, -0.016150932759046555, -0.029262341558933258, 0.005229365546256304, 0.0031327835749834776, 0.008158362470567226, 0.00037130567943677306, -0.027259016409516335, -0.0038201308343559504, 0.01130150817334652, 0.02279644086956978, 0.01198540162295103, -0.006037603132426739, -0.03697168454527855, -0.001156228594481945, -0.010182410478591919, -0.0018288963474333286, -0.002561145694926381, 0.004168985411524773, 0.03274397924542427, -0.009609045460820198, 0.0009239466744475067, -0.023252369835972786, -0.009912997484207153, -0.011501840315759182, 0.010569259524345398, 0.011854149401187897, 0.009194565005600452, 0.02698269672691822, 0.0033642021007835865, -0.010603799484670162, -0.0017503176350146532, 0.01546013168990612, 0.036253251135349274, 0.009823193773627281, -0.02316947467625141, 0.017131870612502098, 0.009650493040680885, -0.025048453360795975, -0.0049392287619411945, -0.02637479081749916, -0.029814980924129486, -0.010182410478591919, -0.013657140545547009, -0.019701652228832245, 0.03495454415678978, -0.01253113429993391, 0.010348202660679817, -0.02265828102827072, -0.0076057217083871365, -0.003516178345307708, 0.0031725047156214714, -0.012572582811117172, 0.002949721412733197, 0.008517579175531864, -0.00263713370077312, 0.0028961843345314264, -0.019093746319413185, -0.01907993108034134, -0.004279513843357563, -0.013546612113714218, -0.006559157744050026, -0.027093224227428436, -0.03459532558917999, 0.013325556181371212, -0.021677343174815178, -0.009878457523882389, -0.00555404182523489, 0.008331063203513622, -0.00840014312416315, 0.02462015673518181, 0.22746703028678894, -0.012462054379284382, 0.0027735671028494835, 0.012772914953529835, -0.012841994874179363, 0.028930755332112312, 0.055650949478149414, 0.011950861662626266, -0.0015223532682284713, 0.0243300199508667, -0.014216689392924309, 0.021276678889989853, 0.008552119135856628, 0.0009317181538790464, 0.0026734007988125086, -0.0265682153403759, -0.028184691444039345, -0.021262861788272858, -0.03208081051707268, -0.002177750924602151, 0.023404346778988838, -0.009463977068662643, 0.011266968213021755, -0.00576819060370326, 0.03219133988022804, -0.0017166411271318793, -0.004030825104564428, 0.020309556275606155, -0.002440255368128419, -0.0038581250701099634, -0.0188865065574646, -0.0038684869650751352, -0.01228935457766056, -0.0012935253325849771, -0.016399621963500977, -0.0017606796463951468, -0.008807715959846973, -0.010313662700355053, -0.014921306632459164, 0.01131532434374094, 0.01703515835106373, 0.0021224869415163994, -0.017808856442570686, 0.008724819868803024, -0.003414285136386752, 0.031417641788721085, -0.007944214157760143, -0.024716868996620178, -0.0024471634533256292, 0.019867444410920143, -0.030782103538513184, -0.012489686720073223, 0.026816904544830322, 0.026208998635411263, -0.00490468880161643, -0.004887418821454048, 0.03069920651614666, 0.0065349796786904335, -0.006704226136207581, -0.018126625567674637, -0.01881742663681507, 0.018264785408973694, -0.01064524706453085, 0.02688598446547985, -0.0024627065286040306, 0.03332425281405449, -0.0013056143652647734, 0.03252292424440384, -0.009864641353487968, -0.01606803759932518, -0.01347753219306469, -0.016427254304289818, -0.02948339655995369, -0.008351786993443966, -0.013118315488100052, -0.029842613264918327, 0.015764083713293076, -0.019190458580851555, 0.027410993352532387, 0.021497733891010284, 0.006269021425396204, 0.013484440743923187, 0.03114132024347782, -0.0004999674274586141, -0.016247645020484924, -0.04404548928141594, 0.014962755143642426, 0.005574766080826521, 0.014769330620765686, -0.028115611523389816, -0.007522825617343187, -0.0032467658165842295, -3.926233694073744e-05, -0.018029913306236267, -0.004690540488809347, -0.005087750963866711, 0.021787870675325394, -0.004110267385840416, -0.012558766640722752, 0.020958909764885902, -0.00679403031244874, -0.0036888786125928164, 0.011529472656548023, 0.02506226859986782, -0.019383883103728294, 0.0019998697098344564, 0.004814884625375271, -0.007419205270707607, 0.02778402529656887, -0.03589403256773949, 0.002291733166202903, -0.02644387260079384, -0.002951448317617178, 0.002177750924602151, -0.0037924989592283964, 0.03771774843335152, -0.018969401717185974, -0.013933461159467697, 0.03069920651614666, -0.009077128022909164, -4.347190770204179e-05, -0.01587461307644844, 0.005298445466905832, -0.01019622664898634, -0.0021017626859247684, -0.013905828818678856, -0.016026588156819344, 0.015335788019001484, -0.0180851761251688, -0.013760760426521301, 0.001193359144963324, -0.009450160898268223, 0.012386065907776356, 0.0003102129267062992, -0.00240916945040226, -0.016593046486377716, -0.007868226617574692, -0.015833165496587753, -0.0020309556275606155, 0.0073432172648608685, -0.000201195856789127, -0.0024368013255298138, -0.030561046674847603, -0.004276059567928314, -0.011612368747591972, -0.03796643763780594, 0.01414760947227478, -0.020185211673378944, -0.003730326658114791, -0.01750490441918373, -0.0004952181479893625, -0.008158362470567226, -0.016178565099835396, -0.015515396371483803, 0.04462575912475586, -9.190895070787519e-05, -0.016924630850553513, -0.019259538501501083, -0.022133272141218185, 0.028543908149003983, -0.04238756373524666, 0.024979373440146446, 0.037496693432331085, -0.019356250762939453, 0.0021846590097993612, -0.0006631691940128803, -0.17806093394756317, 0.030036037787795067, 0.0451231375336647, -0.020116131752729416, 0.02448199689388275, -0.002702759811654687, 0.009995894506573677, 0.006693864241242409, -0.03150053694844246, -0.0022969141136854887, 0.0009403532021678984, 0.016178565099835396, -0.029372869059443474, -0.01453445851802826, -0.011812700890004635, 0.0015543027548119426, -0.0031155135948210955, 0.012061390094459057, 0.035230863839387894, 0.020627325400710106, 0.0156811885535717, 0.005118837114423513, 0.007695525884628296, -0.010134054347872734, 0.009291276335716248, 0.02122141420841217, 0.01655159704387188, -0.00050817069131881, -0.00894587580114603, -0.0015370327746495605, -0.0075780898332595825, 0.006503893528133631, -0.009312001056969166, -0.03998357802629471, 0.02644387260079384, 0.0021052167285233736, -0.01001661829650402, -0.024758316576480865, -0.026360975578427315, 0.01425813790410757, 0.02279644086956978, 0.022202352061867714, 0.015833165496587753, -0.002934178337454796, -0.012807454913854599, 0.026070838794112206, -0.004621460568159819, -0.024274755269289017, 0.016150932759046555, -0.0069494605995714664, -0.015377235598862171, -0.035672977566719055, 0.03315845876932144, -0.0065557039342820644, 0.001436003134585917, -0.003153507597744465, 0.011405128985643387, 0.00990608986467123, -0.0026423148810863495, -0.0011173710227012634, -0.013014695607125759, 0.002714848844334483, 0.025007005780935287, 0.009215288795530796, -0.010182410478591919, -0.03053341433405876, -0.004597282502800226, 0.019356250762939453, -0.000565161753911525, 0.01367095671594143, -0.006103229243308306, -0.030257094651460648, 0.0070530809462070465, -0.025490565225481987, 0.0012270357692614198, -0.01786412112414837, -0.0014688161900267005, 0.019535860046744347, 0.0025162436068058014, 0.009767930023372173, -0.018706897273659706, 0.03719273954629898, -0.01554302778095007, 0.003685424569994211, -0.01187487319111824, 0.007909674197435379, 0.007063442841172218, 0.020917462185025215, -0.018126625567674637, 0.012462054379284382, -0.005909804720431566, -0.045952100306749344, -0.03774537891149521, -0.010831763967871666, 0.019287170842289925, 0.03371110185980797, 0.005191371310502291, 0.007108344696462154, -0.0006286291172727942, -0.022202352061867714, 0.015252891927957535, -0.015819348394870758, -0.0006333783967420459, 0.04501260817050934, 0.029814980924129486, 0.011045912280678749, 0.0033003028947860003, 0.010058065876364708, 0.01971546746790409, 0.0036405224818736315, -0.006096321158111095, 0.011405128985643387, -0.00867646373808384, 0.03268871456384659, 0.003833947004750371, 0.013215027749538422, 0.011135715991258621, -0.01403708104044199, 0.021953662857413292, -0.016510149464011192, 0.05603779852390289, 0.006434813607484102, 0.019259538501501083, -0.01443774625658989, -0.0023331812117248774, -0.005305353552103043, -0.08549356460571289, -0.01703515835106373, 0.029013652354478836, 0.007115252781659365, 0.006175762973725796, 0.017947016283869743, -0.010500178672373295, -0.014071621000766754, -0.027839289978146553, 0.02868206799030304, 0.0005543680163100362, -0.019342435523867607, 0.028847860172390938, -0.01329792384058237, 0.014009449630975723, -0.010811039246618748, 0.009291276335716248, -0.022520121186971664, -0.01328410767018795, 0.017297662794589996, -0.019646387547254562, 0.00028279676917009056, -0.0009800742845982313, -0.03368346765637398, 0.004680178593844175, -0.031804490834474564, -0.04946136847138405, 0.018361497670412064, 0.020364820957183838, 0.01725621521472931, -0.008027110248804092, -0.0011467300355434418, 0.004704356659203768, -0.0054815080948174, -0.0100028021261096, -0.013843657448887825, -0.04600736126303673, -0.0029151812195777893, 0.016703573986887932, -0.03155580163002014, 0.0021414838265627623, -0.0072119650430977345, 0.04622841998934746, -0.002628498710691929, -0.010320570319890976, -0.007619537878781557, -0.03050578385591507, 0.024979373440146446, 0.02442673221230507, -0.023694483563303947, -0.03561771288514137, 0.01142585277557373, -0.010500178672373295, -0.005871810484677553, 0.02970445342361927, 0.009229104965925217, 0.01856873743236065, 0.0018565283389762044, -0.021041806787252426, 0.014976571314036846, 0.0212904941290617, -0.023694483563303947, -0.0037096026353538036, -0.007246505003422499, 0.022451041266322136, 0.024785948917269707, -0.011101176030933857, -0.01318739540874958, 0.006925282534211874, 0.00990608986467123, -0.013836748898029327, 0.005847632419317961, -0.022160904482007027, 0.020060868933796883, -0.004244973883032799, 0.012841994874179363, -0.043327055871486664, -0.03172159194946289, 0.02987024560570717, -0.008261983282864094, -0.009740297682583332, -0.01536341942846775, 0.005820000544190407, 0.005927074700593948, 0.04702974855899811, -0.009643585421144962, -0.027770210057497025, 0.003510997397825122, 0.010928475297987461, 0.004490207880735397, -0.0188865065574646, 0.03733089938759804, 0.014299585483968258, -0.026913616806268692, -0.003719964763149619, 0.009277461096644402, 0.0025283326394855976, 0.01718713529407978, -0.020572060719132423, -0.007239596918225288, -0.01635817438364029, -0.01262784656137228, -0.08317247033119202, 0.03125184774398804, 0.019093746319413185, -0.006966730579733849, -0.0007814689306542277, 0.008234350942075253, 0.01554302778095007, -0.0009144481737166643, -0.003086154581978917, -0.0033503861632198095, -0.03633614629507065, 0.04006647318601608, 0.012040665373206139, 0.00923601258546114, -0.005408973898738623, -0.028087979182600975, -0.002846101066097617, 0.0044349441304802895, -0.015128547325730324, 0.0065073478035628796, 0.007985662668943405, -0.007142885122448206, 0.005478053819388151, 0.005308807361871004, 0.006552249658852816, -0.012116653844714165, -0.0023400892969220877, 0.011246244423091412, -0.006990908645093441, 0.0023331812117248774, 0.03354530781507492, -0.024924108758568764, -0.0025438754819333553, 0.02068259008228779, -0.0069287363439798355, -0.027065593749284744, 0.007135977037250996, 0.0390440858900547, 0.008524487726390362, 0.01413379330188036, -0.027742577716708183, -0.0071843331679701805, -0.0028754600789397955, 0.010762683115899563, -0.023418162018060684, -0.016924630850553513, -0.02234051190316677, 0.020544428378343582, 0.012413698248565197, 0.020876014605164528, 0.0003214384487364441, 0.013208120130002499, 0.004776890855282545, -0.012427514418959618, -0.003733780700713396, 0.020862197503447533, 0.019342435523867607, -0.0034954543225467205, -3.780517727136612e-05, -0.04067437723278999, 0.04263625293970108, -0.007384665310382843, -0.009222196415066719, 0.02992551028728485, 0.025462934747338295, -0.003336569992825389, -0.021912215277552605, 0.004044641274958849, -0.004576558247208595, -0.009367264807224274, -0.011384404264390469, -0.008096190169453621, -0.020572060719132423, 0.00849685538560152, 0.011847241781651974, 0.013235751539468765, 0.01840294525027275, -0.002146664774045348, -0.006655870005488396, 0.020986542105674744, 0.00217602401971817, 0.007509009446948767, -0.016026588156819344, 0.015805533155798912, 0.015377235598862171, -0.010341295041143894, 0.008089282549917698, 0.009802469983696938, -0.00017561462300363928, 0.009291276335716248, 0.00878699216991663, 0.024993188679218292, -0.013774576596915722, 0.0022882791236042976, -0.012538042850792408, 0.022976050153374672, -0.017270030453801155, -0.0006532389088533819, -0.01318739540874958, 0.024095147848129272, 0.008835348300635815, -0.0022053830325603485, 0.020917462185025215, -0.013622600585222244, -0.031168952584266663, 0.01657922938466072, -0.024509627372026443, -0.03172159194946289, -0.018872689455747604, 0.00015089689986780286, 0.02426094003021717, 0.006389911752194166, 0.003581804456189275, 0.012137377634644508, -0.02987024560570717, -0.008241258561611176, -0.0024039882700890303, -0.026899799704551697, -0.025946494191884995, -0.002103489823639393, 0.016247645020484924, -0.016523966565728188, 0.024979373440146446, -0.027037961408495903, 0.027024144306778908, 0.01526670716702938, 0.018706897273659706, -0.01956349052488804, 0.004200071562081575, -0.0003436736296862364, -0.010679787024855614, 0.0003117240557912737, -0.038491446524858475, -0.02119378186762333, -0.00048744663945399225, -0.012510410510003567, -0.029428133741021156, 0.001390237477608025, -0.043133631348609924, 0.06869327276945114, -0.022672096267342567, -0.009512333199381828, 0.024785948917269707, -0.024592524394392967, 0.002511062426492572, 0.008379419334232807, -0.011114992201328278, -0.023003680631518364, 0.013139039278030396, 0.009422528557479382, -0.025103718042373657, 0.0002886253932956606, -0.03500980883836746, -0.029593925923109055, 0.013615692965686321, -0.01045182254165411, 0.0070599885657429695, -0.018485842272639275, -0.015059467405080795, 0.007792238146066666, 0.005094659049063921, -0.006106683053076267, -0.007329401094466448, -0.012876534834504128, 0.011170255951583385, 0.006493531633168459, -0.009802469983696938, -0.03050578385591507, -0.018389130011200905, 0.01878979429602623, 0.004517840221524239, -0.026941249147057533, -0.03412558138370514, 0.019452963024377823, -0.036253251135349274, -0.0032899409998208284, 0.011640001088380814, 0.000382531201466918, 0.006341555621474981, -0.022381961345672607, 0.015045651234686375, -0.023100392892956734, -0.03318609297275543, 0.014893675222992897, 0.006559157744050026, -0.012151193805038929, 0.00034691175096668303, -0.02307276241481304]} +{"id": "test:10000035", "text": "\"Currently there are 3 websites listed for this state.\\nWe are committed to providing for the needs of homeless and owned dogs and cats in our community. Serving over 5,000 companion animals annually through direct intervention and many more through advocacy, we continue to promote the humane values of protection, care, and compassion for all living things.\\nThe Delaware SPCA is dedicated to the prevention of cruelty and the encouragement of humane treatment of all animals. Established in 1873, Delaware SPCA is the oldest animal welfare organization in the state and has provided continuous service to pets and their companions for more than 135 years.\\nFaithful Friends Animal Society is a private animal welfare organization with a No Kill shelter. We are funded through private donations, grants, fees for services and special fundraisers held throughout the year. All cats and dogs in our care stay with us until homes are found.\"", "vector_field": [-0.001419858424924314, 0.011352240107953548, -0.003953071311116219, -0.03127996250987053, -0.02751576155424118, 0.019059568643569946, -0.016090624034404755, 0.0042711724527180195, -0.02392386831343174, -0.022730989381670952, -0.0017048241570591927, 0.0070843808352947235, -0.00650119548663497, 0.011087155900895596, -0.01275056041777134, 0.017058182507753372, 0.013360254466533661, -0.030113589018583298, -0.01611713320016861, -0.0035985209979116917, -0.029344845563173294, 0.007720583584159613, -0.015321879647672176, 0.003760885214433074, -0.008224244229495525, 0.010676275007426739, -0.0045395707711577415, -0.003638283582404256, 0.003345034085214138, 0.0008930031326599419, -0.018449874594807625, -0.0030650384724140167, -0.017614860087633133, -0.0008329449337907135, 0.0019914465956389904, 0.002283039502799511, 0.00032949165324680507, -0.0025597214698791504, 0.009132158011198044, -0.03520321100950241, 0.012299916706979275, 0.021723667159676552, 0.006216229405254126, -0.0047682058066129684, -0.005984280724078417, 0.01639547199010849, -0.01696540229022503, -0.027224170044064522, -0.030060572549700737, -0.0016600911039859056, 0.01100763026624918, 0.008469447493553162, -0.0043076216243207455, -0.009165293537080288, 0.009152039885520935, 0.0026077679358422756, -0.005331510212272406, 0.011292596347630024, 0.006246051751077175, -0.02094166912138462, 0.014804964885115623, 0.008217616938054562, -0.02786037139594555, 0.015600217506289482, -0.03493812680244446, -0.010570241138339043, -0.011776375584304333, 0.006812669336795807, 0.015812285244464874, 0.0014886147109791636, 0.05964399501681328, 0.0035322499461472034, -0.01207459531724453, -0.01880773901939392, 0.00842305738478899, -0.0025630348827689886, 0.017588350921869278, 0.02432149462401867, -0.013903677463531494, 0.001398320309817791, 0.020901905372738838, -0.03902042657136917, -0.012531865388154984, 0.02593851089477539, 0.020371736958622932, -0.011511290445923805, -0.026322882622480392, 0.023897361010313034, -0.006113509181886911, 0.012849966995418072, 0.02812545746564865, 0.040902525186538696, -0.01249210350215435, 0.002395700430497527, -0.01331386435776949, -0.006292441394180059, -0.004261231981217861, 0.03305602818727493, -0.02133929543197155, -0.01932465471327305, -3.694096449180506e-05, 0.010921478271484375, 0.003939817193895578, -0.0062891277484595776, -0.005593281239271164, -0.0033665720839053392, -0.00762117700651288, -0.0077603464014828205, 0.00788626167923212, -0.011345612816512585, -0.014871235936880112, -0.0010644795838743448, 0.024864917621016502, -0.01729675754904747, -0.019417433068156242, -0.006637051235884428, -0.0003394322993699461, -0.020080145448446274, 0.006497881840914488, -0.002195230219513178, 0.03700578585267067, 0.022943057119846344, 0.018264316022396088, -0.021087465807795525, 0.016037607565522194, -0.016713572666049004, 0.00024188953102566302, -0.008880328387022018, -0.012001696974039078, 0.007044618483632803, 0.017985977232456207, 0.0028695387300103903, 0.024135936051607132, 0.012783695943653584, -0.0149375069886446, 0.020981431007385254, -0.0028960471972823143, 0.01536164153367281, -0.00116554310079664, -0.044613707810640335, 0.006726516876369715, 0.022386379539966583, -0.000345852313330397, -0.017097944393754005, 0.010848579928278923, 0.024864917621016502, 0.017031673341989517, -0.0111733078956604, 0.00446667242795229, 0.015547201037406921, 0.012902984395623207, -0.008270633406937122, 0.0185824166983366, -0.0026955772191286087, 0.028708642348647118, -0.009668953716754913, -0.010020190849900246, 0.01030515693128109, 0.02626986615359783, -0.0008341875509358943, -0.005990908015519381, 0.026587966829538345, -0.003724436042830348, 0.007442244794219732, 0.01915234886109829, 0.019245129078626633, 0.011498036794364452, -0.002811551559716463, -0.014977269805967808, -0.0007848984096199274, 0.009085768833756447, 0.052698783576488495, -0.004519689362496138, 0.020901905372738838, -0.01024551223963499, -0.019669264554977417, 0.00025265858857892454, 0.011186562478542328, -0.02531556226313114, -0.010040072724223137, -0.020292211323976517, 0.005344764329493046, 0.01768113113939762, 0.03806612268090248, -0.0071572791785001755, -0.020517533645033836, 0.009688835591077805, 0.004032596480101347, 0.015282116830348969, -0.017097944393754005, 0.005997534841299057, 0.007163906469941139, 0.0235925130546093, -0.002321145497262478, -0.6404439210891724, -0.003959698602557182, -0.008131464011967182, -0.034222397953271866, 0.0073362113907933235, 0.011266088113188744, -9.091567335417494e-05, -0.00977498758584261, -0.038834866136312485, 0.01016598753631115, 0.0026972340419888496, 0.01013947930186987, -0.009483395144343376, -0.017614860087633133, 0.0009219967178069055, 0.0009725284762680531, 0.007303075864911079, -0.0030716657638549805, 0.02982199750840664, -0.02302258089184761, -0.022161057218909264, -0.007402482442557812, -0.016700318083167076, -0.012233645655214787, 0.012333052232861519, -0.00028848639340139925, -0.0015151230618357658, -0.027277186512947083, -0.012207137420773506, 0.0020411501172930002, 0.002698890632018447, 0.009251446463167667, -0.019642755389213562, 0.01701841875910759, 0.03907344117760658, 0.014871235936880112, 0.006484627723693848, 0.013353627175092697, 0.032658401876688004, 0.045461978763341904, -0.029238810762763023, -0.0011961933923885226, 0.053997695446014404, 0.011915544979274273, -0.015852048993110657, -0.01684611476957798, 0.028390541672706604, 0.0068060425110161304, -0.015507438220083714, 0.0024801960680633783, 0.0187017060816288, -0.0001769645605236292, -0.036926258355379105, -0.011305849999189377, 0.006265932694077492, -0.0031395936384797096, 0.021140482276678085, -0.03403683751821518, 0.005951145198196173, 0.025342069566249847, 0.018661942332983017, 0.023963632062077522, 0.009330971166491508, -0.034673042595386505, -0.01701841875910759, 0.019390925765037537, -0.007044618483632803, 0.006620483472943306, 0.006845804862678051, 0.016435233876109123, 0.00977498758584261, 0.019616246223449707, -0.01937767118215561, -0.010848579928278923, 0.009576174430549145, 0.014566388912498951, 0.004635663703083992, 0.00046265515265986323, 0.003252254333347082, 0.00453625712543726, -0.005454111844301224, -0.029450878500938416, -0.003830469911918044, 0.005556832067668438, 0.007998921908438206, 0.0075814141891896725, 0.003378169611096382, 0.006487940903753042, 0.012677662074565887, 0.01578577607870102, 0.02352624200284481, 0.020570550113916397, -0.010835325345396996, -0.024454036727547646, -0.009801496751606464, 0.0116769690066576, -0.006120136473327875, -0.013214457780122757, 0.00471518887206912, -0.03700578585267067, -0.010225631296634674, -0.009423751384019852, 0.011511290445923805, -0.02706511877477169, 0.044057030230760574, 0.03279094398021698, -0.017548589035868645, 0.0017346461536362767, 0.022784005850553513, -0.03472605720162392, 0.0038569781463593245, -0.0174558088183403, -0.014606151729822159, -0.013519304804503918, -0.0005707598756998777, -0.024613087996840477, 0.012969255447387695, -0.0019599678926169872, 0.007793481927365065, -0.015706252306699753, -0.009675581008195877, 0.015348387882113457, -0.004244664218276739, -0.020398246124386787, 0.0212200079113245, 0.012174001894891262, -0.016249675303697586, -0.01925838366150856, 0.0015300341183319688, -0.010159360244870186, 0.010649766772985458, 0.004065732005983591, 0.02520952746272087, -0.01842336729168892, -0.008403176441788673, 0.004334130324423313, 0.020066890865564346, -0.008045312017202377, 0.007508516311645508, 0.011809511110186577, -0.015852048993110657, 0.01611713320016861, -0.022664718329906464, -0.005185713991522789, -0.01465916819870472, -0.012094476260244846, 0.023221394047141075, -0.009609309956431389, -0.008575480431318283, 0.005977653432637453, -0.02600478194653988, -0.025196272879838943, -0.009205056354403496, 0.0366346649825573, -0.006637051235884428, -0.005033290479332209, -0.001409917720593512, -0.018277570605278015, 0.006173153407871723, -0.03785405308008194, -0.0035289363004267216, 0.0071042622439563274, -0.01948370411992073, -0.015083303675055504, -0.003923249430954456, -0.039046935737133026, 0.007680820766836405, 0.029609929770231247, 0.0004663828876800835, -0.027091627940535545, 0.0030948605854064226, -0.010961240157485008, -0.01333374623209238, 0.022028515115380287, 0.018608925864100456, 0.017097944393754005, -0.02381783537566662, -0.019112586975097656, 0.022717734798789024, 0.010596749372780323, 0.014155508019030094, -0.0006672671879641712, 0.005255298689007759, -0.000577801198232919, -0.011087155900895596, 0.0020693151745945215, 0.03199568763375282, 0.03194267302751541, -0.01030515693128109, 0.009218310937285423, -0.0006067947833798826, -0.01131247729063034, -0.009920784272253513, 0.017482317984104156, -0.005029976833611727, -0.0034759193658828735, 0.023221394047141075, 0.003280419623479247, 0.019112586975097656, 0.023208141326904297, 0.024016648530960083, -0.00745549937710166, 0.027542270720005035, -0.01499052345752716, -0.006096941418945789, -0.04858997464179993, 0.006544271484017372, -0.012982509098947048, 0.008442938327789307, -0.0034858600702136755, -0.00731632998213172, 0.009556293487548828, -0.0073892283253371716, -0.020146416500210762, 0.014791710302233696, 0.030140098184347153, -0.01224027294665575, -0.003194267163053155, -0.01373137254267931, -0.01521584577858448, 0.005059798713773489, -0.0006080374005250633, -0.004317562561482191, 0.02380458079278469, 0.002846343908458948, 0.005049858242273331, 0.011643832549452782, 0.037032291293144226, -0.013201204128563404, 0.01016598753631115, -0.018158283084630966, -0.003293673973530531, -0.00373437674716115, 0.035123683512210846, -0.0017843494424596429, -0.010477461852133274, 0.03727087005972862, -0.009337598457932472, 0.010093089193105698, 0.01331386435776949, 0.02387085184454918, 0.016156895086169243, 0.0240961741656065, -0.020623568445444107, -0.004108808469027281, 0.0035090548917651176, 0.02342020906507969, 0.019563229754567146, -0.02083563432097435, 0.006971720140427351, -0.0257264431566, -0.009728598408401012, 0.01573275960981846, 0.006507822312414646, 0.003134623169898987, 0.001576423877850175, -0.011504664085805416, 0.002322802087292075, 0.023778071627020836, 0.0023923867847770452, 0.01221376471221447, 0.013426525518298149, 0.027051864191889763, -0.010232258588075638, 0.015812285244464874, 0.0026525009889155626, -0.008290515281260014, -0.008926717564463615, 0.00800554919987917, -0.01651475951075554, -0.017707638442516327, 0.0073892283253371716, -0.009715343825519085, -0.012704170309007168, 0.01920536532998085, -0.005636357236653566, -0.028841184452176094, 0.01549418456852436, -0.00940386950969696, 0.010351546108722687, -0.02144532836973667, -0.012790323235094547, 0.010033445432782173, 0.021312786266207695, -0.009556293487548828, -0.0025183020625263453, -0.007409109268337488, 0.024520307779312134, -0.014526626095175743, 0.019576484337449074, -0.02015966922044754, 0.009861140511929989, -0.021935734897851944, 0.00562973041087389, 0.01426154188811779, -0.007588041480630636, 0.02201526053249836, -0.01294274628162384, 0.013393389992415905, 0.017230486497282982, 0.016806352883577347, 0.007627804297953844, -0.008489328436553478, -0.02538183331489563, 0.032870467752218246, -0.026309628039598465, -0.0016907415119931102, -0.026296373456716537, -0.0057423911057412624, -0.01453988067805767, -0.0038967409636825323, 0.0059743402525782585, -0.009662327356636524, -0.015255608595907688, -0.006448178552091122, 0.007700702175498009, -0.027992913499474525, 0.010510597378015518, 0.02857610024511814, 0.006378593854606152, 0.0061300769448280334, -0.018264316022396088, -0.0002967702748719603, -0.0014306274242699146, 0.11038114875555038, -0.012445713393390179, -0.001880442607216537, 0.0050001549534499645, 0.008529091253876686, 0.00489743473008275, -0.007660939823836088, 0.0020875397603958845, 0.009900903329253197, 0.007018109783530235, -0.017773909494280815, -0.020954923704266548, 0.02156461775302887, 0.0045594521798193455, 0.014579642564058304, 0.01741604693233967, -0.022253837436437607, -0.012697543948888779, 0.007342838216573, 0.0010164330014958978, 0.020636821165680885, -0.004602528177201748, 0.040292833000421524, 0.02049102447926998, -0.011332359164953232, -0.0042711724527180195, 0.0012227018596604466, 0.012671034783124924, -0.01601109839975834, 0.009920784272253513, -0.011498036794364452, 0.038331206887960434, -0.003953071311116219, 0.01611713320016861, -0.0024437468964606524, -0.009344225749373436, -0.004231410101056099, 0.030590740963816643, 0.03324158489704132, 0.0036714191082865, 0.029344845563173294, 0.026110814884305, -0.013612085022032261, -0.013784389942884445, 0.020650075748562813, -0.013240966945886612, -0.011279341764748096, 0.03048470802605152, 0.015759268775582314, -0.012571628205478191, 0.04002774506807327, -0.002867882139980793, -0.007992295548319817, 0.00042206409852951765, 0.001641038223169744, -0.004552824888378382, 0.023115361109375954, 0.0038569781463593245, -0.02352624200284481, -0.03294999152421951, -0.023459970951080322, -0.04164476320147514, 0.0031545045785605907, -0.023725055158138275, 0.0023658783175051212, -0.030802808701992035, -0.019974110648036003, 0.010126224718987942, -0.03353317826986313, -0.021856209263205528, 0.010311784222722054, -0.014433846808969975, 0.007223550230264664, -0.012425832450389862, 0.013638593256473541, 0.0010098059428855777, 0.019788552075624466, 0.014553134329617023, 0.03472605720162392, 0.029901521280407906, -0.002322802087292075, -0.012743933126330376, 0.007826616987586021, -0.0037907070945948362, 0.029662946239113808, 0.01533513329923153, 0.013459661044180393, -0.02123326249420643, -0.010199123062193394, 0.019443942233920097, 0.007422363851219416, -0.0003276277566328645, 0.016713572666049004, 0.002369191963225603, 0.009158666245639324, -0.0033665720839053392, 0.009397242218255997, 0.02369854599237442, -0.00921168364584446, -0.004426909610629082, 0.03440795838832855, -0.0016435233410447836, -0.04177730530500412, -0.013930185697972775, 0.01622316613793373, 0.009894276037812233, 0.004221469163894653, 0.009012870490550995, -0.0036084617022424936, -0.02269122563302517, 0.0214585829526186, -0.029371352866292, 0.0198283139616251, -0.0052453577518463135, -0.004652231466025114, 0.02010665275156498, -0.006166526116430759, 0.026760272681713104, -0.0008863760158419609, -0.0011058991076424718, -0.02111397311091423, 0.016262928023934364, 0.022439396008849144, 0.04426909610629082, -0.02296956442296505, -0.0038768595550209284, -0.018714960664510727, -0.009808123111724854, 0.011716730892658234, -0.021803192794322968, -0.020862143486738205, 0.02083563432097435, -0.03151853755116463, -0.034673042595386505, -0.017548589035868645, -0.006716576404869556, -0.007415736559778452, 0.021763430908322334, -0.021710414439439774, -0.004933883901685476, -0.0025315561797469854, -0.010099716484546661, -0.012876475229859352, -0.016435233876109123, 0.03334761783480644, -0.031120911240577698, -0.03324158489704132, -0.02173692174255848, 0.02319488674402237, 0.037429921329021454, -0.0333741270005703, -0.018065502867102623, -0.007773600518703461, 0.02078261785209179, -0.008734531700611115, -0.015070049092173576, -0.00585173862054944, -0.010450952686369419, 0.017932960763573647, 0.030617250129580498, 0.018290825188159943, -0.013274102471768856, -0.0010520537616685033, 0.0021239889319986105, 0.008244125172495842, -0.0021455269306898117, -0.010649766772985458, 0.006517763249576092, 0.00644486490637064, 0.023884106427431107, 0.021471837535500526, 0.019006552174687386, -0.004390460439026356, -0.004917316138744354, 0.01016598753631115, -0.003214148571714759, 0.0008954883087426424, -0.03997473046183586, 0.00045934159425087273, -0.024215461686253548, 0.011577561497688293, -0.013771135360002518, 0.020345229655504227, 0.025792714208364487, -0.016209911555051804, 0.007906142622232437, 0.011524545028805733, 0.0016219852259382606, -0.010059953667223454, -0.007985668256878853, 0.021074211224913597, 0.0018207986140623689, 0.01426154188811779, 0.035070668905973434, -0.033824771642684937, -0.009244819171726704, -0.016421979293227196, -0.024944443255662918, 0.002867882139980793, -0.0072633130475878716, -0.004314248915761709, 0.0116769690066576, 0.014579642564058304, 0.011246206238865852, -0.022956309840083122, -0.011650459840893745, -0.007435617968440056, -0.03711181879043579, -0.005556832067668438, -0.03931201994419098, -0.002380789490416646, -0.022028515115380287, -0.03157155215740204, -0.029185794293880463, -0.004281113389879465, -0.017932960763573647, -0.026534950360655785, 0.016779843717813492, 0.00459258770570159, -0.040796492248773575, -0.006050551775842905, -0.010450952686369419, 0.003717808984220028, 0.004817909095436335, 0.04053140804171562, 0.041750796139240265, 0.010550360195338726, -0.03846374899148941, 0.011252833530306816, -0.006965092848986387, -0.0009791555348783731, 0.03021962381899357, 0.007117516361176968, -0.024639597162604332, 0.0031114283483475447, -0.003953071311116219, 0.009138785302639008, 0.0020494339987635612, -0.037933580577373505, 0.026190340518951416, 0.016037607565522194, 0.003946444019675255, -0.012830086052417755, 0.00624936493113637, 0.003820529207587242, 0.012386069633066654, -0.001979849301278591, -0.00650119548663497, -0.02876165881752968, 0.0062559922225773335, -0.02082238160073757, -0.011365494690835476, -0.035627346485853195, 0.00940386950969696, -0.008204362355172634, 0.03432843089103699, 0.016474995762109756, 0.011816137470304966, -0.0024354630149900913, 0.0016849428648129106, 0.0026856365147978067, 0.006613856181502342, -0.016196656972169876, -0.0036150887608528137, -0.004420282784849405, 0.031306467950344086, -0.022373124957084656, 0.006845804862678051, 0.005215535871684551, 0.03037867322564125, -0.00853571854531765, -0.02139231190085411, 0.02307559922337532, 0.004483240190893412, -0.005994221195578575, 0.0006345458095893264, 0.0074687534943223, 0.00434075715020299, -0.05222162976861, -0.010974494740366936, 0.025593901053071022, -9.500583837507293e-05, -0.008104955777525902, -0.04509085789322853, -0.005473993252962828, -0.006892194971442223, -0.008111583068966866, -0.03689974918961525, 0.0015374895883724093, 0.0035720125306397676, -0.009112277068197727, -0.0017263622721657157, -0.023672038689255714, 0.003979580011218786, 0.000932765775360167, -0.005367959383875132, 0.02605779841542244, 0.03485859930515289, -0.02982199750840664, -0.002899360843002796, -0.03207521513104439, 0.010252139531075954, -0.05383864417672157, 0.0003408819902688265, -0.00651444960385561, 0.0017197350971400738, -0.002223395509645343, 0.002808238146826625, -0.02931833639740944, -0.026402408257126808, 0.003618402173742652, 0.010126224718987942, -0.006852432154119015, -0.014725439250469208, -0.002422208897769451, 0.007256685756146908, 0.02263820916414261, -0.0007095150067470968, -0.008131464011967182, 0.033215075731277466, -0.002794983796775341, -0.038278188556432724, 0.0349116176366806, -0.009715343825519085, 0.035574328154325485, -0.014460355043411255, 0.010643139481544495, 0.012578255496919155, -0.013572322204709053, -0.036475617438554764, -0.030643759295344353, -0.008058566600084305, -0.03098836913704872, -0.013969948515295982, -0.030352165922522545, -0.02847006544470787, -0.008833938278257847, 0.014950760640203953, 0.004950451664626598, -0.015971336513757706, 0.02762179635465145, 0.00951653067022562, -0.011073901318013668, -0.015401404350996017, -0.003837096970528364, 0.001170513336546719, -0.005450798198580742, -0.022333361208438873, -0.03801310434937477, 0.020424753427505493, -0.046972960233688354, 0.016779843717813492, -0.001388379605486989, -0.010802189819514751, -0.0041154357604682446, 0.022081531584262848, -0.007919397205114365, -0.010152732953429222, 0.008814056403934956, 0.03408985584974289, 0.017283504828810692, 0.030007556080818176, 0.005086307413876057, 0.011365494690835476, -0.015454421751201153, 0.028788167983293533, -0.017203979194164276, -0.007402482442557812, -0.007004855666309595, -0.026521695777773857, 0.016183404251933098, -0.002796640619635582, -0.017535334452986717, 0.00016008612874429673, 0.01864868961274624, -0.006226170342415571, -0.023208141326904297, 0.02123326249420643, -0.004400401376187801, 0.007024737074971199, -0.018463129177689552, -0.006368652917444706, -0.014977269805967808, -0.007939278148114681, 0.03504415974020958, -0.026773525401949883, 0.0014157164841890335, 0.018993297591805458, 0.006938584614545107, 0.021074211224913597, -0.0011879095109179616, -0.001636067871004343, -0.024851664900779724, 0.0025216154754161835, -0.024970952421426773, -0.021591125056147575, -0.005361332092434168, -0.015096557326614857, -0.00240729795768857, 0.010961240157485008, -0.03589243069291115, 0.04869600757956505, -0.01831733249127865, -0.008860446512699127, -1.9971936126239598e-05, 0.0013395047280937433, -0.014831473119556904, 0.001571453525684774, 0.005997534841299057, -0.013247593306005001, 0.01891377381980419, -0.016978656873106956, 0.0006494568078778684, -0.0257264431566, -0.004141943994909525, -0.022545428946614265, 0.009317717514932156, -0.0252227820456028, 0.017561841756105423, -0.012538492679595947, -0.025302307680249214, -0.02004038169980049, -0.009596056304872036, -0.01684611476957798, -0.013181322254240513, -0.012896357104182243, -0.02583247609436512, -0.0063984752632677555, -0.0066602458246052265, 0.016713572666049004, -0.02296956442296505, -0.01275056041777134, -0.008582107722759247, 0.01288972981274128, 0.010378055274486542, 0.032711416482925415, 0.20220638811588287, 0.00536464573815465, 0.00374100380577147, 0.028231490403413773, -0.01864868961274624, 0.00264753052033484, 0.01736302860081196, -0.007236804347485304, -0.010431071743369102, 0.022850276902318, -0.016859369352459908, -0.0016054174629971385, -0.011213070712983608, 0.007170533295720816, -0.012392696924507618, -0.006033984012901783, -0.0009186831885017455, 0.005580027122050524, -0.009576174430549145, -0.019510213285684586, -0.01605086214840412, 0.005765586160123348, 0.001163886277936399, 0.006159899290651083, 0.02453356236219406, 0.010033445432782173, -0.01662079244852066, -0.031041385605931282, 0.002301264088600874, 0.012849966995418072, -0.019404178485274315, -0.00039120661676861346, -0.0036415972281247377, 0.006421669851988554, -0.017773909494280815, -0.026694001629948616, -0.0012931149685755372, 0.004986900370568037, 0.022267090156674385, 0.014341066591441631, -0.006123450119048357, -0.010073208250105381, -0.034673042595386505, 0.008224244229495525, 0.0029739157762378454, 0.03178362175822258, 0.008058566600084305, -0.01013947930186987, 0.015799030661582947, 0.028841184452176094, -0.027078373357653618, 0.0003750530304387212, 0.012763815000653267, 0.012545119971036911, -0.016700318083167076, -0.016528014093637466, 0.018608925864100456, 0.028947217389941216, 0.0048013413324952126, 0.016488250344991684, -0.007660939823836088, 0.017932960763573647, 0.0058683063834905624, 0.03811913728713989, -0.018767977133393288, 0.023539496585726738, 0.005835170857608318, 0.038993917405605316, 0.034620024263858795, 0.0029921403620392084, -0.0004266202449798584, 0.009881021454930305, -0.009417124092578888, -0.00749526172876358, -0.015176082961261272, -0.01623642072081566, 0.015533946454524994, 0.019735535606741905, 0.006454805377870798, 0.01499052345752716, -0.013214457780122757, -0.0024536876007914543, 0.005715882871299982, 0.03003406524658203, -0.006090314593166113, -0.02312861569225788, 0.01898004487156868, -0.0016617479268461466, 0.0029821996577084064, -0.005556832067668438, 0.0029855133034288883, 0.007071126718074083, -0.0015399748226627707, -0.017707638442516327, 0.003837096970528364, 0.005165832582861185, 0.022717734798789024, 0.034275416284799576, 0.001865531550720334, 0.02078261785209179, 0.01780041866004467, 0.062029752880334854, -0.008018803782761097, 0.020968176424503326, -0.007442244794219732, 0.024679359048604965, 0.010272021405398846, 0.005709255579859018, 0.002874509198591113, -0.012120984494686127, -0.011087155900895596, -0.01356569491326809, -0.031969182193279266, -0.022545428946614265, 0.028947217389941216, 0.025448104366660118, 0.006478000432252884, -0.027780847623944283, 0.010947986505925655, 0.0034891737159341574, -0.018688451498746872, -0.02852308377623558, -0.0169521477073431, 0.027542270720005035, 0.0010827041696757078, -0.00943037774413824, -0.04954427853226662, 0.017150960862636566, -0.00278007285669446, -0.0429171659052372, 0.00233108620159328, 0.0076344311237335205, -0.0048576719127595425, 0.0005181572050787508, -0.031120911240577698, 0.003290360327810049, 0.003134623169898987, -0.012246900238096714, -0.01344640739262104, 0.007541651837527752, 0.01086846087127924, 0.003333436558023095, -0.004059105180203915, -0.006577407009899616, -0.006690068170428276, -0.02376481704413891, 0.02876165881752968, 0.0031197122298181057, 0.008800802752375603, -0.027038609609007835, -0.04705248400568962, 0.017773909494280815, 0.022267090156674385, -0.01644848845899105, 0.031598061323165894, -0.013009017333388329, -0.01263127289712429, -0.019046315923333168, 0.007243431638926268, -0.0057059419341385365, -0.04538245126605034, 0.0198283139616251, -0.005609849002212286, -0.0109811220318079, -0.012299916706979275, -0.005964399315416813, -0.16742731630802155, 0.00651444960385561, -0.006441551260650158, -0.004993527662009001, -0.002422208897769451, 0.006504508666694164, 0.012843339703977108, 0.002746937330812216, -0.012346306815743446, -0.011140172369778156, 0.035468295216560364, 0.015096557326614857, -0.024135936051607132, -0.022876786068081856, 0.0057390774600207806, -0.005046544596552849, -0.0038801732007414103, 0.009284581989049911, 0.017654621973633766, -0.018211299553513527, 0.03154504671692848, -0.020000619813799858, 0.01013947930186987, -0.04530292749404907, 0.01875472255051136, 0.03252585604786873, 0.018714960664510727, -0.010172614827752113, -0.012193882837891579, -0.029291827231645584, -0.0007443073554895818, 0.016196656972169876, 0.016528014093637466, -0.0003785736917052418, -0.008509209379553795, 0.0002841373498085886, -0.016077369451522827, -0.01035817340016365, -0.03194267302751541, 0.03379826247692108, 0.012598136439919472, 0.008635125122964382, 0.008164600469172001, -0.00010266844037687406, 0.00748200761154294, 0.016528014093637466, 0.025633662939071655, -0.0029374666046351194, 0.009410496801137924, -0.014513371512293816, 0.0015573709970340133, 0.03273792564868927, 0.013625338673591614, 0.01558696385473013, -0.00644486490637064, -0.001137377810664475, -0.007945905439555645, 0.013850660994648933, 0.013234339654445648, 0.003949757665395737, -0.029106268659234047, -0.041353169828653336, -0.0007762003224343061, 0.004658858757466078, 0.005354705266654491, -0.023910613730549812, -0.04217493161559105, 0.01970902644097805, -0.023950377479195595, 0.005457425490021706, -0.007680820766836405, 0.002765161916613579, -0.017548589035868645, 0.0017263622721657157, -0.006292441394180059, 0.010835325345396996, -0.023433461785316467, 0.01757509633898735, -0.01128596905618906, -0.02111397311091423, 0.000794010644312948, 0.015030286274850368, -0.029689455404877663, -0.02139231190085411, -0.013366881757974625, 0.010503970086574554, -0.006898821797221899, 0.012783695943653584, 0.014327812939882278, -0.010523851029574871, -0.012598136439919472, -0.028045931831002235, -0.002415581839159131, -0.016594285145401955, 0.015533946454524994, 0.0349116176366806, 0.011517917737364769, -0.0017545274458825588, -0.01813177391886711, -0.0015557141741737723, 0.02762179635465145, -0.009881021454930305, 0.009277954697608948, 0.027780847623944283, 0.015043540857732296, 0.02645542472600937, -0.00831039622426033, 0.03194267302751541, 0.010961240157485008, -0.011345612816512585, 0.00012943574984092265, 0.028098948299884796, 0.027727829292416573, 0.017787164077162743, 0.012724052183330059, 0.00456276535987854, -0.010716037824749947, -0.017707638442516327, -0.0043440707959234715, -0.021326040849089622, 0.04331479221582413, -0.013234339654445648, 0.001824112143367529, 0.008774294517934322, 0.007826616987586021, -0.004281113389879465, -0.1064579039812088, -0.006832550745457411, 0.005374586675316095, 0.030060572549700737, -0.009138785302639008, 0.03464653342962265, 0.009562920778989792, 0.05031302198767662, -0.012386069633066654, 0.006971720140427351, 0.003611775115132332, -0.02381783537566662, 0.011213070712983608, -0.013380136340856552, 0.04432211443781853, -0.006872313562780619, -0.013015644624829292, -0.02740972861647606, -0.015401404350996017, -0.0028778226114809513, 0.02694583125412464, -0.02668074704706669, -0.01831733249127865, -0.029556913301348686, -0.011842646636068821, 0.004509748425334692, -0.0257264431566, 0.022492412477731705, -0.006812669336795807, 0.0169521477073431, -0.007130770944058895, -0.018675196915864944, -0.002735339803621173, 0.003189296927303076, -0.0034659786615520716, -0.015825539827346802, 0.004460045136511326, -0.012823458760976791, 0.025898747146129608, -0.017667876556515694, -0.0029540343675762415, -0.017058182507753372, 0.012584882788360119, -0.029185794293880463, -0.014751947484910488, -0.02307559922337532, 0.020848888903856277, 0.0025365264154970646, 0.019974110648036003, -0.02380458079278469, -0.031306467950344086, -0.03650212287902832, -0.014473608694970608, 0.005749018397182226, 0.023884106427431107, -0.011438393034040928, 0.022558683529496193, 0.027489254251122475, -0.008230871520936489, 0.002884449902921915, -0.007303075864911079, 0.00543423043563962, -0.00284965755417943, -0.006504508666694164, 0.02133929543197155, -0.003863605437800288, -0.038092631846666336, -0.013413271866738796, 0.006799415219575167, -0.036528632044792175, -0.0015639980556443334, 0.0223466157913208, -0.024917935952544212, -0.005251985043287277, -0.01684611476957798, 0.00460584182292223, -0.010782308876514435, -0.02762179635465145, 0.0112329525873065, -0.011186562478542328, -0.0023841029033064842, -0.010040072724223137, -0.003767512273043394, -0.011431765742599964, 0.02380458079278469, -0.01605086214840412, -0.00011017570795956999, 0.01373137254267931, -0.007276567164808512, -0.019046315923333168, 0.00803205743432045, 0.01639547199010849, -0.008787548169493675, 0.0035819532349705696, -0.006385220680385828, 0.00032907744753174484, 0.006925330497324467, -0.024493800476193428, -0.012704170309007168, 0.029185794293880463, 0.0071042622439563274, -0.01583879441022873, -0.06415043026208878, -0.006646991707384586, 0.015096557326614857, -0.02645542472600937, -0.0025083613581955433, -0.011524545028805733, -0.018264316022396088, -0.019417433068156242, 0.012624645605683327, 0.0013958350755274296, -0.05137335881590843, -0.001842336729168892, 0.005265239160507917, 0.016130385920405388, -0.020080145448446274, -0.02218756638467312, 0.015003778040409088, -0.018887264654040337, 0.007853126153349876, 0.02836403250694275, -0.014327812939882278, 0.019298145547509193, 0.026521695777773857, -0.006633737590163946, -0.006295755039900541, 0.017389537766575813, -0.018171537667512894, 0.0015697968192398548, 0.005825229920446873, 0.022943057119846344, -0.01863543502986431, -0.016421979293227196, -0.01265778113156557, 0.03252585604786873, -0.020742855966091156, -0.004456731956452131, -0.0030849198810756207, 0.036528632044792175, -0.00965570006519556, 0.013234339654445648, 0.0093508530408144, -0.023950377479195595, 0.018052248284220695, -0.043818455189466476, -0.011279341764748096, 0.015308625064790249, -0.006335517391562462, 0.0025365264154970646, 0.003691300516948104, 0.01198844239115715, 0.026826543733477592, 0.010961240157485008, -0.009251446463167667, 0.014062728732824326, -0.017217233777046204, -0.00884056556969881, -0.002664098283275962, 0.008992988616228104, 0.018489638343453407, 0.003333436558023095, 0.0020312094129621983, 0.015149574726819992, 0.009383988566696644, -0.0026293061673641205, 0.010881715454161167, 0.017058182507753372, 0.01074917335063219, -0.003813902148976922, 0.00228138267993927, -0.02593851089477539, -0.01729675754904747, 0.021577872335910797, 0.008018803782761097, 0.01768113113939762, 0.00481459591537714, -0.0256071537733078, -0.022426141425967216, -0.00031520193442702293, -0.02307559922337532, 0.018741467967629433, 0.02425522357225418, -0.024586578831076622, 0.006206288933753967, 0.01578577607870102, 0.013552440330386162, -0.00884056556969881, 0.010947986505925655, 0.004546197596937418, -0.001419858424924314, -0.014725439250469208, -0.022704480215907097, 0.005861679092049599, -0.0006428296910598874, -0.0033384067937731743, 0.0130554074421525, -0.012982509098947048, 0.020292211323976517, 0.008926717564463615, 0.02224058285355568, 0.015918320044875145, -0.00387354614213109, 0.01168359536677599, -0.0019848195370286703, -0.00551706925034523, -0.02976897917687893, 0.0030153351835906506, 0.011736612766981125, -0.008184481412172318, -0.005427603609859943, 0.015295370481908321, 0.023844342678785324, -0.010570241138339043, 0.0029788862448185682, 0.010179241187870502, -0.018012486398220062, 0.03493812680244446, 0.027780847623944283, -0.004317562561482191, -0.0207163468003273, 0.011411883868277073, -0.011073901318013668, 0.0063984752632677555, 0.0025265859439969063, 0.0029341531917452812, 0.016713572666049004, -0.0008573824306949973, 0.009390615858137608, -0.018449874594807625, 0.005540264304727316, 0.028178473934531212, -0.018834248185157776, -0.017588350921869278, -0.0361575148999691, 0.0015780807007104158, -0.012995763681828976, -0.006507822312414646, 0.014447100460529327, -0.0032174622174352407, -0.013771135360002518, 0.04437513276934624, 0.03859629109501839, 0.011080528609454632, 0.011749866418540478, 0.025898747146129608, 0.014619405381381512, 0.011338985525071621, 0.028337523341178894, 0.005732450634241104, -0.014433846808969975, 0.017270250245928764, -0.02605779841542244, 0.03443446755409241, -0.004638977348804474, -0.00576889980584383, 0.0006386877503246069, -0.03077630139887333, 0.004509748425334692, -0.0035786395892500877, 0.0011150112841278315, 0.0459391288459301, -0.01428805012255907, -0.009881021454930305, -0.011981816031038761, -0.031863145530223846, -0.007462126202881336, 0.011869154870510101, -0.009907529689371586, -0.0015076675917953253, -0.03178362175822258, -0.0035322499461472034, 0.018038993701338768, -0.018343841657042503, -0.03578639775514603, 0.009728598408401012, 0.01622316613793373, 0.013433152809739113, -0.019006552174687386, 0.03082931786775589, 0.0022432769183069468, -0.027356712147593498, 0.0067861611023545265, -0.006189721170812845, -0.0057954080402851105, 0.0026011408772319555, 0.014672422781586647, 0.003986206836998463, -0.014115745201706886, -0.019059568643569946]} +{"id": "test:10000036", "text": "\"The blog deals with new theories and observations in relation to the composition and structure of the Cosmos.\\nIf you can find observations or experiments that confirms or rejects the theory, you may place them on the blog!\\nIs anyone able to answer these questions!\\nWhat is the length of the coordinates in the two inertial systems S and S' at the velocity v, according to relativity - if the length of the coordinates are identical at v = zero?\\n1) Is the physical length of the coordinates greatest in S?\\n2) Is the physical length of the coordinates greatest in S'?\\n3) Is the physical length of the coordinates in S equal to the physical length of the coordinates in S'?\\nIf the physical lengths are identical, will it then be possible to have any physical length contractions according to relativity? Or, in other words, have you ever experienced a length contraction because of the movement of another coordinate system?\\nIf it isn't possible to have any length contractions according to relativity, how can relativity then explain the relativistic experiments where a Lorentz contraction occurs?\\nIn the quantum field theory, the Casimir effect is a physical force arising from the zero-point field, where the zero-point field is the quantum state with the lowest possible energy.\\nThe existence of the zero-point field, at the same time both confirms the theory about the structure and composition of the Cosmos, and overturns Einstein's principle of relativity.\\nThis can bee seen from the fact, that you cannot - as Einstein postulate - have that \\\"the same laws of electrodynamics and optics will be valid for all frames of reference for which the equations of mechanics hold good\\\" and at the same time have a stationary zero-point field.\\nFor instance will the velocity relative to the zero-point field be different for different frames of reference.\"", "vector_field": [0.0066411783918738365, 0.02633228898048401, 0.006948798429220915, -0.01017881091684103, -0.015668122097849846, 0.020535357296466827, -0.030515924096107483, -0.03210187703371048, -0.0332229807972908, -0.020289260894060135, 0.0022832255344837904, 0.01841619610786438, -0.0006272034370340407, 0.00985751859843731, 0.006056699901819229, 0.0017910331953316927, 0.029969044029712677, 0.007628981024026871, -0.014027480967342854, -0.01201086025685072, 0.011566519737243652, 0.006719792261719704, -0.01425990555435419, 0.002695094794034958, -0.011662223376333714, 0.000801948772277683, 0.013678845018148422, -0.018744323402643204, 0.010964951477944851, -0.0025173586327582598, 0.015312650240957737, 0.01410951279103756, -0.03494565561413765, -0.006528384052217007, -0.021902557462453842, -0.011621207930147648, 0.00821004156023264, -0.015107570216059685, -0.007164132781326771, -0.029750291258096695, 0.014341937378048897, 0.03128155693411827, -0.0283010583370924, -0.007608472835272551, -0.007745192851871252, 0.007382884621620178, 0.022709207609295845, -0.02109590917825699, 0.0030744930263608694, -0.004146036691963673, 0.029695603996515274, 0.02920341119170189, -0.050668466836214066, 0.014382953755557537, 0.007533276919275522, -0.049520015716552734, 0.007225656881928444, 0.0332229807972908, 0.016707194969058037, -0.014423969201743603, 0.007013740483671427, 0.0175411868840456, -0.019646676257252693, 0.02511548064649105, -0.0034658543299883604, -0.014068497344851494, -0.010835067369043827, -0.009570405818521976, -0.0004355816636234522, 0.0005823421524837613, 0.03273078799247742, 0.03207453340291977, 0.0009348235907964408, -0.005834529642015696, 0.04856297746300697, -0.029586227610707283, -0.009707126766443253, -0.005198781378567219, -0.0013928358675912023, 0.02463695965707302, -0.00026874043396674097, -0.005171437282115221, -0.025129152461886406, 0.027138937264680862, 0.002081563463434577, -0.004269084893167019, -0.00353763229213655, 0.006767644546926022, -0.04530903697013855, -0.02264084666967392, 0.005113331135362387, 0.004433149006217718, 0.01759587600827217, 0.018621277064085007, 0.006883856374770403, 0.006764226593077183, -0.0027532007079571486, 0.028957314789295197, -0.016256019473075867, -0.011217883788049221, 0.012338988482952118, -0.002262717578560114, -0.017869316041469574, -0.019277533516287804, -0.01900409162044525, -0.016802899539470673, 0.016748210415244102, -0.017609547823667526, 0.00044348579831421375, -0.013877089135348797, -0.011833123862743378, 0.014423969201743603, -0.015435698442161083, 0.0030146779026836157, 0.008818445727229118, 0.003077910980209708, 0.012838016264140606, -0.008729577995836735, -0.011375111527740955, -0.0008604820468463004, 0.022900614887475967, 0.007704176940023899, 0.01698063500225544, -0.027193626388907433, 0.016707194969058037, -0.0048911613412201405, -0.009768649935722351, 0.0053081572987139225, -0.017021650448441505, -0.015695465728640556, -0.00779304513707757, 0.03314094990491867, -0.02849246747791767, 0.0003580357297323644, -0.011381947435438633, 0.040141019970178604, -0.014711081981658936, -0.00870223343372345, -0.011846795678138733, -0.006900946609675884, 0.011484487913548946, 0.015107570216059685, -0.025361577048897743, 0.006084043998271227, 0.00781355332583189, 0.04448871687054634, 0.00020166212925687432, 0.01937323622405529, 0.01279700081795454, -0.002522485563531518, 0.010445415042340755, 0.01737712323665619, 0.0011322132777422667, 0.01235949620604515, 0.020849812775850296, 0.0034846533089876175, -0.025867441669106483, 0.03866444155573845, -0.02059004455804825, -0.05559038743376732, -0.0031257630325853825, 0.027138937264680862, 0.0009698581416159868, -0.012072383426129818, 0.0033821132965385914, 0.01894940435886383, 0.015463042072951794, -0.011846795678138733, 0.002951444825157523, 0.013535289093852043, -0.014847801998257637, 0.02612720988690853, -0.03896522521972656, 0.017636891454458237, 0.010869246907532215, 0.01041807048022747, 0.01032236684113741, 0.008804773911833763, -0.022941630333662033, -0.024623287841677666, -0.004733933135867119, 0.01591421850025654, 0.03778943419456482, 0.044926222413778305, 0.004306682851165533, -0.012380003929138184, 0.016433754935860634, -0.021123254671692848, 0.007916092872619629, -0.016379065811634064, 0.00920809805393219, 0.0060737901367247105, -0.012619264423847198, 0.00016983198293019086, -0.6304436922073364, -0.038308970630168915, 0.024035392329096794, -0.020138869062066078, 0.0015996251022443175, 0.006794988643378019, 0.008004961535334587, 0.026004161685705185, -0.027234641835093498, 0.03316829353570938, -0.025976816192269325, 0.008592857979238033, -0.015367338433861732, -0.012229612097144127, -0.014054825529456139, 0.0008391195442527533, 0.010780379176139832, 0.002401146572083235, -0.001717546139843762, 0.00730768870562315, -0.028574498370289803, 0.0018559752497822046, -0.00964560266584158, 0.03811756148934364, 0.01793767511844635, 0.01057529915124178, 0.007854568772017956, -0.012338988482952118, -0.016351722180843353, 0.00974130630493164, -0.018935732543468475, 0.01689860224723816, -0.0014065079158172011, 0.023843983188271523, 0.03896522521972656, -0.015162257477641106, -0.005472221411764622, 0.012735476717352867, 0.01518960203975439, 0.020111525431275368, -0.005537163466215134, -0.0022319555282592773, 0.03325032442808151, 0.0016295325476676226, -0.011730583384633064, -0.018744323402643204, 0.034644871950149536, 0.002426781691610813, -0.004952685441821814, -0.00974130630493164, 0.0037461305037140846, 0.009495209902524948, 0.010684674605727196, 0.01009677816182375, -0.0025088137481361628, -0.02142403833568096, 0.02514282427728176, 0.0009502046159468591, -0.012304808013141155, 0.021588101983070374, 0.012434692122042179, -0.0045937951654195786, -0.03368782997131348, -0.006289124023169279, -0.03956679254770279, 0.026646746322512627, -0.037515994161367416, -0.010506939142942429, -0.0022746804170310497, 0.021697478368878365, 0.01602359488606453, 0.02303733490407467, -0.0255119688808918, -0.008715905249118805, 0.0029292278923094273, 0.017281418666243553, 0.02548462525010109, 0.0008920985856093466, -0.015408353880047798, 0.0015868075424805284, -0.00732819689437747, -0.018607603386044502, -0.038281623274087906, -0.01106749102473259, 0.03593004122376442, 0.004453656729310751, -0.017158370465040207, 0.006911200471222401, 0.0030847471207380295, -0.015271633863449097, 0.012899540364742279, 0.01512124203145504, -0.01836150884628296, -0.03275813162326813, -0.02404906414449215, 0.02582642436027527, -0.019783396273851395, 0.0016756756231188774, 0.018156427890062332, -0.0001816881849663332, -0.0026421158108860254, 0.005878963973373175, 0.02059004455804825, -0.0092354416847229, -0.020658405497670174, 0.009378998540341854, -0.0017431811429560184, 0.021861542016267776, 0.04924657568335533, -0.035465192049741745, 0.014396625570952892, -0.01751384325325489, -0.027111593633890152, 0.019605660811066628, -0.015011865645647049, -0.026742449030280113, 0.017267746850848198, 0.014396625570952892, 0.007164132781326771, -0.03185578063130379, 0.010028418153524399, 0.018853699788451195, 0.001209972775541246, -0.010021582245826721, 0.023406479507684708, 0.020822469145059586, 0.006039610132575035, 0.006039610132575035, -0.01078721508383751, -0.009057706221938133, -0.012961064465343952, 0.021027550101280212, -0.0016543130623176694, -0.01711735501885414, -0.0006652286974713206, 0.005509819369763136, 0.029586227610707283, -0.012277464382350445, -0.0034983253572136164, -0.016064610332250595, -0.02232639119029045, -0.006870184559375048, 0.006938544567674398, -0.004665573127567768, -0.031773749738931656, -0.032484691590070724, -0.014752097427845001, 0.0006079771555960178, -0.019865429028868675, -0.0092354416847229, -0.011463979259133339, 0.0014629049692302942, 0.006637760438024998, 0.013548960909247398, -0.0070752645842731, 0.0018559752497822046, -0.018293147906661034, -0.04284807667136192, -0.020234573632478714, -0.03303157538175583, 0.007690505124628544, 0.036367543041706085, -0.028711218386888504, 0.007581128738820553, -0.003267610212787986, -0.04366839677095413, -0.005602105520665646, 0.005509819369763136, -0.011046983301639557, -0.03147296607494354, 0.0077383569441735744, -0.005960995797067881, -0.012557740323245525, -0.013043096289038658, 0.015927890315651894, 0.014136857353150845, -0.014000137336552143, 0.011860467493534088, -0.0020234573166817427, -0.0060737901367247105, 0.006637760438024998, -0.007335032802075148, -0.03201984614133835, -0.0012091182870790362, 0.03387923911213875, 0.020931845530867577, 0.021806854754686356, 0.0003048430662602186, -0.0078408969566226, 0.011662223376333714, 0.022873271256685257, 0.011874139308929443, -0.025156496092677116, -0.008169025182723999, -0.022941630333662033, 0.003100127913057804, -0.018293147906661034, -0.006675358396023512, -0.015271633863449097, 0.03106280416250229, 0.014451313763856888, 0.004197306931018829, 0.008476645685732365, -0.02322874404489994, 0.005516655743122101, -0.022175999358296394, -0.004737351089715958, -0.015326322056353092, 0.034316740930080414, 0.004354534670710564, 0.007211984600871801, -0.016707194969058037, -0.016119299456477165, -0.016064610332250595, 0.003199250204488635, 0.02160177379846573, 0.0007327342173084617, 0.007321360986679792, -0.006986396387219429, 0.03658629581332207, -0.02030293270945549, -0.00042084153392352164, 0.005906308069825172, -0.0235158558934927, -0.024377191439270973, -0.0029121378902345896, 0.0017978692194446921, 0.01235949620604515, -0.0013535289326682687, -0.028711218386888504, -0.0029035930056124926, 0.0004913377924822271, 0.02295530214905739, 0.023638904094696045, -0.002043965272605419, 0.031637027859687805, 0.026373304426670074, 0.0028283968567848206, 0.03565659746527672, -0.006480532232671976, 0.028054961934685707, 0.005687555763870478, -0.009522554464638233, -0.006200255826115608, 0.01417787279933691, -0.01425990555435419, 0.020138869062066078, 0.00033795495983213186, 0.00589263578876853, 0.0011809198185801506, -0.014000137336552143, 0.009515718556940556, -0.027111593633890152, -0.0008750085835345089, 0.003337679198011756, 0.00881160981953144, -0.01602359488606453, -0.0011082872515544295, -0.0018542661564424634, 0.03863709792494774, 0.020371293649077415, -0.004727097228169441, -0.013309700414538383, -0.0029360640328377485, -0.007239328697323799, 0.0049321772530674934, 0.006548892240971327, 0.012872196733951569, -0.02041230909526348, 0.0056260316632688046, -0.01254406850785017, 0.002953154034912586, 0.010014746338129044, 0.0032659012358635664, 0.02078145369887352, -0.0022370824590325356, 0.01873065158724785, -0.014574361965060234, 0.025306887924671173, 0.023885000497102737, -0.0012484253384172916, -0.037215206772089005, 0.010315530933439732, 0.009372161701321602, -0.01057529915124178, 0.006829168647527695, -0.016994306817650795, 0.026195568963885307, 0.0034385102335363626, 0.009050870314240456, -0.011860467493534088, -0.018621277064085007, 0.002968534827232361, -0.019277533516287804, -0.0073145246133208275, -0.019099796190857887, 0.008791102096438408, -0.0003996926243416965, 0.03582066297531128, -0.010089942254126072, 0.005468803457915783, 0.0021926483605057, 0.00756062101572752, 0.0005250906106084585, 0.03114483691751957, 0.010582135058939457, 0.00019012637494597584, -0.037160519510507584, -0.0011416127672418952, -0.00851766113191843, 0.014437641017138958, 0.000185640252311714, -0.018935732543468475, 0.006166075821965933, 0.01396595686674118, 0.011122179217636585, 0.0034060392063111067, -0.0302971713244915, 0.01782829873263836, -0.02646900899708271, -0.017609547823667526, -0.00948837399482727, -0.012516723945736885, -0.040906649082899094, 0.059828709810972214, 0.00730768870562315, -0.012598755769431591, 0.044898878782987595, -0.004768113140016794, -0.021451381966471672, -0.009481538087129593, -0.007129952777177095, 0.017869316041469574, 0.005048389080911875, -0.023297103121876717, -0.013979628682136536, 0.011744256131350994, -0.010766707360744476, 0.008285237476229668, 0.0049492670223116875, 0.034453462809324265, -0.018634948879480362, 0.00992587860673666, -0.006087461952120066, -0.003951210528612137, 0.006531802471727133, 0.03303157538175583, 0.023912344127893448, -0.003426547162234783, -0.003841834608465433, 0.005280813667923212, 0.020029492676258087, 0.0004090921429451555, -0.023351790383458138, -0.01923651620745659, 0.03379720449447632, 0.0014218889409676194, 0.015886874869465828, -0.018375180661678314, 0.002211447339504957, 0.029859667643904686, 0.019810741767287254, 0.012202267535030842, 0.005284231621772051, -0.01873065158724785, -0.0003462863387539983, -0.004679244942963123, -0.002802761737257242, -0.0056089418940246105, -0.00015348965825978667, -0.0020542193669825792, 0.03005107492208481, -0.020152540877461433, 0.011327259242534637, 0.03516440838575363, 0.0018781921826303005, 0.009112394414842129, -0.01759587600827217, 0.01343274861574173, 0.0004364361520856619, -0.020658405497670174, -0.01644742675125599, -0.007929765619337559, -0.016351722180843353, -0.01804705150425434, -0.030597956851124763, 0.028109651058912277, -0.006395081989467144, -0.02171115018427372, -0.03275813162326813, -0.0066821943037211895, 0.0056089418940246105, -0.03415267914533615, 0.0029634078964591026, -0.02542993612587452, -0.0302971713244915, -0.018703307956457138, -0.0049937013536691666, 0.04569185525178909, -4.972124952473678e-05, 0.024377191439270973, -0.008018633350729942, -0.019564645364880562, -0.013514780439436436, -0.029558883979916573, -0.0014731589471921325, -0.01132042333483696, -0.028601842001080513, 0.005947323981672525, 0.019181828945875168, 0.017472827807068825, 0.011026475578546524, -0.0018423032015562057, 0.010377055034041405, 0.00939267035573721, -0.005390189588069916, 0.026961201801896095, -0.005024463403970003, 0.02157443016767502, 0.023351790383458138, -0.00946103036403656, 0.008647545240819454, 0.02450023964047432, -0.016338050365447998, -0.033523764461278915, -0.0136856809258461, 0.012195431627333164, 0.005352591630071402, 0.006924872752279043, -0.011867303401231766, 0.0031257630325853825, 0.007519605103880167, 0.0033291340805590153, -0.0012894413666799664, -0.0012894413666799664, -0.04473481327295303, 0.03237531706690788, -0.00480229314416647, 0.00459037721157074, 0.024062735959887505, 0.04503559693694115, 0.024377191439270973, -0.015818513929843903, -0.021615445613861084, -0.0369417667388916, -0.00906454212963581, 0.01596890576183796, 0.012701296247541904, 0.022093966603279114, 0.0006767644663341343, 0.007512768730521202, -0.002114034490659833, -0.02745339460670948, 0.007109444588422775, 0.027015889063477516, 0.026263929903507233, -0.01982441358268261, -0.03770739957690239, -0.02686549723148346, -0.0034231292083859444, -0.001326184836216271, 0.010069434531033039, 0.0034846533089876175, 0.004880907014012337, 0.012879032641649246, -0.015873203054070473, -0.01150499563664198, 0.0015329739544540644, 0.02821902558207512, -0.0225861594080925, -0.026455337181687355, 0.014970850199460983, -0.009037197567522526, 0.003742712549865246, 0.008825281634926796, -0.01881268434226513, -0.011258899234235287, -0.010028418153524399, 0.005837947595864534, -0.02832840196788311, -0.010493266396224499, -0.01642008312046528, 0.0048843249678611755, 0.017418138682842255, 0.03150030970573425, 0.018142756074666977, 0.018006036058068275, 0.02917606756091118, 0.002546411706134677, 0.001538955490104854, 0.00011824151442851871, 0.014246233738958836, -0.02139669470489025, 0.02202560566365719, 0.0075059328228235245, -0.0033291340805590153, 0.0040024807676672935, 0.00677106250077486, -0.007628981024026871, 0.05012158304452896, -0.01127940695732832, -0.015134913846850395, -0.0066582681611180305, -0.06551626324653625, 0.003968300763517618, -0.0002997160772792995, 0.00306252995505929, 0.015394682064652443, -0.030844051390886307, -0.011511831544339657, -0.0030454399529844522, 0.0006900092121213675, -0.00189015525393188, -0.011737419292330742, 0.005489311646670103, -0.0039033584762364626, 0.025976816192269325, 0.0056670475751161575, 0.01468373741954565, -0.016338050365447998, -0.006316468119621277, -0.016570474952459335, 0.002486596582457423, 0.0007891312707215548, -0.01576382666826248, 0.012352660298347473, -0.00844930112361908, 0.0033034991938620806, -0.03387923911213875, -0.018197443336248398, -0.021465053781867027, 0.015558745712041855, -0.014423969201743603, -0.035492535680532455, 0.005796931684017181, -0.0016611490864306688, -0.01791033148765564, -0.011094835586845875, -0.00575249781832099, 0.018757997080683708, -0.023898672312498093, 0.03005107492208481, -0.012154416181147099, -0.007519605103880167, 0.01644742675125599, -0.0038760146126151085, 0.022463111206889153, 0.03330501541495323, 0.012619264423847198, 0.01966034807264805, -2.4099586880765855e-05, -0.021642789244651794, -0.010623150505125523, -0.006941962521523237, 0.005796931684017181, 0.011607535183429718, 0.014779441989958286, -0.02171115018427372, -0.014957177452743053, 0.0006989814573898911, 0.013760876841843128, -0.039047256112098694, -0.033578455448150635, 0.01987910084426403, -0.007253000512719154, 0.01698063500225544, -0.030543267726898193, -0.008804773911833763, -0.019523628056049347, -0.004939013160765171, -0.0030317679047584534, 0.009119230322539806, -0.018826356157660484, 0.0075401128269732, -0.01836150884628296, 0.029695603996515274, -0.009543062187731266, 0.02410375140607357, 0.008114336989820004, -0.017951346933841705, -0.010165139101445675, -0.006046446040272713, -0.004139200784265995, 0.007854568772017956, -0.005878963973373175, 0.0329495407640934, -0.028601842001080513, 0.00041955977212637663, 0.02367991954088211, 0.013125128112733364, -0.012735476717352867, -0.013555796816945076, -0.013213996775448322, 0.03677770495414734, -0.018990419805049896, -0.013747205026447773, -0.007581128738820553, -0.021164270117878914, 0.008408285677433014, 0.006586490198969841, -0.016625162214040756, -0.033605799078941345, -0.00027771267923526466, 0.0005797786288894713, 0.02083614096045494, 0.03037920407950878, -0.0003887977509293705, -0.03147296607494354, -0.011115343309938908, 0.005653375759720802, -0.010937606915831566, 0.01231847982853651, -0.03303157538175583, -0.03327766805887222, -0.030461234971880913, 0.008661217987537384, -0.0024729245342314243, 0.013330209068953991, 0.006172912195324898, 0.003318880219012499, -0.0013791639357805252, 0.010807722806930542, -0.00651813019067049, 0.019222844392061234, 0.016338050365447998, -0.013774548657238483, -0.011258899234235287, 0.012400511652231216, -0.018744323402643204, -0.00682233227416873, 0.009495209902524948, -0.0012082637986168265, -0.0018217952456325293, -0.02739870548248291, 0.02162911742925644, 0.005147511139512062, 0.025799080729484558, -0.005790095776319504, 0.005865291692316532, 0.03398861363530159, 0.004040078725665808, -0.018484555184841156, -0.021834198385477066, 0.006634342484176159, 0.001196300727315247, -0.01788298785686493, 0.027152609080076218, -0.0369417667388916, 0.030844051390886307, -0.029640914872288704, -0.02016621269285679, 0.02059004455804825, 0.004853563383221626, 0.007936601527035236, -0.009242277592420578, -0.021834198385477066, -0.0012569703394547105, -0.006996650714427233, -0.021724821999669075, -0.023912344127893448, -0.02162911742925644, 0.021888885647058487, -0.01717204414308071, -0.0019192082108929753, 0.011908319778740406, 0.01711735501885414, 0.0030796199571341276, 0.010424906387925148, 0.003308626124635339, -0.012236448004841805, 0.00948837399482727, -0.0008395467884838581, -0.02532055974006653, -0.016310706734657288, 0.00017976555682253093, 0.02508813701570034, 0.031554996967315674, -0.01253039576113224, -0.019646676257252693, -0.0010254007065668702, -0.003886268474161625, -0.005147511139512062, -0.009556734003126621, 0.014232560992240906, 0.01873065158724785, 0.02508813701570034, -0.006439516320824623, 0.012024532072246075, -0.0029411909636110067, 0.027234641835093498, -0.018757997080683708, 0.005397025495767593, -0.019892772659659386, 0.032594069838523865, 0.033414389938116074, -0.009447358548641205, -0.013986465521156788, 0.0010553081519901752, 0.022121310234069824, -0.0013441293267533183, 0.013207160867750645, 0.029039347544312477, 0.00422465056180954, 0.016994306817650795, 0.01663883402943611, 0.002136251423507929, 0.011005966924130917, 0.01129307970404625, -0.005106495227664709, -0.04372308403253555, 0.017773611471056938, 0.003626500256359577, -0.0017021651146933436, -0.017048995941877365, 0.005496147554367781, 0.005855037830770016, -0.01923651620745659, 0.005092823412269354, 0.00047211156925186515, -0.04314886033535004, 0.024869384244084358, -0.005578179843723774, 0.027179954573512077, 0.012045039795339108, 0.009358489885926247, 0.021970918402075768, 0.016597818583250046, 0.004754441324621439, -0.011470815166831017, -0.009679782204329967, -0.0005233815754763782, 0.008941493928432465, 0.0033650232944637537, -0.0027583278715610504, -0.011340931057929993, 0.00675055431202054, -0.006620670203119516, 0.00014857629139441997, -0.004327191039919853, -0.023105695843696594, 0.011211046949028969, -0.04550044611096382, 0.010903427377343178, 0.024760007858276367, -0.0021208703983575106, 0.004323772620409727, -0.01698063500225544, -0.008852626197040081, -0.008777429349720478, 0.002061055274680257, -0.0005302175995893776, -0.021478725597262383, -0.017609547823667526, 0.015572418458759785, 0.003342806128785014, -0.00012742739636451006, -0.011204211041331291, 0.02184787020087242, -0.002225119387730956, -0.002821560949087143, 0.21120519936084747, 0.004200724884867668, -0.012195431627333164, 0.030871396884322166, -0.029148723930120468, -0.02229904569685459, 0.0006823186995461583, -0.006859930697828531, -0.009871190413832664, 0.004928759299218655, -0.002884793793782592, 0.022599831223487854, -0.009474702179431915, -0.007752029225230217, 0.0007472607539966702, -0.017131026834249496, -0.01036338321864605, -0.023488512262701988, 0.025676032528281212, 0.008982510305941105, 0.019537299871444702, -0.001004892634227872, -0.010411234572529793, -0.006613834295421839, 0.014451313763856888, 0.003035186091437936, -0.003995644394308329, -0.026236584410071373, 0.006483950186520815, 0.029941700398921967, -0.012140744365751743, -0.021998262032866478, 0.006774480454623699, 0.00709577277302742, -0.013624156825244427, 0.013200324960052967, 0.014998193830251694, -0.021724821999669075, 0.020521685481071472, 0.0035547222942113876, -0.006709538400173187, -0.026879169046878815, 0.011149522848427296, -0.004627975169569254, 0.012646608054637909, 0.004980029072612524, 0.007868241518735886, -0.010623150505125523, 0.01045908685773611, 0.023392807692289352, -0.02697487361729145, -0.011491323821246624, 0.02827371470630169, 0.029668258503079414, -0.002732692752033472, 0.00015979159798007458, 0.028164338320493698, -0.006309632211923599, 0.013419076800346375, 0.00410502078011632, -0.016707194969058037, 0.021164270117878914, -0.008695397526025772, 0.015394682064652443, -0.008756921626627445, 0.004163126926869154, 0.00600543012842536, -0.011648551560938358, 0.02049434185028076, -0.01138878334313631, -0.006784734316170216, -0.021670134738087654, -0.017705252394080162, 0.0011903193080797791, -0.006060117855668068, -0.003742712549865246, 0.014205217361450195, 0.008367269299924374, 0.038308970630168915, 0.006374574266374111, 0.000777595501858741, 0.015394682064652443, -0.013952285051345825, -0.02311936765909195, -0.01150499563664198, -0.014082169160246849, 0.005759333726018667, -0.010500103235244751, 0.006261779926717281, -0.0008660363382659853, -0.026605729013681412, -0.003776892554014921, -0.01889471709728241, -0.012653443962335587, 0.003585484344512224, -0.019947461783885956, -0.01347376499325037, 0.017131026834249496, -0.020125197246670723, -0.001355237909592688, -0.024459224194288254, 0.045992638915777206, 0.024732664227485657, 0.0048843249678611755, 0.014560689218342304, -0.009351653978228569, 0.006610416341573, 0.005318411625921726, 0.011334095150232315, -0.023488512262701988, 0.008257893845438957, -0.00946103036403656, 0.01852557249367237, -0.006190001964569092, -0.004077676683664322, 0.021929902955889702, -0.0016790935769677162, -0.005619195755571127, 0.021191613748669624, -0.01512124203145504, 0.01684391498565674, -0.011094835586845875, -0.01599625125527382, 0.007382884621620178, 0.0156817939132452, -0.02466430328786373, -0.02109590917825699, 0.007287180982530117, -0.020603718236088753, -0.017705252394080162, 0.023379135876893997, -0.008107501082122326, 0.007731521036475897, 0.02128731831908226, 0.0013372934190556407, -0.01257141213864088, -0.006029355805367231, -0.01950995624065399, -0.010657330974936485, 0.002671168651431799, 0.0019431342370808125, -0.011231555603444576, 0.003534214338287711, -0.01164171565324068, -0.005297903437167406, 0.004925341345369816, 0.04153556376695633, -0.00623785424977541, -0.028164338320493698, -0.01796502061188221, 0.004221232607960701, -0.027439722791314125, -0.0008305745432153344, -0.015285305678844452, 0.012181759811937809, 0.00895516574382782, -0.006207092199474573, -0.04716842994093895, 0.010001074522733688, 0.004289592616260052, -0.015025537461042404, 0.03119952417910099, 0.0010142921237275004, -0.014451313763856888, -0.011751092039048672, -0.017677906900644302, -0.17412669956684113, 0.003817908465862274, 0.013719861395657063, -0.02049434185028076, 0.016543131321668625, 0.027248313650488853, 0.023433823138475418, 0.033414389938116074, -0.014506001025438309, 0.00558843370527029, 0.005438041407614946, -0.005837947595864534, -0.02728932909667492, -0.01998847723007202, -0.022039277479052544, -0.015244290232658386, -0.04137149825692177, -0.011860467493534088, 0.028000274673104286, 0.02038496546447277, 0.011416127905249596, -0.021478725597262383, 0.030762020498514175, -0.01889471709728241, -0.015900546684861183, 0.005383353680372238, -0.012585083954036236, 0.012017696164548397, 0.0184708833694458, -0.028847938403487206, 0.0035683943424373865, -0.016748210415244102, 0.02585376799106598, -0.02059004455804825, 0.0108965914696455, 0.024828368797898293, -0.0005379081121645868, -0.000244387163547799, 0.009987402707338333, -0.0075469487346708775, 0.051570817828178406, 0.014629049226641655, -0.01703532226383686, -0.013412240892648697, -0.0070410845801234245, 0.003095000982284546, 0.0312268678098917, -0.0013449839316308498, 0.010821394622325897, -0.04924657568335533, 0.010069434531033039, -0.0055405814200639725, 0.019646676257252693, -9.047024650499225e-05, 0.012441528029739857, 0.0063643199391663074, -0.005636285524815321, 0.02784988284111023, -0.013665173202753067, -0.016597818583250046, -0.018566587939858437, -0.02224435843527317, 0.013883925043046474, -0.036531608551740646, 0.015162257477641106, -0.02466430328786373, -0.004351116716861725, -0.0192638598382473, -0.028683874756097794, 0.009878026321530342, -0.0012843143194913864, -0.02638697810471058, -0.00974130630493164, 0.0004973193281330168, 0.018375180661678314, 0.01692594774067402, -0.014779441989958286, -0.008435629308223724, -0.008175861090421677, 9.575746662449092e-05, -0.01594156213104725, 0.016392739489674568, 0.0019243352580815554, -0.015750154852867126, -0.020316604524850845, -0.004788621328771114, -0.008175861090421677, -0.011101671494543552, 0.013719861395657063, 0.008777429349720478, 0.01203820388764143, -0.005547417793422937, -0.0020593462977558374, -0.028929971158504486, -0.005643121898174286, -0.005349173676222563, 0.010438579134643078, 0.005947323981672525, 0.029832324013113976, 0.01138878334313631, 0.005479057785123587, -0.007225656881928444, -0.007177804596722126, 0.01827947609126568, 0.020015820860862732, 0.0008536460809409618, -0.02832840196788311, 0.014355609193444252, 0.02819168195128441, -0.007341868709772825, -0.026277601718902588, -0.008592857979238033, 0.01759587600827217, 0.03964882716536522, -0.003937538713216782, 0.026373304426670074, 0.0043408628553152084, -0.01421888917684555, 0.025156496092677116, 0.014382953755557537, 0.04746921360492706, -0.0023242414463311434, -0.00920809805393219, -0.003910194616764784, -0.00047082980745472014, -0.012838016264140606, -0.11801677942276001, -0.04569185525178909, 0.002146505517885089, 0.01669352315366268, 0.03018779493868351, 0.03784412145614624, -0.013166144490242004, -0.008545005694031715, -0.01782829873263836, 0.022066622972488403, -0.02489672787487507, -0.031008116900920868, -0.011860467493534088, 0.026236584410071373, -0.016351722180843353, -0.011101671494543552, 0.009105557575821877, -0.02824637107551098, -0.02367991954088211, 0.012045039795339108, 0.005202199332416058, -0.002322532469406724, -0.006446352228522301, -0.00150135753210634, -0.0014569234335795045, 0.011498159728944302, -0.01998847723007202, 0.014765769243240356, 0.006586490198969841, -0.01687125861644745, -0.01920917257666588, -0.018060723319649696, -0.023611558601260185, -0.02655104175209999, -0.0009416596149094403, 0.003988808486610651, -0.026646746322512627, -0.019591988995671272, 0.034617528319358826, -0.029476851224899292, 0.0016594401095062494, 0.02370726317167282, 0.006586490198969841, 0.008271565660834312, -0.004364788997918367, -0.029066691175103188, 0.0004776658024638891, 0.022175999358296394, 0.0013065313687548041, 0.006261779926717281, -0.020179884508252144, -0.024541256949305534, -0.01703532226383686, -0.00916708167642355, 0.019578317180275917, 0.0006378846592269838, 0.011053819209337234, 0.008230549283325672, -0.006316468119621277, -0.03579331934452057, -0.01886737160384655, 0.019674021750688553, -0.0028882119804620743, 0.00802546925842762, 0.006794988643378019, -0.03191046789288521, -0.013993301428854465, -0.018703307956457138, 0.008011797443032265, -0.0011544302105903625, 0.0013595103519037366, 0.019633004441857338, -0.015230617485940456, 0.002238791435956955, -0.014095840975642204, 0.004238322842866182, -0.031746406108140945, -0.008551841601729393, 0.038281623274087906, -0.03114483691751957, 0.010513775050640106, -0.018963076174259186, 0.020658405497670174, -0.02396703138947487, 0.03603941574692726, 0.024363519623875618, 0.01017881091684103, -0.0003610264975577593, 0.02261350303888321, -0.0183341633528471, -0.001963642193004489, 0.0016825116472318769, 0.02424047142267227, -0.02171115018427372, 0.014547017402946949, -0.02264084666967392, -0.021998262032866478, -0.012468872591853142, 0.010698347352445126, 0.0007053902372717857, -0.049492672085762024, -0.009194426238536835, -0.04153556376695633, 0.020904501900076866, -0.008435629308223724, -0.01827947609126568, -0.02819168195128441, -0.007471752818673849, 0.02202560566365719, -0.005605523474514484, 0.007136788684874773, 0.018006036058068275, -0.019783396273851395, 0.005130421370267868, -0.000650702160783112, -0.03204718977212906, -0.023474838584661484, -0.017281418666243553, 0.02373460680246353, 0.02091817371547222, 0.003259065095335245, 0.00960458628833294, -0.013350716792047024, 0.019072452560067177, 0.013815565034747124, 0.02612720988690853, -0.009365325793623924, 0.023324446752667427, 0.001326184836216271, -0.006586490198969841, -0.012448363937437534, -0.007136788684874773, 0.0056089418940246105, -0.0225861594080925, 0.0003723486152011901, 0.015339993871748447, -0.011094835586845875, 0.003035186091437936, -0.0030146779026836157, 0.04216447472572327, 0.022709207609295845, 0.04530903697013855, -0.03844568878412247, -0.01706266775727272, -0.008162189275026321, -0.0074649169109761715, 0.010636823251843452, 0.019250188022851944, -0.014369281008839607, 0.01468373741954565, 0.022900614887475967, 0.015900546684861183, 0.022599831223487854, 0.006138732191175222, -0.01950995624065399, -0.023775624111294746, 0.022011933848261833, -0.013330209068953991, 0.0018952821847051382, 0.02314671128988266, 0.009795994497835636, 0.013425912708044052, 0.03469955921173096, -0.0027173117268830538, 0.023242415860295296, -0.012154416181147099, 0.027781521901488304, 0.00612505991011858, -0.03199250251054764, 0.019646676257252693, 0.009973730891942978, -0.022845927625894547, -0.022312719374895096, 0.002816433785483241, 0.02069942094385624, -0.0037461305037140846, 0.01562710665166378, -0.007020576391369104, 0.0060737901367247105, 0.009358489885926247, 0.013125128112733364, 0.0066821943037211895, 0.018375180661678314, -0.004197306931018829, -0.008668053895235062, -0.014082169160246849, 0.021957246586680412, 0.006302796304225922, -0.005079151131212711, 0.014287249185144901, 0.008948329836130142, 0.004621138796210289, 0.015039210207760334, -0.001973896287381649, 0.0005648248479701579, 0.01558609027415514, 0.04746921360492706, 0.004009316675364971, -0.020261917263269424, -0.007253000512719154, 0.03997695446014404, 0.016816571354866028, -0.00456986902281642, 0.006542056333273649, 0.003134308150038123, -0.02849246747791767, 0.0034453461412340403, 0.021383022889494896, -0.028601842001080513, -0.04030508175492287, 0.007225656881928444, 0.014136857353150845, -0.002486596582457423, 0.0010074562160298228, 0.01788298785686493, 0.008832117542624474, -0.0015645904932171106, 0.029039347544312477, 0.009754978120326996, -0.020959189161658287, -0.007888749241828918, 0.024486567825078964, 0.0050586434081196785, -0.008257893845438957, -0.002561792731285095, -0.007376048713922501, 0.04131681099534035, 0.02450023964047432, 0.02264084666967392, -0.008148517459630966, 0.03316829353570938, 0.010889755561947823, 0.04621139168739319, -0.003923866432160139, 0.01551773026585579, -0.005403861403465271, -0.017131026834249496, -0.014287249185144901, 0.001287732389755547, -0.014984522014856339, 0.0031582340598106384, 0.09078213572502136, 0.03026982769370079, 0.0007305979961529374, 0.03218390792608261, -0.026441665366292, 0.017924003303050995, 0.01419154554605484, -0.020152540877461433, -0.012243283912539482, -0.05110596865415573, 0.013323373161256313, -0.0008980800630524755, 0.01695329137146473, -0.02317405492067337, 0.004392133094370365, 0.007300852797925472, -0.010903427377343178, 0.010780379176139832, -0.02790457010269165, -0.03210187703371048, 0.011197375133633614, 0.012195431627333164, 0.012127071619033813, 0.0017004561377689242, 0.008045976981520653, -0.016488442197442055, 0.014164200983941555, -0.003937538713216782, -0.0013629284221678972, -0.030926084145903587, -0.016283363103866577, 0.00589263578876853, -0.03973085805773735, -0.018744323402643204, 0.0005109913181513548, -0.022203342989087105, 0.003766638459637761, -0.01669352315366268, 0.006104552187025547, 0.005420951638370752, -0.0010279641719534993, 0.004433149006217718, -0.009898534044623375, -0.01729509048163891, -0.014916162006556988, -0.008750085718929768, 0.021355677396059036, -0.014574361965060234, -0.03997695446014404]} +{"id": "test:10000037", "text": "\"2 New England NE 16 1 plays -1 yards 0:07 End of half.\\n4 Pittsburgh Pit 37 2 plays -1 yards 1:01 End of game.\\n2 8 opp29 1:57 Pit Bleier 0 Bas Draw/SI 4-3 sZ Run-Harris, F.\"", "vector_field": [-0.010413474403321743, -0.0037310307379812002, 0.029319219291210175, -0.009800916537642479, -0.02483641356229782, 0.017805926501750946, 0.0028696220833808184, 0.012153693474829197, -0.021787548437714577, -0.03753305599093437, 0.028567444533109665, 0.014603923074901104, 0.00406515272334218, 0.007698731496930122, 0.00802589301019907, -0.01101210992783308, 0.0015226921532303095, 0.0013686827151104808, 0.0065293037332594395, -0.015870802104473114, -0.008763745427131653, 0.011220935732126236, -0.003811080940067768, -0.0018602947238832712, -0.004287900868803263, 0.0004724696045741439, 0.027871357277035713, -0.0089168855920434, 0.002551161916926503, -0.013998325914144516, 0.015355697833001614, -0.003092370228841901, -0.02798273041844368, -0.030767083168029785, -0.023834047839045525, -0.007684809621423483, -0.01351106446236372, 0.018126126378774643, 0.02660447731614113, 0.0019681884441524744, 0.017583178356289864, -0.016232768073678017, -0.015689820051193237, -0.002559863030910492, -0.023694830015301704, 0.010044547729194164, 0.007434218190610409, -0.05053597688674927, -0.011708197183907032, 0.005008351989090443, -0.002490254119038582, 0.0056313504464924335, -0.02585270255804062, 0.002594667486846447, 0.0009858343983069062, -0.003292495384812355, 0.003616176312789321, 0.018348874524235725, -0.020562434569001198, -0.019393006339669228, -0.025393284857273102, -0.011652510613203049, -0.01337880827486515, -0.00813030544668436, 0.0022414028644561768, -0.03602950647473335, 0.010211608372628689, 0.016706107184290886, -0.028372539207339287, 0.007698731496930122, 0.039537787437438965, 0.01893358863890171, 0.003494360949844122, -0.00863844994455576, 0.018209658563137054, -0.01790337823331356, -0.010141999460756779, -0.0005050987238064408, 0.009543363936245441, 0.0037484329659491777, 0.02943059429526329, -0.026186823844909668, -0.02970902808010578, 0.02819155715405941, 0.037421680986881256, -0.019225945696234703, -0.008986493572592735, 0.026298198848962784, 0.005161491222679615, 0.016650421544909477, 0.023277176544070244, 0.025741327553987503, 0.03176944702863693, 0.03135179728269577, -0.012766251340508461, 0.03889738768339157, -0.013364886865019798, 0.02468327432870865, 0.006504940800368786, -0.01748572662472725, 0.002558122854679823, -0.023847969248890877, 0.018682997673749924, -0.008060697466135025, 0.00030845392029732466, -0.01907280646264553, -0.013204785995185375, -0.018543779850006104, 0.003544827224686742, -0.01570374146103859, -0.006988721899688244, 0.0005481691914610565, 0.004253096878528595, -0.017513569444417953, 0.0007461191271431744, -0.0001517253986094147, 0.009438950568437576, -0.005958511959761381, -0.05050813406705856, -0.030265899375081062, 0.027314485982060432, -0.012961155734956264, 0.01646943762898445, -0.020256156101822853, 0.025824857875704765, -0.0007700471905991435, -0.003574410919100046, -0.016065707430243492, -0.004051231313496828, -0.021523036062717438, 0.025950154289603233, 0.018766527995467186, 0.009132672101259232, -0.010510926134884357, -0.02038145251572132, 0.005815813783556223, -0.017318665981292725, -0.009438950568437576, -0.022330498322844505, -0.02216343581676483, 0.018905745819211006, 0.015550602227449417, -0.0037901983596384525, -0.016037864610552788, 0.003457816317677498, 0.027258800342679024, 0.002145690843462944, 0.008742863312363625, -0.0195461455732584, -0.019017118960618973, -0.0024571900721639395, -6.030948497937061e-05, -0.01816789247095585, -0.007301961537450552, 0.019225945696234703, 0.006651119329035282, 0.005279826000332832, 0.024251699447631836, -0.028790192678570747, 0.014896279200911522, -0.019699284806847572, 0.00529722822830081, 0.010705830529332161, -0.007830987684428692, 0.001058053458109498, 0.019796738401055336, 0.005057078320533037, -0.002025615656748414, -0.0024676313623785973, -0.025017397478222847, 0.010037587024271488, -0.005666154902428389, -0.0022013778798282146, 0.03324515372514725, 0.015439228154718876, 0.021425584331154823, -0.008958650752902031, 0.018836136907339096, -0.016037864610552788, -0.03602950647473335, 0.0016114433528855443, -0.009717386215925217, 0.021230679005384445, 0.044271185994148254, -0.013218708336353302, 0.0016131835291162133, 0.032549068331718445, -0.010155921801924706, 0.004771681968122721, -0.010246412828564644, 0.008241679519414902, 0.02249755896627903, -0.01797298714518547, -0.007267157081514597, -0.6134482622146606, -0.01392871793359518, -0.02564387582242489, 0.0025459411554038525, -0.007691770792007446, 0.008575801737606525, 0.007949323393404484, -0.01873868517577648, -0.005022273864597082, 0.016218846663832664, 0.0017497907392680645, 0.0067694541066884995, 0.0017480505630373955, -0.021453427150845528, -0.005951550789177418, -0.03452595695853233, -0.016204925253987312, -0.020868713036179543, -0.011903101578354836, -0.015996098518371582, -0.005965472664684057, 0.01968536339700222, 0.021801471710205078, 0.002439787844195962, 0.010044547729194164, -0.0011937905801460147, 0.003946817945688963, -0.01898927614092827, 0.012703603133559227, 0.00026973403873853385, -0.02188500203192234, 0.009070024825632572, 0.021161070093512535, -0.01660865545272827, 0.04541277140378952, 0.006960878148674965, -0.00025276688393205404, 0.030070994049310684, 0.0356675423681736, 0.027523312717676163, 0.0005037935916334391, -0.005586104933172464, 0.012912428937852383, 0.004562855698168278, -0.012961155734956264, -0.012432129122316837, 0.017527490854263306, 0.000538162887096405, 0.030572177842259407, -0.04622023180127144, -0.007796183694154024, 0.002436307491734624, 0.02661839872598648, 0.0006956527940928936, 0.023026585578918457, 0.021982453763484955, 0.04925517365336418, -0.009870525449514389, 0.003541346872225404, -0.018084362149238586, 0.012265067547559738, 0.014325487427413464, -0.02757900021970272, -0.006487538572400808, -0.011067796498537064, 0.010928578674793243, -0.013121255673468113, 0.01488235779106617, 0.015550602227449417, -0.004441040568053722, -0.01230683270841837, 0.028247244656085968, -0.020590277388691902, 0.013573712669312954, 0.029374906793236732, -0.016831403598189354, 0.020047329366207123, 0.0035048022400587797, -0.029987463727593422, -2.258206768601667e-05, 0.010622300207614899, -0.0022936093155294657, -0.004228733479976654, -0.012028397992253304, 0.03488792106509209, -0.0001450908021070063, -0.01811220496892929, 0.01804259605705738, -0.006619795225560665, 0.010566613636910915, 0.017443960532546043, 0.005770568270236254, -0.028957253322005272, -0.0023458159994333982, -0.0015531459357589483, 0.013733812607824802, -0.014151466079056263, 0.017750238999724388, 0.008721980266273022, -0.0294862799346447, -0.016706107184290886, -0.006821660790592432, 0.0063169971108436584, -0.02681330405175686, 0.01763886585831642, -0.0008292146376334131, 0.0037240698002278805, -0.028985098004341125, 0.04791868478059769, -0.020604200661182404, 0.008965611457824707, -0.017262978479266167, -0.05159403011202812, -0.014151466079056263, -0.0025668239686638117, -0.034191835671663284, 0.02120283618569374, -0.02015870437026024, 0.011220935732126236, -0.020061250776052475, 0.031017674133181572, -0.008812472224235535, -0.0018950991798192263, -0.009814838878810406, -0.007552552968263626, 0.04535708203911781, 0.02064596489071846, -0.03229847550392151, -0.015731584280729294, -0.030544335022568703, -0.023360706865787506, 0.011958789080381393, 0.020826948806643486, 0.005718361586332321, 0.008165109902620316, 0.011234858073294163, 0.037004027515649796, -0.015787271782755852, 0.007103576324880123, -0.009961016476154327, 0.017583178356289864, 0.011290544643998146, -0.00013856498117092997, -0.009153555147349834, -0.005519976373761892, -0.04897674173116684, -0.0065675885416567326, -0.01674787327647209, 0.018265344202518463, 0.023151881992816925, -0.02230265364050865, 0.005798411555588245, -0.012661837972700596, -0.008631489239633083, -0.021383818238973618, -0.011429762467741966, -0.006731169298291206, -0.016274534165859222, -0.01777808368206024, -0.022887367755174637, -0.000607336638495326, 0.020395373925566673, -0.010274256579577923, -0.017165526747703552, -0.01996379904448986, 0.002726924140006304, -0.007963244803249836, 0.013023803010582924, 0.004051231313496828, -0.0008092020871117711, 0.00275302748195827, 0.001216413453221321, -0.015188636258244514, -0.01426980085670948, 0.0005955901579000056, 0.010677987709641457, -0.018710840493440628, -0.010016703978180885, 0.007301961537450552, 0.00285396003164351, -0.010371708311140537, 0.0003589202824514359, 0.018850058317184448, 0.02468327432870865, 0.012940272688865662, -0.005808853078633547, 0.015898646786808968, 0.004983989056199789, -0.026089372113347054, 0.007810105569660664, 0.010037587024271488, 0.01610747165977955, -0.02586662396788597, 0.008213836699724197, 0.008248641155660152, 0.02674369513988495, 0.0001456346217310056, -0.0039955442771315575, 0.011374074965715408, 0.03474870324134827, 0.02550465799868107, 0.014409017749130726, 0.009438950568437576, -0.00665459968149662, -0.0025877065490931273, -0.020520668476819992, 0.02784351259469986, -0.011917023919522762, 0.0281219482421875, -0.0040303487330675125, 0.0012285950360819697, -0.01299596019089222, -0.007844910025596619, 0.005540858954191208, -0.0045176101848483086, 0.03174160420894623, 0.0026277315337210894, 0.03711540251970291, 0.014826671220362186, 0.013128216378390789, 0.008589724078774452, -0.005561742000281811, 0.017652787268161774, 0.021773627027869225, -0.027676451951265335, 0.001440031686797738, 0.02826116606593132, 0.029931776225566864, -0.0039642201736569405, -0.01756925694644451, 0.0049665868282318115, 0.010030625388026237, 0.021620487794280052, -0.009285811334848404, -0.008144227787852287, -0.00966866035014391, 0.013455377891659737, 0.007517748512327671, 0.025212300941348076, -0.0015809894539415836, -0.0012955934507772326, -0.002352776937186718, 0.01399136520922184, -0.010831126943230629, 0.008854237385094166, 0.006873867474496365, 0.02489210106432438, 0.006557147484272718, -0.028302932158112526, 0.021982453763484955, -0.017917301505804062, 0.02345816045999527, -0.028080184012651443, 0.002848739502951503, 0.023291099816560745, -0.01873868517577648, -0.00485521275550127, -0.00957120768725872, 0.009606012143194675, 0.0027130022644996643, 0.00713490042835474, 0.003960739821195602, 0.0026520946994423866, -0.0044480012729763985, 0.004099957179278135, -0.006136014126241207, -0.0026242511812597513, -0.02113322727382183, -0.00655366713181138, -0.011582901701331139, -0.01358067337423563, -0.009919251315295696, 0.009000415913760662, -0.004287900868803263, 0.003490880597382784, 0.0014713556738570333, 0.022887367755174637, -0.029792558401823044, 0.015815116465091705, 0.005843657534569502, -0.017235135659575462, -0.020325765013694763, 0.034470271319150925, 0.014715297147631645, -0.010831126943230629, -0.009703464806079865, 0.014659609645605087, -0.01145760528743267, -0.0007978907087817788, 0.023834047839045525, -0.008944728411734104, 0.011297505348920822, 0.009766112081706524, 0.017541414126753807, 0.022135592997074127, 0.0024606704246252775, 0.022678541019558907, -0.02111930400133133, 0.000721756077837199, -0.04786299914121628, 0.007712653372436762, 0.00897953286767006, 0.0017402195371687412, 0.001728908158838749, 0.028706662356853485, 0.019184181466698647, 0.014283722266554832, -0.015286088921129704, -0.03098982945084572, -0.0035639696288853884, 0.011207014322280884, -0.013016842305660248, 0.006090768612921238, 0.033440060913562775, -0.03444242477416992, -0.009599051438272, -0.010503965429961681, -0.03190866485238075, 0.028247244656085968, -0.005245022010058165, -0.014248917810618877, -0.018641231581568718, -0.0006177779287099838, -0.019184181466698647, 0.06777111440896988, 0.023611299693584442, 0.023346785455942154, -0.0005481691914610565, -0.02263677679002285, 0.02098008804023266, 0.00795628409832716, -0.019170258194208145, -0.006334399338811636, 0.0059028249233961105, 0.014144504442811012, -0.0020900038070976734, 0.006449253764003515, 0.005429484881460667, 0.006069886032491922, -0.01012111734598875, 0.002163092838600278, -0.04354725405573845, 0.01381734386086464, 0.006501460447907448, 0.010747596621513367, 0.010559652000665665, -0.004287900868803263, 0.03307809308171272, -0.03060002066195011, -0.021829314529895782, 0.022748149931430817, -0.011401918716728687, 0.021425584331154823, -0.006296114530414343, -0.019713208079338074, 0.010023664683103561, -0.025212300941348076, 0.016037864610552788, -0.0035152435302734375, 0.013051646761596203, 0.021342052146792412, -0.030154524371027946, 0.022274810820817947, 0.008325210772454739, 0.002730404492467642, -0.001976889558136463, -0.0028626611456274986, -0.01182653196156025, 0.03380202502012253, -0.005307669751346111, 0.001385214738547802, 0.009766112081706524, 0.018961433321237564, -0.04711122438311577, 0.017541414126753807, 0.006473616696894169, -0.00356745021417737, 0.01556452363729477, 0.0013478000182658434, 0.0003547872765921056, -0.009480716660618782, -0.005432965699583292, 0.018362797796726227, -0.00017761116032488644, 0.005878461990505457, -0.018836136907339096, 0.026701929047703743, 0.0074968659318983555, 0.007103576324880123, -0.04652651026844978, -0.03394124284386635, 0.011179170571267605, -0.03658637776970863, 0.0011955308727920055, -0.01947653852403164, -0.004472364205867052, -0.009452872909605503, 0.007998049259185791, 0.03263259679079056, 0.012181537225842476, 0.01790337823331356, 0.012063202448189259, 0.0058993445709347725, -0.009376303292810917, 0.004914380144327879, -0.020130859687924385, 0.009870525449514389, -0.0046916319988667965, 0.020673809573054314, 0.02415424771606922, 0.02373659424483776, -0.015174714848399162, -0.01617708057165146, -0.015132949687540531, 0.0029862169176340103, -0.011777806095778942, 0.022929133847355843, 0.01207712385803461, 0.00016162289830390364, -0.014687453396618366, 0.025476815178990364, 0.005972433835268021, 0.007900596596300602, -0.006334399338811636, 0.021996375173330307, -0.03182513639330864, 0.012849781662225723, 0.013935678638517857, 0.02256716787815094, 0.022469716146588326, 0.02017262578010559, 0.015327854081988335, -0.005495613440871239, -0.0025285391602665186, 0.010295138694345951, -0.03655853122472763, 0.010086312890052795, 0.0056278700940310955, -0.004058192018419504, 0.004945704247802496, 0.012571346014738083, -0.015968255698680878, 0.0054155634716153145, -0.015035497024655342, 0.024251699447631836, -0.02867881953716278, 0.011930945329368114, -0.02510092779994011, -0.02235834114253521, 0.004127800930291414, -0.0021335091441869736, -0.023666987195611, -0.01196574978530407, 0.006905191112309694, -0.006960878148674965, -0.0031045516952872276, 0.023346785455942154, -0.010434356518089771, -0.0274397823959589, 0.005293747875839472, 0.022372262552380562, 0.011805649846792221, -0.012480854988098145, 0.005245022010058165, -0.008812472224235535, -0.008561880327761173, -0.01729082129895687, 0.014631765894591808, -0.021342052146792412, -0.0240150298923254, -0.025699563324451447, 0.017457883805036545, 0.01323959045112133, -0.004496727604418993, -0.005951550789177418, 0.003219406120479107, -0.014395096339285374, 0.005485171917825937, -0.003814561292529106, -0.025142692029476166, -0.013984404504299164, 0.001563587342388928, 0.00881943292915821, 0.024028951302170753, 0.03318946808576584, 0.004872614983469248, -0.0072601959109306335, 0.028901567682623863, 0.0125156594440341, 0.0006778155220672488, -0.00027604232309386134, -0.014395096339285374, -0.040094658732414246, -0.010002782568335533, -0.007587357424199581, 0.03129610791802406, -0.008276483975350857, 0.001363462070003152, 0.009035220369696617, 0.008798549883067608, 0.0010415214346721768, 0.008471389301121235, -0.04312960058450699, -0.05098147317767143, -0.013531947508454323, 0.025741327553987503, 0.0014339409535750747, 0.03884170204401016, -0.0466378852725029, -0.019295554608106613, -0.0029531526379287243, 0.0015174715081229806, -0.00966866035014391, -0.000329989125020802, 0.01755533553659916, -0.015327854081988335, 0.019100649282336235, -0.003323819488286972, 0.0030645267106592655, -0.01241820678114891, -0.025685641914606094, -0.022525401785969734, -0.02511484920978546, -0.005787970498204231, -0.0046951123513281345, 0.019393006339669228, -0.007517748512327671, -0.014938045293092728, -0.02723095566034317, 0.020840870216488838, -0.02017262578010559, -0.01762494444847107, -0.011276623234152794, -0.0013425793731585145, 0.0022970899008214474, -0.015731584280729294, -0.005965472664684057, -0.027147425338625908, 0.039064448326826096, 0.016010019928216934, -0.019977720454335213, 0.01788945682346821, -0.017318665981292725, -0.02716134674847126, 0.04076290503144264, -0.018877901136875153, 0.03959347680211067, -0.0010711051290854812, 0.03828483074903488, -0.0018307110294699669, -0.014910201542079449, -0.01866907626390457, -0.002234441926702857, -0.0018916187109425664, 0.01762494444847107, 0.007907558232545853, 0.008485310710966587, 0.014409017749130726, 0.009585129097104073, -0.005993316415697336, 0.010977305471897125, -0.01495196670293808, -0.036669906228780746, 0.041876643896102905, 0.0044862860813736916, -0.020423216745257378, -0.040233876556158066, -0.004754279740154743, -0.029597654938697815, 0.0011241819011047482, -0.026103293523192406, 0.008575801737606525, 0.010246412828564644, -0.00884727668017149, -0.02098008804023266, 0.037143245339393616, -0.009223164059221745, 0.035639695823192596, 0.022330498322844505, -0.020882634446024895, -0.01333704311400652, -0.003212445415556431, -0.012091045267879963, -0.00952944252640009, -0.003325559664517641, 0.03279965743422508, -0.024237778037786484, 0.022246968001127243, -0.000690867193043232, -0.004127800930291414, -0.02125852182507515, 0.004910899791866541, 0.0035918131470680237, 0.03380202502012253, -0.017945144325494766, -0.0030175407882779837, -0.018070440739393234, 0.028289008885622025, 0.01865515485405922, 9.310174937127158e-05, -0.0222887322306633, -0.0281219482421875, -0.017652787268161774, -0.0020064732525497675, 0.047640252858400345, 0.0267576165497303, -0.014019208960235119, -0.005781009327620268, -0.01605178602039814, 0.004733397159725428, -0.015884723514318466, -0.0038041200023144484, 0.02387581206858158, -0.037755802273750305, -0.007677848916500807, -0.022400107234716415, 0.00048552127555012703, 0.00774745736271143, 0.016274534165859222, -0.006334399338811636, 0.005029234569519758, 0.016065707430243492, -0.01687316969037056, 0.02317972481250763, -0.043602943420410156, 0.02737017348408699, -0.039064448326826096, -0.011805649846792221, -0.009306694380939007, -0.006574549712240696, 0.015272167511284351, -0.026910755783319473, 0.0209244005382061, -0.005332032684236765, 0.01673395186662674, 0.00877766776829958, -0.021578723564743996, 0.005513015668839216, 0.027049973607063293, -0.02400110848248005, 0.012473894283175468, -0.022330498322844505, -0.013949600048363209, 0.008394819684326649, 0.03098982945084572, 0.005262424238026142, -0.0007156652864068747, 0.010316021740436554, 0.012334676459431648, 0.016441594809293747, 0.00048552127555012703, 0.018627310171723366, 0.004169566091150045, -0.0023388550616800785, -0.0026764576323330402, 0.005453848280012608, -0.016274534165859222, -0.017109839245676994, -0.031017674133181572, 0.0032629116903990507, -0.0012686200207099319, 0.02777390368282795, -0.021007930859923363, 0.0034299727994948626, 0.018975354731082916, 0.003252470400184393, 0.017917301505804062, 0.0022501039784401655, 0.0003330345207359642, -0.0033899478148669004, 0.015397462993860245, -0.0014748361427336931, -0.03424752131104469, -0.024725040420889854, 0.017611023038625717, 0.0384240485727787, 0.019448693841695786, -0.013051646761596203, -0.013559791259467602, -0.015592367388308048, -0.02104969695210457, -0.009961016476154327, 4.744817488244735e-05, 0.028205478563904762, 0.02853960171341896, 0.018780449405312538, 0.003363844472914934, 0.03694834187626839, 0.00936934258788824, -0.002791312290355563, -0.0041730464436113834, -0.0024241257924586535, 0.011568979360163212, -0.006946956738829613, 0.0027147424407303333, 0.016817482188344002, -0.012056241743266582, -0.0157733503729105, 0.018961433321237564, -0.00029714248375967145, -0.00843658484518528, 0.038730327039957047, -0.008701098151504993, -0.0377279594540596, -0.007155783008784056, -0.013643321581184864, 0.019643599167466164, 0.012341637164354324, -0.001783725107088685, -0.011784766800701618, -0.009278850629925728, 0.009821799583733082, 0.014687453396618366, -0.00808854028582573, 0.014422940090298653, -0.002425866201519966, 0.0013321380829438567, -0.00478560384362936, -0.013302238658070564, 0.0034056096337735653, -0.0022588050924241543, -0.04246135801076889, 0.01563413254916668, 0.023764438927173615, -0.019504381343722343, 0.042489200830459595, 0.02379228174686432, 0.00672420859336853, -0.014701374806463718, -0.0070409285835921764, -0.01811220496892929, -0.019198102876544, 0.010879852809011936, -0.00822775810956955, 0.018084362149238586, 0.009383263997733593, 0.01633021980524063, -0.025156613439321518, -0.021871080622076988, 0.015119027346372604, -0.002281427849084139, -0.009063063189387321, 0.001381734386086464, 0.024947788566350937, -0.013573712669312954, -0.022469716146588326, -0.026228589937090874, 0.0018898784182965755, 0.01145760528743267, -0.020492825657129288, -0.015926489606499672, -0.012432129122316837, 0.007343726698309183, -0.0016044824151322246, -0.012571346014738083, -0.057803135365247726, 0.007559513673186302, -0.0037449526134878397, -0.012362520210444927, 0.01699846424162388, 0.2063761204481125, -0.008387858048081398, -0.009014337323606014, 0.03238200768828392, -0.01406097412109375, 0.021662253886461258, 0.005158010870218277, 0.00018946640193462372, -0.0185159370303154, -0.006783375982195139, -0.0028591807931661606, 0.006181260105222464, -0.03992759808897972, -0.0035552685149013996, 0.029235688969492912, -0.023332864046096802, -0.03884170204401016, -0.020061250776052475, -0.030850613489747047, 0.007051369640976191, -0.01210496760904789, -0.001159856328740716, 0.017819847911596298, -0.016622576862573624, 0.04493943229317665, 0.006195181515067816, -0.034470271319150925, 0.02195461094379425, 0.010761518031358719, 0.01000974327325821, -0.019462615251541138, 0.0027095219120383263, -0.0070165651850402355, 0.028901567682623863, -0.031546700745821, 0.006644158158451319, 0.03210357204079628, 0.015508837066590786, -0.0006251739105209708, 0.02749546989798546, 0.01631629839539528, 0.005426004528999329, 0.01564805395901203, -0.004858693107962608, -0.009035220369696617, 0.02326325513422489, 0.006849504075944424, -0.0038806896191090345, -0.019100649282336235, 0.0018185294466093183, -0.029207846149802208, -0.017262978479266167, 0.04789084196090698, 0.02873450517654419, 0.00842962320894003, -0.012049280107021332, 0.014311566017568111, 0.010545730590820312, 0.0028295970987528563, 0.018641231581568718, -0.01018376462161541, 0.026868989691138268, -0.021704018115997314, 0.021286366507411003, -0.027147425338625908, 0.03856326639652252, 0.00696435896679759, 0.018293188884854317, 0.024585822597146034, -0.00877766776829958, 0.01159682311117649, -0.025421127676963806, -0.008735902607440948, -0.016789639368653297, -0.048865366727113724, -0.02440483868122101, 0.030878456309437752, 0.010497004725039005, -0.014701374806463718, 0.00635876227170229, -0.023137958720326424, -0.004430599045008421, -0.02379228174686432, -0.02104969695210457, 0.005248502362519503, -0.048726148903369904, 0.024919943884015083, -0.015258245170116425, -0.006999162957072258, 0.027356252074241638, 0.01701238751411438, 0.006734649650752544, 0.0008753304718993604, -0.007824026979506016, 0.006191701162606478, -0.024585822597146034, 0.020673809573054314, 0.044271185994148254, -0.00842962320894003, 0.0012225043028593063, -0.023472081869840622, 0.003577891504392028, 0.018140049651265144, 0.005359876435250044, -0.01591256819665432, -0.016845325008034706, 0.021871080622076988, 0.04931086301803589, 0.016274534165859222, 0.00042156819836236537, -0.004771681968122721, -0.043463725596666336, 0.020826948806643486, -0.01406097412109375, 0.005540858954191208, 0.02127244509756565, -0.014395096339285374, -0.016079628840088844, -0.011401918716728687, -0.021439505741000175, -0.015689820051193237, 0.004503688309341669, -0.0036753437016159296, 0.013183903880417347, -0.026576632633805275, -0.020534591749310493, -0.03416398912668228, 0.0039677005261182785, 0.015383541584014893, -0.015968255698680878, 0.00021241555805318058, -0.03781149163842201, -0.002852219855412841, -0.022372262552380562, -0.013956560753285885, 0.03341221436858177, 0.01265487726777792, -0.014381174929440022, -0.012620072811841965, 0.014144504442811012, 0.02970902808010578, -0.01210496760904789, -0.0067694541066884995, 0.006790336687117815, 0.009619933553040028, -0.03516635671257973, 0.02125852182507515, -0.013782539404928684, -0.034609485417604446, -0.015787271782755852, -0.016706107184290886, 0.0006412708899006248, -0.0012538281735032797, 0.000989314867183566, 0.017304742708802223, -0.01094250101596117, -0.02819155715405941, -0.031658075749874115, -0.006891269702464342, -0.010587495751678944, -0.03135179728269577, 0.004110398702323437, 0.0012651396682485938, -0.023416394367814064, -0.020701652392745018, -0.01326743420213461, -0.17864398658275604, 0.018543779850006104, 0.02311011590063572, 0.004054711665958166, 0.001712376018986106, 0.007392453029751778, 0.016023941338062286, 0.004531531594693661, -0.0164555162191391, -0.022483637556433678, 0.022455792874097824, 0.01516079343855381, -0.010705830529332161, -0.02510092779994011, -0.024112483486533165, -0.011506332084536552, -0.007232352625578642, 0.021801471710205078, 0.008561880327761173, 0.002230961574241519, 0.02532367594540119, -0.03324515372514725, -0.010559652000665665, -0.0019977721385657787, 0.015202558599412441, 0.02146734856069088, 0.0017315185396000743, 0.014938045293092728, -0.008561880327761173, -0.01660865545272827, -0.02593623287975788, -0.006574549712240696, 0.0011372334556654096, 0.005683557130396366, -0.012320755049586296, -0.006191701162606478, -0.002439787844195962, 0.01755533553659916, -0.02497563138604164, 0.017095917835831642, 0.023694830015301704, -0.016302376985549927, -0.01722121238708496, 0.004531531594693661, -0.014478626661002636, 0.014353331178426743, -0.0009005636093206704, 0.004618542734533548, 0.009988860227167606, -0.010608378797769547, -0.009411107748746872, -0.006463175639510155, -0.007183626294136047, -0.005231100134551525, -0.006525823380798101, 0.01155505795031786, -0.02387581206858158, -0.00895168911665678, 0.015620211139321327, 0.013246551156044006, 0.0002268811222165823, 0.0006886919145472348, -0.006508421152830124, 0.015926489606499672, -0.015829037874937057, -0.01426980085670948, -0.008882081136107445, -0.003946817945688963, -0.030767083168029785, 0.020130859687924385, -0.011749962344765663, -0.03925935551524162, -0.020868713036179543, -0.024571901187300682, 0.029347063973546028, 0.010643183253705502, -0.034804392606019974, 0.00877766776829958, -0.00843658484518528, 0.0010101974476128817, -0.016135316342115402, 0.052707768976688385, -0.043797846883535385, -0.014464705251157284, 0.020590277388691902, 0.020075174048542976, 0.0034769587218761444, 0.03483223542571068, 0.014576079323887825, -0.018947510048747063, 0.031156891956925392, -0.04020603373646736, 0.0037171090953052044, -0.0030175407882779837, 0.01873868517577648, 0.01584295928478241, 0.00632743863388896, 0.00203953729942441, 0.007559513673186302, -0.005620909389108419, 0.012940272688865662, -0.0005503444699570537, -0.02326325513422489, 0.0036440198309719563, 0.011478488333523273, 0.0017262977780774236, 0.0026051087770611048, 0.014645688235759735, 0.01189614087343216, 0.014283722266554832, -0.010169843211770058, -0.0010954681783914566, 0.023666987195611, 0.01968536339700222, 0.004649866838008165, 0.030488647520542145, 0.003376025939360261, -0.026367807760834694, -0.002305791014805436, -0.017040230333805084, 0.05329248309135437, -0.028302932158112526, -0.01292635127902031, 0.022539323195815086, -0.0012042319867759943, -0.02407071739435196, -0.08425447344779968, -0.024947788566350937, 0.017805926501750946, 0.02600584179162979, 0.004040789790451527, 0.015341775491833687, -0.015592367388308048, 0.03416398912668228, 0.006661560386419296, 0.028790192678570747, -0.011589862406253815, -0.00043679511873051524, -0.01811220496892929, 0.01652512513101101, 0.03761658817529678, 0.009174437262117863, -0.0030227613169699907, -0.04137546196579933, -0.004736877512186766, 0.03903660550713539, -0.010643183253705502, 0.02447444759309292, 0.011861336417496204, -0.024655431509017944, 0.005377278663218021, -0.010712792165577412, -0.017875535413622856, 0.014687453396618366, -0.011151326820254326, 0.023917578160762787, 0.019713208079338074, -0.0008057216764427722, -0.025880545377731323, 0.013030764646828175, 0.021578723564743996, -0.029347063973546028, -0.017123760655522346, -0.009800916537642479, 0.027203112840652466, -0.04521786421537399, -0.011102600954473019, -0.004548933822661638, -0.027133503928780556, -0.013392729684710503, 0.004917860496789217, -0.03388555720448494, -0.019365163519978523, 0.02496170997619629, -0.020840870216488838, -0.04911595582962036, -0.0003463036846369505, -0.012801054865121841, -0.01333704311400652, -0.015341775491833687, 0.029597654938697815, -0.001214673276990652, 0.03369065001606941, 0.038591109216213226, -0.023555612191557884, -0.006062924861907959, -0.027314485982060432, 0.01797298714518547, -0.016692185774445534, 0.017541414126753807, 0.0075316703878343105, -0.0008762005600146949, -0.013622438535094261, -0.012536541558802128, -0.015327854081988335, -0.019977720454335213, -0.01699846424162388, 0.026980364695191383, -0.019044963642954826, 0.010239452123641968, -0.025769172236323357, -0.01734650880098343, 0.0033203389029949903, -0.010023664683103561, 0.032075729221105576, -0.004872614983469248, -0.02702212892472744, -0.025281909853219986, -0.0036857849918305874, 0.0025372400414198637, 0.021843235939741135, 0.006950437091290951, -0.0033046770840883255, 0.0006190831190906465, 0.011847415007650852, -0.021105382591485977, -0.004747319035232067, 0.03758874163031578, 0.015829037874937057, -0.03232631832361221, 0.010657104663550854, 0.013879991136491299, -0.009014337323606014, -0.032688286155462265, -0.00808854028582573, -0.005892383400350809, -0.03046080283820629, -0.03408046066761017, -0.07244881987571716, 0.009320615790784359, -0.02621466852724552, 0.00512668676674366, 0.019866347312927246, 0.01653904654085636, 0.004350549075752497, -0.02859528735280037, -0.020799104124307632, 0.021717939525842667, -0.02098008804023266, 0.024669352918863297, -0.007239313330501318, -0.005363356787711382, 0.009494638070464134, -0.02674369513988495, 0.014381174929440022, 0.02038145251572132, -0.007051369640976191, 0.012863703072071075, -0.0008953429642133415, 0.011617706157267094, -0.005760126747190952, 0.005370317492634058, 0.0006978280725888908, -0.0035013218875974417, -0.018613388761878014, 0.02703605219721794, 0.0011624667095020413, -0.028205478563904762, 0.030961986631155014, -0.009961016476154327, -0.0051440889947116375, 0.016636500135064125, -0.005300709046423435, 0.0015505356714129448, 0.01145760528743267, 0.0067729344591498375, 0.01155505795031786, -0.0070757330395281315, -0.048865366727113724, -0.021996375173330307, 0.003619656665250659, -0.003374285763129592, 0.03441458195447922, -0.011805649846792221, -0.0008801160729490221, 0.006010718643665314, 0.01816789247095585, 0.005826255306601524, 0.01962967775762081, 0.00593414856120944, 0.008387858048081398, -0.019866347312927246, -0.008241679519414902, -0.01474313996732235, 8.162717131199315e-05, -0.006236947141587734, 0.014193231239914894, 0.000435489957453683, 0.01947653852403164, 0.026520946994423866, 0.019170258194208145, 0.00337776611559093, 0.025281909853219986, 0.02489210106432438, 0.0053877197206020355, 0.019727129489183426, 0.007517748512327671, -0.010712792165577412, 0.00157837918959558, -0.01145760528743267, 0.000609076872933656, 0.02956981211900711, 0.019393006339669228, 0.0026590556371957064, -0.002913127653300762, 0.02120283618569374, -0.03744952380657196, 0.022483637556433678, 0.025616033002734184, -0.008735902607440948, -0.021564800292253494, 0.00945983361452818, 0.023541690781712532, 0.019518302753567696, -0.029235688969492912, 0.03541694954037666, 0.01823750138282776, -0.002183975651860237, 0.005203256383538246, -0.0030227613169699907, 0.002357997465878725, -0.015453149564564228, 0.009884446859359741, 0.016010019928216934, 0.004263537935912609, -0.006438812240958214, -0.0003325994766782969, 0.021731862798333168, 0.006097729317843914, -0.008011970669031143, -0.017443960532546043, -0.008081579580903053, -0.029681185260415077, -0.000892732641659677, -0.015870802104473114, -0.03338437154889107, -0.012905468232929707, -0.016914933919906616, 0.01619100384414196, 0.0024693715386092663, -0.0006186480750329792, 0.010935540311038494, -0.02132813073694706, 0.029235688969492912, -0.03661422058939934, 0.0035065424162894487, -0.03210357204079628, -0.014673531986773014, 0.02113322727382183, 0.013002920895814896, 0.010364747606217861, 0.015926489606499672, 0.02915215864777565, -0.004653347190469503, 0.007559513673186302, -0.02091047912836075, -0.006337879691272974, -0.005603507161140442, 0.016441594809293747, 0.008151188492774963, -0.012522620148956776, -0.004983989056199789, -0.00813030544668436, 0.020632043480873108, -0.0014348110416904092, 0.024850334972143173, -0.002758248010650277, 0.053598761558532715, 0.008053735829889774, -0.020214390009641647, 0.020256156101822853, 0.02202421985566616, 0.010371708311140537, -0.018362797796726227, 0.011381035670638084, -0.019587911665439606, -0.05585408955812454, 0.018752606585621834, -0.016358064487576485, -0.008575801737606525, -0.031156891956925392, 0.011401918716728687, -0.021161070093512535, 0.000734372646547854, 0.017040230333805084, 0.00964081659913063, 0.02792704477906227, 0.02078518271446228, 0.025142692029476166, -0.00039698759792372584, -0.00017358690092805773, -0.01912849396467209, 0.0015679378993809223, 0.01584295928478241, -0.005318111274391413, -0.008756784722208977, -0.04354725405573845, -3.415180981392041e-05, 0.0028591807931661606, -0.014116661623120308, -0.008937767706811428, 0.02709173783659935, -0.01598217710852623, -0.023082273080945015, -0.010552691295742989, 0.028289008885622025, 0.009543363936245441, 0.010698869824409485, 0.026312120258808136, -0.030572177842259407, -0.019086727872490883, 0.02188500203192234, -0.020618122071027756, 0.002763468772172928, 0.01244605053216219, -0.03107336163520813]} +{"id": "test:10000038", "text": "\"One of the biggest games in the world is now available on smartphones around the world thanks to Tencent. The Chinese company\u2019s official port of PlayerUnknown\u2019s Battlegrounds (PUBG) is now available on iOS and Android worldwide.\\nWe tried PUBG on iPhone X when it released in China last month and came away very impressed. It successfully squeezed much of the PC and Xbox game down to a much smaller screen -- and in places looked even better than the Xbox One version of the game.\\nIt comes a week after rival Fortnite began inviting players to test its own mobile version. Unlike PUBG, Fortnite will allow players to play across platforms -- meaning iPhone players can kill or be killed by gamers on PC or PlayStation 4.\\nFortnite recently surpassed pioneer PUBG in popularity. Both games feature 100 players landing on a deserted island filled with weapons, with everyone fighting to be the last person left standing. While PUBG adopts a realistic setting, Fortnite has a more cartoon-like look -- and an emphasis on building structures sets it apart from its rival.\\nBut Tencent may find its main competition for PUBG on mobile could come from other Chinese companies. NetEase was quick to copy PUBG and released similar games for smartphone while the original was still on PC. NetEase says just one of its clones, Knives Out, has over 100 million registered players -- far more than the 40 million playing PUBG on PC and Xbox.\"", "vector_field": [0.002726710867136717, -0.02143443375825882, 0.010403350926935673, -0.01619025506079197, -0.01214923057705164, -0.008363221772015095, -0.020532067865133286, -0.0042077661491930485, -0.0010756449773907661, -0.03190971165895462, 0.008834020234644413, 0.0009465022012591362, -0.013201989233493805, 0.015510211698710918, 0.011044160462915897, 0.02326531894505024, 0.01825653947889805, 0.0056463186629116535, 0.010037173517048359, -0.01033142302185297, -0.010429506190121174, 0.012221157550811768, -0.008493999019265175, 0.009245969355106354, -0.017406485974788666, 0.003334826324135065, 0.014398603700101376, -0.008579004555940628, 0.034368328750133514, -0.013967037200927734, 0.003782739397138357, 0.012077302671968937, -0.03449910506606102, 0.006365594919770956, -0.035937659442424774, 0.0010486721294000745, -0.019158905372023582, -0.021591365337371826, 0.02146058902144432, -0.004596829414367676, 0.027515586465597153, 0.017877284437417984, -0.01112916599959135, 0.015065568499267101, -0.012371552176773548, 0.0038775529246777296, 0.010521050542593002, -0.0008655835990794003, -0.001011890941299498, -0.0075524006970226765, -0.015562523156404495, -0.0074020070023834705, 0.0010110735893249512, -0.01855732873082161, -0.019538158550858498, -0.008853636682033539, -0.02879720740020275, 0.02224525436758995, 0.013116983696818352, 0.005868640728294849, -0.01188113633543253, 0.008454766124486923, -0.019721247255802155, -0.008775170892477036, -0.026809388771653175, 0.006015765015035868, -0.014817091636359692, -0.014960946515202522, -0.00718622375279665, 0.009082498028874397, 0.020741311833262444, 0.037245433777570724, 0.026351667940616608, 0.022075243294239044, 0.020859012380242348, -0.006702347192913294, -0.018138840794563293, 0.012744267471134663, 0.006578108295798302, 0.00835668295621872, 0.0018145379144698381, -0.031491223722696304, 0.007617789786309004, 0.02662630006670952, 0.003929864149540663, 0.007375851273536682, -0.004073719494044781, 0.004250268917530775, 0.0098017742857337, -0.0156671442091465, 0.015091723762452602, -0.005002239719033241, 0.010808761231601238, 0.007781261578202248, -0.03499605879187584, 0.014097815379500389, -0.001246473053470254, 0.02769867517054081, -0.00928520318120718, -0.015562523156404495, 0.00040459289448335767, 0.0034034845884889364, -0.0024880419950932264, -0.00231639645062387, -0.011998835951089859, 0.02845718525350094, 0.024782337248325348, -0.01670028828084469, 0.012992745265364647, 0.029346471652388573, -0.011684969998896122, -0.003638884052634239, -0.009984862059354782, -0.018452705815434456, -0.009075959213078022, 0.0016935686580836773, 0.003609459148719907, -0.007081601768732071, -0.029424939304590225, -0.013757793232798576, 0.03444679453969002, -0.0003169311094097793, 0.033897530287504196, -0.015078646130859852, 0.01859656162559986, -0.0007282672449946404, -0.01677875407040119, -0.021865999326109886, 0.03261591121554375, -0.011534576304256916, 0.03823934122920036, -0.005587468855082989, -0.004302579909563065, -0.019564313814044, -0.0031206782441586256, -0.0022068703547120094, -0.018936581909656525, -0.011874597519636154, -0.01034450065344572, -0.013273917138576508, 0.016020243987441063, 0.018688105046749115, 0.0024308268912136555, -0.021800609305500984, -0.006424444727599621, 0.025514692068099976, 0.004145646933466196, 0.020309746265411377, 0.01807345077395439, -0.002765944227576256, 0.020584378391504288, -0.007872805930674076, -0.0005259707686491311, 0.005528619047254324, 0.012227696366608143, -0.022075243294239044, 0.004812612198293209, 0.027306342497467995, 0.014463991858065128, -0.004139108117669821, -0.010096023790538311, 5.3639210818801075e-05, -0.005486116278916597, -0.005420727655291557, 0.023396095260977745, 0.03321748599410057, 0.025802401825785637, 0.03086349181830883, 0.015745611861348152, -0.01096569374203682, 0.005737863015383482, 0.01751110702753067, -0.0417703352868557, 0.012247313745319843, 0.013679327443242073, 0.0018145379144698381, 0.021552132442593575, 0.029111072421073914, 0.008827481418848038, -0.0110964709892869, 0.0006330448668450117, -0.0018341544782742858, 0.014607847668230534, 0.025043891742825508, -0.024455392733216286, -0.004299310501664877, 0.0019812791142612696, -0.012724651023745537, 0.00758509524166584, -0.016647975891828537, 0.026312433183193207, 0.01959047093987465, 0.013940881937742233, -0.009344052523374557, -0.6398681998252869, -0.004390854854136705, 0.018714260309934616, 0.012345396913588047, 0.0005778730846941471, 0.013483161106705666, 0.032537441700696945, 0.005064358934760094, -0.025292368605732918, 0.034080617129802704, -0.0004806073266081512, -0.007657023146748543, 0.01292735617607832, -0.035179149359464645, -0.029137227684259415, -0.011122627183794975, 0.008716320618987083, -0.0045674042776227, -0.011449570767581463, -0.003720619948580861, -0.00536514725536108, 0.001043768017552793, -0.00040765799349173903, -0.0235530287027359, -0.014333214610815048, 0.0235530287027359, 0.011848442256450653, -0.019655859097838402, -0.0031713545322418213, 0.02144751138985157, -0.022572197020053864, 0.024690793827176094, 0.03473450616002083, 0.0005349617567844689, 0.04124722629785538, -0.027803298085927963, -0.020741311833262444, 0.017838051542639732, 0.013836259953677654, 0.021512899547815323, -0.0053716860711574554, -0.012979667633771896, 0.005584199447184801, -0.00926558580249548, -0.005679013207554817, 0.0008655835990794003, 0.022362953051924706, -0.015248657204210758, -0.004878000821918249, 0.0025354488752782345, 0.007369312457740307, -0.007297384552657604, 0.015902545303106308, -0.004603368230164051, 6.370295159285888e-05, 0.02195754274725914, 0.001405858201906085, -0.011161860078573227, 0.009723307564854622, -0.01906736008822918, -0.03219742327928543, 0.02272913046181202, -0.004995700903236866, -0.018975816667079926, -0.02900645136833191, 0.024468470364809036, -0.023422250524163246, 0.016804909333586693, 0.01779881864786148, -0.001204787753522396, -0.010887227021157742, 0.03580687940120697, -0.02769867517054081, -0.00277902209199965, 0.018452705815434456, 0.016844142228364944, 0.020218202844262123, -0.0012105093337595463, -0.0182696171104908, 0.014594769105315208, 0.0018308850703760982, -0.008696704171597958, 0.014333214610815048, -0.011920369230210781, 0.020741311833262444, 0.004184880293905735, -0.0062086619436740875, 0.000843514921143651, 0.013744715601205826, -0.02327839657664299, 0.00042134878458455205, 0.0004883722285740077, 0.000485920172650367, -0.0018455975223332644, -0.014856324531137943, 0.02512235939502716, -0.02169598825275898, 0.01204460859298706, 0.006891974713653326, -0.02457309328019619, -0.011397259309887886, -0.004681834485381842, 0.018204228952527046, 0.007140451576560736, 0.025540847331285477, 0.016922609880566597, -0.0261293463408947, -0.027254031971096992, 0.04522285982966423, -0.03562379255890846, 0.002435731003060937, -0.014594769105315208, -0.019930491223931313, -0.005969993304461241, 0.031229667365550995, -0.020715156570076942, -0.0004675295786000788, -0.00745431799441576, 0.04027947038412094, -0.02115979976952076, 0.01622948795557022, -0.015758689492940903, 0.0021006136666983366, -0.004227383062243462, -0.009036725386977196, 0.02297760732471943, 0.01490863598883152, -0.01650412194430828, -0.0011541114654392004, -0.00018094293773174286, 0.008010121993720531, -0.013372000306844711, 0.012567718513309956, -0.0026302626356482506, 0.01904120482504368, 0.016373343765735626, 0.021316733211278915, -0.0058294073678553104, 0.006748119369149208, -0.03588534891605377, 0.006349247880280018, -0.016307955607771873, 0.020205125212669373, -0.00425680773332715, 0.0209244005382061, -0.017733430489897728, -0.00994562916457653, -0.004387585446238518, -0.02011357992887497, -0.006260972935706377, 0.016151022166013718, -0.01721031963825226, 0.005401110742241144, -0.007428162265568972, -0.014620925299823284, 0.005110131110996008, -0.014097815379500389, -0.019708169624209404, -0.008951719850301743, -0.01854425109922886, 0.01344392728060484, 0.013045055791735649, -0.014280903153121471, -0.01857040636241436, 0.009847545996308327, -0.02008742466568947, -0.02564546838402748, 0.031988177448511124, 0.01449014712125063, -0.017942674458026886, 0.01059297751635313, -0.00031529637635685503, -0.006950824521481991, 0.022323720157146454, -0.0222321767359972, -0.024468470364809036, -0.0391809418797493, 0.0015129322418943048, 0.004812612198293209, -0.009115192107856274, 0.013215066865086555, -0.0038612058851867914, 0.004080258309841156, 0.0015799556858837605, -0.0024079408030956984, -0.001445908797904849, -0.007761645130813122, 0.038265496492385864, -0.02250680886209011, 0.016281798481941223, 0.0124107850715518, 0.03109889104962349, -0.009487908333539963, 0.007212379481643438, -0.012430401518940926, 0.0034590649884194136, 0.0010862706694751978, 0.02198369801044464, 0.007624328602105379, 0.017118776217103004, 0.016281798481941223, 0.01807345077395439, 0.004246999509632587, -0.03366212919354439, 0.003965827636420727, -0.015209423378109932, -0.007061985321342945, -0.007807416841387749, 0.018426550552248955, 0.00978869665414095, 0.02454693801701069, -0.01726263016462326, -0.011076854541897774, 0.0010110735893249512, -0.005963454023003578, 0.027646364644169807, 0.006620611064136028, -0.012417323887348175, -0.00888633169233799, -0.0005594824906438589, -0.005911143030971289, -0.015758689492940903, 0.009311358444392681, 0.011874597519636154, -0.02067592367529869, 0.019734324887394905, 0.0028035426512360573, 0.03847474232316017, 0.01424167025834322, -0.022833751514554024, -0.03211895376443863, 0.005986340343952179, 0.0098017742857337, -0.019655859097838402, 0.0045608654618263245, 0.0032154920045286417, 0.015784844756126404, -0.006728502456098795, 0.024952348321676254, -0.021630600094795227, -0.0003402258444111794, 0.02276836335659027, 0.02746327593922615, -0.01296005118638277, 0.025436224415898323, 0.02952956035733223, 0.005479577463120222, -0.025200825184583664, -0.012973128817975521, 0.018465783447027206, -0.008180133067071438, 0.012933894991874695, -0.035414546728134155, -6.33453555565211e-06, 0.0011344949016347528, -0.016334110870957375, 0.0005729689146392047, 0.004508554469794035, 0.022022930905222893, 0.023853816092014313, -0.007663561962544918, -0.016111789271235466, 0.008866715244948864, -0.005453421734273434, 0.003148468444123864, 0.0025419876910746098, 0.003736967220902443, -0.00952060241252184, -0.01964278146624565, -0.007689717225730419, -0.02217986434698105, 0.0024373657070100307, 0.011743820272386074, 0.005358608439564705, 0.002185618970543146, -0.002684208331629634, 0.031726621091365814, -0.03214510902762413, 0.0032759765163064003, 0.023095307871699333, -0.013051594607532024, -0.0006367229507304728, 0.0157063789665699, 0.010867610573768616, -0.001687029842287302, -0.02850949577987194, -0.024390004575252533, -0.007467395626008511, -0.0046556792221963406, 0.005512272007763386, -0.013097367249429226, -0.013980114832520485, -0.00372715899720788, 0.0026482443790882826, -0.020178968086838722, -0.009625224396586418, 0.002988266060128808, -0.014751702547073364, -0.01858348399400711, 0.010939538478851318, 0.024926193058490753, 0.02480849251151085, -0.03729774430394173, -0.02093747816979885, 0.008722859434783459, 0.020558223128318787, -0.005996148567646742, -0.022859908640384674, -0.0026057418435811996, 0.006257703527808189, -0.006565030664205551, -0.03496990352869034, 0.0024880419950932264, -0.02508312463760376, -0.00208917073905468, 0.018675027415156364, -0.007532784249633551, 0.003488489892333746, 0.018099606037139893, -0.000698842282872647, -0.012875045649707317, -0.02510928176343441, -0.026587067171931267, -0.017105696722865105, 0.10483124852180481, -0.01676567643880844, -0.005263794679194689, -0.00549592450261116, -0.022049088031053543, -0.009572912938892841, -0.010305267758667469, -0.039102472364902496, 0.002829698147252202, -0.010043712332844734, -0.034368328750133514, 0.01723647490143776, 0.0008304371731355786, -0.011998835951089859, 0.01779881864786148, -0.006963902153074741, 0.011044160462915897, -0.013770870864391327, -0.004479129333049059, -0.007415084633976221, 0.007016213145107031, 0.01643873192369938, 0.012973128817975521, 0.017955752089619637, 5.26813664691872e-06, -0.009245969355106354, 0.015130957588553429, 0.003326652804389596, -0.004822420421987772, -0.001277532777749002, -0.02374919503927231, -0.007696256041526794, 0.005234369542449713, 0.0037173505406826735, 0.007114296313375235, 0.016281798481941223, 0.0014638906577602029, 0.015523290261626244, 0.0032040488440543413, -0.00860515981912613, 0.022062165662646294, -0.0019093515584245324, -0.000514119048602879, -0.008337065577507019, 0.02323916181921959, -0.01069759950041771, -0.014934791252017021, -0.005342260934412479, -0.03196202218532562, -0.013535471633076668, 0.02374919503927231, -0.009376747533679008, -0.024219993501901627, -0.006045190151780844, -0.017432641237974167, 0.006225009448826313, -0.004135838709771633, -0.002867296803742647, -0.023932283744215965, -0.042999643832445145, -0.008964798413217068, -0.03559763729572296, -0.012476174160838127, 0.011292637325823307, -0.007951272651553154, -0.014960946515202522, -0.02639090083539486, -0.018151918426156044, 0.0038448586128652096, -0.006708886008709669, -0.004678565077483654, -0.027646364644169807, -0.01424167025834322, -0.009984862059354782, 0.014882479794323444, -0.02380150556564331, 0.003303766716271639, 0.015837155282497406, -0.001100165769457817, 0.03034038096666336, 0.014463991858065128, 0.0037990864366292953, -0.004204496741294861, -0.03975636139512062, -0.009886779822409153, 0.008729398250579834, 0.016595665365457535, 0.0030650978442281485, -0.015614833682775497, -0.005306297447532415, 0.032328199595212936, -0.004152185749262571, 0.012495790608227253, 0.011874597519636154, -0.006565030664205551, -0.019995881244540215, -0.012181924656033516, -0.003831780981272459, 0.005306297447532415, -0.005384763702750206, 0.009939090348780155, -0.01854425109922886, -0.015640988945961, -0.01696184277534485, 0.014673235826194286, -0.008964798413217068, 0.01152803748846054, -0.0011402163654565811, -0.024363849312067032, 0.007827033288776875, 0.00027340670931153, -0.01595485582947731, 0.021892154589295387, -0.005577660631388426, 0.01424167025834322, 0.020963633432984352, 0.006293667480349541, 0.007513167802244425, -0.0008144986350089312, -0.008029738441109657, -0.028091007843613625, -0.04336582124233246, -0.005329183302819729, 0.016804909333586693, 0.008258599787950516, -0.019250448793172836, -0.01426782552152872, -0.02981727197766304, -0.039834827184677124, -0.0032400127965956926, -1.5248861018335447e-05, -0.010030634701251984, 0.023893050849437714, 0.01412397064268589, -0.02325223945081234, -0.01755034178495407, 0.0016837604343891144, 0.028378719463944435, 0.0183219276368618, -0.022075243294239044, -0.043261200189590454, -0.002352360403165221, 0.01986510306596756, 0.02012665756046772, -0.0019240640103816986, -0.028169475495815277, -0.030209602788090706, 0.03193586692214012, 0.012005374766886234, -0.008853636682033539, -0.01931583695113659, -0.0012072398094460368, -0.0024520782753825188, 0.008337065577507019, -0.03337441757321358, -0.04514439404010773, -0.03321748599410057, -0.03337441757321358, 0.0248215701431036, 0.021329810842871666, 0.025292368605732918, -0.006175967864692211, 0.015379434451460838, 0.006126926280558109, -0.004747223574668169, 0.012253852561116219, -0.021094411611557007, -0.006931207608431578, -0.024363849312067032, -0.002241199603304267, 0.026613222435116768, 0.032589755952358246, -0.012502329424023628, -0.0012186828535050154, -0.03549301624298096, 0.02402382716536522, -0.012613490223884583, 0.016791831701993942, -0.0248477254062891, -0.0313081331551075, -0.0248215701431036, -0.013319688849151134, -0.022376030683517456, 0.014987101778388023, -0.019564313814044, -0.03687925636768341, -0.01217538584023714, -0.023134540766477585, 0.01986510306596756, -0.004662218037992716, 0.011181476525962353, -0.02563239075243473, -0.014280903153121471, 0.012142691761255264, 0.004171802196651697, -0.016595665365457535, 0.004721067845821381, -0.020335901528596878, -0.010887227021157742, -0.01517019048333168, 0.00798396673053503, 0.008422071114182472, -0.0037500448524951935, -0.015824077650904655, -0.003632345236837864, 0.0418488010764122, -0.0392594076693058, -0.02796022966504097, 0.006774274632334709, -0.012515407055616379, 0.012914278544485569, 0.012901200912892818, 0.0049041565507650375, -0.012181924656033516, -0.015811000019311905, 0.013116983696818352, -0.00824552122503519, 0.002573047298938036, -8.824927135719918e-06, -0.032589755952358246, 0.010893765836954117, 0.029137227684259415, 0.040436405688524246, 0.0020450332667678595, 0.006094231735914946, 0.011534576304256916, 0.011403798125684261, -0.02427230402827263, -0.006931207608431578, 0.013823182322084904, 0.001904447446577251, 0.03238051012158394, -7.581008685519919e-05, -0.003318479284644127, -0.0007184589048847556, 0.004299310501664877, -0.006306745111942291, -0.03512683883309364, -0.02510928176343441, 0.01775958575308323, 0.018230384215712547, -0.006813507992774248, -0.015091723762452602, -0.013234683312475681, -0.00745431799441576, -0.010403350926935673, -0.017393408343195915, -0.009174042381346226, 0.005737863015383482, -0.0222583319991827, -0.03504837304353714, 0.026247045025229454, 0.003029133891686797, 0.002852584235370159, 0.02746327593922615, 0.003267802996560931, -0.014424758963286877, -0.009245969355106354, -0.01123378798365593, -0.025501612573862076, -0.0005656126886606216, 0.0013371999375522137, -0.030183447524905205, -0.010684521868824959, 0.010507972911000252, 0.0028722009155899286, 0.00443662703037262, -0.010651827789843082, 0.008493999019265175, 0.026547834277153015, -0.0156802237033844, 0.010468739084899426, 0.012201541103422642, 0.004786456469446421, 0.020532067865133286, -0.003864475293084979, -0.009259046986699104, -0.026456288993358612, 0.017694195732474327, 0.002917973091825843, 0.009056341834366322, -0.006019034888595343, -0.011338409967720509, -0.013175833970308304, 0.0019812791142612696, -0.035989969968795776, -0.010632211342453957, -0.007101218681782484, 0.008938642218708992, -0.027358654886484146, -0.0013045056257396936, 0.007022751960903406, -0.012266930192708969, -0.023409172892570496, 0.01018102839589119, 0.0016191889299079776, 0.008637853898108006, 0.029634183272719383, -0.031543534249067307, -0.00196002796292305, 0.004240460693836212, 0.00041889669955708086, -0.018740415573120117, 0.028300251811742783, -0.001768765738233924, -0.024625403806567192, 0.02850949577987194, -0.014346292242407799, 0.012103457935154438, -0.005244177766144276, -0.006686000153422356, -0.0036683089565485716, 0.002739788731560111, -0.013823182322084904, 0.006640227977186441, 0.014516303315758705, 0.004165263380855322, -0.026286277920007706, -0.01164573710411787, 0.005538427270948887, 0.01932891458272934, -0.027358654886484146, 0.033583663403987885, 0.014712469652295113, 0.013849337585270405, -0.01137764286249876, 0.02898029424250126, -0.018426550552248955, -0.019171983003616333, -0.022872986271977425, -0.01591562293469906, 0.0023670727387070656, -0.017576497048139572, -0.007264690473675728, -0.0247954148799181, 0.000289345218334347, 0.003678117413073778, 0.006646766792982817, -0.012430401518940926, 0.008722859434783459, 0.020035114139318466, 0.03363597393035889, 0.011397259309887886, -0.014764780178666115, 0.022532964125275612, 0.024219993501901627, -0.027358654886484146, 0.0033904067240655422, -0.031177356839179993, 0.023134540766477585, 0.017380330711603165, 0.040436405688524246, 0.005842484999448061, -0.00639828946441412, -0.039834827184677124, -0.008879792876541615, -0.040148694068193436, -0.000767091813031584, 0.004766840022057295, 0.032772842794656754, 0.02168291062116623, 0.010985310189425945, 0.023723039776086807, 0.04059333726763725, -0.003030768595635891, 0.002340917242690921, -0.006872357800602913, -0.035440701991319656, -0.018439628183841705, -0.024193838238716125, -0.0032661682926118374, 0.014856324531137943, -0.028300251811742783, -0.015745611861348152, 0.020466679707169533, 0.018387317657470703, -0.006453869864344597, 0.02173522114753723, 0.009357130154967308, 0.02590702287852764, -0.024886958301067352, 0.0015529828378930688, -0.005025125574320555, 0.010723755694925785, 0.03156968951225281, -0.006551953032612801, -8.607816562289372e-05, 0.009226352907717228, 0.015222501941025257, -0.008572465740144253, -0.007532784249633551, 0.019760480150580406, -0.006090962328016758, 0.014163203537464142, 0.0038121643010526896, -0.010645288974046707, -0.008670548908412457, 0.0038579364772886038, 0.021029023453593254, -0.017707273364067078, -0.015366356819868088, 0.018230384215712547, -0.006859280169010162, -0.026220889762043953, -0.0008680357132107019, 0.014777857810258865, -0.02068900130689144, 0.003887361381202936, -0.002780656795948744, -0.007774722762405872, 0.01622948795557022, -0.01007640641182661, -0.005669204983860254, -0.018910426646471024, -0.03588534891605377, -0.013483161106705666, -0.02746327593922615, 0.006643497385084629, 0.028404874727129936, 0.02616857923567295, 0.008932103402912617, 0.014790935441851616, -0.0038219725247472525, -0.0032089529559016228, -0.011397259309887886, -0.0018357891822233796, 0.016857219859957695, -0.011841903440654278, 0.009102114476263523, 0.002259181346744299, 0.029084917157888412, -0.035152994096279144, 0.009690613485872746, 0.027280187234282494, 0.006025573704391718, 0.005401110742241144, 0.23100538551807404, 0.01084799412637949, 0.02640397846698761, 0.036173056811094284, -0.0010625672293826938, 0.027541741728782654, -0.0051232087425887585, -0.002507658675312996, 0.006225009448826313, -0.001300418865866959, -0.011475726030766964, 0.02932031638920307, -0.004842037335038185, 0.004263346549123526, -0.004547787830233574, -0.009304819628596306, -0.05074167251586914, -0.008108205161988735, 0.007375851273536682, 0.022323720157146454, 0.013875492848455906, -0.02401074953377247, -0.03274668753147125, -0.006058267783373594, 0.01413704827427864, -0.012881584465503693, -0.007434701081365347, 0.015575600787997246, 0.0444905087351799, -0.01412397064268589, -0.011953064240515232, 0.009102114476263523, 0.01775958575308323, 0.008683626540005207, -0.023330707103013992, 0.002942493883892894, 0.011155321262776852, -0.0031697198282927275, 0.019224293529987335, 0.02064976841211319, 0.0039233253337442875, -0.0030650978442281485, 0.013391616754233837, -0.0016028417740017176, 0.02407613955438137, -0.0021709066350013018, -0.022284487262368202, -0.0015211058780550957, 0.0011835363693535328, -0.017615729942917824, -0.022415263578295708, -0.006689269561320543, 0.027123253792524338, 0.02198369801044464, 0.0011663718614727259, 0.0028509495314210653, 0.0071535296738147736, 0.005335722118616104, -0.004923772998154163, 0.002996439579874277, 0.00033695640740916133, 0.015797922387719154, -0.011920369230210781, 0.013372000306844711, -0.041691869497299194, 0.026547834277153015, -0.014634002931416035, 0.03800394386053085, 0.023213006556034088, -0.0015897640259936452, -0.03562379255890846, 0.0004182836855761707, -0.01647796481847763, -0.0030389423482120037, -0.019734324887394905, -0.033609818667173386, 0.036748480051755905, -0.0053324527107179165, 0.023997671902179718, 0.01775958575308323, 0.012181924656033516, -0.010135256685316563, 0.009291741997003555, 0.00543053587898612, -0.011619580909609795, -0.018151918426156044, 0.01239770743995905, -0.02432461641728878, -0.023670729249715805, -0.011593425646424294, 0.0024716949556022882, -0.013483161106705666, -0.002821524627506733, -0.03481297194957733, 0.01986510306596756, 0.00047692921361885965, 0.01911967061460018, 0.02197062037885189, -0.006702347192913294, -0.014647080563008785, 0.004014869220554829, 0.011998835951089859, 0.019786637276411057, -0.021604442968964577, -0.029451094567775726, 0.01279657892882824, 0.00017195197870023549, 0.035205304622650146, 0.007931655272841454, 0.01881888322532177, 0.0030503852758556604, -0.008278216235339642, -0.0022150438744574785, -0.026861699298024178, 0.013679327443242073, 0.039887137711048126, 0.003004613099619746, 0.0005521262646652758, 0.002038494450971484, -0.020205125212669373, 0.014738624915480614, -0.016896452754735947, 0.011220710352063179, 0.004472590517252684, -0.019211215898394585, 0.00021537420980166644, 0.019263526424765587, 0.028326407074928284, -0.00763740623369813, -0.010619133710861206, -0.0017540532862767577, -0.018426550552248955, -0.00438431603834033, -0.0020107042510062456, 0.020728234201669693, 0.02142135612666607, 0.02246757596731186, -0.03899785131216049, -0.003736967220902443, 0.012181924656033516, 0.014333214610815048, -0.003697733860462904, 0.012332318350672722, 0.03473450616002083, 0.015627911314368248, -0.01851809397339821, -0.0004524084215518087, 0.00660426402464509, -0.00615308154374361, -0.016608742997050285, -0.003678117413073778, -0.02717556618154049, -0.038527052849531174, 0.00928520318120718, 0.021251345053315163, -0.015000179409980774, -0.013025439344346523, -0.02330455183982849, 0.00823244359344244, 0.009893318638205528, -0.027829453349113464, -0.006421175319701433, 0.01931583695113659, -0.021055178716778755, -0.014346292242407799, -0.001678856206126511, -0.1649889051914215, 0.034368328750133514, 0.022402185946702957, 0.013103906065225601, 0.02064976841211319, 0.002048302674666047, -0.018151918426156044, -0.012214618735015392, 0.020322823897004128, -0.006421175319701433, 0.029948048293590546, -0.0010830012615770102, -0.003665039548650384, -0.00040459289448335767, 0.02122518979012966, 0.0030193256679922342, 0.009245969355106354, 0.03588534891605377, 0.008408993482589722, 0.0019387764623388648, 0.04357506334781647, -0.00021394382929429412, 0.01201191358268261, 0.013025439344346523, -0.007218918297439814, 0.0011385816615074873, 0.0078662671148777, 0.01910659298300743, 0.01855732873082161, -0.010298728942871094, -0.012181924656033516, -0.02850949577987194, 0.007912038825452328, -0.0015775036299601197, 0.005309566855430603, -0.0008549579652026296, 0.02040129154920578, 0.006430984009057283, 0.005214753095060587, 0.030706558376550674, 0.0208720900118351, 0.02534468099474907, -0.02351379580795765, 0.012633106671273708, 0.015758689492940903, 0.03156968951225281, -0.012914278544485569, -0.022702975198626518, 0.031334288418293, -0.01204460859298706, 0.008932103402912617, -0.004420279525220394, -0.0022215829230844975, 0.007192762568593025, 0.017380330711603165, 0.0047995345667004585, 0.00864439271390438, -0.01752418652176857, 0.0392594076693058, 0.008199749514460564, -0.028849517926573753, -6.048459545127116e-05, -0.00548284687101841, -0.0028231593314558268, -0.029346471652388573, -0.029660338535904884, -0.005538427270948887, 0.028954138979315758, -0.02929416112601757, -0.010560283437371254, -0.025161592289805412, -0.020466679707169533, 0.03729774430394173, -0.011253404431045055, 0.012979667633771896, -0.006679460871964693, 0.003913516644388437, 0.007362773641943932, 0.008016660809516907, 0.016281798481941223, -0.001969836186617613, -0.0005120756686665118, -0.014947868883609772, 0.02926800586283207, -0.017707273364067078, 0.009507524780929089, 0.00451509328559041, 0.01910659298300743, -0.025671623647212982, -0.012515407055616379, 0.03060193546116352, -0.008775170892477036, -0.025253135710954666, -0.008520154282450676, -0.01646488718688488, 0.013587783090770245, 0.0031043309718370438, -0.028300251811742783, -0.013613938353955746, -0.03499605879187584, 0.025737013667821884, -0.028927983716130257, 0.004662218037992716, 0.008990953676402569, 0.04357506334781647, 0.017733430489897728, 0.02410229481756687, -0.012613490223884583, 0.017628807574510574, 0.0013437388697639108, -0.02720172144472599, 0.010056789964437485, 0.013666248880326748, 0.021355966106057167, -0.01292735617607832, 0.01188113633543253, -0.002409575507044792, -0.008637853898108006, -0.005499193910509348, -0.019420459866523743, 0.07255536317825317, -0.0065911863930523396, 0.01619025506079197, 0.030706558376550674, 0.011011465452611446, -0.016321033239364624, -0.08992261439561844, -0.02510928176343441, 0.021852921694517136, 0.028431029990315437, -0.026573989540338516, 0.02879720740020275, -0.012057686224579811, 0.03457757085561752, 0.01857040636241436, 0.020793622359633446, -0.006659844424575567, -0.018949659541249275, -0.014320136979222298, 0.009742924012243748, 0.02115979976952076, -0.013914726674556732, 0.019786637276411057, -0.014673235826194286, -0.00874901469796896, 0.03162200003862381, 0.006303475704044104, -0.012456557713449001, 0.004080258309841156, -0.028064852580428123, -0.018426550552248955, -0.029372626915574074, -0.044281262904405594, 0.007094679865986109, 0.01043604500591755, 0.025475457310676575, 0.019446615129709244, 0.0011941620614379644, 0.01880580559372902, -0.03494374826550484, 0.004253538325428963, -0.00013200353714637458, -0.03844858705997467, -0.004233921878039837, 0.0104752779006958, -0.029921893030405045, -0.0017704004421830177, -0.0062184701673686504, 0.014620925299823284, -0.03549301624298096, -0.022323720157146454, -0.02330455183982849, -0.019995881244540215, -0.0017704004421830177, 0.022284487262368202, -0.025527769699692726, 0.0030585587956011295, -0.015209423378109932, 0.000590542156714946, -0.021839844062924385, 0.004361429717391729, 0.011044160462915897, 0.007362773641943932, 0.006473486311733723, 0.001824346138164401, 0.0025942986831068993, 0.0003034446854144335, -0.016831064596772194, -0.027280187234282494, 0.012436940334737301, -0.006630419287830591, 0.0069900574162602425, -0.0075197066180408, -0.01596793346107006, 0.007578556425869465, -0.0014197533018887043, -0.029660338535904884, -0.0006248712888918817, -0.013823182322084904, 0.011299176141619682, -0.036486923694610596, -0.0111160883679986, -0.0055711218155920506, 0.009115192107856274, 0.018949659541249275, -0.003681386820971966, -0.005057820118963718, -0.013260838575661182, -0.003704272909089923, -0.019158905372023582, 0.019956646487116814, 0.00928520318120718, -0.0017965559381991625, 0.006444061640650034, -0.005146094597876072, -0.03980867192149162, -0.012103457935154438, 0.0016142848180606961, -0.00016326754121109843, -0.012639645487070084, -0.0006604263908229768, -0.011188015341758728, 0.030131136998534203, 0.007107757497578859, 0.0038677447009831667, 0.006686000153422356, 0.00850053783506155, 0.0062478953041136265, -0.07218918204307556, 0.017903439700603485, 0.015562523156404495, 0.027123253792524338, 0.007624328602105379, 0.00633290084078908, 0.010828377678990364, -0.020270513370633125, 0.004253538325428963, 0.004008330404758453, -0.0196297038346529, 0.020558223128318787, -0.0046556792221963406, 0.003956019412726164, -0.032275889068841934, -0.005244177766144276, 0.006793891545385122, 0.0031206782441586256, -0.0006526614888571203, 0.011286098510026932, 0.004452974069863558, -0.0012088745133951306, 0.01728878542780876, 0.0003690377634484321, -0.004188149701803923, 0.0027218067552894354, -0.011580348014831543, 0.011194554157555103, -0.007107757497578859, -0.048492297530174255, 0.00811474397778511, -0.016098711639642715, -0.009278664365410805, -0.007630867417901754, -0.009278664365410805, -0.023853816092014313, -0.02227140963077545, 0.02193138748407364, -0.004246999509632587, 0.013391616754233837, -0.013483161106705666, -0.021499821916222572, 0.03292977437376976, -0.011769975535571575, -0.0063917506486177444, -0.0020532067865133286, -0.011279559694230556, 0.004498746246099472, 0.00765048386529088, -0.0021153262350708246, 0.023369939997792244, 0.01332622766494751, -0.00024500349536538124, -0.035414546728134155, -0.0091348085552454, -0.03287746384739876, -0.014437836594879627, 0.00028137597837485373, -0.03368828445672989, -0.01834808476269245, 0.027515586465597153, 0.027881763875484467, 0.004224113188683987, -0.021251345053315163, 0.012103457935154438, 0.018897349014878273, -0.02824794128537178, -0.0018178073223680258, 0.009572912938892841, 2.8301068596192636e-05, 0.021238267421722412, 0.0021414817310869694, -0.0031239476520568132, 0.0035963812842965126, -0.0027365193236619234, 0.0032253002282232046, -0.0032645333558321, 0.008683626540005207, -0.027097098529338837, 0.03109889104962349, 0.015209423378109932, -0.007722411770373583, -0.047759946435689926, -0.004995700903236866, 0.017432641237974167, 0.01934199221432209, -0.00633290084078908, 0.037010032683610916, -0.022859908640384674, 0.007487012073397636, -0.020754389464855194, 0.012724651023745537, -0.01721031963825226, 0.02932031638920307, 0.011678431183099747, 0.02722787670791149, -0.01400627102702856, 0.0030389423482120037, -0.00013179919915273786, 0.0045216321013867855, 0.009775618091225624, -0.0005014499765820801, 0.0024373657070100307, -0.0017638616263866425, -0.012757346034049988, 0.0078662671148777, -0.0032808806281536818, -0.00706852413713932, 0.000623236526735127, 0.0012505599297583103, 0.013391616754233837, 0.01282273419201374, -0.006633689161390066, 0.005597277078777552, -0.017367253080010414, 0.018243461847305298, -0.0019714708905667067, 0.0010543935932219028, -0.020479757338762283, 0.04263346642255783, 0.03742852061986923, 0.025776246562600136, -0.005388033110648394, -0.008016660809516907, 0.04263346642255783, -0.0038840919733047485, 0.009095575660467148, -0.0062478953041136265, 0.00952060241252184, -0.018478861078619957, -0.02640397846698761, 0.004364699125289917, -0.0010805490892380476, -0.009599069133400917, -0.004181610886007547, -0.019132748246192932, -0.0037631227169185877, 0.027044788002967834, 0.01162611972540617, 0.0654933750629425, 0.010030634701251984, -0.00016551527369301766, -0.006228278856724501, 0.012351935729384422, 0.02062361314892769, -0.00040561461355537176, 0.01800806261599064, -0.023605339229106903, -0.024468470364809036, 0.010481816716492176, 0.01413704827427864, 0.018491938710212708, -0.008905948139727116, -0.0235007181763649, 0.02329147420823574, -0.012234235182404518, -0.01569330133497715, -0.03313902020454407, 0.027855608612298965, 0.0339759960770607, -0.0157325342297554, 0.030706558376550674, 0.004289502277970314, -0.022899141535162926, 0.010841455310583115, 0.02146058902144432, 0.018478861078619957, -0.023213006556034088, -0.03800394386053085, 0.02010050229728222, -0.006708886008709669, -0.035179149359464645, -0.007166607305407524, 0.0365130789577961, 0.016791831701993942, -0.001092809485271573, -0.01647796481847763, 0.004711259622126818, 0.0130581334233284, 0.013417772017419338, 0.005414188839495182, -0.016334110870957375, -0.020466679707169533, 0.014987101778388023, -0.017707273364067078, -0.03107273578643799, 0.008265138603746891, -0.02093747816979885]} +{"id": "test:10000039", "text": "\"Volunteers are vital to Thames Reach in our efforts to end street homelessness and support vulnerable and socially excluded people across London.\\nAs well as helping individuals to take steps forward in their lives, volunteering also offers opportunities to learn, meet new people and experience something different.\\nBenefacto is a social enterprise helping to get people who work in a professional capacity to volunteer in their local communities. Since last June, they\u2019ve been arranging for professional and corporate volunteers to dedicate some of their time to helping Thames Reach clients at iReach, a weekly digital skills workshop at our Employment Academy in Camberwell.\\niReach gives people with little or no computer skills the opportunity to learn at their own pace. Funded by the Worshipful Company of Information Technologists, the workshop teaches practical digital skills, so clients can access services and benefits online, and can also help to reduce social isolation.\\nBenefacto volunteer Molly spent an afternoon volunteering at an iReach session earlier this month. \u201cYou take it for granted but lots of people can\u2019t use a computer,\u201d she said. \u201cIt\u2019s so important for applying for jobs and staying in contact with friends and family.\\n\u201cYou can see people\u2019s progress throughout the session. As a volunteer you want to get to a stage where you\u2019re almost in the way. There was a lot of people at the session, everyone was enjoying it, it was amazing to see,\u201d she added.\\nStevie Back, volunteer coordinator at Benefacto, said: \u201cDoing this gives volunteers a chance to touch down with perhaps an unfamiliar reality and realise the importance of digital inclusion.\\n\u201cWe\u2019ve spent a long time seeking out the right charities for our volunteers. People working in a professional or corporate environment will tend to have digital skills they can impart through sessions like iReach, which is a fantastic project.\\n\u201cWe\u2019ve really enjoyed working with Thames Reach and will continue to do so. We\u2019ve had 23 volunteers helping out with iReach. They share their expertise and make contact with people in their communities they might otherwise never have met,\u201d she said.\\nThames Reach digital support worker Chris Hamm, who runs the iReach sessions, said: \u201cWhen our service users first arrive, some are apprehensive about using the equipment. Some have almost no previous experience of using computers.\\n\u201cAfter a while, though, you can see how self-confident our learners become. The transformation is lovely to watch and the knowhow that our volunteers share with them plays a big part in that,\u201d he said.\"", "vector_field": [-0.016571514308452606, -0.023611344397068024, 0.0016782573657110333, -0.025504065677523613, -0.022236058488488197, 0.034477464854717255, -0.004840733949095011, 0.010314644314348698, -0.03336089476943016, -0.0208743903785944, -0.014842194505035877, -0.000481264985864982, -0.020152704790234566, -0.003635656787082553, -0.013820942491292953, -0.022467542439699173, 0.02471429668366909, -0.025367898866534233, -0.018954435363411903, -0.02471429668366909, -0.02829548716545105, 0.011349513195455074, 0.01627194695174694, 0.013283083215355873, -0.033333662897348404, -0.022385843098163605, 0.03869863972067833, -0.0035539567470550537, 0.010593786835670471, -0.028458887711167336, 0.015972379595041275, -0.008946167305111885, -0.009701893664896488, -0.011104412376880646, -0.0015659196069464087, -0.01937655359506607, -0.0031165205873548985, 0.00143911421764642, 0.023516027256846428, 0.00013669882901012897, 0.019567187875509262, -0.007312163710594177, -0.010832078754901886, 0.010580169968307018, -0.03935224190354347, 0.01124738808721304, -0.009756360203027725, -0.009082334116101265, -0.010178477503359318, 0.02512279711663723, 0.017320433631539345, -0.0009063610923476517, -0.018137434497475624, 0.005589652340859175, -0.002001653891056776, -0.00287993042729795, -0.03303409740328789, 0.01512814499437809, 0.013997959904372692, -0.012009922415018082, 0.006474737077951431, 0.023733895272016525, -0.016380881890654564, 0.0008467880543321371, -0.006873025558888912, 0.002406750340014696, -0.024196863174438477, 0.004912221804261208, -0.0019488891121000051, 0.0050824303179979324, 0.013997959904372692, 0.010423578321933746, 0.026429999619722366, 0.012561398558318615, 0.02505471371114254, 0.0004753076646011323, -0.0018569764215499163, 0.014215826988220215, 0.029412055388092995, 0.0022518604528158903, -0.0016399604501202703, -0.022876042872667313, -0.0340144969522953, 0.016871081665158272, 0.0175655335187912, -0.010709528811275959, -0.0123231066390872, 0.03404173254966736, 0.028867388144135475, 0.002830569865182042, -0.018191901966929436, 0.01137674693018198, -0.004207557998597622, 0.022876042872667313, 0.00804065726697445, -0.0037616111803799868, -0.010212519206106663, 0.01830083504319191, -0.0038637365214526653, -0.025885332375764847, -0.03891650587320328, -0.012486507184803486, -0.006151340901851654, 0.0006038151914253831, -0.03532170131802559, -0.0020714392885565758, -0.009409134276211262, 0.011152070946991444, 0.031781360507011414, -0.03646550327539444, -0.015005595050752163, 0.027941452339291573, 0.0016118759522214532, -0.0062568699941039085, 0.016557898372411728, -0.014229443855583668, 0.002362496219575405, -0.018518701195716858, -0.00018574019486550242, -0.013861793093383312, 0.03194475919008255, 0.012983515858650208, 0.015386862680315971, -0.007815981283783913, 0.0004510529397521168, 0.010872929356992245, -0.028186554089188576, -0.001121675013564527, 0.0008667875663377345, -0.017783399671316147, 0.0012416720855981112, -0.011737588793039322, 0.035893600434064865, 0.002403346123173833, 0.00038211842183955014, 0.027328701689839363, -0.020642906427383423, 0.008312990888953209, -0.005790498573333025, -0.00844234973192215, 0.012813307344913483, 0.012813307344913483, 0.004285853821784258, -0.0019522933289408684, -0.024128779768943787, 0.011608229950070381, 0.008149590343236923, -0.007570880930870771, 0.023216459900140762, 0.007387055549770594, 0.028186554089188576, -0.006243253592401743, -0.007986189797520638, -0.03175412863492966, 0.024142395704984665, 0.0018944223411381245, -0.022249676287174225, 0.00925254262983799, -0.002796528162434697, -0.00595730310305953, -0.010512086562812328, 0.005269660148769617, 0.0230394434183836, 0.0020884601399302483, 0.051825132220983505, 0.016666831448674202, -0.02410154603421688, 0.007312163710594177, -0.0046058460138738155, 0.0011412490857765079, 0.018654868006706238, 0.01599961332976818, -0.028186554089188576, 0.007802364882081747, 0.016380881890654564, 0.006338570266962051, -0.01783786714076996, 0.015917913988232613, 0.0012442253064364195, -0.02011185511946678, 0.006961533799767494, 0.035022132098674774, 0.002629723632708192, 0.01925400272011757, -0.020139088854193687, -0.003269708249717951, 0.016517048701643944, 0.015305162407457829, 0.018341684713959694, -0.016380881890654564, 0.011737588793039322, 0.003038224531337619, 0.004670525435358286, -0.021282890811562538, -0.6361718773841858, 0.00867383275181055, 0.01802850142121315, -0.0070738717913627625, -0.003567573381587863, -0.0246462132781744, -0.004422021098434925, -0.0062024034559726715, -0.008721491321921349, 0.018055733293294907, 0.00867383275181055, 0.041966646909713745, -0.010552936233580112, -0.014828577637672424, -0.006069640628993511, -0.019294852390885353, -0.0070057883858680725, -0.012139281257987022, 0.024156011641025543, -0.0023846232797950506, -0.04098624363541603, 0.012023539282381535, -0.02965715527534485, 0.0024458984844386578, 0.008149590343236923, 0.024264946579933167, 0.013249041512608528, -0.031100526452064514, -0.022780725732445717, 0.021255657076835632, -0.0006238147034309804, 0.030337991192936897, 0.01749745011329651, 0.02796868607401848, 0.06029471382498741, -0.0197305865585804, -0.002401644131168723, 0.028404420241713524, -0.019281236454844475, 0.01702086627483368, -0.02728785201907158, 0.008244907483458519, 0.04125857725739479, 0.01606769673526287, 0.015400479547679424, 0.0054228478111326694, 0.020915240049362183, 0.001116568804718554, 0.00029446097323670983, -0.003278218675404787, 0.005572631489485502, -0.014134126715362072, -0.0253270473331213, 0.005977727938443422, 0.025558531284332275, -0.011145262978971004, -0.0008616812992841005, -0.026484467089176178, 0.007754706311970949, 0.017265966162085533, -0.009164034388959408, 0.00699217151850462, -0.01797403395175934, -0.03679230436682701, -0.005776881705969572, 0.02776443585753441, 0.012547781690955162, 0.011329088360071182, 0.018477851524949074, 0.020574823021888733, 0.016694065183401108, 0.018123818561434746, -0.004163303878158331, -0.015550263226032257, 0.0007387055666185915, 0.002806740812957287, 0.011458446271717548, -0.030065657570958138, -0.01714341528713703, 0.014896661974489689, 0.0029905661940574646, -0.02389729581773281, 0.0003487149951979518, 0.005504548083990812, 0.029793323948979378, 0.004833925981074572, -0.013344358652830124, -0.035212766379117966, 0.01306521613150835, -0.0025684486608952284, -0.009906143881380558, 0.010859312489628792, -0.024482812732458115, -0.011077179573476315, -0.008857658132910728, 0.014392843469977379, 0.0004757331917062402, 0.010947820730507374, 0.010457620024681091, -0.0314817912876606, -0.018259985372424126, -0.016925549134612083, 0.021664157509803772, 0.005347955971956253, 0.024401113390922546, 0.013541800901293755, -0.028268253430724144, -0.0008655110141262412, 0.03867140784859657, -0.036901235580444336, -0.021732240915298462, -0.016530664637684822, 0.011056753806769848, -0.022399459034204483, 0.017456600442528725, -0.02510918118059635, 0.009878910146653652, -0.016871081665158272, -0.0031846039928495884, -0.007843215018510818, -0.0028339740820229053, 0.041421979665756226, 0.0036799111403524876, -0.009136800654232502, -0.002762486459687352, 0.009912951849400997, -0.00199314346536994, -0.017783399671316147, -0.011213346384465694, -0.012432040646672249, 0.008660216815769672, -0.024673447012901306, 0.012037156149744987, -0.006886642426252365, 0.03317026421427727, 0.019349319860339165, 0.014474543742835522, -0.007965764962136745, -0.014556244015693665, -0.024224095046520233, -0.0071283383294939995, 0.012214173562824726, 0.015509412623941898, -0.007169188465923071, -0.018423384055495262, -0.044199783354997635, -0.00562369404360652, -0.0002678658638615161, -0.006396441254764795, -0.020234404131770134, -0.03327919542789459, 8.574261300964281e-05, -0.036302100867033005, 0.012779265642166138, 0.017538299784064293, -0.01742936670780182, -0.017647232860326767, -0.013330741785466671, -0.011438021436333656, -0.026035115122795105, -0.00786363985389471, 0.017470216378569603, -0.00629772013053298, -0.01702086627483368, -0.025150030851364136, -0.015032828785479069, 0.01478772796690464, 0.029275888577103615, 0.006120703183114529, -0.022031808272004128, 0.014011576771736145, 0.004078199155628681, -0.009150417521595955, 0.03910714015364647, 0.02207265794277191, 0.03194475919008255, -0.02376112900674343, -0.004350533243268728, 0.006801537703722715, 0.02012547105550766, -0.0040884120389819145, -0.010028693825006485, -0.022440308704972267, -0.00479647982865572, 0.018940819427371025, -0.0011829501017928123, 0.028513353317975998, 0.0017403834499418736, -0.016094930469989777, 0.02140543982386589, 0.006947917398065329, -0.016149397939443588, 0.016625981777906418, -0.00232845451682806, 0.01046442799270153, 0.012649907730519772, 0.002888440852984786, 0.016734914854168892, 0.006757283583283424, 0.011601421982049942, 0.014883045107126236, 0.015305162407457829, 0.033388130366802216, 0.009940185584127903, 0.010498469695448875, -0.01876380294561386, 0.009436368010938168, -0.013024366460740566, 0.018341684713959694, -0.0007004085928201675, -9.350838081445545e-05, -0.026865733787417412, -0.0176880843937397, 0.011615038849413395, -0.015032828785479069, 0.005222001578658819, -0.006358995568007231, 0.030201824381947517, -0.01198949757963419, 1.7526172086945735e-05, 0.0004940305952914059, -0.033251963555812836, -0.0015446435427293181, 0.009586151689291, -0.0031403496395796537, 0.020343339070677757, 0.025626614689826965, 0.013017557561397552, 0.0065972874872386456, 0.0031369454227387905, 0.002488450612872839, 0.00452414620667696, 0.02220882475376129, 0.0070602549239993095, -0.003186305984854698, -0.031509026885032654, 0.01796041801571846, -0.024264946579933167, 0.03262559324502945, 0.00017818718333728611, 0.015237079001963139, 0.019022518768906593, 0.023584110662341118, -0.02120118960738182, -0.004711375571787357, 0.01620386354625225, 0.014488160610198975, 0.025068331509828568, -0.00472839642316103, 0.026620633900165558, 0.017048098146915436, 0.03796333819627762, -0.02457812987267971, 0.00979720987379551, 0.004173516295850277, -0.02195010893046856, -0.00867383275181055, 0.016585132107138634, 0.020098237320780754, 0.0005906240548938513, -0.004037349484860897, 0.043791282922029495, -0.004735204856842756, -0.008387882262468338, -0.0099878441542387, 0.014025193639099598, -0.002037397585809231, -0.013296700082719326, -0.023447943851351738, -0.03254389390349388, 0.009157225489616394, -0.01708894968032837, -0.006852600257843733, -0.0034586398396641016, 0.021936491131782532, -0.0036696987226605415, 0.01944463700056076, 0.0012238002382218838, -0.0024629193358123302, 0.025218114256858826, 0.0032373685389757156, -0.015414095483720303, -0.0013301806757226586, 0.02208627574145794, 0.017987649887800217, -0.014174976386129856, -0.03134562447667122, 0.008292566053569317, -0.017647232860326767, -0.021555224433541298, -0.006658562459051609, 0.012915432453155518, 0.002782911527901888, 0.001424646470695734, -0.0009761466644704342, -0.013814134523272514, 0.0046637170016765594, -0.0023863252718001604, 0.01499197818338871, -0.01951272040605545, -0.006624520756304264, -0.030882658436894417, -0.0033190688118338585, -0.03390556573867798, 0.025286197662353516, -0.021092256531119347, -0.011894180439412594, -4.180749965598807e-05, -0.028322720900177956, -0.02747848443686962, 0.012091622687876225, 0.0028101447969675064, 0.025340665131807327, -0.021650541573762894, -0.006627925205975771, 0.01864125207066536, 0.0020339933689683676, 0.0027403593994677067, 0.024537280201911926, -0.009293392300605774, 0.005518164485692978, -0.015972379595041275, -0.00725769717246294, 0.0035301274619996548, 0.12091623246669769, 0.011104412376880646, -0.014351993799209595, 0.0024135587736964226, -0.014678794890642166, -0.008571707643568516, -0.006998979952186346, -0.016789382323622704, -0.0034637460485100746, 0.016163013875484467, 0.04468998312950134, -0.01390945166349411, 0.016625981777906418, -0.011948647908866405, 0.000327226152876392, 0.016707681119441986, -0.011839713901281357, 0.0037616111803799868, -0.006147936452180147, -0.02174585871398449, 0.0006659413920715451, -0.019676120951771736, 0.012234598398208618, 0.017592767253518105, 0.02802315354347229, 0.013112874701619148, 0.025504065677523613, 0.016176629811525345, -0.012043964117765427, -0.011281429789960384, 0.014120509847998619, 0.003043330740183592, 0.01032826118171215, 0.00857851654291153, 0.0002195691631641239, -0.021105872467160225, -0.03126392513513565, 0.004425425082445145, 0.04076837748289108, 0.008687449619174004, 0.006961533799767494, 0.00867383275181055, 0.022712642326951027, -0.017320433631539345, 0.0029105679132044315, -0.010219328105449677, 0.00636580353602767, 0.03262559324502945, -0.0007518967613577843, -0.012227789498865604, 0.03891650587320328, -0.00492924265563488, -0.030882658436894417, -0.02930312231183052, -0.012888199649751186, 0.006358995568007231, 0.002022078726440668, 0.01458347775042057, -0.004816905129700899, -0.04022371023893356, -0.0045615918934345245, -0.02377474494278431, 0.007918106392025948, -0.0026365320663899183, 3.9360751543426886e-05, -0.008680641651153564, -0.012200556695461273, -0.008524049073457718, -0.012404806911945343, -0.008285757154226303, -0.017606383189558983, -0.03545786812901497, -0.02762826904654503, -0.014678794890642166, 0.014038809575140476, -0.004275641404092312, 0.02147352322936058, 0.025980649515986443, 0.0016297479160130024, 0.023407094180583954, 0.002570150652900338, 0.005405826959758997, -0.013453291729092598, -0.004908817820250988, -0.0106686782091856, 0.0230394434183836, -0.0021871812641620636, -0.004599038045853376, 0.007645772770047188, 0.021487141028046608, -0.003220347687602043, 0.018464235588908195, 0.01350775919854641, 0.003211837261915207, 0.016585132107138634, 0.036029767245054245, 0.03679230436682701, 0.02445557899773121, 0.002362496219575405, 0.0020101643167436123, 0.004350533243268728, -0.031018825247883797, -0.017306815832853317, 0.0021871812641620636, 0.002980353543534875, 0.007114721927791834, 0.003826290601864457, -0.010627828538417816, -0.006845792289823294, -0.028513353317975998, 0.014474543742835522, -0.027859752997756004, -0.004609250463545322, 0.012629481963813305, -0.015836212784051895, 0.005582843907177448, 0.009490834549069405, 0.028241019695997238, 0.00616495730355382, -0.0020935663487762213, -0.021037789061665535, -0.03289793059229851, 0.018872736021876335, -0.01471964456140995, -0.036710601300001144, 0.0029378014151006937, 0.010035502724349499, -0.020084621384739876, 0.0029973743949085474, -0.001945484895259142, -0.016448965296149254, 0.0027795073110610247, 0.00804065726697445, -0.017265966162085533, -0.06307251751422882, -0.03464086726307869, -0.007250888738781214, 0.00280163437128067, -0.004486700054258108, 0.006304528564214706, -0.025476831942796707, -0.014365610666573048, 0.019499104470014572, -0.013146916404366493, 0.02079268917441368, -0.03679230436682701, -0.0050041344948112965, 0.0015965572092682123, -0.011894180439412594, 0.021255657076835632, -0.007387055549770594, 0.018137434497475624, -0.016462581232190132, 0.0018756993813440204, -0.015332396142184734, -0.005514760501682758, -0.01809658482670784, -0.016040463000535965, 0.011615038849413395, 0.036765068769454956, 0.009497642517089844, -0.006760688032954931, 0.013521376065909863, -0.003962457645684481, 0.009967419318854809, -0.026920201256871223, 0.0054228478111326694, -0.011499296873807907, 0.007727473042905331, 0.025231730192899704, 0.0078091733157634735, 0.03025628998875618, -0.008462774567306042, 0.0015191122656688094, -0.0078091733157634735, 0.014610710553824902, -0.023053061217069626, -0.019362935796380043, -0.0239653792232275, -0.047032054513692856, -0.012575015425682068, -0.01804211735725403, 0.012486507184803486, 0.012404806911945343, -0.03404173254966736, 0.0190497525036335, 0.033388130366802216, -0.007366630714386702, 0.009484026581048965, 0.005051793064922094, 0.014692410826683044, -0.003402471076697111, 0.02517726458609104, 0.008898508735001087, -0.00164166244212538, -0.01572727970778942, -0.021378207951784134, -0.034477464854717255, -0.01884550228714943, 0.02024802193045616, 0.0007378545124083757, -0.01594514772295952, 0.008244907483458519, -0.0015446435427293181, -0.018328068777918816, 0.008490007370710373, -0.024401113390922546, -0.006352187134325504, -0.002325050299987197, -0.014665178023278713, 0.0057973070070147514, -0.0054228478111326694, -0.017987649887800217, -5.428060467238538e-05, 0.002840782515704632, -0.007741089444607496, -0.00864659994840622, 0.004983709659427404, -0.00030595005955547094, -0.00568837346509099, -0.009674659930169582, -0.014079660177230835, 0.013521376065909863, 0.023516027256846428, 0.015618346631526947, -6.467929051723331e-05, 0.021854791790246964, -0.015754513442516327, 0.0070602549239993095, 0.006583670619875193, 0.0075368392281234264, 0.02775081992149353, 0.02459174580872059, -0.01715703308582306, -0.015591112896800041, 0.009266159497201443, 0.004047561902552843, -0.006250062026083469, -0.02126927301287651, 0.0035743818152695894, -0.00911637581884861, -0.0022739877458661795, -0.01830083504319191, -0.002299519022926688, -0.03052862361073494, 0.017783399671316147, -0.013562225736677647, 0.010232944041490555, -0.018545934930443764, -0.006239849142730236, -0.021228423342108727, 0.005998153239488602, -0.025572149083018303, 0.00966104306280613, 0.0007531733135692775, -0.013684775680303574, -0.028785686939954758, 0.001294436864554882, -0.030555857345461845, 0.009878910146653652, 0.0062568699941039085, 0.022930510342121124, -0.007427905686199665, 0.015032828785479069, 0.011642271652817726, 0.0175655335187912, -0.00972912646830082, 0.003799057099968195, 0.019812287762761116, 0.025408748537302017, 0.017048098146915436, -0.0035641691647469997, 0.009395517408847809, 0.020847156643867493, -0.0013803922338411212, -0.022671792656183243, -0.004837329965084791, 0.0014382631052285433, -0.014270293526351452, -0.016353648155927658, 0.02106502279639244, -0.00016648534801788628, -0.02471429668366909, -0.004299470689147711, 0.014038809575140476, -0.03450470045208931, 0.02099693939089775, -0.011615038849413395, 0.0021344164852052927, -0.0026552551425993443, 0.01124738808721304, -0.021568840369582176, -0.006736858747899532, 0.03820843994617462, -0.01252054888755083, 0.028404420241713524, -0.002730146748945117, 0.043655116111040115, -0.02248115837574005, 0.007741089444607496, -0.0021888832561671734, 0.020956089720129967, -0.03287069499492645, 0.013957109302282333, -0.000442968012066558, 0.017797017470002174, 0.015563879162073135, 0.007836406119167805, -0.03518553450703621, -0.026988284662365913, 0.011642271652817726, 0.010069544427096844, -0.03518553450703621, -0.021310124546289444, 0.015019211918115616, -0.003942032344639301, 0.005204980727285147, 0.001319117029197514, -0.027587419375777245, 0.014351993799209595, -0.011887372471392155, 0.0010042310459539294, 0.026525316759943962, -0.00979040190577507, -0.0014620923902839422, 0.0062466575764119625, 0.01384817622601986, -0.013650733977556229, -0.023202843964099884, -0.01951272040605545, 0.007638964336365461, 0.0011199729051440954, -0.02768273651599884, -0.02080630697309971, -0.008278949186205864, -0.02263094298541546, -0.025694698095321655, 0.03254389390349388, 0.02796868607401848, 0.014937511645257473, 0.024973014369606972, 0.011553763411939144, 0.010750378482043743, 0.0024952588137239218, 0.0049020093865692616, 0.023257311433553696, -0.010355494916439056, 0.0012569909449666739, -0.011676313355565071, -0.0036560818552970886, -0.030909892171621323, 0.04144921153783798, -0.006416866090148687, -0.03415066376328468, -0.016054080799221992, -0.000481264985864982, -0.07380247116088867, 0.002393133705481887, 0.0004629675531759858, 0.017442982643842697, 0.0015625155065208673, -0.008421923965215683, 0.01877741888165474, 0.019825903698801994, -0.013092449866235256, 0.02222244255244732, 0.012030348181724548, -0.023475177586078644, 0.02256285957992077, -0.01567281223833561, -0.0016263436991721392, -0.00884404219686985, -0.056482039391994476, 0.02120118960738182, 0.04577932134270668, -0.007380247116088867, -0.0023948356974869967, 0.006944512948393822, -0.007373439148068428, 0.006352187134325504, 0.007441522553563118, 0.013466908596456051, -0.0034892773255705833, -0.009068717248737812, 0.011812481097877026, -0.032516662031412125, -0.028349952772259712, 0.029330356046557426, -0.003683315357193351, -0.010219328105449677, 0.003523319261148572, -0.0006221126532182097, 0.005072217900305986, 0.0029531202744692564, -0.025340665131807327, -0.03300686180591583, 0.004912221804261208, -4.7791399993002415e-05, 0.02472791261970997, 0.004299470689147711, -0.011056753806769848, 0.013419250026345253, 0.012350340373814106, -0.016149397939443588, -0.01748383231461048, 0.023270927369594574, 0.0075232223607599735, 0.01640811376273632, 0.01843700185418129, -0.01127462089061737, 0.019703354686498642, -0.011233771219849586, 0.002527598524466157, -0.010049118660390377, 0.01727958209812641, -0.022603709250688553, -0.002268881304189563, -0.013214999809861183, -0.014637944288551807, 0.020030153915286064, 0.005865390412509441, -0.0023863252718001604, -0.007829598151147366, -0.0009957206202670932, -0.014529011212289333, -0.019294852390885353, 0.006205807439982891, 0.011213346384465694, -0.01939016953110695, 0.018791034817695618, -0.004262024536728859, -0.02039780467748642, 0.004415212664753199, 0.004364150110632181, 0.007802364882081747, 0.001569323823787272, 0.21100427210330963, -0.021487141028046608, -0.014229443855583668, 0.00023190928914118558, -0.0013003941858187318, 0.014066043309867382, 0.02606234885752201, -0.015972379595041275, -0.012717991136014462, 0.015141761861741543, -0.0278325192630291, -0.01158099714666605, -0.03317026421427727, -0.002137820702046156, -0.029875023290514946, -0.01292904932051897, -0.019608037546277046, -0.01889996975660324, -0.0016748531488701701, -0.0023182418663054705, 0.006389632821083069, 0.01259544026106596, 0.004091816022992134, -0.006954725831747055, 0.03390556573867798, 0.015972379595041275, -0.003072266234084964, -0.015427712351083755, 0.02343432791531086, 0.014283910393714905, -0.005480718798935413, -0.0022518604528158903, -0.020207172259688377, 0.015441329218447208, -0.017538299784064293, -0.0018059138674288988, 0.008408308029174805, -0.033251963555812836, 0.030011190101504326, 0.017851483076810837, -0.0023216460831463337, 0.02829548716545105, -0.01131547149270773, -0.0071623800322413445, 0.014556244015693665, 0.04052327573299408, 0.006056023761630058, -0.005593056324869394, 0.018750185146927834, 0.00030829041497781873, -0.048448190093040466, -0.019689736887812614, 0.018287217244505882, 0.0026892968453466892, -0.009109566919505596, 0.001972718397155404, 0.024618979543447495, 0.012677140533924103, -0.011485680006444454, 0.013650733977556229, -0.009701893664896488, 0.028268253430724144, -0.004922434221953154, 0.039134375751018524, -0.024401113390922546, -0.0007157273939810693, -0.007053446490317583, -0.010232944041490555, -0.008748725056648254, -0.0024629193358123302, -0.0078091733157634735, 0.007686622906476259, -0.003999903332442045, 0.015155378729104996, -0.02174585871398449, -0.04044157639145851, 0.019022518768906593, -0.008285757154226303, 0.0376637727022171, 0.050871964544057846, -0.008748725056648254, 0.009191267192363739, -0.019880371168255806, -0.023870062083005905, 0.0004467977269086987, -0.031781360507011414, 0.024346645921468735, -0.00010233795183012262, -0.005664544180035591, -0.0007467904943041503, 0.007856831885874271, -0.008306181989610195, -0.00622963672503829, -0.028731221333146095, -0.005116472020745277, 0.009375092573463917, -0.006267082877457142, 0.03131839260458946, -0.0176880843937397, 0.03153625875711441, -0.021187573671340942, 0.06056704744696617, -0.014828577637672424, 0.025544915348291397, 0.023801978677511215, 0.005436464678496122, 0.010239752940833569, 0.0205339714884758, -0.007318972144275904, -0.03276176378130913, 0.0012297575594857335, -0.040686678141355515, 0.0039045866578817368, -0.006073044613003731, 0.011948647908866405, 0.012670332565903664, -0.013895834796130657, -0.0029667369090020657, 0.027110835537314415, 0.012295872904360294, 0.004667121451348066, -0.032244328409433365, -0.0017905950080603361, 0.01424306072294712, -0.016054080799221992, 0.007216847036033869, -0.0340144969522953, 0.010750378482043743, -0.03733697161078453, -0.005582843907177448, -0.00246462132781744, 0.013854984194040298, 0.007053446490317583, -0.04313768073916435, -0.012867774814367294, -0.010763995349407196, -0.011397171765565872, -0.03665613755583763, -0.012241406366229057, 0.01783786714076996, -0.003077372442930937, 0.005232213996350765, -0.0053547644056379795, -0.003245878964662552, -0.027846135199069977, -0.021432673558592796, 0.02200457453727722, -0.004888392519205809, -0.012452465482056141, -0.000914871518034488, -0.023216459900140762, 0.014338376931846142, 0.013119683600962162, -0.014202210120856762, 0.029058022424578667, -0.004197345580905676, -0.0005212639807723463, -0.018940819427371025, 0.004578612744808197, -0.0057053943164646626, -0.02309391088783741, 0.018055733293294907, 0.012949474155902863, -0.005548802204430103, 0.009436368010938168, 0.0014995383098721504, -0.1738579273223877, 0.020016537979245186, -0.010559745132923126, -0.011703547090291977, 0.036846768110990524, -0.024128779768943787, 0.020574823021888733, -0.0034926815424114466, -0.028377186506986618, -0.019893987104296684, 0.02559938095510006, -0.007033021654933691, -0.04057774320244789, -0.003249283181503415, 0.002820357447490096, -0.008435540832579136, 0.006447503808885813, 0.009245733730494976, 0.0417487807571888, 0.022603709250688553, 0.037173569202423096, -0.01939016953110695, 0.01681661605834961, 0.00853766594082117, 0.024469196796417236, -0.021854791790246964, 0.0015471966471523046, -0.003308856161311269, 0.018259985372424126, -0.012452465482056141, 0.0029309929814189672, 0.004394787363708019, 0.04959199205040932, -0.013548608869314194, -0.01245927345007658, 0.007475564256310463, -0.0007008341490291059, 0.00020425039110705256, -0.003822886385023594, 0.0213373564183712, 0.032298795878887177, 0.021868407726287842, -0.01958080381155014, 0.012670332565903664, -0.010512086562812328, 0.038862038403749466, 0.014270293526351452, -0.014910277910530567, -0.012677140533924103, 0.013051599264144897, 0.018532318994402885, -0.0075232223607599735, 0.016244713217020035, 0.027805285528302193, -0.0012978409649804235, -0.002978651551529765, -0.004469679202884436, -0.028268253430724144, -0.003826290601864457, 0.012534165754914284, 0.01050527859479189, -0.025776399299502373, 0.0024493024684488773, -0.010028693825006485, 0.0026433405000716448, -0.017265966162085533, -0.023461561650037766, -0.02391091175377369, -0.01830083504319191, 0.008592132478952408, -0.011730780825018883, -0.00033169411472044885, -0.015468562953174114, -0.009640618227422237, 0.006386228837072849, 0.01898166909813881, -0.01884550228714943, -0.014842194505035877, 0.02234499156475067, 0.002335262717679143, 0.0035471483133733273, 0.02431941218674183, -0.02938482165336609, 0.013875409960746765, -0.0160813145339489, -0.013582650572061539, 0.001299543073400855, 0.0026416382752358913, 0.0008969996124505997, -0.007101105060428381, 0.014120509847998619, -0.03044692426919937, -0.021963724866509438, 0.002272285521030426, 0.007945340126752853, 0.02903078868985176, 0.025844482704997063, 0.00519817229360342, -0.021759474650025368, 0.008905316703021526, 0.04765842482447624, -0.021841173991560936, -0.017442982643842697, 0.014937511645257473, 0.035348933190107346, 0.007203230168670416, 0.01565919630229473, 0.015523029491305351, 0.0246462132781744, 0.014161360450088978, -0.0173476655036211, 0.001116568804718554, -0.010995479300618172, 0.0028816326521337032, 0.013698392547667027, 0.0050586010329425335, -0.006181978154927492, -0.020030153915286064, 0.014964745379984379, -0.026361916214227676, 0.053595300763845444, -0.013460100628435612, -0.012214173562824726, 0.003945436794310808, -0.013657542876899242, -0.0030246078968048096, -0.10424939543008804, -0.01158099714666605, 0.012234598398208618, 0.002435685833916068, 0.003999903332442045, 0.010750378482043743, -0.0014833684545010328, -0.007421097252517939, -0.01809658482670784, 0.026361916214227676, -0.02357049472630024, -0.02167777344584465, -0.004010115750133991, 0.007455138955265284, 0.013357975520193577, -0.0005353062297217548, -0.014814961701631546, -0.005725819151848555, 0.009422751143574715, 0.03646550327539444, -0.0013642223784700036, -0.018409768119454384, -0.0025156838819384575, -0.0034926815424114466, -0.012030348181724548, 0.008224482648074627, -0.019825903698801994, -0.01620386354625225, -0.008265332318842411, 0.023284543305635452, -0.009633810259401798, -0.02525896392762661, 0.050681330263614655, -0.00959295965731144, -0.00732578057795763, -0.01239119004458189, 0.004782863426953554, -0.003659486072137952, 0.013895834796130657, -0.016707681119441986, 0.005545398220419884, 0.010634636506438255, 0.03322472795844078, -0.017728934064507484, -0.00911637581884861, -0.019703354686498642, 0.006910471245646477, -0.0014901767717674375, 0.012173322960734367, -0.017511066049337387, -0.030610324814915657, -0.0104371951892972, -0.04093177616596222, -0.029221421107649803, 0.004272237420082092, -0.012813307344913483, -0.0002236116270069033, 0.014365610666573048, -0.005732627585530281, 0.00588921969756484, 0.0011242281179875135, -0.013208191841840744, -0.009906143881380558, 0.035430632531642914, 0.04085007682442665, -0.02700190059840679, -0.02965715527534485, -0.019962070509791374, 0.024087928235530853, 0.004190537147223949, 0.01006273552775383, 0.012432040646672249, -0.011431213468313217, -0.0049326466396451, -0.019213153049349785, -0.006086661480367184, 0.008435540832579136, -0.01708894968032837, 0.009075525216758251, 0.005085834767669439, -0.01258863229304552, -0.010083160363137722, 0.03254389390349388, -0.026852117851376534, 0.012377573177218437, -0.007557264529168606, 0.009885719045996666, 0.01681661605834961, -0.014610710553824902, -0.018940819427371025, 0.0016663427231833339, 0.02079268917441368, 0.005436464678496122, 0.01171716395765543, 0.0016765552572906017, 0.003567573381587863, 0.0025105776730924845, -0.010913779027760029, 0.0007421097252517939, -0.010866120457649231, -0.017606383189558983, 0.007387055549770594, -0.04782182350754738, 0.012009922415018082, -0.011015904136002064, -0.015958763659000397, 0.0058619859628379345, -0.004377766512334347, 0.010723145678639412, 0.0023522835690528154, 0.0017633616225793958, -0.004847542382776737, -0.021487141028046608, 0.02984778955578804, 0.0049462635070085526, -1.3629989553010091e-05, -0.005654331762343645, -0.006964938249439001, 0.023842828348279, -0.008170015178620815, -0.006893450394272804, 0.017374899238348007, 0.00519817229360342, -0.02749210223555565, 0.01742936670780182, 8.483837882522494e-05, 0.0021531395614147186, 0.01729319989681244, -0.020152704790234566, -0.0045309546403586864, -0.007666198071092367, 0.011301854625344276, -0.011526530608534813, -0.012234598398208618, 0.006035598926246166, 0.01776978373527527, 0.0027335509657859802, -0.0028322720900177956, -0.013957109302282333, 0.004374362528324127, -0.016762148588895798, 0.019621653482317924, -0.006985363084822893, 0.0041871326975524426, -0.01681661605834961, -0.019267620518803596, -0.0031897102016955614, -0.0025395131669938564, 0.0016501728678122163, -0.007924915291368961, -0.0054228478111326694, 0.040741145610809326, -0.003245878964662552, 0.02017993852496147, -0.01790595054626465, 0.0058960276655852795, 0.0003219071077182889, -0.0018110200762748718, -0.018246367573738098, 0.009204884059727192, 0.004313087556511164, -0.018940819427371025, 0.024346645921468735, -0.015849830582737923, 0.019403787329792976, 0.01279969047755003, -0.01668044738471508, 0.0015403883298859, -0.011240579187870026, 0.008775957860052586, 0.015386862680315971, -0.01579536311328411, -0.027505718171596527, 0.014814961701631546, 0.01951272040605545, 0.0176880843937397, 0.000987210194580257, 0.005208384711295366, -0.02067013829946518, -0.014828577637672424, -0.008789574727416039, 0.019962070509791374, 0.02647084929049015, -0.029085254296660423, -0.0029871619772166014, 0.019676120951771736, 0.028213785961270332, 0.020983323454856873, -0.003090989077463746, 0.0003559488395694643, 0.00898020900785923, -0.008796383626759052, 0.0010893354192376137, 0.01046442799270153, -0.004340320825576782, -0.004667121451348066, 0.0198531374335289, 0.01471964456140995, -0.006427078973501921, 0.008721491321921349, -0.00466031301766634, 0.01903613656759262, -0.003475660691037774, 0.013637117110192776, -0.023012209683656693, -0.0057734777219593525, -0.019839521497488022, -0.001251884619705379, 0.007679814472794533, -0.0321626290678978, -0.003509702393785119, 0.03491320088505745, 0.012220981530845165, -0.004214366432279348, 0.025898948311805725, 0.0027029134798794985, -0.017524683848023415, 0.026212133467197418, 0.012173322960734367, -0.022862426936626434, -0.0021207998506724834, 0.02249477617442608, 0.009129992686212063, -0.011601421982049942, 0.02769635245203972, -0.016721298918128014, 0.0397062748670578, 0.0032918353099375963, -0.004987113643437624, -0.018328068777918816, -0.019880371168255806, 0.008701066486537457, -0.017102565616369247, 0.009225308895111084, -0.046841420233249664, -0.001362520270049572, 0.001217842916958034, -0.01850508525967598, 0.019008902832865715, 0.013024366460740566, -0.011928222142159939, 0.07097020000219345, -0.0018076159758493304, -0.004759034141898155, 0.01097505446523428, -0.006464524660259485, 0.0143792275339365, 0.014842194505035877, 0.016598748043179512, -0.009279776364564896, -0.029684389010071754, 0.014066043309867382, 0.0025480235926806927, 0.02290327660739422, -0.025027479976415634, -0.01714341528713703, 0.0038637365214526653, -0.0017463407712057233, 0.014270293526351452, -0.005276468116790056, 0.019008902832865715, 0.019471870735287666, -0.025435982272028923, 0.007169188465923071, 0.0020271851681172848, -0.021705007180571556, 0.030283523723483086, 0.0079317232593894, 0.005661139730364084, -0.01700724847614765, -0.020697372034192085, 0.001930166152305901, -0.019158685579895973, -0.04422701522707939, -0.006784516852349043, 0.016367264091968536, -0.014460927806794643, -0.007965764962136745, -0.013078832998871803, 0.004667121451348066, 0.014937511645257473, 0.0015378352254629135, 0.0002448876912239939, -0.017524683848023415, -0.04711375758051872, 0.01884550228714943, 0.0041224537417292595, -0.014569860883057117, -0.01137674693018198, -0.027805285528302193]} +{"id": "test:10000040", "text": "\"Volunteers are needed to help with the Assembly for Children at the 99th International Assembly in July. Below are the dates, times, and volunteers needed for each session. If you will be attending the Assembly and are interested in helping with the Assembly for Children, register online by clicking here. A background check and pastoral referral must be completed and can be found on the website.\\n\u201cHands-on\u201d laborers needed for take down of Assembly for Children props and d\u00e9cor.\"", "vector_field": [-0.010128876194357872, -0.008052019402384758, -0.006822036579251289, -0.03094448894262314, -0.0284979660063982, 0.033417899161577225, -0.023443609476089478, -0.013234078884124756, -0.026051443070173264, 0.01231999322772026, -0.008421686477959156, 0.007547928486019373, -0.015794863924384117, -0.02310754917562008, 0.00024196386220864952, -0.006610318552702665, 0.015996500849723816, 0.000906524364836514, -0.006586794275790453, -0.018765641376376152, -0.022610178217291832, 0.013052606023848057, -0.0011123616714030504, 0.00567942950874567, -0.0006788430619053543, -0.00017937252414412796, 0.020607255399227142, -0.015176511369645596, 0.009389542043209076, -0.008004970848560333, -0.004553625360131264, 0.0010695138480514288, -0.004856080282479525, -0.03497721999883652, -0.016574524343013763, -0.015055529773235321, 0.002518776571378112, -0.02662946842610836, 0.03253069892525673, 0.0018533760448917747, 0.041402705013751984, 0.020607255399227142, -0.011325253173708916, -0.02079544961452484, -0.02079544961452484, 0.0287130456417799, -0.01695091277360916, -0.02484162338078022, -0.015109298750758171, 0.014800122939050198, 0.017811229452490807, -0.0027388965245336294, -0.017609592527151108, -0.0022079201880842447, -0.010720343329012394, -0.001511433976702392, -0.0014929505996406078, 0.025338994339108467, -0.0025658251252025366, -0.03035302273929119, 0.017717132344841957, -0.007682352792471647, -0.02115839533507824, 0.023793112486600876, -0.027745189145207405, -0.018631218001246452, -0.030218597501516342, 0.027489783242344856, 0.0030480725690722466, -0.017596149817109108, 0.01719287596642971, 0.008939220570027828, 0.039359454065561295, -0.0010216252412647009, 0.033364128321409225, -0.006089423783123493, 0.011997374705970287, 0.010303627699613571, 0.0006586794042959809, -0.0006011289660818875, -0.007655467838048935, -0.03035302273929119, -0.016574524343013763, 0.01541847549378872, 0.005918032955378294, 0.0387142188847065, 0.019962018355727196, -0.0004020128690171987, -0.017636477947235107, -0.012723266147077084, -0.006368354428559542, 0.015109298750758171, 0.014786680229008198, 0.007991529069840908, -0.012548514641821384, 0.02115839533507824, -0.027664534747600555, 0.04927997291088104, -0.0033001182600855827, -0.03271889314055443, -0.010357397608458996, 0.009786093607544899, -0.004775425884872675, -0.002110462635755539, -0.00015920886653475463, -0.009194626472890377, 0.002777543617412448, -0.008811517618596554, 0.008361196145415306, 0.016708949580788612, -0.025298666208982468, 0.024330811575055122, 0.0028010678943246603, -0.020593812689185143, -0.010747228749096394, -0.018416138365864754, -0.010619524866342545, 0.004076418932527304, -0.005417302250862122, -0.015593226999044418, 0.01521683856844902, 0.020647583529353142, 0.009671833366155624, -0.004140270408242941, 0.019545303657650948, -0.005393777973949909, 0.002458285540342331, 0.005649184342473745, -0.02115839533507824, -0.03659031167626381, -0.011762131936848164, 0.0283366572111845, 0.02283870056271553, 0.004664525389671326, 0.004143631085753441, 0.011392464861273766, 0.004167155362665653, 0.021749863401055336, -0.02092987485229969, -0.002239846158772707, 0.016668621450662613, 0.0027355358470231295, -0.02496260590851307, -0.006623760797083378, -0.011009356006979942, 0.0021289458964020014, 0.013610467314720154, -0.001984439790248871, -0.008078904822468758, 0.005941557232290506, 0.027489783242344856, 0.045085933059453964, -0.0007586575229652226, -0.004106664564460516, 0.023725900799036026, 0.016251906752586365, -8.92661846592091e-05, 0.018631218001246452, 0.005891148000955582, -0.006072620861232281, -0.006922855041921139, -0.0030480725690722466, 0.013966691680252552, -0.0008779591880738735, 0.016668621450662613, 0.013502927497029305, -0.014974874444305897, -0.02305378019809723, -0.002365869004279375, -0.0030077453702688217, 0.006230569444596767, 0.02082233503460884, -0.008112510666251183, 0.009154299274086952, 0.0030379907693713903, 0.01057247631251812, 0.009382820688188076, 0.007675631437450647, -0.019639400765299797, -0.013966691680252552, -0.016440100967884064, -0.0007720999419689178, -0.00018420339620206505, 0.01709877885878086, -0.03441263735294342, 0.016426658257842064, -0.0067884307354688644, -0.003790767164900899, 0.006936297286301851, -0.01336850319057703, 0.01751549541950226, -0.014625371433794498, 0.015687324106693268, 0.003076637862250209, -0.6564210653305054, -0.004990504588931799, 0.005756723694503307, 0.002239846158772707, 0.00971888191998005, 0.022247232496738434, 0.006633842829614878, -0.00264984043315053, -0.014988317154347897, 0.008112510666251183, 0.014114558696746826, 0.037746362388134, -0.004637640900909901, -0.026051443070173264, -0.015512572601437569, -0.02278492972254753, 0.011795738711953163, -0.002251608297228813, 0.008972826413810253, 0.00770251639187336, -0.022153135389089584, 0.013711285777390003, -0.013032442890107632, 0.014638814143836498, -0.002829632954671979, -0.016708949580788612, 0.02119872346520424, -0.01521683856844902, 0.00312536652199924, 0.012044423259794712, 0.0036059336271137, 0.011977211572229862, -0.01347604300826788, 0.012924903072416782, 0.027718305587768555, -0.0016055310843512416, -0.02839042618870735, 0.027772074565291405, 0.020472832024097443, 0.04030042514204979, -0.021467572078108788, 0.0012224216479808092, 0.03634835034608841, 0.009866748936474323, -0.006045735906809568, 0.00032156831002794206, 0.0015727650607004762, -0.009161020629107952, 0.012743430212140083, -0.02676389180123806, 0.00434190733358264, 0.017784344032406807, 0.0023977947421371937, 0.0021843959111720324, 0.0037806853652000427, -0.03250381350517273, 0.0031640136148780584, -0.0033622896298766136, -0.02294624038040638, 0.003086719661951065, -0.007682352792471647, 0.003295077243819833, -0.029707785695791245, 0.0005960880662314594, -0.015660438686609268, 0.009060202166438103, -0.006200323812663555, 0.018886623904109, 0.018913509324193, 0.005723117385059595, -0.0010056622559204698, 0.04008534550666809, -0.019518418237566948, 0.015579784289002419, 0.0033421257976442575, 0.005592053756117821, -0.021857403218746185, -0.018080078065395355, -0.022139692679047585, 1.7472542822360992e-05, 0.00855611078441143, -0.01691058650612831, -0.001692066784016788, 0.0028750011697411537, 0.016346003860235214, -0.0013551656156778336, 0.003568967105820775, -0.015539457090198994, 0.001144287409260869, -0.013166867196559906, 0.011815901845693588, 0.012978672981262207, 0.029492706060409546, -0.040972549468278885, -0.030030403286218643, 0.019733497872948647, -0.013133260421454906, 0.018093520775437355, 0.0192764550447464, -0.001321559539064765, 0.0031774560920894146, -0.011432792991399765, -0.010108712129294872, 0.0013610466849058867, 0.022489197552204132, 0.01748860999941826, -0.012669497169554234, -0.003568967105820775, 0.03419755771756172, -0.02646815776824951, -0.012037701904773712, -0.02274460345506668, -0.013180308975279331, -0.0037000307347625494, 0.010901816189289093, -0.017623035237193108, -0.019693169742822647, 0.008394801989197731, 0.007890710607171059, -0.012992115691304207, -0.008300704881548882, 0.013576861470937729, 0.014141443185508251, -0.013334897346794605, -0.005165256559848785, 0.020244309678673744, 0.02675044909119606, -0.02265050634741783, 0.006516221445053816, -0.010364118963479996, -0.02508358657360077, 0.021938057616353035, 0.0006746423314325511, -0.01705845259130001, 0.023336069658398628, 0.016332561150193214, 0.0024246794637292624, -0.012609005905687809, 0.0028766815084964037, -0.008293983526527882, -0.016332561150193214, 0.005800411570817232, 0.004261252470314503, -0.005323205143213272, 0.004546904470771551, -0.025997672230005264, -0.008146116510033607, -0.001618973445147276, -0.01241409033536911, 0.00384789751842618, -0.011923441663384438, 0.003558885073289275, -0.023537706583738327, 0.011587380431592464, -0.018160732463002205, -0.011506726033985615, 0.003989043179899454, -0.01712566427886486, -0.0043015796691179276, -0.031589727848768234, 0.02649504318833351, 0.016776161268353462, -0.03212742507457733, 0.003427821444347501, -0.010175924748182297, 0.013247521594166756, -0.003663063980638981, 0.009564293548464775, -0.0059281145222485065, -0.035326723009347916, 0.018120404332876205, -0.010928701609373093, -0.014316195622086525, 0.02086266130208969, -3.9907234167912975e-05, -0.012158684432506561, -0.007763007190078497, 0.0041167461313307285, 0.004684689454734325, -0.0037605217657983303, -0.004617477301508188, 0.005870984401553869, -0.011278204619884491, -0.0073395706713199615, 0.009322330355644226, 0.029116317629814148, 0.02325541526079178, 0.03602573275566101, -0.006721218582242727, 0.027637651190161705, 0.013355061411857605, -0.00264311907812953, 0.009134136140346527, -0.0006700214580632746, 0.0034883124753832817, 0.019894806668162346, -0.014692583121359348, 0.014128001406788826, 0.004399037454277277, 0.007043837103992701, 0.014396850019693375, -0.009960846044123173, 0.027825843542814255, -0.011486561968922615, -0.00568951154127717, -0.02265050634741783, 0.01135885901749134, -0.025379320606589317, 0.006280978675931692, 0.006180160213261843, 0.00971216056495905, -0.02083577774465084, -0.005030831787735224, 0.008267099037766457, -0.005390417296439409, 0.008925777859985828, -0.009994451887905598, 0.015337820164859295, -0.03191234543919563, -0.01048510055989027, 0.004654443822801113, 0.0029892618767917156, 0.04344595596194267, 0.0015332779148593545, -0.02703274041414261, 0.003999125212430954, 0.0068590035662055016, 0.010868210345506668, 0.0004830875841435045, 0.010001173242926598, -0.010088548995554447, -0.0008922417764551938, 0.010834604501724243, 0.03441263735294342, 0.020029231905937195, -0.025204569101333618, 0.012125077657401562, -0.01734074391424656, 0.02084922045469284, 0.01716599240899086, 0.014719468541443348, 0.005098044406622648, 0.01703156717121601, -0.005709675140678883, 0.005185420159250498, -0.0142489830031991, 0.015270608477294445, 0.012709824368357658, -0.00432846462354064, 0.016924027353525162, 0.002706970786675811, 0.035514917224645615, -0.028229117393493652, 0.016829930245876312, -0.0005112327053211629, -0.0021457490511238575, -0.0036059336271137, -0.010034779086709023, 0.013583582825958729, 0.01338194590061903, 0.023860326036810875, 0.01340210996568203, 0.027516668662428856, 0.02274460345506668, -0.008804796263575554, -0.0043015796691179276, -0.004990504588931799, -0.027906499803066254, -0.008320868015289307, -0.03417067602276802, -0.002720413263887167, -0.02263706363737583, 0.014665698632597923, -0.03328347206115723, 0.010028057731688023, 0.008152837865054607, -0.015700766816735268, -0.03462771698832512, 0.0032329061068594456, 0.0190882608294487, -0.02652192860841751, -0.023846883326768875, 0.003437903244048357, 0.003952076658606529, 0.00874430499970913, -0.003548803273588419, -0.0008603159803897142, 0.014410292729735374, -0.017542380839586258, -0.008999711833894253, -0.017528938129544258, -0.00022852142865303904, 0.0032917167991399765, 0.0018937032436951995, 0.011728526093065739, -0.01914202980697155, 0.002041570143774152, -0.0008262898190878332, 0.0064086816273629665, -0.03064875677227974, -0.013146703131496906, 0.005551726557314396, -0.0038344550412148237, -0.039171259850263596, 0.03083695098757744, 0.015781421214342117, -0.006368354428559542, 0.0026817661710083485, -0.004395676776766777, 0.0003581149212550372, 0.02507014572620392, 0.01347604300826788, -0.0056391023099422455, -0.016292233020067215, 0.0025658251252025366, -0.004523380193859339, -0.008112510666251183, 0.023806555196642876, 0.018483350053429604, -0.00432846462354064, -0.015888961032032967, -0.004227646626532078, -0.011090010404586792, 0.0029069269075989723, 0.09942026436328888, 0.014746353030204773, 0.01699124090373516, 0.01707189530134201, 0.0009972607949748635, -0.022448869422078133, -0.010787555947899818, -0.009127414785325527, 0.02662946842610836, 0.01730041578412056, -0.011130337603390217, -0.01931678131222725, -0.0018214501906186342, -0.0067077758722007275, 0.0009947402868419886, 0.02084922045469284, 0.0025540629867464304, 0.004130188841372728, 0.013227357529103756, 0.003921831026673317, -0.0010661532869562507, -0.023524263873696327, -0.011641150340437889, 0.027718305587768555, 0.03468148782849312, -0.01899416372179985, 0.015996500849723816, 0.012367041781544685, -0.003535360796377063, -0.014719468541443348, -0.006539745721966028, 0.008173001930117607, 0.004506576806306839, 0.02286558598279953, -0.00423436751589179, 0.008018413558602333, -0.0032984379213303328, 0.027960268780589104, 0.03309527784585953, 0.0013501247158274055, 0.005410580895841122, -0.008307426236569881, -0.01058591902256012, -0.018026307225227356, -0.005087962374091148, -0.008058740757405758, -0.0008346913382411003, 0.037504397332668304, -0.02450556308031082, -0.02076856419444084, 0.05549037829041481, 0.003807570319622755, -0.004916571080684662, -0.028309771791100502, -0.011076567694544792, -0.003505115397274494, -0.01224606018513441, -0.0014568241313099861, -0.0032614711672067642, -0.02099708653986454, -0.01529749296605587, -0.025930460542440414, 0.012756872922182083, 0.006049096584320068, -0.02895500883460045, -0.03489656746387482, -0.014060788787901402, 0.013993577100336552, -0.015095856972038746, -0.013086212798953056, 0.01347604300826788, -0.012239338830113411, -0.014396850019693375, 0.0013148384168744087, 0.015741093084216118, -0.0006078502046875656, 0.02099708653986454, -0.010874931700527668, -0.008253656327724457, -0.016708949580788612, -0.02511047199368477, -0.025863248854875565, -0.00970543920993805, 0.004281416069716215, -0.008804796263575554, 0.04820457845926285, 0.010458216071128845, -0.01049182191491127, 0.0017063493141904473, 0.020311521366238594, -0.0046779680997133255, 0.009893633425235748, 0.029627131298184395, -0.012461138889193535, 0.024102289229631424, 0.01756926439702511, 0.025244897231459618, 0.006368354428559542, -0.017542380839586258, -0.012750151567161083, -0.01416832860559225, -0.03048744611442089, -0.003679867135360837, -0.004022649023681879, -0.011284925974905491, 0.01253507286310196, -0.023793112486600876, 0.011668034829199314, -0.0288743544369936, -0.014074231497943401, 0.019518418237566948, -0.023752786219120026, -0.013260964304208755, 0.015767978504300117, 0.002656561555340886, 0.011076567694544792, 0.011224434711039066, 0.0019054653821513057, 0.0386604480445385, -0.001752557698637247, 0.010014615952968597, -0.04212859645485878, 0.03484279662370682, 0.03653654456138611, -0.010800997726619244, 0.006274257320910692, -0.012467860244214535, -0.019451206550002098, 0.04371480643749237, 0.010283464565873146, -0.021561669185757637, 0.02294624038040638, 0.0025086947716772556, -0.013240800239145756, -0.035219185054302216, -0.015700766816735268, -2.0846278857789002e-05, 0.02105085551738739, -0.016413215547800064, -0.008945941925048828, 0.013523091562092304, -0.023658689111471176, -0.003915109671652317, -0.012084750458598137, 0.007581534329801798, -0.01724664680659771, 0.015862075611948967, 0.008031856268644333, 0.01731385849416256, 0.042746949940919876, -0.010787555947899818, 0.02481473796069622, -0.011708362959325314, 0.003310200059786439, -0.012783757410943508, -0.0019508337136358023, -0.03613327071070671, 0.020244309678673744, 0.04266629368066788, 0.021534783765673637, 0.0003137968888040632, -0.0034059775061905384, 0.0012896338012069464, -0.011520168744027615, -0.0007611779728904366, 0.007144655101001263, 0.02088954672217369, -0.0052627138793468475, 0.006183520890772343, 0.009376099333167076, 0.014840450137853622, 0.01738107018172741, -0.008072183467447758, -0.0026179146952927113, 0.012568678706884384, 0.014033904299139977, 0.0142624257132411, -0.012716545723378658, -0.0023692294489592314, -0.040999431163072586, 0.009987730532884598, -0.007749564945697784, -0.0020264473278075457, 0.009147577919065952, -0.016319118440151215, 0.00216423231177032, 0.03997780755162239, -0.018389252945780754, 0.01250818744301796, -0.021400360390543938, -0.00953068770468235, -0.013993577100336552, 0.03968207538127899, -0.02075512334704399, -0.004267973825335503, 0.021306263282895088, -0.01699124090373516, -0.033659860491752625, 0.019814152270555496, 0.003395895706489682, 0.007924316450953484, 0.005491235293447971, 0.009786093607544899, -0.005481153726577759, -0.009355936199426651, 0.01726008951663971, -0.007205146364867687, 0.008455293253064156, 0.007870546542108059, -0.0192764550447464, 0.003078317968174815, -0.027180608361959457, -0.009886912070214748, -0.017461726441979408, -0.004886325914412737, -0.0019088260596618056, -0.010438052006065845, 0.01145295612514019, 0.005494595970958471, -0.0010014615254476666, -0.0017760819755494595, -0.03180480748414993, -0.006133111659437418, 0.0019088260596618056, 0.010854767635464668, 0.02457277476787567, 0.01432963740080595, -0.020593812689185143, -0.015835190191864967, -0.0003864700556732714, 0.0283635426312685, 0.005424023140221834, 0.027207491919398308, -0.02275804616510868, -0.018416138365864754, 0.0031623332761228085, 0.013886037282645702, -0.0008804796379990876, -0.020607255399227142, 0.024223271757364273, 0.017717132344841957, 0.009557572193443775, -0.015015201643109322, -0.0023104187566787004, 0.004587231669574976, 0.015122741460800171, -0.01057919766753912, -0.005541644524782896, -0.0004202862037345767, -0.020163655281066895, 0.005440826527774334, 0.009046760387718678, -0.02696552872657776, 0.015189954079687595, 0.011123616248369217, 0.021534783765673637, 0.004590592347085476, -0.023806555196642876, -0.007729401346296072, 0.004856080282479525, 0.011795738711953163, 0.013644073158502579, -0.00015091235400177538, 0.016776161268353462, 0.009503803215920925, 0.01918235793709755, 0.002233124803751707, -0.0030077453702688217, -0.0020751762203872204, 0.026226194575428963, -0.02279837243258953, -0.023779671639204025, -0.02091643214225769, 0.029143203049898148, 0.02683110348880291, 0.005229108035564423, -0.005168616771697998, 0.002972458954900503, -0.01434308011084795, -0.019787266850471497, 0.02101052924990654, -0.009571014903485775, 0.00568279018625617, -0.00472165597602725, 0.008535947650671005, -0.03242315724492073, 0.0011745329247787595, -0.020405618473887444, 0.0009569334797561169, -0.011681477539241314, -0.021951500326395035, 0.009140857495367527, 0.004146991763263941, 0.009503803215920925, -0.005363532342016697, 0.018873181194067, 0.00624737236648798, 0.017811229452490807, -0.01547224447131157, 0.016399772837758064, -0.021682649850845337, -0.002365869004279375, 0.0018970639212056994, -0.008334310725331306, 0.0025826282799243927, 0.004157073795795441, 0.015888961032032967, 0.007736122235655785, -0.01354997605085373, 2.2828513465356082e-05, 0.01704500988125801, 0.003078317968174815, -0.029331397265195847, -0.01696435548365116, 0.004452807363122702, -0.004419201053678989, -0.0035320003516972065, 0.029196973890066147, -0.03387494012713432, 0.03989715129137039, -0.02879370003938675, -0.007964643649756908, 0.011970490217208862, -0.02090298943221569, 0.00840824469923973, -0.011614265851676464, 0.027341917157173157, 0.015700766816735268, -0.0388755276799202, -0.025150800123810768, 0.01691058650612831, 0.002239846158772707, -0.03285331651568413, -0.011594101786613464, -0.02473408356308937, -0.009315609000623226, -0.007836940698325634, 0.0388486422598362, -0.033364128321409225, -0.016238464042544365, 0.018039749935269356, 0.024129174649715424, 0.024478677660226822, 0.024438349530100822, -0.007084164302796125, 0.02107774093747139, 0.018846295773983, -0.035380493849515915, -0.0284710805863142, -0.003269872860983014, -0.03285331651568413, 0.039386339485645294, 0.002549022203311324, -0.009241675026714802, -0.014840450137853622, -0.02511047199368477, -0.0284979660063982, -0.0003459326981101185, 0.00032156831002794206, 0.02488195151090622, -0.004079779610037804, 0.004399037454277277, 0.03683227673172951, 0.010901816189289093, -0.005024110898375511, 0.03046056255698204, -0.026387503370642662, -0.016453543677926064, 0.027852728962898254, 0.0020684548653662205, -0.004701492376625538, 0.005592053756117821, -0.025701940059661865, 0.02482818067073822, 0.04237056151032448, 0.018026307225227356, -0.008394801989197731, 0.02841731160879135, 0.012205732986330986, 0.01047837920486927, -0.015539457090198994, 0.0034580668434500694, -0.02113150991499424, 0.015888961032032967, 0.002762420801445842, -0.027664534747600555, -0.01418177131563425, -0.008099067956209183, -0.008220050483942032, -0.004536822438240051, -0.014302752912044525, 0.015512572601437569, -0.0192495696246624, -0.012481302954256535, -0.013012278825044632, -0.03086383454501629, -0.0001341093156952411, -0.025661611929535866, 0.028121577575802803, 0.0035320003516972065, -0.026239637285470963, -0.003794127842411399, 0.01146639883518219, -0.008085626177489758, 0.0032816349994391203, -0.0033454864751547575, -0.0376119390130043, -0.016009941697120667, 0.008132674731314182, -0.023860326036810875, -0.006882527843117714, -0.0021289458964020014, -0.00289012398570776, -0.022059038281440735, 0.00024637466412968934, -0.01258212048560381, 0.0006397760007530451, -0.014813565649092197, 0.01744828373193741, 0.0030514332465827465, -0.020513158291578293, -0.00385125819593668, 0.01750205270946026, -0.02086266130208969, -0.011741968803107738, -0.0037773246876895428, -0.04763999581336975, -0.019894806668162346, 0.005417302250862122, 0.0094365905970335, -0.04640329256653786, -0.0006120509351603687, 0.012091471813619137, 0.016789603978395462, -0.011405907571315765, 0.0015030325157567859, 0.2087341547012329, -0.013180308975279331, 0.00761514063924551, 0.005249271634966135, -0.021628880873322487, -0.007151376456022263, 0.031670380383729935, 0.009819700382649899, -0.0077428435906767845, 0.006475893780589104, -0.0023120990954339504, -0.0190479326993227, -0.023954423144459724, 0.0017895244527608156, -0.004167155362665653, -0.03293396905064583, -0.029143203049898148, -0.031751036643981934, -0.02681766077876091, 0.005988605786114931, 0.024169500917196274, -0.011405907571315765, -0.010323791764676571, -0.02863239124417305, 0.02133314684033394, -0.009127414785325527, -0.007420225068926811, 0.021857403218746185, -0.013516370207071304, -0.0006771627813577652, -0.0015845271991565824, -0.008623323403298855, 0.005111486651003361, 0.003666424658149481, -0.013805382885038853, -0.033391013741493225, -0.008347753435373306, -0.02695208601653576, 0.008878729306161404, 0.02277148887515068, -0.02321508899331093, 0.029707785695791245, 0.006519582122564316, -0.024263598024845123, -0.028309771791100502, 0.02274460345506668, -0.0036697853356599808, -0.01059936173260212, -0.01137902308255434, 0.00471829529851675, -0.03409001976251602, -0.009665112011134624, 0.002416278002783656, 0.01933022402226925, 0.023578034713864326, 0.005460990127176046, 0.03513852879405022, 0.005511399358510971, -0.007238752208650112, 0.006291060242801905, -0.009389542043209076, 0.0288743544369936, -0.023416725918650627, 0.007937759160995483, -0.001569404499605298, 0.010827883146703243, -0.010048221796751022, -0.012992115691304207, 0.004311661701649427, -0.020311521366238594, 0.0019676366355270147, 0.0042310068383812904, -0.02309410646557808, -0.0035723275505006313, -0.016587967053055763, -0.014934547245502472, 0.011782296001911163, 0.03037990815937519, 0.013092933222651482, 0.004775425884872675, -0.020190540701150894, 0.00847545638680458, -0.014302752912044525, -0.009658390656113625, -0.006687612272799015, -0.004446086008101702, 0.029600245878100395, 0.003073277184739709, -0.005665987264364958, -0.007245473563671112, 0.0033555682748556137, 0.002233124803751707, -0.005726478062570095, -0.011809180490672588, -0.015700766816735268, 0.02492227777838707, 0.026279963552951813, 0.02285214327275753, -0.01712566427886486, -0.0014122960856184363, -0.02500293217599392, 0.036912932991981506, -0.0031052029225975275, 0.0048628016375005245, -0.016480427235364914, 0.0060423752292990685, 0.031052028760313988, -0.0035185578744858503, 0.013751612976193428, -0.012817363254725933, -0.009483639150857925, -0.029546476900577545, 0.002249927958473563, 0.015095856972038746, 0.012541793286800385, -0.018039749935269356, -0.014611928723752499, -0.03417067602276802, 0.025231454521417618, -0.002117183757945895, 0.0055584474466741085, -0.029412051662802696, -0.014477504417300224, 0.02673700638115406, -0.014921104535460472, -0.03446640819311142, -0.01879252679646015, 0.01728697307407856, -0.0005355970934033394, -0.023322628811001778, 0.008939220570027828, -0.014625371433794498, 0.020203983411192894, -0.011157223023474216, 0.005988605786114931, -0.029922865331172943, -0.0016668621683493257, -0.02269083261489868, 0.005400498863309622, 0.0025725464802235365, 0.010733786039054394, -0.019760381430387497, 0.00014534634829033166, -0.02258329465985298, -0.009665112011134624, 0.002943893661722541, 0.035407379269599915, 0.005592053756117821, 0.00673802150413394, -0.013254242949187756, -0.02072823792695999, 0.0026666433550417423, 0.025312108919024467, -0.03581065312027931, 0.03040679171681404, -0.009208069182932377, -0.025540629401803017, -0.02892812341451645, -0.016157809644937515, 0.002036529127508402, -0.007521043531596661, -0.01233343593776226, 0.009866748936474323, 0.004315022379159927, -0.015351262874901295, -0.031186453998088837, -0.1722782701253891, -0.008878729306161404, 0.0072118672542274, -0.002834673970937729, 0.0004868682590313256, 0.012682938948273659, 0.03298773989081383, -0.014894220046699047, -0.013254242949187756, -0.011110174469649792, 0.012568678706884384, -0.013677679933607578, -0.033471666276454926, 0.004926653113216162, 0.008038577623665333, -0.016829930245876312, -0.02470719814300537, 0.03274577483534813, 0.029304511845111847, 0.009806257672607899, 0.015606669709086418, 0.011688198894262314, -0.013805382885038853, -0.010028057731688023, 0.029734671115875244, -0.002103741280734539, -0.007420225068926811, -0.02482818067073822, 0.013234078884124756, -0.009839863516390324, -0.0094231478869915, 0.02076856419444084, 0.0050476351752877235, -0.0021978383883833885, -0.004691410344094038, -0.02086266130208969, -0.0094365905970335, -0.02661602571606636, -0.006452369503676891, 0.014302752912044525, -0.0023927537258714437, 0.01914202980697155, -0.016130924224853516, -0.0002614973927848041, -0.018227944150567055, 0.0384722538292408, 0.031831689178943634, -0.023457052186131477, 0.0017744016367942095, -0.023483937606215477, 0.01747516728937626, -0.02844419702887535, 0.009860027581453323, -0.0020600534044206142, 0.010377561673521996, 0.037773247808218, -0.00520558375865221, -0.03513852879405022, 0.011318531818687916, 0.02469375729560852, 0.0025809479411691427, -0.013657515868544579, 0.01253507286310196, -0.016829930245876312, -0.014665698632597923, -0.008025134913623333, 0.012676217593252659, 0.007090885657817125, -0.0192764550447464, 0.00953740905970335, 0.0006443968159146607, -0.02841731160879135, -0.002039889805018902, -0.02297312393784523, 0.015337820164859295, -0.013859152793884277, 0.000302874919725582, 0.007205146364867687, 0.02271771803498268, -0.006553187966346741, 0.008616602048277855, 0.0568883940577507, -0.004348628222942352, 0.023725900799036026, -0.015700766816735268, -0.011392464861273766, -0.020365292206406593, 0.0022011990658938885, -0.012145241722464561, -0.009107250720262527, -0.00018693388847168535, -0.009752487763762474, -0.000846033392008394, -0.00156268326099962, 0.039009951055049896, 0.025244897231459618, 0.0005154334357939661, -0.01242081169039011, -0.0017130705527961254, 0.007413504179567099, 0.00856283213943243, -0.015028644353151321, -0.019975461065769196, 0.0285248514264822, 0.017609592527151108, -0.0035924911499023438, -0.00615663593634963, 0.026374060660600662, 0.025796037167310715, -0.01691058650612831, -0.00849562045186758, 0.005054356064647436, 0.021521341055631638, 0.019625958055257797, 0.02117183804512024, 0.015015201643109322, -0.00970543920993805, 0.004610755946487188, -0.00021350370661821216, 0.001870179083198309, 0.02513735741376877, -0.025312108919024467, -0.013509648852050304, 0.0032967575825750828, 0.01337522454559803, 0.008710699155926704, -0.09350559115409851, 0.006869085133075714, 0.011930163018405437, 0.004936735145747662, 1.8824663129635155e-05, 0.0016618212684988976, 0.011668034829199314, 0.019854478538036346, 0.005087962374091148, 0.0058272965252399445, -0.01688370108604431, -0.009799536317586899, 0.0008388920687139034, 0.008099067956209183, 0.013791940174996853, 0.0035252789966762066, -0.029627131298184395, -0.023806555196642876, 0.002219682326540351, 0.020607255399227142, -0.021709535270929337, -0.0037134732119739056, -0.0022432066034525633, -0.04000469297170639, -0.007057279348373413, -0.006032293662428856, -0.03274577483534813, 0.012192290276288986, 0.012689660303294659, 0.013731448911130428, 0.010370840318500996, -0.022032154724001884, -0.004735098220407963, -0.009947403334081173, -0.03223496302962303, -0.004388955421745777, -0.05909295380115509, -0.022475754842162132, -0.0016004901845008135, -0.024102289229631424, 0.000904003856703639, 0.0094298692420125, 0.020459389314055443, -0.025285223498940468, 0.027583880349993706, -0.03089071996510029, 0.007836940698325634, -0.017864998430013657, 0.01715254969894886, -0.007420225068926811, -0.02468031458556652, -0.010377561673521996, -0.020472832024097443, -0.016708949580788612, -0.01531093567609787, -0.010834604501724243, -0.014947989955544472, 0.0071849822998046875, -0.01719287596642971, -0.01748860999941826, -0.009960846044123173, 0.005985245108604431, -0.0194243211299181, 0.003237946890294552, 0.029519591480493546, -0.00028208113508298993, -0.021749863401055336, -0.01321391575038433, 0.01728697307407856, -0.020378734916448593, -0.020069558173418045, 0.022085923701524734, -0.01896727830171585, 0.013025721535086632, 0.00840824469923973, -0.01914202980697155, -0.011862950399518013, 0.017757458612322807, 0.0017239925218746066, 0.021991826593875885, -0.025822920724749565, -0.010774113237857819, 0.019800709560513496, 0.00425117090344429, 0.006092784460633993, 0.001448422553949058, -0.004688049666583538, -0.024330811575055122, -0.003925191704183817, 0.0023776311427354813, 0.0021894369274377823, 0.04145647585391998, 0.01918235793709755, -0.0022045597434043884, -0.005904590245336294, 0.011862950399518013, -0.006818675901740789, -0.027341917157173157, -0.02711339481174946, 0.022341329604387283, -0.04169844090938568, -0.009591178968548775, -0.0771058201789856, 0.01137230172753334, 0.005434105172753334, -0.0005528202163986862, 0.0015946091152727604, -0.006990067195147276, 0.029707785695791245, 0.005645823664963245, -0.00033942153095267713, -0.013926364481449127, -0.023524263873696327, 0.02476096898317337, -0.0019054653821513057, 0.012272944673895836, -0.013321454636752605, 0.011721804738044739, 0.03269200772047043, -0.016036827117204666, 0.016211578622460365, 0.008240213617682457, -0.0009367698221467435, -0.00132071936968714, 0.007426946423947811, 0.02469375729560852, -0.002705290447920561, 0.016534198075532913, 0.0036597035359591246, -0.0006725419079884887, -0.007487437222152948, 0.005057716742157936, 0.03051433153450489, -0.007749564945697784, -0.0037638824433088303, 0.019464649260044098, 0.00264311907812953, -0.022516081109642982, -0.016130924224853516, 0.02857862040400505, 0.010357397608458996, 0.020499715581536293, -0.027126837521791458, -0.02291935496032238, 0.005044274497777224, -0.0192092414945364, 0.0020818973425775766, 0.010458216071128845, 0.011815901845693588, 0.019746940582990646, 0.0003883604076690972, -0.005961720831692219, 0.035299837589263916, 0.009019874967634678, -0.008267099037766457, 0.005941557232290506, 0.039547648280858994, -0.02658914029598236, 0.0028010678943246603, 0.012051144614815712, -0.0037134732119739056, -0.004483052529394627, 0.027852728962898254, -0.006838839966803789, 0.027772074565291405, 0.006835479289293289, 0.003017827169969678, 0.004644361790269613, 0.0012501466553658247, 0.013939807191491127, 0.0016130923759192228, -0.031320877373218536, -0.0287130456417799, 0.023309186100959778, 0.025365877896547318, 0.010807719081640244, 0.016574524343013763, 0.027315031737089157, -0.008851844817399979, 0.017824672162532806, -0.029600245878100395, 0.005168616771697998, -0.00028061086777597666, -0.0018416139064356685, -0.01740795560181141, 0.029358282685279846, 0.03438575193285942, 0.004953538067638874, 0.00029909421573393047, 0.0005313963629305363, -0.016749275848269463, -0.003069916507229209, -0.020580371841788292, 0.03398248180747032, 0.01258884184062481, -0.008838402107357979, 0.015848632901906967, 0.029196973890066147, 0.013953249901533127, -0.024088846519589424, 0.03274577483534813, 0.03376740217208862, 0.015485687181353569, 0.0030531135853379965, -0.007675631437450647, -0.0284710805863142, -0.033552322536706924, 0.005101404618471861, -0.027933383360505104, -0.005655905231833458, 0.01432963740080595, -0.001745836460031569, -0.00873758364468813, 9.341443364974111e-05, 0.021884286776185036, 0.016413215547800064, -0.020324964076280594, 0.021601995453238487, 0.022233789786696434, -0.02863239124417305, -0.012051144614815712, 0.024277040734887123, 0.0006280138622969389, -0.01224606018513441, 0.016346003860235214, -0.008542669005692005, 0.03978961333632469, -0.01707189530134201, -0.014060788787901402, -0.02482818067073822, -0.011177386157214642, 0.023497380316257477, 0.008831680752336979, -0.01234687864780426, -0.015337820164859295, -0.0035756882280111313, 0.005171977449208498, 0.003200980369001627, 0.007171540055423975, 0.03414379060268402, 0.025809478014707565, 0.05973818898200989, -0.001691226614639163, 0.0021961580496281385, 0.004704853054136038, 0.014222098514437675, 0.015458802692592144, 0.010182646103203297, -0.003864700673148036, 0.013012278825044632, -0.02074168063700199, 0.0035454428289085627, 0.008334310725331306, 0.010619524866342545, -0.01894039288163185, -0.04191351681947708, -0.024115731939673424, 0.02492227777838707, 0.04016600176692009, -0.00868381466716528, -0.005665987264364958, 0.029949748888611794, -0.040784355252981186, 0.039009951055049896, 0.01258212048560381, -0.0019205881981179118, 0.0037235550116747618, 0.008428407832980156, -0.019720055162906647, -0.004042813088744879, -0.01911514438688755, 0.013610467314720154, 0.0047552622854709625, -0.014732911251485348, -0.004580510314553976, -0.009066923521459103, -0.012958508916199207, -0.008703977800905704, -0.012777036055922508, 0.016668621450662613, 0.0287130456417799, -0.009981009177863598, 0.02320164628326893, -0.010780834592878819, -0.004358710255473852, -0.0283366572111845, 0.012965230271220207, 0.005709675140678883, -0.005736560095101595, -0.016144366934895515]} +{"id": "test:10000041", "text": "\"Jomsom Muktinath Trek is an exciting trek in the Annapurna and Mustang regions of Nepal. The 13-day trek begins from Kathmandu with the tour of UNESCO World Heritage Sites. Next, we fly to Pokhara, the beautiful Lake City and begin trekking from there. We trek in the beautiful Annapurna region and gradually enter the Mustang region.\\nOur trail passes through quaint villages, cascading waterfalls, lush forests and surprisingly an almost-barren leeward side of the Himalayas. The diverse Nepalese landscape combined with the rich culture of its people makes this trek all the more rewarding.\\nOn our Jomsom Muktinath Trek, we also walk on trails along the Kali Gandaki gorge which is the deepest gorge on earth and then visit Muktinath Temple which is an important place of pilgrimage for Hindus and Buddhists alike. Dipping our bodies into the hot spring at Tatopani will also be a pleasing experience. The trek ends with our flight to Pokhara from Jomsom followed by a drive to Kathmandu.\\nDay 13: Departure to your onward destination.\\n\u2022 An experienced, English-speaking and government-licensed trek leader and assistant trek leader (4 trekkers: 1 assistant guide).\\n\u2022 Domestic flight from Pokhara to Kathmandu.\\n\u2022 Porter service (2 trekkers: 1 porter).\\n\u2022 Staff costs including their salary, insurance, equipment, food and accommodation.\\n\u2022 All necessary paperwork and trekking permits (ACAP, TIMS).\\n\u2022 Medical kit (carried by your trek leader).\"", "vector_field": [-0.01588447391986847, -0.001557622803375125, 0.00823080725967884, 0.005955038126558065, -0.018153684213757515, 0.041003189980983734, -0.03596632927656174, -0.022954441606998444, -0.03570399433374405, -0.005400852300226688, -0.008637428283691406, 0.0071289935149252415, -0.002185590798035264, -0.019019395112991333, -0.02072458155453205, -0.004577771294862032, 0.027676498517394066, -0.02060653083026409, -0.019753936678171158, -0.02540728822350502, -8.059673564275727e-05, 0.013051239773631096, -0.011601830832660198, 0.002256093779578805, 0.011523129418492317, 0.024331707507371902, -0.003810437396168709, -0.015845123678445816, -8.613039244664833e-05, 0.0156483706086874, -0.013602145947515965, 0.004118682816624641, -0.02901441417634487, -0.032057516276836395, -0.02182639390230179, -0.003185748588293791, 0.0072732786647975445, -0.012421632185578346, 0.008060288615524769, -0.008801388554275036, 0.015294216573238373, 0.009057166986167431, -0.01688135229051113, 0.010204888880252838, -0.02333482913672924, 0.0015297495992854238, -0.007286395411938429, -0.012362606823444366, -0.014244870282709599, 0.020226141437888145, 0.002464323304593563, -0.002747974591329694, -0.014402272179722786, 0.02498754858970642, -0.013156174682080746, -0.020711464807391167, -0.017970047891139984, 0.010158980265259743, 0.011011573486030102, -0.014874477870762348, -0.00017431029118597507, 0.00839476753026247, -0.010670536197721958, 0.03420867770910263, -0.035992562770843506, -0.024908848106861115, -0.0354941263794899, -0.00012235536996740848, 0.022246133536100388, -0.0016150089213624597, 0.04278707876801491, 0.024672744795680046, 0.01199533510953188, 0.010926314629614353, 0.014441623352468014, -0.04666965827345848, -0.01379889901727438, -0.003033265471458435, -0.007804510183632374, -0.0007521678344346583, 0.021747693419456482, -0.03192634880542755, -0.030772069469094276, 0.009326061233878136, 0.0033874197397381067, -0.0009345737053081393, -0.0044203693978488445, 0.006846982054412365, 0.018245501443743706, -0.0002576226252131164, -0.0007058490882627666, 0.012460982427001, 0.015464735217392445, 0.0002449156891088933, -0.03276582434773445, -0.005640234332531691, -0.014454740099608898, 0.011883842758834362, 0.013877599500119686, -0.022954441606998444, -0.019622769206762314, -0.017248623073101044, 0.022836390882730484, -0.0050827693194150925, -0.04231487214565277, -0.016868235543370247, 0.007745484355837107, 0.0019790008664131165, 0.01504499651491642, -0.0009837617399170995, -0.010873846709728241, 0.03134920820593834, -0.03489075228571892, -0.03607126697897911, -0.0005263125640340149, -0.019058745354413986, 0.017891347408294678, 0.014271104708313942, 0.02372833341360092, -0.024672744795680046, 0.02072458155453205, 0.03441854566335678, 0.016500964760780334, -0.03940293937921524, 0.026181180030107498, 0.01760277710855007, 0.005984550807625055, -0.007673341780900955, 0.005066373385488987, -0.011431312188506126, -0.001292007160373032, 0.01249377429485321, 0.0042662471532821655, 0.006932241376489401, 0.0016805930063128471, 0.03449724614620209, -0.01732732355594635, -0.0005168848438188434, -0.019858870655298233, 0.0014403911773115396, 0.013057798147201538, -0.002129844157025218, 0.011385402642190456, -0.01219208724796772, 0.016422264277935028, 0.0017560147680342197, 0.016855118796229362, -0.009457229636609554, 0.029827658087015152, -0.011024690233170986, 0.03273959085345268, -0.012611825950443745, 0.015845123678445816, 0.009995019994676113, 0.014166169799864292, -0.024541577324271202, 0.042209938168525696, -0.007804510183632374, -0.027099357917904854, 0.007778276689350605, 0.0006546115037053823, 0.0043875775299966335, 0.005643513519316912, -0.01730109006166458, 0.026076246052980423, 0.026535334065556526, 0.008060288615524769, 0.005427085794508457, -0.032687123864889145, -0.013352926820516586, 0.024161189794540405, -0.010172097012400627, -0.016173044219613075, 0.017406025901436806, -0.006118998397141695, 0.014205520041286945, 0.0018625889206305146, 0.014690842479467392, -0.01567460410296917, -0.013346368446946144, -0.022233016788959503, 0.009778591804206371, -0.0017101059202104807, 0.03510062023997307, -0.038510993123054504, -0.026115596294403076, -0.01495317928493023, -0.011359169147908688, 0.008998140692710876, 0.013956300914287567, 0.002631562761962414, 0.0073847719468176365, -0.033946339040994644, 0.013215200044214725, -0.6312865018844604, -0.012802019715309143, 0.02759779803454876, -0.027519095689058304, -0.0023036422207951546, -0.03811749070882797, 0.016802651807665825, 0.009030933491885662, -0.01567460410296917, 0.029145581647753716, -0.02086886577308178, 0.023256128653883934, 0.014100585132837296, -0.011949426494538784, 0.0027955230325460434, -0.027912601828575134, -0.006673184223473072, -0.02585325948894024, -0.012342930771410465, -0.02270522154867649, -0.01064430270344019, -0.00034636614145711064, 0.02182639390230179, 0.008598078042268753, 0.00022524045198224485, -0.004702381324023008, 0.011778907850384712, -0.018193034455180168, -0.016238627955317497, 0.037251777946949005, -0.029827658087015152, 0.012697085738182068, -0.003198865335434675, -0.0009026014595292509, 0.03816995769739151, -0.026194296777248383, -0.04207877069711685, 0.022875741124153137, 0.0035972888581454754, 0.04981769621372223, 0.0006111619877628982, -0.002115087816491723, 0.01693381927907467, 0.00012030587095068768, 0.0023085610009729862, 0.008440676145255566, 0.029591554775834084, -0.009424437768757343, 0.00944411288946867, -0.01979328691959381, 0.019019395112991333, 0.016435381025075912, -0.010952548123896122, 0.004131799563765526, 0.009575281292200089, 0.015333566814661026, 0.03352660313248634, -0.025026898831129074, 0.03441854566335678, -0.008119313977658749, -0.0015781178371980786, 0.03166401386260986, 0.002129844157025218, 0.004413811024278402, -0.016041874885559082, 0.017681477591395378, -0.018324201926589012, 0.000575500656850636, 0.019963806495070457, -0.01972770318388939, 0.01441538892686367, 0.015071230940520763, -0.014074351638555527, -0.003761249128729105, 0.01898004487156868, 0.03861593082547188, -0.012251113541424274, -0.028122469782829285, -0.0037448531948029995, 0.014087468385696411, -0.009424437768757343, -0.013090590015053749, -2.1545407435041852e-05, -0.012972538359463215, 0.033500369638204575, -0.02606312930583954, -0.0069191246293485165, 0.019137445837259293, 0.019176796078681946, 0.003774366108700633, -0.02231171727180481, -0.0036858273670077324, -0.0029644020833075047, 0.010624627582728863, -0.000704209494870156, -0.009575281292200089, -0.002341353101655841, -0.005197541322559118, 0.014782660640776157, -0.04239357262849808, -0.004092448856681585, -0.012756111100316048, 0.008145547471940517, -0.005994388367980719, -0.0002373325260123238, -0.008998140692710876, -0.057136885821819305, -0.014061234891414642, 0.045725248754024506, -0.023715216666460037, -0.016710832715034485, -0.019753936678171158, 0.01484824437648058, 0.0036530354991555214, -0.0029463665559887886, -0.01286760438233614, 0.025040017440915108, -0.008473468013107777, 0.028489740565419197, 0.006587924901396036, 0.01186416670680046, 0.0031939465552568436, 0.013628379441797733, -0.024371057748794556, -0.005702539347112179, 0.017432259395718575, 0.0169075857847929, -0.04415122792124748, 0.016343561932444572, -0.007489706389605999, -0.02216743305325508, 0.016710832715034485, 0.01663213223218918, -0.008361974731087685, -0.0020429452415555716, -0.007719250861555338, 0.010985339991748333, -0.019229263067245483, 0.02720429189503193, -0.02778143249452114, 0.007916003465652466, -0.010165538638830185, 0.01765524409711361, -0.03898319974541664, 0.0012813496869057417, -0.028725843876600266, -0.031086871400475502, -0.01756342686712742, -0.027309227734804153, -0.03447101265192032, -0.011490337550640106, -0.0031447585206478834, -0.006699417717754841, -0.0009731043828651309, 0.005738610401749611, 0.005666467826813459, -0.010690211318433285, -0.008998140692710876, -0.029093114659190178, -0.017642127349972725, 0.002097052289173007, 0.017550310119986534, -0.010827938094735146, 0.00656169094145298, -0.009988461621105671, -0.021931329742074013, -0.026194296777248383, 0.006886332295835018, 0.011693648062646389, -0.03365777060389519, -0.0009370330953970551, -0.004489232785999775, -0.003044742625206709, 0.030640900135040283, -0.011155858635902405, -0.005820590537041426, -0.009286710992455482, -0.010204888880252838, 0.007443797308951616, -0.0012772507034242153, 0.0013854645658284426, -0.009890085086226463, -0.008958790451288223, -0.013562795706093311, 0.005302476231008768, 0.004990951623767614, 0.00145104865077883, 0.0027381370309740305, -0.018061866983771324, 0.039953846484422684, 0.001956046326085925, 0.01937354914844036, 0.005951758939772844, -0.009437554515898228, 0.017576543614268303, 0.003944884520024061, 0.010067162103950977, 0.010814821347594261, 0.014664608985185623, 0.03586139529943466, 0.021642759442329407, -0.010834496468305588, 0.008886648342013359, 0.013011889532208443, 0.03360530361533165, -0.021616525948047638, -0.005712376907467842, -0.016868235543370247, 0.00886041484773159, -0.009208010509610176, -0.0008927638409659266, -0.046774592250585556, -2.4145716452039778e-05, 0.020226141437888145, -0.006161627825349569, 0.021301722154021263, -0.00030414634966291487, 0.0033841405529528856, -0.01984575390815735, -0.014376038685441017, -0.003590730484575033, 0.00792912021279335, 0.013641497120261192, 0.006266562733799219, -0.0032513327896595, -0.0012223239755257964, -0.0008509539184160531, 0.011123066768050194, -0.0004619581450242549, -0.016684599220752716, 0.0018921018345281482, 0.014703959226608276, -0.017851997166872025, 0.018599655479192734, 0.024410409852862358, -0.0032726475037634373, -0.0012133062118664384, -0.014808894135057926, 0.04464966803789139, -0.015241749584674835, 0.023557815700769424, 7.726628973614424e-05, 0.007758601102977991, -0.03510062023997307, 0.008421001024544239, -0.0011346052633598447, 0.006214095279574394, 0.023400412872433662, 0.01492694579064846, 0.001741258311085403, -0.011811699718236923, 0.01702563650906086, -0.01756342686712742, 0.0031234435737133026, -0.010034370236098766, -0.006879773922264576, -0.007443797308951616, 0.017458492890000343, 0.011955984868109226, 0.019045628607273102, -0.0008234906126745045, -0.0036858273670077324, 0.027702732011675835, -0.014756427146494389, 0.022574054077267647, -0.0009460509172640741, 0.009995019994676113, -0.004318714141845703, -0.007975028827786446, 0.0016338643617928028, 0.003757969941943884, -0.00828983262181282, 0.008263599127531052, -0.013215200044214725, -0.002126564970239997, -0.0021314837504178286, -8.797494956525043e-05, 0.015936940908432007, 0.027624031528830528, 0.01693381927907467, -0.0035185879096388817, -0.03428737819194794, 0.016041874885559082, 0.021603409200906754, -0.011660856194794178, -0.01312338188290596, -0.011195208877325058, -0.014887594617903233, -0.01207403652369976, 0.03596632927656174, -0.005650071892887354, -0.020409777760505676, 0.012001893483102322, -0.021419772878289223, 0.0017166642937809229, -0.01391694974154234, 0.012251113541424274, -0.024436643347144127, 0.039534106850624084, 0.005709097720682621, -0.03460218012332916, 0.012428190559148788, -0.021327955648303032, -0.016028758138418198, 0.025892609730362892, 0.024541577324271202, -0.016225511208176613, -0.004948321729898453, -0.005620559211820364, -0.02517118491232395, 0.004089169669896364, -0.008329182863235474, -0.009503139182925224, -0.03402503952383995, -0.01753719337284565, 0.012854487635195255, -0.014087468385696411, 0.018691474571824074, 0.006938799750059843, -0.013575912453234196, 0.0054041314870119095, -0.01751095987856388, -0.01114274188876152, -0.0005603343015536666, 0.09680216014385223, -0.011011573486030102, -0.013025006279349327, 0.00487289996817708, -0.009660541079938412, 0.00344644533470273, -0.002697146963328123, -0.050316136330366135, 0.0025512222200632095, 0.0003613275184761733, 0.0015904149040579796, 0.0006443639867939055, 0.008886648342013359, 0.004987672436982393, 0.020173674449324608, 0.01270364411175251, -0.0090834004804492, -0.009457229636609554, -0.015582786872982979, 0.01235604751855135, 0.0158582404255867, -0.048689648509025574, 0.014126819558441639, 0.02423989027738571, -0.02843727357685566, 0.0033431504853069782, 0.012067478150129318, 0.002402018290013075, 0.013969417661428452, -0.04745666682720184, -0.0007730728248134255, 0.017406025901436806, 0.010716444812715054, -0.003757969941943884, -0.004682706203311682, 0.008519376628100872, 0.00958183966577053, 0.006781397853046656, -0.003728457260876894, -0.014192403294146061, 0.001860949327237904, 0.02093445137143135, -0.00038161760312505066, -0.001166577567346394, 0.010086837224662304, -0.0033349525183439255, -0.015609020367264748, 0.020698348060250282, -0.01369396410882473, 0.011490337550640106, 0.008414442650973797, -0.0067223720252513885, -0.030667133629322052, 0.0013141417875885963, 0.002123285783454776, 0.02119678631424904, 0.02263963781297207, 0.0012010091450065374, -0.003925209399312735, -0.009706449694931507, -0.027073124423623085, -0.0064567564986646175, -0.0021380423568189144, 0.003330033738166094, -0.007253603544086218, -0.02477768063545227, -0.006860098801553249, 0.009935993701219559, -0.035808928310871124, -0.012933188118040562, -0.03903566673398018, -0.029932592064142227, -0.016802651807665825, 0.00022503550280816853, 0.02357093244791031, -0.0029152140486985445, 0.019557183608412743, 3.046074925805442e-05, 0.012579034082591534, 0.014310454949736595, 0.006099323276430368, -0.003049661638215184, -0.012618384324014187, -0.011680531315505505, -0.026679620146751404, 0.017694594338536263, 0.033290497958660126, 0.025630272924900055, -0.010486900806427002, 0.02624676376581192, 0.03462841361761093, 0.0027348578441888094, 0.007266720291227102, -0.009037491865456104, 0.014664608985185623, 0.009896643459796906, 0.004141637124121189, -0.013903832994401455, -0.002075737342238426, 0.0013239793479442596, 0.01705187000334263, -0.022364184260368347, 0.007653666660189629, -0.0063649388030171394, 0.024371057748794556, 0.019530950114130974, -0.008250482380390167, -0.014139936305582523, -0.0050827693194150925, 0.026837021112442017, 0.01723550632596016, -0.04506940767168999, 0.014048118144273758, -0.00034493146813474596, -0.01389071624726057, 0.05398884788155556, 0.002757812151685357, 0.00028324141749180853, 0.002506952965632081, -0.03541542589664459, -0.01282169483602047, -0.0452268086373806, 0.02126237191259861, 0.0074503556825220585, -0.006351822055876255, 0.013575912453234196, 0.006187861785292625, -0.018009398132562637, 0.008329182863235474, -0.00288242194801569, 0.0011690369574353099, 0.034759584814310074, -0.01615992747247219, -0.013103706762194633, -0.021459123119711876, -0.008880089968442917, -0.047194331884384155, 0.0183766707777977, 0.005489390809088945, 0.01781264692544937, -0.0226134043186903, 0.01303156465291977, 0.017825763672590256, -0.017524076625704765, 0.010388524271547794, -0.019150562584400177, -0.015359800308942795, -0.00918833538889885, -0.011929751373827457, 0.009588398039340973, -0.005682863760739565, 0.0022101846989244223, -0.010381965897977352, 0.0005488570895977318, 0.008198014460504055, -0.014612141996622086, -0.03234608843922615, -0.006971591617912054, 0.007476589642465115, 0.02305937558412552, 0.012697085738182068, -0.023229895159602165, 0.02540728822350502, -0.005610721185803413, -0.004217058885842562, -0.0075946408323943615, -0.01709122210741043, 0.010408199392259121, -0.04643355682492256, -0.0004217878740746528, 0.029801422730088234, -0.003712061094120145, 0.00798814557492733, 0.0077389259822666645, 0.013549678958952427, 0.03491698578000069, -0.008224248886108398, -0.020711464807391167, -0.021905096247792244, -0.044098760932683945, -0.01471707597374916, -0.006991267204284668, -0.014677725732326508, 0.012172412127256393, -0.049660295248031616, -0.005450040102005005, 0.06122933328151703, 0.014835127629339695, 0.0021249253768473864, 0.011129625141620636, 0.014402272179722786, -0.023597165942192078, 0.011726440861821175, -0.02123613841831684, -0.013812015764415264, -0.014887594617903233, 0.009208010509610176, -0.02759779803454876, -0.016776418313384056, -0.008499701507389545, -0.008375092409551144, 0.010185213759541512, 0.0066895801573991776, 0.008381650783121586, -0.011391961015760899, 0.0247120950371027, -0.023951319977641106, -0.03719931095838547, 0.007975028827786446, -0.02486949786543846, -0.0203048437833786, 0.009758916683495045, -0.010952548123896122, -0.01421863678842783, 0.02368898317217827, 0.0046171220019459724, -0.008821063674986362, 0.017878230661153793, -0.0367533415555954, 0.010880405083298683, 0.009542489424347878, -0.010309823788702488, 0.029932592064142227, 0.008119313977658749, 0.012283905409276485, 0.003036544658243656, 0.0018494721734896302, -0.013457860797643661, 0.025210535153746605, -0.006358380429446697, 0.011844491586089134, 0.013143057003617287, -0.010349174030125141, 0.0024413687642663717, -0.02927675098180771, -0.016028758138418198, 0.00015822169370949268, -0.008132430724799633, 0.004597446881234646, 0.02420054003596306, 0.004167870618402958, 0.0020429452415555716, 0.00490241264924407, 0.01595005765557289, -0.023229895159602165, 0.016317328438162804, 0.011070598848164082, 0.0213541891425848, 0.0006390352500602603, -0.009391645900905132, -0.010880405083298683, -0.0063682179898023605, -0.016920702531933784, -0.005089327692985535, -0.001260854653082788, -0.011732999235391617, 0.01336604356765747, -0.01057215966284275, -0.006551853381097317, -0.0029676812700927258, -0.012178970500826836, 0.022718338295817375, -0.007922561839222908, 0.015517202205955982, 0.012605267576873302, 0.004495791159570217, -0.016173044219613075, -0.004679427016526461, -0.004551537800580263, 0.027702732011675835, 0.004085890483111143, -0.00812587235122919, 0.01574018783867359, 0.02528923563659191, -0.007391330320388079, -0.015005646273493767, -0.01023768074810505, -0.010093395598232746, -0.0006771560292690992, 0.00029799784533679485, 0.01699940301477909, 0.007437238935381174, -0.015582786872982979, -0.025092484429478645, -0.01625174470245838, -0.029171815142035484, 0.014494090341031551, 0.011713323183357716, 0.010637744329869747, -0.03074583411216736, -0.01898004487156868, -4.186111254966818e-05, 0.0008591519435867667, 0.02372833341360092, 0.01078202947974205, -0.0034070948604494333, -0.008381650783121586, 0.021747693419456482, -0.018297968432307243, 0.016081225126981735, 0.01648784801363945, 0.04499070718884468, -0.0021101690363138914, 0.016146810725331306, 0.0023610282223671675, -0.025892609730362892, 0.009844176471233368, 0.0008837460190989077, -0.04026864841580391, -0.031060637906193733, 0.009942552074790001, 0.006338705308735371, -0.022967558354139328, -0.03483828529715538, 0.023085609078407288, 7.39870811230503e-05, 0.009693332947790623, 0.01084105484187603, -0.016960052773356438, 0.027912601828575134, -0.016317328438162804, 0.0023741452023386955, 0.019360432401299477, -0.005154911894351244, 0.002010153140872717, -0.011765791103243828, -0.012001893483102322, 0.016120577231049538, -0.012802019715309143, -0.0034169326536357403, -0.016317328438162804, -0.00018988651572726667, -0.004987672436982393, -0.008086522109806538, -0.02450222708284855, -0.027466628700494766, -0.022901974618434906, 0.026102479547262192, -0.004007189534604549, -0.012769227847456932, 0.014507207088172436, 0.028988180682063103, 0.011687089689075947, 0.015923824161291122, -0.01856030523777008, -0.015294216573238373, 0.030247395858168602, -0.024410409852862358, -0.020711464807391167, -0.03197881579399109, 0.013510328717529774, 0.04661719128489494, 0.013149615377187729, 0.0057254936546087265, -0.002987356623634696, 0.010500017553567886, -0.061124399304389954, -0.02021302469074726, 0.00911619234830141, 0.0101524218916893, 0.04047851637005806, 0.04058345407247543, 0.012264230288565159, -0.004758127965033054, 0.0112017672508955, 0.023072492331266403, -0.027545329183340073, 0.009358854033052921, 0.0171174556016922, -0.016802651807665825, 0.007496264763176441, -0.007693017367273569, -0.04548914358019829, -0.0008706291555427015, 0.01739290915429592, 0.009896643459796906, 0.010204888880252838, 0.008106197230517864, 0.0020462244283407927, -0.0017756900051608682, 0.0026282835751771927, -0.0070962016470730305, 0.015189281664788723, 0.02363651618361473, 0.04666965827345848, -0.017484726384282112, -0.003079174319282174, 0.007699575740844011, -0.005453319288790226, 0.008807946927845478, -0.0009845815366134048, 0.0032267386559396982, 0.015294216573238373, 0.02096068486571312, 0.0028070001862943172, -0.02540728822350502, 0.007758601102977991, -0.011693648062646389, 0.03984890878200531, 0.0017494563944637775, -0.025276118889451027, 0.02507936768233776, -0.00045376011985354125, -0.01482201088219881, -0.02300690859556198, 0.0026791112031787634, -0.0004072363954037428, -0.03481205180287361, -0.0010739399585872889, -0.015241749584674835, 0.0007988965371623635, -0.01078202947974205, 0.0032874038442969322, -0.01751095987856388, -0.009109633974730968, -0.0005320511409081519, -0.011628064326941967, -0.011949426494538784, 0.012447865679860115, 0.028253639116883278, 0.00859151966869831, 0.0346546471118927, 0.014179286547005177, -0.001967523479834199, 0.03659593686461449, -0.009168660268187523, -0.03428737819194794, 0.0011059121461585164, 0.026115596294403076, 0.019465366378426552, -0.003220180282369256, -0.027256758883595467, -0.030535966157913208, -0.01240195706486702, -0.013969417661428452, 0.0063124713487923145, 0.19370925426483154, 0.004876179154962301, -0.015845123678445816, 0.024462876841425896, 0.0035546591971069574, 0.015110581181943417, 0.021367305889725685, 0.013641497120261192, -0.00046236804337240756, -0.005495949182659388, -0.02056717872619629, 0.002867665607482195, -0.002405297476798296, 0.0008165222825482488, 0.01048034243285656, -0.019452249631285667, -0.0460662841796875, -0.026587801054120064, -0.010224564000964165, -0.005751727148890495, -0.002160996664315462, -0.009732683189213276, 0.00030127703212201595, -0.0002742235956247896, -0.011647739447653294, 0.01223799679428339, -0.007050292566418648, -0.01804875023663044, 0.03473334759473801, 0.011621505953371525, -0.003718619467690587, 0.02191821299493313, -0.02883077785372734, -0.020160557702183723, 0.003436607774347067, -0.006338705308735371, 0.016054991632699966, 0.002402018290013075, -0.01667148247361183, 0.025813909247517586, -0.00344644533470273, 0.012782344594597816, 0.019386665895581245, 0.022665871307253838, 0.0008935836376622319, 0.005079490132629871, -0.014100585132837296, 0.011077157221734524, 0.005217216443270445, 0.006846982054412365, -0.03940293937921524, 0.002925051609054208, 0.01567460410296917, 0.007338862866163254, 0.012815136462450027, -0.03008999302983284, 0.01576642133295536, 0.0015379475662484765, -0.0017838880885392427, 0.011923193000257015, -0.02107873558998108, 0.016946936026215553, 0.0014854803448542953, 0.02736169472336769, -0.011031248606741428, -0.00386290461756289, -0.013549678958952427, 0.00480403658002615, 0.007797951810061932, -0.006023901514708996, -0.001974082086235285, -0.02201003022491932, 0.026141829788684845, -0.0003322245611343533, -0.010440991260111332, -0.011496895924210548, 0.030142461881041527, 0.0007124074618332088, 0.028752077370882034, 0.018966926261782646, 0.008775155059993267, -0.03250348940491676, 0.024843264371156693, 0.007069968152791262, 0.02489573135972023, -0.025722092017531395, 0.026863254606723785, -0.009371970780193806, -0.013077473267912865, 0.004256409127265215, -0.0029398081824183464, 0.005256567150354385, 0.014310454949736595, -0.03832735866308212, -0.015097464434802532, 0.01570083759725094, 0.0236627496778965, 0.0024266124237328768, -0.029565321281552315, 0.008073405362665653, -0.011942868120968342, 0.0024561253376305103, -0.025512222200632095, -0.003305439604446292, -0.015635253861546516, 0.015543435700237751, 0.013667730614542961, -0.0035710553638637066, 0.01021144725382328, 0.0019576859194785357, -0.0003518997982610017, -0.017406025901436806, 0.002069178968667984, 0.0025577805936336517, -0.010939431376755238, 0.02284950762987137, -0.002328236121684313, -0.004659751430153847, 0.004830270539969206, -0.008735804818570614, 0.010355732403695583, -0.022246133536100388, -0.007083084899932146, -0.01378578133881092, -0.01732732355594635, -0.005079490132629871, 0.009476904757320881, 0.020396661013364792, 0.02423989027738571, -0.03113934025168419, 0.021931329742074013, -0.0169075857847929, 0.0041547538712620735, -0.007811068557202816, 0.0005418887594714761, 0.0036858273670077324, -0.01514993142336607, -0.036176200956106186, 0.00363008095882833, 0.005741889588534832, 0.015989407896995544, 0.017169922590255737, 0.02035731077194214, 0.018717708066105843, 0.009627748280763626, -0.0096080731600523, -0.00015576228906866163, -0.00815210584551096, -0.02462027780711651, 0.0006902728346176445, -0.01166741456836462, -0.01693381927907467, 0.008683336898684502, -0.006305912975221872, 0.029407918453216553, -0.004715498071163893, -0.02252158708870411, -0.04813874140381813, -0.005089327692985535, -0.0014617060078307986, 0.00313492096029222, 0.011569038964807987, 0.004102286417037249, 0.005089327692985535, -0.011385402642190456, 0.009535931050777435, -0.164117693901062, 0.006728930398821831, 0.010355732403695583, 0.00872268807142973, 0.02498754858970642, 0.02140665613114834, 0.02843727357685566, 0.0010747597552835941, 0.003149677300825715, -0.004246571566909552, 0.027046890929341316, 0.009588398039340973, -0.04840108007192612, -0.011614947579801083, -0.006755164358764887, 0.020501594990491867, -0.004784361459314823, -0.004767965525388718, 0.022810157388448715, 0.020816398784518242, 0.009883526712656021, -0.013864482752978802, -0.0006591203855350614, -0.014153053052723408, -0.010290148667991161, 0.017668360844254494, 0.011313260532915592, -0.0075946408323943615, -7.147643918870017e-05, 0.007443797308951616, -0.030981937423348427, 6.978560122661293e-05, 0.015359800308942795, -0.01888822577893734, 0.01534668356180191, -0.007404447067528963, -0.0004541700182016939, -0.00875547993928194, 0.014494090341031551, 0.010008136741816998, 0.016802651807665825, -0.009778591804206371, 0.0204753614962101, 0.005463157314807177, -0.0031627940479665995, 0.04771900549530983, -0.026797670871019363, -0.001741258311085403, 0.01136572752147913, 0.006260004360228777, 0.029565321281552315, 0.01702563650906086, -0.009772033430635929, 0.002864386420696974, 0.005843544844537973, -0.014874477870762348, 0.026797670871019363, -0.003443166147917509, 0.019871987402439117, -0.013090590015053749, -0.009371970780193806, -0.0024249728303402662, -0.010054045356810093, -0.010519692674279213, -0.011221442371606827, -0.037881385535001755, -0.016356678679585457, 0.004826991353183985, -0.02480391412973404, 0.006610879208892584, 0.0020921332761645317, -0.005653351079672575, 0.012500332668423653, -0.008991582319140434, 0.01718303933739662, 0.01972770318388939, 0.03612373396754265, 0.02305937558412552, 0.0019462087657302618, -0.010218005627393723, -0.02315119467675686, 0.06689579784870148, -0.025249885395169258, 0.02180016040802002, 0.01856030523777008, -0.02252158708870411, -0.010972223244607449, -0.014900711365044117, -0.02294132485985756, -0.003538263263180852, 0.009857293218374252, -0.02180016040802002, 0.019530950114130974, -0.003590730484575033, 0.03140167519450188, 0.013097148388624191, 0.009109633974730968, -0.0010509855346754193, -0.0037842036690562963, -0.020527828484773636, 0.0240824893116951, -0.012310138903558254, -0.03819619119167328, 0.011155858635902405, 0.051549118012189865, 0.01609434187412262, 0.021485356613993645, 0.02330859564244747, 0.0035743345506489277, 0.004358064848929644, -0.009699891321361065, -0.023138077929615974, 0.005138515494763851, 0.020645881071686745, -0.005545137450098991, 0.032267387956380844, 0.024095606058835983, -0.012060919776558876, 0.01484824437648058, 0.03562529385089874, 0.040347348898649216, -0.025577805936336517, -0.002690588589757681, -0.008853856474161148, -0.02065899781882763, -0.014730192720890045, -0.09953045845031738, -0.008598078042268753, 0.03730424493551254, 0.003149677300825715, -0.018612772226333618, -0.007338862866163254, 0.002097052289173007, 0.01732732355594635, -0.01100501511245966, 0.008827622048556805, -0.019347315654158592, -0.026614034548401833, 0.05036860331892967, 0.03027362935245037, 0.023505348712205887, -0.03544165939092636, -0.008525935001671314, -0.019111212342977524, 0.01657966524362564, 0.015399151481688023, -0.02465962804853916, -0.011326377280056477, 0.012054361402988434, -0.033106863498687744, 0.018966926261782646, -0.011732999235391617, -0.02201003022491932, 0.027125591412186623, 0.005856661591678858, 0.013562795706093311, 0.002108529442921281, -0.0009739241795614362, -0.00922112725675106, -0.013667730614542961, 0.0010124548571184278, 0.02943415194749832, -0.05430364981293678, -0.0021527986973524094, 0.044938236474990845, -8.131405775202438e-05, -0.01939978264272213, -0.01139851938933134, 0.004958159290254116, -0.010991898365318775, -0.01676330156624317, -0.016553431749343872, -0.014599025249481201, 0.007902886718511581, -0.0025397450663149357, -0.02783389948308468, -0.01579265668988228, -0.005151632707566023, -0.023859502747654915, -0.0017461772076785564, 0.010696769692003727, 0.015333566814661026, -0.015490968711674213, -0.0065452950075268745, 0.005515624303370714, 0.009129309095442295, -0.01070332806557417, -0.006220653653144836, 0.017497843131422997, 9.294499614043161e-05, 0.011477220803499222, 0.01399565115571022, -0.0008985024178400636, 0.007378213573247194, 0.02354469895362854, -0.01916367933154106, -0.007102760020643473, 0.007299512624740601, -0.029985059052705765, 0.001619107904843986, -0.04165903106331825, 0.008342299610376358, -0.03840605914592743, 0.002816837979480624, 0.015661487355828285, -0.011228000745177269, -0.01402188464999199, -0.02170834317803383, 0.008834180422127247, -0.03775021806359291, 0.019635885953903198, -0.00427936390042305, 0.009391645900905132, -0.025722092017531395, -0.0029430873692035675, -0.022574054077267647, 0.025813909247517586, 0.0054926699958741665, -0.00042137797572650015, -0.005063094198703766, 0.011044365353882313, 0.007155227474868298, -0.004253129940479994, 0.012716760858893394, 0.00022728995827492326, -0.01618616096675396, -0.004787640646100044, -0.00500734755769372, -0.07733679562807083, 0.010034370236098766, 0.005000789184123278, -0.015661487355828285, 0.006138673517853022, -0.0016666564624756575, 0.024423526600003242, -0.0014805614482611418, 0.012710202485322952, 0.01930796541273594, -0.020436011254787445, -0.0037382948212325573, 0.0011042725527659059, 0.00924736075103283, -0.01124767679721117, 0.010755795054137707, -0.007502823136746883, 0.008250482380390167, 0.013136498630046844, -0.0077061341144144535, -0.03318556398153305, -0.01295942161232233, 0.000437773996964097, 0.019386665895581245, -0.03895696625113487, -0.0017002682434394956, -0.025394171476364136, 0.019544066861271858, -0.0052598463371396065, 0.0023544698487967253, 0.027624031528830528, -0.024305474013090134, 0.01259870920330286, 0.014428505674004555, -0.025354821234941483, -0.003908813465386629, -0.01648784801363945, 0.018088100478053093, -0.001995396800339222, 0.04708939790725708, -0.028332339599728584, 0.017760179936885834, 0.015425384975969791, -0.012592150829732418, -0.01597629114985466, -9.991330443881452e-05, 0.020081857219338417, 0.002595491474494338, 0.008735804818570614, -0.0006677283090539277, 0.03693697601556778, 0.020895101130008698, 0.024030020460486412, -0.0015494248364120722, 0.009424437768757343, 0.0046204011887311935, 0.010919755324721336, -0.03273959085345268, 0.008952232077717781, -0.029932592064142227, 0.04577771574258804, -0.006358380429446697, 0.005751727148890495, -0.006407568231225014, 0.008952232077717781, -0.016776418313384056, -0.02291509136557579, -0.013838249258697033, 0.0002674602437764406, 0.007148669101297855, -0.015386034734547138, -0.01210027001798153, 0.02480391412973404, 0.007188019342720509, 0.004095728043466806, 0.004184266552329063, 0.0061353943310678005, -0.025669625028967857, -0.007069968152791262, 0.00689289066940546, 0.008198014460504055, -0.02841104008257389, -0.029302984476089478, 0.012067478150129318, 0.00262008560821414, -0.010834496468305588, 0.0008214410627260804, 0.004335110075771809, -0.02140665613114834, 0.010093395598232746, -0.011044365353882313, 0.009752358309924603, 0.010368849150836468, -0.01636979542672634, 0.007876652292907238, 0.027965068817138672, -0.010965664871037006, -0.01676330156624317, 0.015687720850110054, 0.03777645155787468, 0.000572221411857754, 0.012316697277128696, -0.010034370236098766, -0.026076246052980423, -0.008821063674986362, -0.0037022235337644815, -0.040557220578193665, -0.04092448949813843, -0.011673972941935062, -0.004895854275673628, 0.01139851938933134, 0.006305912975221872, 0.007148669101297855, 0.008034054189920425, -0.0236627496778965, -0.007673341780900955, 0.023767685517668724, -0.02864714339375496, -0.01699940301477909, 0.0609145313501358, 0.0106770945712924, 0.009234244003891945, 0.007712692487984896, 0.006925683002918959, 0.03415621072053909, -0.0022364184260368347, 0.001726501970551908, 0.016500964760780334, -0.01001469511538744, 0.023807035759091377, -0.005676305387169123, 0.0380125567317009, -0.02717805840075016, -0.003525146283209324, -0.013431627303361893, -0.018848875537514687, 0.019950689747929573, 0.03344789892435074, 0.007673341780900955, 0.05608753859996796, -0.016081225126981735, -0.01877017505466938, 0.030247395858168602, -0.012742994353175163, 0.034969452768564224, 0.00507621094584465, -0.003121803980320692, 0.010847613215446472, -0.015281099826097488, 0.01781264692544937, -0.0016297653783112764, 0.013575912453234196, 0.0043318308889865875, -0.044807069003582, 0.017156805843114853, -0.01772082969546318, -0.0010485261445865035, 0.007351979613304138, 0.01017865538597107, 0.02783389948308468, -0.02423989027738571, -0.003534984076395631, 0.010113070718944073, -0.007607757579535246, 0.024817030876874924, 0.03071960061788559, 0.02212808094918728, 0.01891445927321911, 0.0029742398764938116, 0.022114964202046394, -0.00023569293261971325, -0.019649002701044083, -0.009483463130891323, 0.010277030989527702, -0.01615992747247219, 0.013720197603106499, -0.03050973266363144, 0.01849472150206566, 0.009962227195501328, 0.00742412218824029, 0.006014063488692045, -0.010788587853312492, -0.027125591412186623, 0.00902437511831522, -0.0042990390211343765, -0.005335268098860979, -0.006528899073600769, -0.0183766707777977]} +{"id": "test:10000042", "text": "\"After the martyrdom of St. Boniface, Vergilius was made Bishop of Salzburg (766 or 767) and laboured successfully for the upbuilding of his diocese as well as for the spread of the Faith in neighbouring heathen countries, especially in Carinthia. He died at Salzburg, 27 November, 789. In 1233 he was canonized by Gregory IX. His doctrine that the earth is a sphere was derived from the teaching of ancient geographers, and his belief in the existence of the antipodes was probably influenced by the accounts which the ancient Irish voyagers gave of their journeys. This, at least, is the opinion of Rettberg (\\\"Kirchengesch. Deutschlands\\\", II, 236).\"", "vector_field": [-0.0028729664627462626, -0.013446083292365074, -0.01798146776854992, -0.02669207565486431, -0.0045320503413677216, 0.03388199582695961, -0.03913770616054535, -0.014753340743482113, -0.019635548815131187, -0.04431338235735893, 0.019582191482186317, 0.02012910693883896, -0.008330435492098331, -0.00449536694213748, 0.010024535469710827, 0.002332722069695592, 0.030893975868821144, 0.014046354219317436, 0.013219313696026802, 0.0013672852655872703, 0.0054524666629731655, 0.021809866651892662, -0.0040351590141654015, 0.015086825005710125, -0.0013789571821689606, -0.01031133159995079, 0.005199018865823746, -0.008303756825625896, 0.005926014389842749, -0.012405612505972385, -0.014166409149765968, 0.0008820657385513186, -0.03033372201025486, -0.00902408268302679, -0.034388892352581024, -0.01140515971928835, -0.004235249478369951, 0.0015415307134389877, 0.002082608873024583, 0.007870227098464966, 0.026545342057943344, 0.0011538553517311811, 0.009190824814140797, -0.02561158686876297, -0.03334841877222061, 0.016740906983613968, -0.004828851204365492, -0.01874181255698204, 0.00775684230029583, 0.0023944166023284197, 0.021663133054971695, 0.02362402155995369, 0.003374860156327486, 0.0002924239670392126, 0.00859055295586586, -0.0237840935587883, 0.02270360477268696, 0.016981014981865883, -0.021823206916451454, -0.04097853973507881, -0.005799290258437395, 0.009957837872207165, -0.021089540794491768, 0.010344679467380047, -0.03028036467730999, -0.013119268231093884, -0.01707439124584198, -0.007783521432429552, -0.03081393986940384, 0.00021447202016133815, 0.02018246427178383, 0.025424836203455925, 0.021116219460964203, 0.0018841857090592384, 0.04506038501858711, -0.008390462026000023, -0.02184988558292389, 0.014686644077301025, -0.03329506143927574, 0.01627402938902378, 0.01511350367218256, -0.03302827477455139, -0.011471856385469437, 0.004802172537893057, 0.03113408572971821, -0.00993115920573473, -0.02878635562956333, 0.0176613237708807, -0.010971629992127419, 0.002202663104981184, 0.015380291268229485, 0.027399061247706413, 0.01140515971928835, 0.0039484528824687, -0.0017257807776331902, -0.027745885774493217, -0.012438960373401642, 0.03441556915640831, -0.0009479288128204644, -0.017914772033691406, -0.01020461693406105, -0.013506109826266766, -0.010091232135891914, -0.012925847433507442, -0.021783187985420227, -0.008877349086105824, -0.0064829327166080475, -0.014686644077301025, 0.04466020315885544, -0.016300708055496216, -0.012198852375149727, 0.03604297339916229, 0.013832924887537956, -0.018541721627116203, 0.020916128531098366, -0.021716490387916565, 0.03796384111046791, 0.019422119483351707, 0.019288726150989532, -0.020222481340169907, 0.011185060255229473, 0.019288726150989532, -0.003631643019616604, -0.02313046343624592, 0.021302970126271248, 0.0040351590141654015, -0.011925394646823406, -0.011291774921119213, -0.0073499917052686214, 0.004728805739432573, -0.018941903486847878, 0.007343322038650513, 0.01559372153133154, -0.006876443978399038, 0.021222934126853943, 0.005159000400453806, -0.01820823736488819, 0.020369214937090874, -0.004758819472044706, -0.025678282603621483, 0.04231247678399086, 0.006896453443914652, -0.04263262078166008, -0.017567947506904602, 0.0012413949007168412, 0.018341630697250366, -0.005759272258728743, 0.02239679917693138, 0.020982826128602028, -0.016727566719055176, 0.019795622676610947, -0.02841285429894924, 0.026118483394384384, -0.00985779333859682, 0.019101975485682487, -0.00939091481268406, -0.03209451958537102, 0.03761701658368111, -0.0026945522986352444, -0.005936018656939268, -0.0075700911693274975, 0.024624472483992577, 0.014633286744356155, 0.002562826033681631, 0.01599390245974064, 0.01506014633923769, 0.013439413160085678, -0.007850217632949352, -0.021516401320695877, -0.016620852053165436, 2.5955753244488733e-06, 0.03996474668383598, -0.028813034296035767, 0.021289631724357605, -6.544627103721723e-05, 0.01888854429125786, -0.02039589360356331, 0.013085920363664627, -0.009490960277616978, 0.007936923764646053, -0.01820823736488819, -0.008290417492389679, 0.026078464463353157, 0.03684333339333534, -0.004618756007403135, 0.009137467481195927, -0.008490507490932941, -0.005522498395293951, 0.0204092338681221, -0.025371478870511055, 0.000729913532268256, 0.004015149548649788, 0.0030447107274085283, -0.0032481360249221325, -0.6428508162498474, -0.031214121729135513, -0.004005145281553268, -0.017514590173959732, 0.028012672439217567, -0.007476715836673975, 0.006386222317814827, 0.0015331936301663518, -0.01810152269899845, 0.00933755747973919, -0.009057430550456047, 0.03214787691831589, 0.0022710273042321205, 0.00404182868078351, -0.021916581317782402, -0.011365141719579697, 0.005865986924618483, -0.004762154538184404, 0.022943712770938873, -0.004872204270213842, -0.015366951934993267, -0.0029263237956911325, 0.00902408268302679, 0.00944427214562893, 0.005805959925055504, 0.03260141611099243, 0.03174769505858421, -0.03118744306266308, 0.014686644077301025, 0.01615397445857525, -0.01574045419692993, 0.04530049487948418, -0.008944046683609486, 0.009757747873663902, 0.03526929020881653, -0.011238417588174343, -0.02399752289056778, 0.031000690534710884, 0.016834283247590065, 0.021783187985420227, -0.006449584383517504, -0.02307710610330105, 0.018941903486847878, -0.011525213718414307, -0.015553703531622887, 0.011765322647988796, 0.03281484544277191, 0.012685739435255527, -0.0005669231177307665, 0.00488554360345006, 0.02539815753698349, -0.016634192317724228, 0.02862628363072872, 0.0072699557058513165, -0.000714489899110049, -0.007216597907245159, 0.01723446324467659, -0.011205068789422512, -0.009304209612309933, 0.007029846776276827, -0.006736380979418755, 0.007810200098901987, 0.013599486090242863, 0.0002861711254809052, 0.004688787739723921, 0.007223267573863268, -0.03060051053762436, 0.009450942277908325, 0.04618089273571968, -0.0038484076503664255, -0.0015340272802859545, 0.017567947506904602, -0.025264762341976166, -0.02341059036552906, 0.010964960791170597, 0.019782282412052155, 0.01140515971928835, 0.011952073313295841, 0.005152330733835697, 0.010137919336557388, -0.00641623605042696, -0.007876897230744362, -0.001193039701320231, 0.013712869957089424, 0.013386055827140808, -0.008203711360692978, -0.03182773292064667, 0.0018558396259322762, -0.007416688837110996, -0.004235249478369951, 0.0006969819660298526, 0.04183225706219673, -0.011825350113213062, -0.010111240670084953, 0.017848074436187744, 0.034335535019636154, -0.013646173290908337, 0.007076534908264875, 0.022530192509293556, -0.028706319630146027, -0.0016249017789959908, 0.008730616420507431, 0.012472309172153473, -0.0010804887861013412, -0.01001119613647461, 0.0009237512131221592, -0.028599604964256287, -0.017567947506904602, 0.042872726917266846, -0.03881756216287613, 0.013726209290325642, -0.0076567973010241985, 0.00219432613812387, 0.0009212500881403685, -0.021449703723192215, -0.015020128339529037, 0.03001357801258564, -0.023063767701387405, -0.008363783359527588, -0.02862628363072872, 0.0016715895617380738, -0.0031547604594379663, -0.0043386295437812805, -0.02255687117576599, -0.02518472634255886, 0.015326933935284615, -0.006116100121289492, -0.020475929602980614, -0.041912294924259186, -0.018074844032526016, 0.001296419883146882, 0.0003589123662095517, 0.03225459158420563, 0.0013556132325902581, 0.020115766674280167, 0.005362425930798054, 0.025584908202290535, 0.003908434882760048, 0.02297039143741131, -0.0026211857330054045, -0.010484742932021618, 0.009737738408148289, 0.008857340551912785, -0.014980110339820385, -0.0044486792758107185, -0.03484242781996727, 0.016727566719055176, 0.005622543394565582, -0.010858245193958282, -0.001520687947049737, 0.006079417187720537, -0.02737238258123398, -0.0271589532494545, -0.011905386112630367, 0.00985779333859682, -1.9123233869322576e-05, -0.0026161835994571447, -0.014086372219026089, -0.0005652557010762393, -0.02238345891237259, 0.017167767509818077, 0.00372501858510077, -0.021409684792160988, -0.004058502614498138, 0.0046387650072574615, -0.015393630601465702, -0.01145851705223322, 0.028973108157515526, 0.002396083902567625, -0.02518472634255886, 0.013032562099397182, -0.008270408026874065, 0.0038317334838211536, 0.025838356465101242, -0.007370000705122948, 0.00488554360345006, -0.03382863849401474, 0.001046306686475873, 0.003374860156327486, -0.0007811867399141192, -0.009070769883692265, -0.027038898319005966, -0.02060932293534279, -0.011138372123241425, 0.026025107130408287, -0.01219218224287033, -0.004325290210545063, 0.012318906374275684, -0.00641623605042696, 0.016940997913479805, 0.009504299610853195, 0.021676473319530487, 0.013899621553719044, -0.029613396152853966, 0.010824897326529026, 0.015500345267355442, -0.004652104806154966, -0.0006611324497498572, 0.007696815300732851, -0.009277530014514923, 0.01820823736488819, 0.017701340839266777, 0.02679879032075405, -0.007196588907390833, 5.320115087670274e-05, -0.04220576211810112, -0.0009304209379479289, 0.00400181021541357, 0.02894642949104309, -0.016740906983613968, -0.00043144519440829754, -0.013119268231093884, -0.03823062777519226, -0.022957053035497665, 0.006042733788490295, 0.02266358584165573, 0.021276291459798813, -0.027932636439800262, 0.014886735007166862, -0.002869631629437208, -0.00029221552540548146, -0.0020892785396426916, 0.023597342893481255, 0.008277078159153461, -0.03588290140032768, 0.014646626077592373, -0.013966318219900131, -0.006329529918730259, 0.0063161905854940414, -0.02921321615576744, 0.003825063817203045, 0.013205974362790585, 0.007143231574445963, -0.0012205521343275905, 0.01794145070016384, 0.01982230134308338, 0.011738643981516361, -0.023744074627757072, 0.024837903678417206, -0.006773063912987709, 0.002762916497886181, 0.013359377160668373, 0.021062862128019333, -0.008730616420507431, 0.003191443858668208, 0.021543079987168312, 0.031107407063245773, -0.007029846776276827, -0.013819585554301739, 0.030840618535876274, -0.008110336028039455, -0.011338463053107262, 0.002846287563443184, -0.010624806396663189, 0.03174769505858421, -0.03193444758653641, 0.008450489491224289, -0.0004981420352123678, -0.0004078928614035249, 0.015206879936158657, 0.029880184680223465, 0.0021342989057302475, 0.0012147162342444062, -0.020155785605311394, 0.029666755348443985, 0.010684833861887455, -0.0015765465795993805, -0.003965127281844616, -0.020982826128602028, -0.01842166669666767, -0.016620852053165436, -0.0064295753836631775, 0.0006390390917658806, -0.009297539480030537, 0.008263738825917244, -0.0007378337904810905, -0.017541268840432167, 0.014139730483293533, 0.016247350722551346, 0.009437602944672108, -0.018928563222289085, -0.028199423104524612, 0.01998237334191799, 0.013766228221356869, 0.007189919240772724, -0.01691431924700737, 0.0014564922312274575, 0.009844454005360603, -0.024837903678417206, -0.009731069207191467, -0.0018041494768112898, 0.009204164147377014, -0.021022843196988106, -0.02309044636785984, 0.008510516956448555, -0.016834283247590065, 0.006276172585785389, -0.041912294924259186, -0.002416093135252595, 0.003708344418555498, 0.006212810520082712, 0.0043619731441140175, -0.0055625163950026035, -0.007169910240918398, 0.04369977116584778, 0.004131869412958622, -0.010511421598494053, 0.003584955120459199, -0.004048498347401619, -0.009517638944089413, -0.021783187985420227, 0.0036983396857976913, -0.0271589532494545, -0.008237060159444809, 0.012752436101436615, 0.03001357801258564, -0.017861412838101387, 0.01253900583833456, 0.029826827347278595, 0.002998023061081767, -0.0006627998664043844, -0.014033014886081219, -0.026278555393218994, -0.02825278230011463, 0.08318430185317993, -0.0025061338674277067, 0.01025797426700592, 0.02325051836669445, 0.027932636439800262, -0.002371072769165039, -0.022690264508128166, -0.009244182147085667, 0.02234344184398651, -0.01620733179152012, 0.016834283247590065, -0.016954336315393448, -0.005996046122163534, -0.0031114076264202595, 0.015527023933827877, -0.009324218146502972, 0.025958409532904625, -0.0183683093637228, 0.006819752044975758, -0.008270408026874065, -0.01339272502809763, 0.016780924052000046, -0.0012555680004879832, 0.02619851939380169, 0.013299349695444107, -0.03601629287004471, 0.03366856649518013, 0.019048618152737617, -0.006096091121435165, -0.00567590119317174, -0.0012639050837606192, 0.003558276453986764, 0.026385270059108734, 0.00907744001597166, -0.0027695863973349333, 0.013766228221356869, 0.0017174435779452324, 0.006903123110532761, -0.0011346800019964576, 0.0010729854693636298, 0.02078273519873619, 0.0033982039894908667, 0.0013464424991980195, -0.02190324291586876, 0.0015498677967116237, -0.03542936220765114, -0.010531431064009666, 0.021743169054389, 0.007456706836819649, 0.005048950668424368, 0.028012672439217567, -0.01654081605374813, -0.004245253745466471, -0.030360400676727295, -0.013599486090242863, -0.011031657457351685, 0.00848383828997612, 0.01205211877822876, -0.010738191194832325, 0.05338415130972862, -0.022356780245900154, -0.016500798985362053, -0.003795050084590912, -0.011385150253772736, -0.00810366589576006, -0.018928563222289085, -0.010498082265257835, -0.009577666409313679, -0.043886519968509674, -0.0038884258829057217, -0.006936471443623304, -0.019395440816879272, -0.017314499244093895, -0.006439580116420984, 0.007129892241209745, -0.0016198995290324092, 0.028973108157515526, -0.017407875508069992, 0.000266162067418918, -0.010624806396663189, -0.0022810320369899273, -0.020155785605311394, -0.01991567574441433, -0.021436363458633423, 0.009851123206317425, 0.02447774074971676, -0.0020942806731909513, -0.0024761201348155737, 0.002784593030810356, -0.001156356418505311, 0.0037216837517917156, 0.0042252447456121445, -0.01872847229242325, -0.03436221182346344, 0.004185226745903492, -0.0005073128268122673, 0.0235173050314188, 0.005689240526407957, 0.0055925301276147366, -0.0016557490453124046, 0.0247578676789999, 0.0176613237708807, 0.008523856289684772, -0.030520474538207054, 0.0183683093637228, 0.028039351105690002, 0.020169124007225037, 0.005859317258000374, -0.01707439124584198, 0.0005093971267342567, 0.02062266319990158, -0.015046807006001472, 0.038684166967868805, 0.006996498443186283, -0.021302970126271248, 0.028866391628980637, 0.0402848906815052, 0.03377528116106987, 0.015553703531622887, -0.009524309076368809, -0.0035649461206048727, -0.008337104693055153, 0.021489722654223442, -0.013032562099397182, 0.0076567973010241985, -0.011031657457351685, -0.010671494528651237, 0.01203211024403572, -0.007983611896634102, 0.004842190537601709, 0.01036468893289566, 0.011558562517166138, 0.019302066415548325, -0.010531431064009666, -0.02495795674622059, -0.00404182868078351, 0.00046729473979212344, -0.007883566431701183, -0.03350849077105522, -0.0005531669012270868, -0.005936018656939268, 0.003845072817057371, -0.01707439124584198, 0.006783068645745516, -0.019955694675445557, -0.02062266319990158, -0.013025892898440361, -0.012112146243453026, 0.003051380394026637, 0.01600724086165428, -0.023717395961284637, -0.015647077932953835, -0.0026295229326933622, 0.005555846728384495, -0.018555060029029846, -0.012012100778520107, -0.00026470309239812195, -0.004081846680492163, 0.010151258669793606, 0.009084109216928482, 0.013926300220191479, 0.009150806814432144, 0.00942426361143589, 0.005385769996792078, -0.0018174889264628291, 0.01707439124584198, -0.015980562195181847, -0.02711893431842327, -0.03948453068733215, 0.010771539993584156, 0.00681641697883606, 0.011685286648571491, 0.007990281097590923, -0.008477168157696724, 0.005362425930798054, 0.02669207565486431, -0.0038283986505120993, -0.03468235582113266, -0.02427764981985092, -0.04660775139927864, -0.015473666600883007, -0.0018358305096626282, 0.009631023742258549, 0.0013589480658993125, 0.014566590078175068, -0.015607060864567757, 0.04930230230093002, 0.0005919344839639962, -0.0037650365848094225, -0.00720992824062705, -0.006963150110095739, -0.019635548815131187, 0.04812844097614288, 0.016674209386110306, -0.013399395160377026, -0.017101069912314415, 0.015873847529292107, 0.0235173050314188, 0.014099711552262306, 0.002259355504065752, -0.012518997304141521, -0.022370120510458946, -0.013832924887537956, 0.0056158737279474735, -0.021543079987168312, 0.014126391150057316, 0.014113051816821098, -0.004605416674166918, 0.017034372314810753, -0.004648769740015268, -0.02062266319990158, -0.006526285782456398, -0.009417593479156494, -0.002332722069695592, 0.0033932016231119633, 0.0026511994656175375, -0.01060479786247015, 0.0181948971003294, -0.0025111360009759665, -0.023597342893481255, -0.004632095340639353, 0.0009862795704975724, 0.018181558698415756, 0.02411757782101631, 0.03164098039269447, 0.01654081605374813, -0.013052571564912796, -0.01626068912446499, -0.004472023341804743, 0.009757747873663902, -0.0035049188882112503, -0.0004777161229867488, -0.0013122602831572294, 0.008277078159153461, -0.01648745872080326, 0.002876301296055317, 0.019568853080272675, -0.03065386787056923, -0.018221577629446983, 0.04065839201211929, 0.03769705444574356, -0.010371359065175056, -0.011245086789131165, -0.010424716398119926, -0.0013697863323614001, 0.016887640580534935, -0.01095162145793438, -0.003708344418555498, -0.01578047312796116, -0.02039589360356331, 0.004462018609046936, 0.001970891607925296, -0.010844905860722065, 0.019502155482769012, 0.007483385503292084, 0.000546497234608978, 0.013446083292365074, -0.03185440972447395, 0.007216597907245159, 0.030733903869986534, -0.017594626173377037, 0.006916462443768978, -0.010324670933187008, 0.005952693056315184, -0.01352611929178238, 0.03270813077688217, -0.015807151794433594, 0.013412734493613243, -0.016767585650086403, 0.02293037436902523, -0.0066496748477220535, 0.006496272049844265, -0.01874181255698204, -0.0030480455607175827, 0.018274934962391853, 0.00928420014679432, -0.01001119613647461, -0.002709559164941311, -0.005365760996937752, 0.0024210952688008547, 0.021169576793909073, 0.003808389650657773, 0.018008146435022354, -0.0077635119669139385, -0.019195349887013435, 0.0019675567746162415, 0.00912412814795971, 0.007423358503729105, -0.004955575335770845, -0.03148090839385986, -0.020702699199318886, -0.011585241183638573, 0.007976941764354706, 0.010351349599659443, -0.011325123719871044, 0.013992996886372566, -0.007016507443040609, 0.021769847720861435, -0.04332626610994339, 0.009537648409605026, 0.02550487220287323, 0.00902408268302679, -0.00813701469451189, -0.010984969325363636, 0.013192635029554367, -0.007456706836819649, 0.026385270059108734, 0.004188561346381903, -0.008957386016845703, 0.00037746242014691234, 0.011878707446157932, 0.011605249717831612, 0.027799243107438087, -0.04962245002388954, -0.0013322693994268775, -0.015313594602048397, 0.013112599030137062, -0.020689358934760094, -0.02742573991417885, 0.025478193536400795, 0.02335723303258419, -0.0053690955974161625, 0.011865368112921715, -0.024184273555874825, 0.0062928469851613045, 0.02915985882282257, 0.01610061712563038, 0.005749267525970936, -0.02222338691353798, -0.026118483394384384, -0.005962697323411703, -0.0010938281193375587, -0.012145494110882282, -0.02309044636785984, -0.015487005934119225, -0.005755937192589045, 0.0007395012071356177, 0.013346037827432156, 0.004895547870546579, -0.021596437320113182, 0.011591910384595394, 0.010831566527485847, -0.01567375659942627, 0.015913866460323334, -0.00015392380009870976, -0.009050761349499226, 0.007883566431701183, -0.002607846399769187, -0.020742718130350113, -0.006386222317814827, -0.009357566945254803, 0.04946237802505493, 0.0042019011452794075, 0.00824372936040163, -0.006516281049698591, 0.0031647649593651295, -0.03244134411215782, -0.01852838136255741, -0.0008220385643653572, 0.014073032885789871, 0.023383911699056625, 0.015326933935284615, 0.013912960886955261, -0.033481813967227936, -0.005118982400745153, 0.01670088805258274, -0.004662109073251486, -0.019315404817461967, -0.01799480803310871, -0.003059717593714595, 0.01229222770780325, -0.008303756825625896, -0.03999142348766327, -0.009250851348042488, 0.012665729969739914, 0.014406517148017883, 0.012165503576397896, 0.007696815300732851, -0.008904027752578259, -0.019128654152154922, 0.0025228080339729786, 0.0008403802057728171, 0.01964888907968998, 0.024877920746803284, 0.033855315297842026, -0.038897596299648285, 0.00843048095703125, -0.011671947315335274, 0.010998308658599854, -0.026011766865849495, -0.030093614012002945, 0.004908887203782797, -0.01718110591173172, 0.01964888907968998, -0.012699078768491745, 0.01012458000332117, 0.01361282542347908, -0.013366046361625195, 0.011765322647988796, 0.011825350113213062, -0.00883066188544035, 0.004878873936831951, 0.00977108720690012, -0.014593268744647503, -0.013646173290908337, 0.009764417074620724, -0.035189252346754074, 0.004075177013874054, 0.000550665776245296, -0.006883114110678434, 0.03593625873327255, -0.006746385246515274, 0.021062862128019333, -0.00770348496735096, 0.02229008451104164, -0.0291065014898777, 0.007009837776422501, -0.019888997077941895, 0.02018246427178383, 0.035616111010313034, -0.025411495938897133, 0.011798671446740627, 0.012518997304141521, 0.007670136634260416, -0.03257473558187485, -0.020262500271201134, -0.026558682322502136, 0.004028489347547293, -0.02662537805736065, 0.010104571469128132, -0.014753340743482113, -0.027559135109186172, -0.0002507384342607111, -0.0025511542335152626, 0.006466258782893419, -0.00256949570029974, 0.18984588980674744, 0.0002367737761233002, -0.008463828824460506, 0.015527023933827877, -0.05922679230570793, 0.014046354219317436, -0.0041418736800551414, -0.01755460910499096, 0.006896453443914652, 0.0017157761612907052, -0.014593268744647503, -0.01476668007671833, 0.027825921773910522, 0.009691051207482815, 0.020489269867539406, -0.02587837353348732, -0.018114861100912094, -0.013225983828306198, 0.006162787787616253, -0.03462899848818779, 0.013699530623853207, 0.009057430550456047, -0.0006698863580822945, -0.01604725979268551, 0.03446892648935318, 0.0007816036231815815, -0.010424716398119926, -0.011792001314461231, 0.038524094969034195, 0.01516686100512743, 0.0010704842861741781, 0.013099259696900845, -0.0067063672468066216, -0.0028046020306646824, -0.0342554971575737, -0.004795502871274948, -0.009677711874246597, 0.0003405707539059222, -0.001695767161436379, 0.01911531388759613, 0.002381077269092202, -0.014406517148017883, 0.011785331182181835, -0.03238798677921295, -0.0007269955822266638, 0.05826636031270027, -0.009697720408439636, -0.0012980872998014092, -0.018181558698415756, 0.019035277888178825, -0.04052500054240227, 0.006032729055732489, 0.020582644268870354, 0.022103331983089447, 0.019848980009555817, -0.0413520410656929, 0.018648436293005943, -0.007770181633532047, -0.013012553565204144, -0.006436245050281286, -0.012585693970322609, 0.009484291076660156, -0.015046807006001472, 0.000885400571860373, 0.00784354843199253, 0.010531431064009666, -0.027932636439800262, 0.0014281460316851735, 0.01583383046090603, -0.026612039655447006, 0.013792906887829304, -0.012512327171862125, -0.016180653125047684, 0.012852480635046959, 0.0019141993252560496, -0.022850336506962776, 0.01449989341199398, -0.011578571051359177, -0.006739715579897165, 0.010558109730482101, -0.00799695122987032, 0.019622210413217545, 0.014673304744064808, -0.012972535565495491, -0.020489269867539406, -0.0404716432094574, 0.021889902651309967, -0.006836425978690386, -0.020155785605311394, 0.010878254659473896, -0.005118982400745153, -0.01900859922170639, -0.008063647896051407, -0.007576760835945606, 0.01712774857878685, -0.009484291076660156, -0.007216597907245159, 0.025118030607700348, -0.02088944986462593, 0.00931754894554615, -0.011638598516583443, 0.02051594853401184, 0.013652843423187733, -0.028279460966587067, -0.005002263002097607, 0.012332245707511902, 0.011478526517748833, 0.011832019314169884, 0.010404706932604313, -0.017954789102077484, -0.01540696993470192, -0.016567494720220566, 0.00269121746532619, -0.006883114110678434, 0.008163693360984325, 0.002692884998396039, 0.0023927490692585707, -0.006829756312072277, 0.007936923764646053, -0.013092589564621449, 0.0018458350095897913, -0.010984969325363636, 0.035029180347919464, 0.015073485672473907, -0.0032414663583040237, -0.009957837872207165, -0.04772825911641121, 0.022103331983089447, -0.03500249981880188, -0.00942426361143589, 0.01932874508202076, -0.03270813077688217, -0.010297992266714573, 0.014699983410537243, -0.0186350978910923, 0.0307072252035141, -0.011065005324780941, -0.002434434602037072, -0.021996617317199707, -0.014179748483002186, 0.017207784578204155, -0.012505657970905304, 0.03596293553709984, -0.011485195718705654, 0.003925108816474676, 0.0008362116059288383, 0.014033014886081219, -0.020849432796239853, -0.011551892384886742, -0.02071603760123253, -0.0015965555794537067, -0.020769396796822548, -0.01210547611117363, -0.01755460910499096, 0.018715133890509605, 0.011691955849528313, -0.02239679917693138, -0.04815511777997017, -0.006686358246952295, 0.0004810509562958032, -0.016180653125047684, 0.0063762180507183075, 0.02825278230011463, -0.03708344325423241, -0.025438174605369568, -0.016514137387275696, -0.16935661435127258, 0.02318382076919079, 0.009057430550456047, -0.019248707219958305, 0.01970224641263485, -0.003448226721957326, 0.04127200320363045, -0.007149901241064072, -0.004341964144259691, -0.002314380370080471, -0.007936923764646053, 0.017301160842180252, -0.02003573067486286, -0.02502465434372425, 0.004892213270068169, -0.007810200098901987, -0.026345251128077507, 0.01991567574441433, 0.021302970126271248, 0.01798146776854992, 0.021983278915286064, -0.00902408268302679, 0.007936923764646053, -0.02405088022351265, 0.027692528441548347, -0.006152783520519733, -0.009651033207774162, -0.0005294061847962439, 0.0091841546818614, -0.005759272258728743, -0.007116552907973528, 0.016527477651834488, 0.027879279106855392, -0.02497129701077938, 0.0017291156109422445, -0.00572925852611661, -0.03468235582113266, -0.013992996886372566, -0.007983611896634102, 0.029506681486964226, 0.034442249685525894, 0.009697720408439636, 0.003991805948317051, -0.004235249478369951, -0.03190776705741882, 0.012025440111756325, -0.013259331695735455, 0.0009320883546024561, 0.016460780054330826, 0.00293966312892735, 0.0068697743117809296, -0.004622091073542833, 0.0025278101675212383, 0.013019222766160965, -0.004008479882031679, -0.011571901850402355, -0.004698792472481728, 0.0078969057649374, 0.012752436101436615, 0.018995260819792747, -0.022023295983672142, -0.027479097247123718, -0.0026028442662209272, -0.017407875508069992, 0.0033765274565666914, -0.04788833111524582, -0.014086372219026089, 0.0186350978910923, -0.007976941764354706, -0.0020876110065728426, 0.006152783520519733, -0.021569758653640747, 0.00492889666929841, -0.003825063817203045, -0.00036787474527955055, 0.012472309172153473, -0.012732426635921001, 0.012112146243453026, 0.0028879730962216854, 0.007236607372760773, -0.011578571051359177, 0.02277030050754547, 0.006936471443623304, 0.0019041948253288865, 0.0016899311449378729, -0.013652843423187733, -0.02678545005619526, 0.003474905388429761, -0.003484909888356924, -0.011024987325072289, -0.00939091481268406, -0.022023295983672142, -0.02625187672674656, 0.02018246427178383, 0.020062409341335297, 0.005555846728384495, -0.014286463148891926, 0.01307258103042841, -0.013079250231385231, -0.019342083483934402, -0.0042252447456121445, -0.006149448454380035, -0.04828851297497749, 0.018808508291840553, 0.03708344325423241, -0.0064862677827477455, 0.005932684056460857, 0.011345132254064083, 0.020902790129184723, 0.013652843423187733, 0.00819037202745676, -0.004765489138662815, -0.015126843005418777, -0.010758200660347939, -0.015500345267355442, 0.01272575743496418, 0.011018318124115467, -0.01867511495947838, -0.0022009958047419786, -0.021489722654223442, 0.056025344878435135, 0.015140182338654995, -0.0005911007174290717, -0.009264190681278706, -0.028492890298366547, -0.011445177718997002, -0.10527428984642029, -0.0011863700347021222, 0.018755150958895683, 0.028012672439217567, 0.0026662060990929604, 0.023984184488654137, 0.01012458000332117, 0.004328624811023474, -0.0075434125028550625, 0.01787475310266018, -0.020822754129767418, -0.03652318939566612, -0.005309068597853184, 0.014406517148017883, 0.020422572270035744, -0.0006432076334021986, 0.00022572711168322712, -0.011331792920827866, 0.013299349695444107, 0.008964055217802525, -0.004315285477787256, -0.0013856268487870693, -0.009010743349790573, -0.013806246221065521, -0.00030534647521562874, -0.013205974362790585, -0.015020128339529037, 0.014873395673930645, 0.007256616372615099, 0.012892499566078186, 0.008890688419342041, -0.014019675552845001, 0.015620400197803974, -0.005405778996646404, -0.02007574960589409, -0.00543579226359725, -0.011725304648280144, 0.010051214136183262, -0.0027345705311745405, -0.00982444453984499, 0.007396679371595383, 0.0036916700191795826, 0.00493890093639493, -0.030200328677892685, 0.00010952871525660157, -0.014579929411411285, -0.017594626173377037, 0.005752602592110634, 0.025278102606534958, -0.008770634420216084, -0.004785498138517141, -0.02195660024881363, -0.012365594506263733, -0.006673018913716078, 0.03193444758653641, -0.015073485672473907, -0.0015915533294901252, 0.0031931111589074135, -0.016033919528126717, 0.021396346390247345, 0.005522498395293951, 0.03257473558187485, 0.00429527647793293, 0.013519449159502983, -0.002481122501194477, -0.018381649628281593, -0.01814153976738453, -0.026505324989557266, -0.00414854334667325, -0.005749267525970936, -0.002591172233223915, -0.007923584431409836, -0.005065625067800283, 0.005335747264325619, -0.006096091121435165, 0.009197494015097618, -0.015180200338363647, 0.010678163729608059, 0.00805030856281519, -0.0413520410656929, -0.0077368333004415035, -0.012892499566078186, 0.033535171300172806, -0.016340725123882294, 0.04890212416648865, 0.004728805739432573, 0.015927204862236977, 0.0035249278880655766, 0.01986231841146946, -0.010911603458225727, 0.0006248659919947386, 0.008910697884857655, 0.04796836897730827, -0.015473666600883007, -0.017194446176290512, 0.02126295305788517, 0.02422429248690605, -0.008123675361275673, -0.010704842396080494, -0.0005377432680688798, -0.03236130625009537, 0.010044544003903866, -0.07123222202062607, 0.009537648409605026, 0.002276029670611024, 0.001598222996108234, 0.00765012763440609, 0.0051756747998297215, 0.02110288105905056, -0.013292680494487286, -0.018448345363140106, 0.00755675183609128, -0.002968009328469634, 0.00632286025211215, -0.0014039684319868684, 0.001549034146592021, -0.021836545318365097, -0.009110788814723492, 0.023770753294229507, 0.027479097247123718, 0.013899621553719044, 0.005515828728675842, 0.006559634115546942, 0.017314499244093895, -0.005822634324431419, 0.01594054512679577, -0.03044043853878975, 0.018821848556399345, 0.003031371394172311, -0.012372263707220554, -0.003908434882760048, -0.023117125034332275, 0.0034448918886482716, -0.0247578676789999, -0.0006782234995625913, 0.0291065014898777, 0.002371072769165039, -0.011972082778811455, 0.003511588554829359, 0.011325123719871044, 0.011771991848945618, 0.06605654954910278, -0.007430028170347214, -0.0034448918886482716, 0.0100712226703763, -0.010638145729899406, 0.020809413865208626, 0.007563421502709389, 0.005225697532296181, 0.06082751974463463, 0.01584716886281967, 0.0012388938339427114, 0.003089731093496084, 0.027639171108603477, -0.01140515971928835, -0.011665277183055878, -0.004128534346818924, -0.009971177205443382, 0.0013497773325070739, 0.022049974650144577, 0.0010488077532500029, -0.01794145070016384, 0.030467117205262184, 0.014753340743482113, 0.007790191099047661, -0.016087278723716736, 0.043246231973171234, 0.005335747264325619, -0.013379385694861412, 0.0055925301276147366, -0.01530025526881218, -0.01938210241496563, -0.0315876230597496, -0.011018318124115467, -0.0037850455846637487, 0.020689358934760094, 0.010491413064301014, -0.0016015578294172883, 0.03081393986940384, -0.010678163729608059, 0.012012100778520107, 0.00770348496735096, 0.00011713632557075471, -0.026878826320171356, -0.003246468724682927, 0.011478526517748833, 0.033054955303668976, -0.0122455395758152, -0.012672399170696735, 0.012585693970322609, 0.006419570650905371, 0.009684381075203419, 0.008497177623212337, 0.0023560659028589725, 0.012225531041622162, -0.039164382964372635, -0.007109883241355419, -0.00048396893544122577, -0.015553703531622887, 0.007343322038650513, 0.0053724306635558605, 0.009104118682444096, -0.008443820290267467, 0.01259236317127943, 0.0024827898014336824, 0.0023110455367714167, -0.022850336506962776, 0.04431338235735893, -0.015180200338363647, -0.03636311739683151, -0.02862628363072872, 0.01219218224287033, 0.008850670419633389, 0.047808293253183365, 0.0006536290165968239, 0.01465996541082859, -0.022156691178679466, 0.006282842252403498, 0.018221577629446983, 0.005492484662681818, -0.005829303991049528, 0.022957053035497665, 0.015233558602631092, -0.010351349599659443, 0.013152617029845715, -0.007049855776131153, -0.000972940179053694, -0.018488364294171333, 0.030893975868821144, -0.02931993082165718, -0.007590100169181824, -0.023237179964780807, 0.02996022067964077, -0.004301946144551039, -0.016354065388441086, 0.00724994670599699, 0.0022510183043777943, -0.012198852375149727, -0.007396679371595383, 0.025478193536400795, -0.03436221182346344, 0.06536290049552917, 0.022850336506962776, -0.0018274934263899922, 0.01001119613647461, -0.01664753071963787, 0.03174769505858421, -0.007043186109513044, 0.026278555393218994, 0.0003086813085246831, -0.041912294924259186, 0.006479598116129637, -0.014886735007166862, 0.04095185920596123, -0.014593268744647503, -0.02835949696600437, 0.0144198564812541, -0.01782139576971531, 0.020542627200484276, -0.01151187438517809, -0.0022726948373019695, 0.02495795674622059, -0.033588528633117676, -0.004005145281553268, 0.01599390245974064, -0.026025107130408287, 0.006602987181395292, 0.017167767509818077, 0.005078964401036501, -0.0017007694113999605, -0.024197613820433617, -0.010271313600242138, 0.010478073731064796, -0.03772373124957085, -0.025651603937149048, 0.026825468987226486, -0.025051333010196686, 0.004905552603304386, -0.03892427682876587, 0.019248707219958305, 0.026665396988391876, -0.012498987838625908, 0.01750124990940094, -0.01874181255698204, -0.017007693648338318, 0.02051594853401184, -0.02421095222234726, -0.001304756966419518, -0.003448226721957326, -0.010704842396080494]} +{"id": "test:10000043", "text": "\"How to unblock ABC Go UK outside US? Watch ABC in the UK on Apple TV, iPhone, iPad, Android, PC or Mac. ABC is geoblocked outside US. In order to bypass geographic restrictions and access ABC in UK, you have to spoof/change your online location. VPN and Smart DNS allows you to watch ABC in the UK.\\nIf you try to watch ABC shows online in the UK, you get the following error message: \u201cYou appear to be outside the United States or its territories. Due to International rights agreements, we only offer this video to viewers located within the US.\u201d That basically means you cannot watch any of ABC\u2019s shows in UK due to geo-restrictions. VPN enables you to trick ABC into thinking you are in USA by giving you an American IP address.\\nVPN allows you to unblock all US channels in the UK. Watch ABC, NBC, CBS, Netflix, Vudu, Hulu, Showtime, or HBO Go.\\nVPN apps can be installed on Android, iPhone, iPad, PC, or Mac.\\nThanks to VPN\u2019s data encryption, your ISP will not be able to see what you are doing online.\\nYour Internet speed will drop by around 10 to 15% when using VPN.\\nI am currently using ExpressVPN to unblock WatchABC in the UK. You can use any of the three VPN providers in the table below to watch ABC outside US.\\nThere is an alternative way to watch ABC outside US. Smart DNS also allows you to remove geoblocks and access US channels in the UK. You do not get a new IP address when using Smart DNS. Therefore, you will still appear to be in the UK for all websites apart from the ones you are trying to unblock.\\nSmart DNS can be setup on Apple TV, Mac, Smart TV, PS3, PS4, Xbox, Amazon Fire TV, Roku, Chromecast, Android, iPhone, iPad, or PC.\\nNo effect on Internet speed with Smart DNS. No traffic encryption either.\\nYou can still use all local UK apps and sites while unblocking US channels with Smart DNS. Your IP address is left unchanged.\\nNo need to install or download software. Smart DNS can be configured directly on your device. All you need is access to your device\u2019s network settings.\\nSome ISPs uses transparent proxies or DNS hijacking. Either method might prevent Smart DNS from functioning properly.\\nThere are quite a few Smart DNS services available. Not all of them help you to unblock ABC outside US. Unlocator allows you to unblock and watch ABC in the UK. You can check the list of 214 channels they allow you to unblock.\\nSo, you can either use VPN or Smart DNS proxy to unblock and watch ABC in the UK. Keep in mind that you still need a valid American cable subscription in order to sign into ABC. Even if you are outside the US.\"", "vector_field": [0.008336379192769527, -0.02406599186360836, 0.008557922206819057, -0.038308098912239075, -0.0254838727414608, 0.015697965398430824, -0.015267536044120789, -0.01839447021484375, 0.015596686862409115, 0.00568101741373539, -0.01754627376794815, -0.010672084055840969, -0.011437992565333843, -0.01053282804787159, -0.02098969742655754, 0.0026616910472512245, 0.023711523041129112, -0.013609122484922409, 0.015938498079776764, -0.0016821508761495352, -0.037902988493442535, 0.036358512938022614, 0.02487620897591114, -0.018862877041101456, -0.022838005796074867, 0.005462638568133116, 0.002595227910205722, -0.012849543243646622, 0.00686785951256752, -0.013267312198877335, -0.0010024859802797437, -6.384221342159435e-05, -0.009570694528520107, -0.02005288377404213, -0.028509529307484627, -0.0026363718789070845, -0.0030224912334233522, -0.057879913598299026, 0.00783632230013609, -0.005737985949963331, 0.01272294670343399, -0.0015500157605856657, -0.015609347261488438, -0.004630266688764095, -0.004595452453941107, 0.03099082037806511, -0.00469039985910058, -0.010621445253491402, -0.010861978866159916, 0.01774882711470127, -0.0012200736673548818, -0.0058645824901759624, -0.02204044908285141, -0.031826358288526535, -0.0006507850484922528, -0.008519943803548813, -0.020356714725494385, -0.003946645651012659, 0.00783632230013609, -0.011513950303196907, -0.007532490883022547, 0.006063971668481827, -0.02820569835603237, -0.0069184978492558, -0.014102849178016186, -0.005067024379968643, -0.01928064599633217, -0.014254764653742313, -0.02572440728545189, 0.026104195043444633, 0.025369936600327492, 0.018255213275551796, 0.0019005297217518091, -0.006741262972354889, 0.03772575408220291, -0.013849656097590923, -0.021242890506982803, 0.0039498102851212025, -0.0131660345941782, 0.02139480598270893, 0.0019052771385759115, -0.03342147171497345, 0.005557585973292589, 0.027522075921297073, 0.013583803549408913, 0.019356602802872658, -0.008513613604009151, 0.004253642167896032, 8.436469215666875e-05, 0.008906062692403793, -0.005348701495677233, 0.015748603269457817, 0.0009850789792835712, -0.0033358174841850996, -0.002000224543735385, 0.04043491929769516, -0.03585212677717209, 0.03863724693655968, -0.03572553023695946, -0.010551817715168, 0.014913066290318966, 0.017394358292222023, -0.031142735853791237, -0.0071590314619243145, -0.034257009625434875, -0.004864470101892948, 0.002573073608800769, -0.017115844413638115, 0.008469305001199245, -0.009773248806595802, -0.02645866572856903, 0.004703059326857328, 0.02957293950021267, -0.029395705088973045, 0.016470203176140785, -0.010830329731106758, -0.017153823748230934, -0.007741375360637903, -0.011596238240599632, -0.00638995785266161, 0.00784265249967575, 0.01359646301716566, 0.026180153712630272, -0.003253529779613018, 0.017204463481903076, -0.03301636502146721, 0.007874301634728909, -0.01484976802021265, 0.009203564375638962, -0.0237874798476696, 0.029623577371239662, 0.011450652033090591, 0.008804786019027233, 0.033320195972919464, -0.014052210375666618, 0.018786918371915817, -0.0031522526405751705, -0.005940540228039026, -0.005636708810925484, -0.004737873561680317, 0.030940182507038116, 0.0024623016361147165, 0.008355367928743362, 0.011874750256538391, -0.013406568206846714, 0.02504078485071659, 0.006038652267307043, -0.013976252637803555, 0.01277991570532322, 0.018052659928798676, -0.02228098176419735, -0.01616637222468853, -0.024483760818839073, -0.020128842443227768, -0.006238041911274195, 0.03734596446156502, 0.017115844413638115, 0.009596013464033604, -0.006829880643635988, -0.012514063157141209, 0.00405108742415905, 0.019508518278598785, -0.005013220943510532, -0.023002581670880318, 0.0012145350920036435, 0.013444546610116959, -0.013912954367697239, -0.007253978867083788, -0.019837670028209686, 0.01221656147390604, -0.00036633858690038323, -0.011311396025121212, -0.035269781947135925, 0.011267087422311306, 0.018938833847641945, 0.026635902002453804, 0.005079684313386679, 0.010254315100610256, -0.022091086953878403, 0.004753698129206896, -0.0016916455933824182, 0.006402617320418358, 0.003617494599893689, 0.01725510135293007, -2.6011623049271293e-05, 0.01791340298950672, 0.012311508879065514, 0.008874413557350636, 0.016153711825609207, -0.03825745731592178, -0.004630266688764095, 0.00895670149475336, -0.0032218806445598602, -0.00824776105582714, -0.6295390129089355, -0.00553859630599618, 0.03288976848125458, 0.0029765998478978872, 0.0190780907869339, 0.02888931892812252, 0.012836883775889874, -0.020964378491044044, -0.019419901072978973, 0.014469978399574757, -0.014507957734167576, 0.005715831648558378, -0.019698413088917732, -0.013052097521722317, -0.012552041560411453, -0.005272743757814169, 0.007773024495691061, -0.006054476834833622, 0.01633094623684883, 0.011216448619961739, -0.015191578306257725, 0.018242554739117622, -0.029851451516151428, -0.007323606871068478, 0.006206392776221037, 0.00629817508161068, 0.0019290139898657799, -0.03840937465429306, -0.011431663297116756, 0.014343381859362125, -0.04301748797297478, 0.01993894763290882, 0.021939171478152275, 0.01843244954943657, 0.058842048048973083, 0.014229445718228817, -0.012874863110482693, 0.019533837214112282, 0.005794954020529985, 0.028585486114025116, 0.015090301632881165, -0.03802958503365517, 0.034788716584444046, -0.010551817715168, -0.009855536743998528, 0.006855200044810772, -0.01036825217306614, 0.018888195976614952, 0.0010444209910929203, -0.015457430854439735, 0.008026217110455036, -0.0029845121316611767, -0.027699312195181847, 0.006570357829332352, 0.02154672145843506, 0.037776391953229904, 0.02820569835603237, -0.014153487049043179, 0.05610756203532219, -0.001589577179402113, -0.02204044908285141, 0.012273529544472694, -0.011558258906006813, 0.0036776280030608177, -0.01981235109269619, -0.0005032210610806942, 0.009615003131330013, 0.010279634967446327, -0.003076294669881463, 0.021939171478152275, 0.0034086103551089764, 0.025838343426585197, -0.007608449086546898, -0.011267087422311306, -0.010969585739076138, 0.02325577475130558, 0.03205423057079315, -0.005747480783611536, -0.022268321365118027, -0.005196786019951105, -0.00041658157715573907, 0.012862203642725945, 0.009741599671542645, -0.008722498081624508, 0.026357388123869896, 0.01414082758128643, -0.013495185412466526, 0.013507844880223274, 0.027572715654969215, -0.013178694061934948, 0.031421247869729996, 0.01444465946406126, -0.01596381701529026, -0.015318174846470356, 0.006032322533428669, 0.008887073025107384, -0.02045799233019352, 0.02273673005402088, -0.0027645507361739874, -0.0032471998129040003, 0.005209445487707853, 0.02268609032034874, 0.005190455820411444, 0.002641119295731187, 0.01484976802021265, 0.006526048760861158, -0.014356041327118874, 0.020432673394680023, 0.0540313795208931, -0.04167556390166283, -0.008349038660526276, -0.02419258840382099, 0.016862651333212852, -0.003908666782081127, 0.023243114352226257, -0.016419565305113792, -0.0003004688478540629, -0.010292294435203075, 0.020850442349910736, -0.007640098221600056, 0.01124176848679781, -0.027775269001722336, 0.007538821082562208, -0.011146821081638336, -0.011640546843409538, 0.01714116521179676, 0.024091310799121857, -0.0059531996957957745, -0.020318737253546715, 0.010735382325947285, -0.008412336930632591, -0.021420124918222427, -0.00786164216697216, 0.007621108554303646, 0.013216673396527767, 0.020230118185281754, 0.02701568976044655, -0.009203564375638962, -0.0036744631361216307, 0.00010058486805064604, 0.011108841747045517, -0.022027788683772087, 0.002486038487404585, -0.02369886264204979, -0.008342708460986614, -0.00965931173413992, -0.004373908508569002, -0.006124105304479599, -0.02807910181581974, -0.004978406708687544, 0.007032434921711683, -0.0003825587627943605, 0.039320867508649826, 0.012615339830517769, -0.016153711825609207, -0.012868532910943031, 0.013090076856315136, -0.018280532211065292, -0.023546947166323662, 0.0014004736440256238, 0.026408027857542038, 0.0127419363707304, 0.004307445604354143, -0.00814648438245058, 0.006564028095453978, 0.007095733191817999, -0.0023135507944971323, 0.012324168346822262, -0.013039438053965569, -0.025218021124601364, 5.1330920541659e-05, 0.003810554277151823, -0.014735830947756767, 0.011400014162063599, -0.005722161382436752, 0.019799690693616867, -0.006614666432142258, 0.021812574937939644, -0.0077603645622730255, 0.008716167882084846, 0.023850778117775917, -0.005772799719125032, -0.015862539410591125, -0.014482637867331505, 0.03532041981816292, 0.005829768255352974, 0.014267424121499062, 0.032028909772634506, -0.055449262261390686, -0.002129985950887203, 0.023838119581341743, 0.022306300699710846, 0.025129402056336403, -0.016951270401477814, -0.0026711858808994293, -0.0035510314628481865, -0.003443424589931965, -0.007234989199787378, 0.02014150097966194, 0.018305853009223938, 0.01977437175810337, 0.00568101741373539, 0.04899284243583679, -0.01377369835972786, 0.015356154181063175, -0.0029876772314310074, 0.02997804805636406, -0.015077642165124416, 0.017596911638975143, 0.00895670149475336, 0.005190455820411444, -0.01892617531120777, 0.007266638334840536, 0.004104891326278448, -0.02883867919445038, 0.020799802616238594, -0.010153038427233696, 0.036282554268836975, 0.01612839289009571, 0.03957406058907509, -0.01247608382254839, -0.0035162174608558416, 0.019837670028209686, 0.028737403452396393, -0.03889044001698494, -0.0055544208735227585, 0.015470091253519058, -0.019293304532766342, -0.012254539877176285, -0.04856241121888161, -0.018533725291490555, -0.005984848830848932, -0.005630378611385822, 0.040257684886455536, 0.015571367926895618, 0.00729195773601532, 0.0022407579235732555, -0.019799690693616867, 0.021445445716381073, -0.011412673629820347, -0.00293703842908144, 0.005747480783611536, 0.031421247869729996, -0.020432673394680023, 0.03501658886671066, 0.02014150097966194, 0.032636575400829315, -0.0038770174141973257, -0.01811595819890499, 0.0021173262502998114, -0.012381136417388916, -0.00024231358838733286, -0.018951494246721268, 0.008437655866146088, 0.018090637400746346, -0.019179368391633034, 0.010741711594164371, 0.026230791583657265, 0.030054006725549698, 0.016343606635928154, 0.0118937399238348, -0.004155529662966728, 0.014368701726198196, -0.002857915824279189, 0.0001991916651604697, 0.026230791583657265, 0.0037472560070455074, -0.02528131939470768, 0.0008090306655503809, -0.013317951001226902, -0.02130618877708912, -0.02179991453886032, 0.021369487047195435, -0.0019353438401594758, 0.0266612209379673, -0.022875985130667686, -0.018128616735339165, 0.015115620568394661, 0.022065768018364906, 0.04291620850563049, -0.01628030836582184, -0.02426854707300663, 0.028939956799149513, -0.018343830481171608, 0.0037440911401063204, -0.010140378959476948, -0.010545487515628338, -0.013355929404497147, -0.009393459185957909, 0.0056905122473835945, -0.028129739686846733, -0.007722385693341494, -0.012526722624897957, 0.002376849064603448, -0.0007128964643925428, -0.030054006725549698, 0.03164912015199661, -0.0009439350687898695, 0.00532654719427228, -0.01148230116814375, -0.0028705752920359373, -0.005981684196740389, -0.04309344291687012, -0.033598706126213074, 0.03666234388947487, 0.0015064981998875737, -0.012305178679525852, 0.001987564843147993, 0.014723171480000019, -0.02402801252901554, -0.024002693593502045, -0.013520505279302597, -0.026914414018392563, 0.00010622236732160673, -0.01609041355550289, 0.0019511684076860547, -0.03397849574685097, 0.00895670149475336, 0.03050975315272808, -0.010501178912818432, -0.01195070892572403, -0.017318399623036385, -0.029345065355300903, -0.004256806802004576, 0.07737577706575394, -0.020192140713334084, -0.01333061046898365, 0.030231241136789322, -0.017407016828656197, -0.0039023368153721094, 0.007488182280212641, -0.052562862634658813, 0.023521628230810165, -0.011596238240599632, 0.000450999999884516, 0.03144656866788864, -0.003259859513491392, 0.00504487007856369, 0.03099082037806511, 0.017356378957629204, 0.025711746886372566, -0.0009542210027575493, -0.02312917821109295, 0.005617719143629074, -0.009121277369558811, -0.00899468082934618, 0.017647549510002136, 0.02349630743265152, 0.01948319934308529, 0.0072033400647342205, 0.002131568267941475, -0.000805865740403533, 0.00435491930693388, -0.0401817262172699, -0.03894107788801193, 0.00913393683731556, 0.014697852544486523, 0.0007742166635580361, 0.02754739671945572, 0.021458104252815247, 0.0037029471714049578, 0.015204237774014473, 0.0027075824327766895, 0.012558371759951115, 0.0249648280441761, 0.020356714725494385, -0.01592583768069744, -0.009678301401436329, 0.02896527573466301, -0.0058487579226493835, -0.02487620897591114, 0.032231464982032776, 0.0016679087420925498, -0.02021745964884758, -0.00883010495454073, -0.013077417388558388, -0.012444434687495232, -0.008317389525473118, 0.019660433754324913, -0.009045318700373173, -0.02143278531730175, 0.010868308134377003, 0.012089964933693409, -0.017812125384807587, -0.015495410189032555, 0.016482863575220108, -0.007323606871068478, 0.02115427330136299, 0.0025999753270298243, -0.026180153712630272, -0.007621108554303646, 0.002641119295731187, 0.013988912105560303, -0.014267424121499062, 0.014026890508830547, -0.05291733145713806, -0.006000673398375511, 0.0012960315216332674, 0.03139593079686165, 0.0010301789734512568, 0.011532939970493317, 0.0035510314628481865, 0.007095733191817999, 0.03873852640390396, 0.007823662832379341, -0.029421024024486542, -0.001797670149244368, -0.016596799716353416, 0.01644488424062729, 0.012488743290305138, 0.013140715658664703, 0.017837444320321083, -0.012931831181049347, 0.024331845343112946, 0.018862877041101456, 0.029598258435726166, 0.0017691858811303973, -0.00925420317798853, 0.008108505047857761, 0.009102287702262402, 0.024243228137493134, 0.016267647966742516, 0.0027075824327766895, -0.0131660345941782, 0.015735942870378494, -0.007747705094516277, -0.014913066290318966, 0.01766020990908146, 0.00454797875136137, -0.006412112154066563, -0.010380911640822887, -0.003220298094674945, -0.004092231392860413, -0.0023182982113212347, 0.029522301629185677, -0.02045799233019352, 0.0037630805745720863, -0.008963031694293022, 0.004060582257807255, 0.00568418251350522, 0.011995017528533936, 0.0012770420871675014, -0.00855159293860197, -0.012590020895004272, -0.005035375244915485, -0.040612153708934784, 0.011843101121485233, -0.00899468082934618, -0.01229884847998619, 0.02257215417921543, -0.005731656216084957, -0.033775944262742996, 0.029724854975938797, -0.0057348208501935005, 0.00026268771034665406, -0.005054364912211895, -0.020609907805919647, -0.0010270139900967479, -0.030459115281701088, -0.011988687328994274, 0.029826132580637932, 0.047549642622470856, -0.004430877044796944, -0.00644059618934989, -0.004247311968356371, 0.009355480782687664, 0.014356041327118874, -0.006741262972354889, 0.02204044908285141, -0.02130618877708912, 0.011887410655617714, -0.010488519445061684, -0.03043379634618759, -0.004247311968356371, 0.02828165516257286, 4.804237687494606e-05, -0.01166586671024561, -0.01065309438854456, -0.014115508645772934, -0.02861080691218376, -0.02638270892202854, -0.010102399624884129, 0.016381585970520973, 0.01811595819890499, 0.007937599904835224, 0.01852106675505638, 0.01579924114048481, 0.028003143146634102, 0.0013893964933231473, -0.010640434920787811, -0.002055610530078411, 0.010298624634742737, -0.03126933425664902, -0.013229332864284515, 0.01201400626450777, -0.012064645066857338, 0.006855200044810772, 0.007886961102485657, -0.019343944266438484, 0.01717914268374443, -0.009551704861223698, 0.0027787929866462946, 0.006025992799550295, -0.013052097521722317, -0.003411775454878807, -0.003946645651012659, 0.0005048034945502877, 0.002873740391805768, -0.016837332397699356, -0.00715270172804594, -0.021496083587408066, 0.013406568206846714, 0.032839130610227585, -0.031421247869729996, 0.02883867919445038, 0.005117663182318211, 0.030737627297639847, 0.015406792983412743, 0.0010871473932638764, -0.024698974564671516, -0.013267312198877335, -0.03286444768309593, 0.025066103786230087, 0.021584700793027878, 0.024319184944033623, -0.02268609032034874, -0.014925725758075714, -0.006741262972354889, -0.008216111920773983, 0.016951270401477814, -0.0001277833362109959, -0.01993894763290882, 0.007905950769782066, -0.018584365025162697, -0.0028041121549904346, -0.03894107788801193, -0.0047790175303816795, -0.011957038193941116, 0.004744203295558691, 0.017698189243674278, 0.02883867919445038, 0.008380687795579433, -0.010165697894990444, -0.031699761748313904, -0.006620996166020632, -0.0048011718317866325, 0.016153711825609207, 0.01217225193977356, 0.01417880691587925, 0.014558596536517143, -0.006589347030967474, -0.022103747352957726, -0.0254838727414608, -0.001913189422339201, 0.01022899616509676, 0.00786164216697216, 0.00926053337752819, 0.030408475548028946, -0.015584027394652367, -0.03316827863454819, -0.005845592822879553, -0.004367578774690628, -0.03342147171497345, -0.001505706924945116, 0.004307445604354143, 0.009323831647634506, 0.007659087423235178, 0.0030035015661269426, -0.029648898169398308, 0.03205423057079315, -0.023470988497138023, 0.006943817250430584, -0.015064981766045094, -0.018445108085870743, -0.039042357355356216, 0.008228771388530731, 0.00554176140576601, -0.002354694763198495, -0.008988350629806519, 0.010115059092640877, 0.0006974675343371928, -0.010279634967446327, 0.0015373560599982738, 0.011672195978462696, 0.0068045612424612045, -0.0009993209969252348, -0.022762048989534378, -0.006905838381499052, 0.008659199811518192, 0.009279522113502026, 0.017115844413638115, -0.01107719261199236, -0.014482637867331505, 0.02033139578998089, -0.03349743038415909, -0.025509191676974297, -0.0007390070240944624, 0.005918385926634073, 0.014368701726198196, 0.001440035062842071, -0.012241880409419537, -0.03286444768309593, -0.003987789154052734, -0.021698638796806335, 0.03762447461485863, 0.00828574039041996, -0.02535727620124817, -0.015166259370744228, 0.010222665965557098, 0.005108168348670006, -0.0037251017056405544, 0.005630378611385822, 0.02861080691218376, -0.005190455820411444, -0.0411691777408123, -0.011286077089607716, -0.006988126318901777, 0.012121614068746567, 0.01669807732105255, 0.003395950887352228, 0.013963592238724232, 0.02471163496375084, -0.021926511079072952, -0.03197827190160751, 0.02021745964884758, 0.023812798783183098, -0.0037472560070455074, 0.005978519096970558, -0.005750645417720079, -0.02292662486433983, 0.027446119114756584, 0.01077969092875719, 0.0008228771621361375, -0.0384853333234787, -0.0023341227788478136, -0.011906399391591549, 0.0004660333215724677, -0.01640690490603447, -0.0064880698919296265, 0.020875761285424232, -0.007747705094516277, -0.02500280551612377, 0.004421382211148739, -0.006829880643635988, -0.0016061929054558277, -0.003794729709625244, 0.002068270230665803, 0.007703396491706371, 0.010931606404483318, -0.008507284335792065, 0.018685640767216682, -0.008349038660526276, 0.01373571902513504, -0.03253529593348503, -0.0074058943428099155, 0.01592583768069744, 0.0016133140306919813, -0.010197347030043602, -0.018166596069931984, -0.01669807732105255, 0.001216117525473237, 0.01373571902513504, -0.021179592236876488, 0.003217133227735758, 0.0077603645622730255, 0.02645866572856903, 0.013444546610116959, 0.03666234388947487, -0.004522659350186586, 0.015495410189032555, -0.026990370824933052, -0.028939956799149513, -0.023597585037350655, 0.012039326131343842, 0.00447835074737668, 0.03557361289858818, 0.02455971948802471, -0.008057866245508194, -0.029421024024486542, -0.004585957620292902, -0.04873964935541153, -0.006785571575164795, -0.005563915707170963, 0.014507957734167576, 0.012627999298274517, 0.02828165516257286, 0.013355929404497147, 0.016343606635928154, -0.004449866712093353, 0.026180153712630272, -0.009792238473892212, -0.0350925475358963, 0.03220614790916443, -0.01047585904598236, -0.01105820294469595, -0.00357635086402297, -0.029876770451664925, 0.007083073724061251, 0.014292743057012558, -0.007690736558288336, -0.006193733308464289, 0.030332518741488457, -0.010013782419264317, -0.021572042256593704, -0.005465803202241659, 0.014685193076729774, -0.010627775453031063, -0.01572328433394432, -0.011520280502736568, -0.05122093856334686, 0.0022201859392225742, 0.012817894108593464, 0.0037251017056405544, -0.02480025216937065, 0.012792575173079967, 0.013355929404497147, 0.003275684081017971, 0.0033579717855900526, 0.003326322650536895, -0.022179704159498215, -0.006931157782673836, -0.0006764999707229435, 0.013723059557378292, 0.008450315333902836, -0.019419901072978973, 0.015951156616210938, -0.0019179368391633034, 0.0007117096101865172, -0.003864357713609934, 0.0029940069653093815, 0.00012570635590236634, 0.012431775219738483, 0.024660995230078697, -0.016103073954582214, 0.011229108087718487, -0.03390254080295563, -0.0031126912217587233, -0.017128504812717438, 0.02248353697359562, -0.016622118651866913, -0.014026890508830547, 0.0033168280497193336, 0.03704213351011276, 0.012609010562300682, 0.020483311265707016, 0.013001459650695324, 0.00407324219122529, -0.031092097982764244, -0.0048454804345965385, -0.0035573614295572042, 0.030484434217214584, -0.02876272238790989, -0.022344280034303665, -0.016723396256566048, -0.006766582373529673, -0.02317981608211994, 0.009374469518661499, 0.010330273769795895, -0.0006337736849673092, -0.00867185927927494, 0.20336459577083588, -0.019622456282377243, -0.014811788685619831, 0.018052659928798676, -0.014571256004273891, 0.01624232903122902, 0.01049484871327877, 0.015913179144263268, -0.016558820381760597, -0.006658975500613451, -0.020128842443227768, 0.007969249039888382, -0.009425108321011066, 0.0005340789211913943, -0.0053075579926371574, -0.02719292603433132, -0.020369375124573708, -0.03342147171497345, -0.0007943929522298276, 0.018280532211065292, 0.0029765998478978872, -0.009868196211755276, -0.0173816978931427, -0.004111221060156822, 0.00693748751655221, 0.008937711827456951, 0.0016742385923862457, 0.0327378511428833, 0.015457430854439735, 0.014824449084699154, -0.018445108085870743, -0.00629817508161068, 0.0049530877731740475, 0.019521178677678108, -0.0020018068607896566, -0.003304168349131942, -0.006146259605884552, -0.008051536977291107, 0.005086014047265053, 0.016963928937911987, -0.012507732957601547, 0.042435143142938614, -0.006478575058281422, -0.028509529307484627, -0.004915108438581228, -0.01636892557144165, 0.011089852079749107, -0.020280757918953896, 0.002009719144552946, 0.0048676347360014915, -0.026559943333268166, -0.020673206076025963, -0.010096070356667042, -0.012912841513752937, -0.019799690693616867, -0.033041682094335556, 0.01774882711470127, 0.017077866941690445, -0.017596911638975143, 0.024686316028237343, 0.003310498083010316, 0.015887858346104622, -0.020888419821858406, 0.02754739671945572, -0.03342147171497345, 0.01839447021484375, 0.0003018534916918725, -0.00589623162522912, 0.008716167882084846, 0.009722610004246235, -0.017558932304382324, -0.003487733192741871, -0.011121501214802265, 0.0033801263198256493, -0.0053803506307303905, -0.043169401586055756, 0.029243789613246918, 0.04258705675601959, 0.033649347722530365, 0.030737627297639847, 0.0012271946761757135, 0.00909595750272274, -0.006481740158051252, -0.011064533144235611, 0.0006068719085305929, -0.012077304534614086, 0.02828165516257286, 0.002174294786527753, -0.024926848709583282, -0.019305964931845665, 0.011697515845298767, -0.030408475548028946, -0.01802733913064003, -0.00483282096683979, -0.015900518745183945, 0.0062475367449223995, -0.004158694762736559, -0.0018752104369923472, -0.02034405618906021, 8.861754031386226e-05, -0.007773024495691061, 0.026483984664082527, 0.013862315565347672, 0.012248210608959198, 0.006196897942572832, 0.004158694762736559, 0.030560392886400223, 0.009374469518661499, 0.017596911638975143, 0.008747817017138004, -0.019749052822589874, -0.041067901998758316, 0.012665978632867336, 0.0043106102384626865, 0.02373684197664261, 0.04775219410657883, -0.00939978938549757, -0.020837781950831413, -0.013621781952679157, 0.014748490415513515, 0.00230880337767303, -0.022053107619285583, 0.0010499595664441586, 0.0007568096043542027, 0.013039438053965569, -0.006823550909757614, 0.0043802387081086636, 0.009709950536489487, -0.03172507882118225, -0.052613500505685806, -0.0150143438950181, 0.003294673515483737, 0.018483087420463562, -0.006500729825347662, 0.0038833473809063435, 0.036358512938022614, 0.03846001252532005, -0.049321990460157394, -0.004152365028858185, 0.017609572038054466, 0.01831851154565811, -0.01960979588329792, 0.0283576138317585, 0.00147405790630728, 0.015368813648819923, -0.006741262972354889, -0.0007896455936133862, -0.007234989199787378, -0.023812798783183098, -0.03400381654500961, -0.009355480782687664, -0.006614666432142258, -0.0071590314619243145, -0.00276929815299809, 0.0016473367577418685, 0.008399676531553268, -0.021774595603346825, -0.02459769695997238, 0.00454797875136137, 0.025344617664813995, -0.004984736908227205, -0.020078202709555626, 0.0347633957862854, -0.013216673396527767, -0.009893515147268772, 0.001375154359266162, -0.15616942942142487, 0.026762496680021286, 0.012381136417388916, -0.007279298268258572, 0.011330385692417622, 0.002025543712079525, 0.03157316520810127, -0.006703284103423357, -0.030155284330248833, -0.004684070125222206, -0.017153823748230934, 0.008621220476925373, -0.030940182507038116, -0.006564028095453978, 0.008842764422297478, -0.007494512014091015, -0.004582792986184359, 0.047423046082258224, 0.028914637863636017, -0.014533276669681072, 0.03375062346458435, 0.01795138232409954, 0.033117640763521194, 0.007171690929681063, 0.00622854707762599, -0.006114610470831394, -0.017407016828656197, 0.007779354229569435, -0.02390141785144806, -0.026154834777116776, 0.018685640767216682, 0.0008003271650522947, 0.02228098176419735, -0.028129739686846733, 0.008982020430266857, -0.022951943799853325, -0.006880518980324268, 0.011551929637789726, 0.0016789858927950263, 0.023344391956925392, 0.010096070356667042, 0.009899845346808434, -0.0066020069643855095, 0.005367691162973642, 0.0021901193540543318, 0.03590276464819908, -0.00994415394961834, -0.005174631252884865, 0.01458391547203064, -0.005570245441049337, 0.0035162174608558416, -0.010134048759937286, 0.007304617203772068, -0.0039498102851212025, -0.014735830947756767, 0.016748715192079544, -0.0008410754380747676, 0.004858140368014574, -0.029471661895513535, -0.011735494248569012, 0.0024717964697629213, -0.017331060022115707, 0.007766694761812687, -0.01644488424062729, -0.04565069451928139, -0.015001683495938778, 0.013115395791828632, 0.0204200129956007, -0.038181502372026443, -0.007779354229569435, -0.017672870308160782, 0.0011306648375466466, -0.005462638568133116, -0.015191578306257725, 0.010184687562286854, 0.004532154183834791, -0.029142512008547783, -0.004240982234477997, 0.01657148078083992, 0.009355480782687664, 0.012121614068746567, 0.010608785785734653, -0.018381809815764427, 0.029395705088973045, -0.000304820598103106, 0.016963928937911987, -0.017065206542611122, 0.016179030761122704, -0.02787654660642147, 0.007469192612916231, 0.02106565609574318, 0.015242217108607292, 0.00638995785266161, -0.00644059618934989, 0.011311396025121212, 0.011381024494767189, 0.04706857353448868, 0.01049484871327877, -0.0018293192842975259, -0.01960979588329792, 0.031699761748313904, 0.0068425401113927364, -0.010197347030043602, 0.016596799716353416, 0.03706745058298111, 0.0032060560770332813, 0.019533837214112282, -0.01095692627131939, 0.012564701028168201, 0.0030715472530573606, -0.03856129199266434, -0.008937711827456951, 0.01936926320195198, 0.01749563403427601, -0.03759915754199028, 0.0033896209206432104, -0.0006824341835454106, -0.002680680714547634, -0.01105820294469595, -0.009488406591117382, 0.036358512938022614, 0.014799129217863083, -0.004003613721579313, 0.002802529837936163, -0.020065544173121452, 0.011374694295227528, -0.08963031321763992, -0.03167444095015526, 0.03683957830071449, 0.0007445455994457006, -0.0017691858811303973, 0.00926053337752819, -0.013014119118452072, 0.0024116630665957928, -0.010659424588084221, 0.026889093220233917, -0.0018562210025265813, -0.023243114352226257, 0.010342933237552643, -0.009652982465922832, 0.00939978938549757, 0.008716167882084846, 0.009906175546348095, -0.013279971666634083, 0.010874638333916664, 0.02474961429834366, -0.009741599671542645, 0.012007676996290684, -0.00798190850764513, -0.004146034829318523, -0.0202554389834404, -0.009735269472002983, -0.017710847780108452, 0.0009368140017613769, 0.008412336930632591, 0.014191466383635998, -0.005956364795565605, 0.006988126318901777, -0.009121277369558811, -0.018255213275551796, 0.0031016140710562468, 0.002277154242619872, -0.0013435052242130041, 0.005883571691811085, 0.008317389525473118, -0.021812574937939644, -0.0058867367915809155, -0.018748939037322998, 0.005348701495677233, -0.04557473585009575, -0.007735045626759529, -0.037573836743831635, 0.0020302911289036274, 0.029243789613246918, 0.008159143850207329, -0.006076631601899862, -0.0006891596131026745, 0.006817220710217953, -0.02949698083102703, -0.010583466850221157, -0.011165809817612171, -0.020635226741433144, -0.006975466385483742, -0.015976477414369583, -0.021255550906062126, -0.0004355710407253355, -0.010159368626773357, -0.03916895389556885, -0.03491531312465668, 0.004250477068126202, 0.0030446455348283052, 0.030054006725549698, -0.0006713569746352732, 0.0008027008734643459, 0.01417880691587925, -0.002747143851593137, -0.04666346684098244, -0.020711185410618782, -0.02268609032034874, 0.036282554268836975, -0.026078876107931137, 0.004712554160505533, 0.000942352635320276, 0.014254764653742313, 0.020508630201220512, 0.019217347726225853, -0.011191129684448242, -0.013799017295241356, 0.007640098221600056, -0.02471163496375084, 0.0025366770569235086, -0.008785796351730824, 0.004323270171880722, -0.001440035062842071, -0.007247649133205414, -0.019711073487997055, -0.0011955455411225557, 0.04529622197151184, -0.0019495858578011394, 0.0015096630668267608, -0.011032884009182453, -0.0076021188870072365, -0.0127419363707304, 0.00840600673109293, 0.028104420751333237, -0.0011472806800156832, -0.027420800179243088, -0.0068045612424612045, -0.07094467431306839, 0.002125238534063101, 0.014558596536517143, 0.011501290835440159, -0.0004193508648313582, -0.010836658999323845, -0.002106249099597335, -0.023445669561624527, 9.079341543838382e-05, -0.0015682140365242958, -0.03529509902000427, -0.00021877455583307892, -0.01762223057448864, -0.0035541963297873735, 0.009165585972368717, 0.017837444320321083, 0.017774146050214767, -0.010545487515628338, -0.008279410190880299, 0.028104420751333237, -0.003169659525156021, -0.018533725291490555, 0.0003473491233307868, 0.013685080222785473, 0.0038010594435036182, 0.0317503996193409, -0.018014680594205856, 0.028053781017661095, -0.0007718429551459849, -0.03889044001698494, 0.01148863136768341, -0.00895670149475336, -0.023154497146606445, 0.011526609770953655, -0.00838701706379652, -0.0049973963759839535, 0.0008236683788709342, 0.023635564371943474, 0.015659986063838005, 0.0212682094424963, -0.008235101588070393, -0.008583242073655128, 0.03192763403058052, -0.011229108087718487, 0.0007113140309229493, -0.01512828003615141, -0.020394694060087204, 0.01721712201833725, 0.004661915823817253, -0.011735494248569012, 0.021318849176168442, 0.005620884243398905, -0.035345740616321564, -0.02544589340686798, 0.00798190850764513, -0.034940630197525024, 0.007393234875053167, -0.007741375360637903, -0.009273192845284939, -0.014875086955726147, 0.025749726220965385, 0.019470538944005966, 0.043042805045843124, 0.015115620568394661, 0.020470652729272842, 0.03772575408220291, -0.03281380981206894, -0.0028911472763866186, 0.004453031346201897, -0.015356154181063175, 0.02091374062001705, -0.012387466616928577, 0.016596799716353416, 0.012393795885145664, 0.002383178798481822, 0.011279746890068054, -0.0029940069653093815, 0.01891351491212845, -0.008925052359700203, 0.016520841047167778, 0.0043960632756352425, 0.0010705315507948399, -0.02673717774450779, -0.0008125912281684577, 0.0027582210022956133, -0.007811003364622593, -0.022027788683772087, -0.01924266666173935, 0.0016046104719862342, 0.016153711825609207, -0.022027788683772087, 0.0075135016813874245, -0.003595340298488736, 0.01693861000239849, -0.0012382719432935119, -0.008070525713264942, 0.019913626834750175, -0.01515359990298748, -0.008007227443158627, 0.028863999992609024, 0.0024195753503590822, -0.010526497848331928, -0.004896119236946106, -0.020027564838528633, -0.028053781017661095, 0.013128056190907955, 0.007285628002136946, -0.027800587937235832, -0.013849656097590923, -0.00011077192903030664, -0.011577248573303223, 0.003060470102354884, 0.021951831877231598, -0.001916354289278388, -0.0212682094424963, 0.004361249040812254, 0.0018340665847063065, -0.010444209910929203, -0.025775045156478882, 0.012039326131343842, -0.02009086310863495, 0.010273304767906666, 0.006855200044810772, -0.03147188574075699, 0.040612153708934784, 0.0015326087595894933, 0.013254652731120586, -0.016103073954582214, 0.005918385926634073, -0.033446792513132095, -0.009298511780798435, -0.015267536044120789, -0.017596911638975143, 0.004595452453941107, -0.012950820848345757, 0.012988799251616001, 0.0037282665725797415, 0.04433409124612808, -0.003975129686295986, 0.07474256306886673, 0.020926399156451225, 0.002099919132888317, -0.0035162174608558416, 0.0029908420983701944, 0.03787766769528389, 0.002880070125684142, 0.029142512008547783, -0.005149312317371368, -0.035750847309827805, -0.009146596305072308, 0.025661109015345573, 0.010456870310008526, -0.015280196443200111, -0.003826378844678402, 0.011336715891957283, 0.011539270170032978, -0.021723957732319832, -0.02726888284087181, 0.03357338905334473, 0.02338237129151821, 0.00554176140576601, 0.010134048759937286, -0.010001122951507568, 0.009532715193927288, -0.015584027394652367, 0.01900213211774826, 0.003772575408220291, -0.030484434217214584, -0.029395705088973045, 0.026914414018392563, -0.0034086103551089764, -0.030231241136789322, -0.0333961546421051, 0.017888084053993225, -0.00034576666075736284, -0.021888533607125282, -0.0009004175080917776, 0.019305964931845665, 0.0190780907869339, 0.014001571573317051, 0.016520841047167778, 0.007811003364622593, -0.036358512938022614, 0.009779579006135464, -0.02447110041975975, 0.002164799952879548, -0.014723171480000019, 0.019597135484218597]} +{"id": "test:10000044", "text": "\"Stitch can replicate data from all your sources (including Shippo) to a central warehouse. From there, it's easy to use Slemma to perform the in-depth analysis you need.\\nIntegrate Shippo and Slemma to turn your data into actionable insights.\"", "vector_field": [-0.007479815278202295, -0.0037158015184104443, -0.018155425786972046, -0.02173692174255848, -0.022866470739245415, 0.023458795621991158, -0.025263316929340363, 0.0008333864388987422, 0.00016691404744051397, -0.02834891341626644, -0.0008673933916725218, 0.025593917816877365, 0.011061310768127441, -0.0031165897380560637, 0.020703798159956932, -0.010255474597215652, 0.01900947466492653, -0.010489649139344692, -0.0029891710728406906, -0.021915996447205544, 0.012576558627188206, -0.024850068613886833, 0.0053481366485357285, -0.012376821599900723, -0.007610677741467953, 0.011929134838283062, 0.025290867313742638, -0.018733976408839226, -0.016102954745292664, -0.00874711386859417, 0.013933394104242325, -0.013223983347415924, -0.012569671496748924, -0.03325280547142029, -0.01848602667450905, -0.012280396185815334, 0.01796257682144642, -0.023334819823503494, -0.002088631736114621, -0.00794127769768238, -0.006350266747176647, 0.0035005672834813595, 0.0015910104848444462, -0.024629667401313782, -0.021571621298789978, 0.0076382276602089405, -0.004383888095617294, -0.0057235052809119225, -0.009539174847304821, -0.01698455400764942, 0.0041910381987690926, 0.022935345768928528, -0.0347956046462059, 0.014284656383097172, 0.012066883966326714, 0.009931761771440506, -0.0017941914265975356, -0.006649872753769159, 0.0036641452461481094, -0.0011932578636333346, 0.014684131368994713, 0.0037536825984716415, -0.01272808387875557, -0.021406322717666626, -0.009594274684786797, -0.031985510140657425, -0.022095071151852608, -0.00023912508913781494, -0.0020094255451112986, 0.007810414768755436, 0.003454076824709773, 0.03085595928132534, 0.0003153179422952235, 0.004483756609261036, 0.00754180271178484, 0.0023296938743442297, 0.010620511136949062, 0.0034127519465982914, -0.01760442741215229, 0.010441436432301998, 0.012383708730340004, -0.0034712955821305513, -0.008388964459300041, 0.008802213706076145, 0.028114737942814827, 0.005778605118393898, 0.006580997724086046, 0.020524723455309868, 0.0022763158194720745, 0.015097380615770817, 0.010944223031401634, 0.0033387113362550735, -0.004242694471031427, 0.01516625564545393, -0.015097380615770817, 0.015744805335998535, 0.006315829232335091, 0.029010113328695297, 0.014367306604981422, -0.04201369732618332, -0.005303367972373962, -0.005069193430244923, -0.030718211084604263, -0.010882236063480377, -0.02414754405617714, 0.0036882515996694565, 0.008581814356148243, -0.013761207461357117, 0.027729040011763573, 0.022287921980023384, -0.01095111109316349, 0.03898319974541664, 0.006036886014044285, -0.04171064496040344, 0.0001613179629202932, -0.011109523475170135, 0.016943227499723434, -0.013774982653558254, -0.029368262737989426, -0.006343379151076078, 0.017315153032541275, -0.0059714550152421, 0.01150211039930582, -0.002043863059952855, 0.019739549607038498, 0.06314451992511749, -0.013134445995092392, -0.008933075703680515, -0.0013189545134082437, 0.004717931617051363, 0.01139879785478115, 0.008891751058399677, 0.025001592934131622, 0.01519380509853363, -0.030304960906505585, 0.013444382697343826, -0.012569671496748924, 0.010028187185525894, 0.008017039857804775, -0.016419779509305954, 0.021612947806715965, -0.0019956505857408047, -0.0054170116782188416, -0.0033438769169151783, -0.007879289798438549, 0.014560156501829624, 0.021185923367738724, -0.024794967845082283, -0.0032767239026725292, 0.0016960446955636144, 0.018210526555776596, -0.04317079484462738, -0.03259160742163658, 0.016846803948283195, 0.0013757763663306832, -0.006997691001743078, 0.034602753818035126, 0.01294159609824419, 0.0026258560828864574, -0.003891432425007224, -0.020703798159956932, 0.0003945241041947156, -0.008299427106976509, 0.004487200640141964, 0.008017039857804775, 0.03463030606508255, 0.0005759664345532656, -0.020937973633408546, -0.0184309259057045, 0.009057050570845604, -0.001983597408980131, 0.023017995059490204, -0.026806116104125977, 0.019918624311685562, -0.005389461759477854, 0.014339756220579147, -0.011529659852385521, 0.017948802560567856, -0.03603535518050194, -0.04344629496335983, 0.011515885591506958, 0.013320407830178738, 0.007459152955561876, -0.0033679830376058817, -0.012610996142029762, -0.019105900079011917, -0.00887797586619854, -0.013947169296443462, 0.020083924755454063, -0.007865514606237411, 0.011240385472774506, -0.0031355302780866623, 0.0008097106474451721, -0.010517198592424393, -0.6510607600212097, -0.033500757068395615, 0.0024536687415093184, -0.011164623312652111, 0.015372879803180695, 0.01493208110332489, -0.006591328885406256, -0.006422585342079401, -0.00497621251270175, 0.018610000610351562, 0.016571303829550743, 0.012783183716237545, 0.0010658391984179616, -0.011577872559428215, 0.00045629628584720194, -0.015317779965698719, 0.002355522010475397, -0.027439763769507408, -0.004652500152587891, 0.030249860137701035, -0.023279719054698944, 0.0025363184977322817, 0.002680955920368433, 0.03388645499944687, 8.275751315522939e-05, 0.0057441676035523415, 0.021833347156643867, -0.019560474902391434, -0.0146428057923913, 0.013671670109033585, -0.0402504988014698, 0.021351221948862076, -0.004931443836539984, 0.00676007242873311, 0.04352894425392151, -0.0037089139223098755, -0.016777928918600082, 0.026310216635465622, 0.021778246387839317, 0.0184309259057045, -0.012321721762418747, -0.02194354683160782, 0.029147861525416374, 0.025070467963814735, 0.005403236951678991, -0.0008936519734561443, 0.013926506973803043, 0.015731029212474823, -0.008189226500689983, -0.010255474597215652, 0.0009866331238299608, -0.01906457543373108, 0.013843856751918793, 0.02037319913506508, 0.01349259540438652, 0.0099799744784832, 0.0478818379342556, -0.017879927530884743, 0.011536547914147377, -0.018692651763558388, 0.02089664712548256, 0.0159238800406456, -0.006773847620934248, -0.016226928681135178, -0.021654272451996803, 0.021599171683192253, -0.012108209542930126, -0.021516522392630577, 0.01404359471052885, -0.0184309259057045, 0.02969886176288128, 0.015111155807971954, 0.0045423004776239395, 0.02586941607296467, -0.016254479065537453, 0.032564058899879456, 0.0038811012636870146, 0.017563102766871452, 0.013602795079350471, -0.010055736638605595, -0.003061489900574088, -0.01588255539536476, -0.017273828387260437, -0.035704754292964935, 0.02367919497191906, 0.02074512280523777, -0.03300485759973526, -0.031875308603048325, 0.006043773610144854, 0.009855999611318111, 0.005007205996662378, 0.0318477600812912, -0.0013800810556858778, -0.055926427245140076, -0.006240067072212696, 0.012383708730340004, -0.005844036117196083, 0.013898957520723343, 0.008299427106976509, -0.03267425671219826, -0.021778246387839317, -0.0014584262389689684, 0.02812851406633854, 0.007720877416431904, 0.022136395797133446, 0.01095111109316349, -0.006257285829633474, 0.022811369970440865, 0.022095071151852608, -0.02095174789428711, 0.006405366584658623, -0.01849980093538761, -0.024230193346738815, -0.011508997529745102, -0.011199060827493668, -0.03179265931248665, 0.013100008480250835, -0.020676247775554657, 0.015620829537510872, -0.01870642602443695, 0.010131499730050564, -0.00653622904792428, 0.013919619843363762, -0.015000955201685429, -0.004042957443743944, -0.000146574413520284, 0.0057648299261927605, -0.010861573740839958, 0.010138386860489845, -0.005795823875814676, 0.0214338731020689, -0.019243650138378143, 0.013396169990301132, -0.012734971009194851, 0.01519380509853363, -0.003436858067288995, 0.013527032919228077, -0.016681503504514694, 0.017315153032541275, -0.039313800632953644, 0.010482761077582836, 0.0041462695226073265, 0.011860259808599949, -0.031296759843826294, -0.003147583454847336, -0.038322001695632935, -0.0021867784671485424, -0.02106194756925106, 0.0006401062128134072, -0.03033251129090786, -0.01516625564545393, -0.025029143318533897, -0.02964376099407673, 0.03620065376162529, -0.037743452936410904, -0.013761207461357117, -0.01283828355371952, -0.030084561556577682, -0.012783183716237545, -0.03851484879851341, -0.00204902864061296, 0.034134406596422195, -0.01984974928200245, 0.019505374133586884, -0.01175694726407528, -0.02535974234342575, -0.0008979566628113389, -0.007934389635920525, -0.029533561319112778, -0.02027677372097969, 0.014284656383097172, -0.021626722067594528, -0.014780555851757526, 0.0055375429801642895, 0.005926685873419046, 0.03366605564951897, -0.024230193346738815, -0.010200374759733677, 0.014629031531512737, -0.029588662087917328, -0.007789752446115017, 0.011302373372018337, -0.015469305217266083, -0.0018596225418150425, 0.02420264296233654, 0.0017855820478871465, 0.04746858775615692, -0.00742471544072032, -0.02022167481482029, 0.009201687760651112, -0.0055823116563260555, 0.011591647751629353, 0.011453897692263126, 0.007355840411037207, -0.0015040559228509665, 0.014057368971407413, -0.031159009784460068, 0.023348594084382057, 0.008099689148366451, -0.003409308148548007, 0.016158053651452065, 0.008354526944458485, -0.02435416914522648, -0.013189545832574368, 0.014022931456565857, -0.016254479065537453, 0.004469981882721186, -0.027481090277433395, 0.017576877027750015, 0.009070825763046741, 0.011323035694658756, -0.021723147481679916, 0.0063261603936553, -0.005286149214953184, 0.02405111864209175, 0.01006262470036745, -0.002909964881837368, 0.005830261390656233, 0.003774345153942704, 0.009318775497376919, 0.0019095569150522351, -0.010985548608005047, 0.023665418848395348, 0.002799765206873417, -0.01438108179718256, 0.0010589517187327147, 0.0015135261928662658, 0.013898957520723343, -0.0015514073893427849, -0.018981926143169403, 0.01661262847483158, 0.019037025049328804, 0.01472545601427555, 0.00987666193395853, 0.04554009065032005, -0.020442074164748192, 0.018210526555776596, -0.009256787598133087, 0.030112111940979958, -0.016461104154586792, 0.0051862807013094425, 0.03080086037516594, 0.02620001696050167, -0.014821880497038364, 0.03581495210528374, 0.0005884499987587333, 0.04416259378194809, -0.01516625564545393, -0.004511306528002024, -0.005251712165772915, -0.02629644051194191, 0.005313699599355459, -0.012025559321045876, 0.0058337049558758736, 0.006060992367565632, -0.015496854670345783, -0.019946174696087837, 0.015731029212474823, 0.017631977796554565, 0.01607540436089039, 0.004063619766384363, 0.01943649910390377, -0.005757942795753479, 0.017852377146482468, 0.010475873947143555, -0.013816307298839092, -0.019216099753975868, -0.002806652570143342, -0.018610000610351562, 0.015414205379784107, 0.006126423366367817, 0.017094753682613373, -0.006904709618538618, -0.014463731087744236, 0.0009702753159217536, -0.013444382697343826, 0.021488972008228302, 0.004456206690520048, 0.008264989592134953, 0.020827773958444595, -0.011447010561823845, -0.03283955901861191, 0.03176511079072952, 0.0002957316464744508, -0.011350585147738457, 0.0007468623225577176, -0.021075723692774773, 0.010723823681473732, -0.01891305111348629, 0.03270180895924568, 0.015648379921913147, 0.020001273602247238, 0.0003570733533706516, -0.0014007434947416186, -0.0015359105309471488, 0.016309579834342003, 0.029781511053442955, -0.03452010452747345, 0.014394856058061123, -0.010324349626898766, -0.004962437320500612, 0.012211522087454796, -0.008685125969350338, -0.042729996144771576, 0.03746795281767845, 0.002569034229964018, -0.02964376099407673, -0.008141014724969864, 0.0058095986023545265, -0.01698455400764942, -0.003750238800421357, -0.02509801834821701, -0.01487698033452034, -0.009966199286282063, 0.01734270341694355, 0.0004782501782756299, -0.002252209698781371, 0.0106618357822299, 0.01932629942893982, 0.008203001692891121, 0.002577643608674407, -0.04143514484167099, -0.015731029212474823, 0.024547018110752106, 0.04909403622150421, 0.012590333819389343, -0.008471613749861717, 0.01487698033452034, -0.006343379151076078, -0.014119356870651245, -0.004669718910008669, -0.03540170565247536, 0.016543753445148468, -0.014491281472146511, 0.0008002403774298728, -0.004476869013160467, 0.025277093052864075, 0.009001950733363628, 0.0006603382062166929, -0.007645115256309509, -0.01932629942893982, -0.028266264125704765, 0.007748427335172892, -0.021227248013019562, -0.011488335207104683, -0.012790070846676826, -0.010276136919856071, 0.013809419237077236, 0.021984871476888657, 0.00432190066203475, 0.020152799785137177, 0.026392865926027298, 0.010785810649394989, -0.005516880191862583, -0.028955012559890747, 0.013065570965409279, -0.016433553770184517, 0.032922208309173584, -0.04598088935017586, 0.017246278002858162, -0.004211700987070799, 0.0022074407897889614, -0.009518512524664402, 0.014284656383097172, -0.01577235385775566, 0.011274822987616062, 0.005506549030542374, 0.018114101141691208, -0.0034954017028212547, -0.005382574163377285, -0.002856586826965213, 0.016998328268527985, -0.011956684291362762, -0.03537415340542793, 0.0030546023044735193, -0.0030718210618942976, -0.025442391633987427, 0.009353213012218475, 0.015372879803180695, 0.028266264125704765, 0.017287602648139, -0.005475555546581745, -0.008671351708471775, -0.02840401418507099, -0.015841228887438774, 0.015469305217266083, 0.00016142557433340698, -0.011991121806204319, -0.013706107623875141, -0.008485388942062855, -0.014477506279945374, 0.011805159971117973, -0.03190285712480545, 0.0025879747699946165, -0.011240385472774506, -0.015097380615770817, -0.015455530025064945, -0.001128687639720738, 0.009559838101267815, 0.01354080718010664, 0.01713607832789421, -0.00631238566711545, -0.024023568257689476, 0.0023813501466065645, -0.0034919579047709703, -0.013100008480250835, -0.0035332830157130957, -0.030415160581469536, -0.009422088041901588, -0.007934389635920525, -0.041986145079135895, -0.012321721762418747, 0.0019095569150522351, 0.016791703179478645, 0.03385890647768974, 0.010682499036192894, -0.009821562096476555, -0.015248904936015606, -0.0009909378131851554, 0.02754996530711651, 0.035181306302547455, 0.009029501117765903, 0.009249900467693806, -0.027949439361691475, 0.018844176083803177, -0.025469942018389702, -0.0024037344846874475, 0.013244645670056343, 0.02005637437105179, -0.031875308603048325, -0.00566840497776866, -0.006081654690206051, -0.003042549127712846, 0.0007313654641620815, 0.0078035276383161545, 0.0018010789062827826, 0.014188231900334358, 0.004831574857234955, 0.00631238566711545, 0.002286646980792284, -0.02037319913506508, 0.0056236363016068935, 0.0058130426332354546, 0.007672665175050497, -0.010110837407410145, -0.013485707342624664, 0.007500477600842714, 0.030360061675310135, -0.02870706282556057, -0.007052790839225054, 0.002314196899533272, 0.013251532800495625, -0.01516625564545393, 0.006002448499202728, 0.007879289798438549, 0.03429970517754555, -0.020882872864603996, -0.013885182328522205, -0.03322525694966316, 0.001985319424420595, -0.011378135532140732, 0.012163309380412102, -0.007583127822726965, -0.0068082851357758045, 0.016047853976488113, 0.012121984735131264, 0.007328290492296219, -0.03204060718417168, 0.004256469663232565, -0.016254479065537453, -0.01588255539536476, -0.00015776659711264074, 0.019147224724292755, 0.021144596859812737, 0.0146152563393116, 0.00720431562513113, -0.0014747839886695147, -0.013313520699739456, -0.003355930093675852, -0.020042598247528076, -0.01540043018758297, 0.00877466332167387, 0.02586941607296467, 0.04672474041581154, 0.02934071235358715, 0.02986416220664978, 0.005172505974769592, -0.002300421940162778, -0.022714946419000626, -0.019519150257110596, 0.011729397810995579, 0.001188092166557908, -0.0007020936463959515, 0.003030496183782816, 0.01608917862176895, 0.006918484810739756, 0.013651007786393166, 0.04008519649505615, 0.0005234493291936815, 0.03369360789656639, -0.005162174813449383, -0.0029082430992275476, -0.012225296348333359, -0.035126205533742905, -0.00872645154595375, 0.018637550994753838, 0.02011147327721119, 0.016777928918600082, -0.04468604177236557, 0.008836651220917702, 0.022618521004915237, -0.016736604273319244, 0.017191177234053612, 0.009814674966037273, 0.04344629496335983, -0.003705470124259591, 0.015111155807971954, 0.01692945323884487, 0.01472545601427555, -0.02063492313027382, 0.008154789917171001, -0.028899911791086197, 0.011047535575926304, 0.01608917862176895, 0.0023727407678961754, 0.002860030625015497, 0.0063674855045974255, 0.022191496565937996, 0.01375431939959526, 0.013719881884753704, -0.022852694615721703, -0.008065251633524895, 0.0037915639113634825, 0.0035746078938245773, -0.0086438013240695, -0.0254148431122303, -0.022136395797133446, 0.003095927182585001, -0.023872043937444687, -0.0012173641007393599, 0.005875030066817999, 0.01849980093538761, -0.030415160581469536, -0.021502748131752014, -0.011639859527349472, -0.01848602667450905, 0.03231610730290413, 0.01519380509853363, 0.023954693228006363, 0.03027741052210331, 0.03427215665578842, -0.02545616775751114, 0.007844852283596992, -0.005041643511503935, -0.011343698017299175, 0.011998009867966175, 0.019312525168061256, -0.01139191072434187, -0.036228202283382416, 0.023445019498467445, -0.0024330061860382557, -0.018692651763558388, -0.0425371453166008, 0.01155032217502594, 0.017480451613664627, 0.0006263311952352524, -0.0454849898815155, -0.015262680128216743, -0.012796958908438683, 0.018375826999545097, -0.022439446300268173, 0.02551126666367054, 0.006136754527688026, -0.018954375758767128, 0.006704972591251135, 0.01969822496175766, -0.010772036388516426, 0.02190222218632698, -0.002598305931314826, -0.0007782864850014448, 0.01042766124010086, -0.004132494796067476, -0.002004259964451194, -0.02016657404601574, 0.017466677352786064, 0.022976670414209366, -0.01770085282623768, 0.0009754409547895193, 0.019946174696087837, 0.034712955355644226, 0.0010598127264529467, 0.00420481339097023, -0.019574249163269997, 0.026447966694831848, -0.004473425447940826, -0.002984005492180586, -0.010758261196315289, -0.008692014031112194, 0.012528345920145512, -0.027467314153909683, -0.008368302136659622, -0.004993431270122528, -0.00026043326943181455, -0.0041256071999669075, 0.019216099753975868, 0.006539672613143921, 0.009008838795125484, 0.002279759617522359, 0.0025053247809410095, -0.0001719720457913354, 0.015786129981279373, 0.004239250905811787, 0.017521778121590614, -0.03143450990319252, -0.018059002235531807, 0.020717572420835495, -0.006264172960072756, -0.0008725590305402875, 0.006567222997546196, -0.011915359646081924, -0.0035367265809327364, 0.03724755346775055, -0.01775595173239708, 0.023389920592308044, 0.008347638882696629, 0.01870642602443695, -0.03523640334606171, 0.005671849008649588, 0.005113962106406689, -0.01666772924363613, 0.016433553770184517, -0.011178398504853249, -0.029368262737989426, 0.01220463402569294, 0.023458795621991158, 0.009022613056004047, -0.00877466332167387, -0.015386654995381832, 0.0005548734916374087, 0.018155425786972046, -0.008382076397538185, -0.017273828387260437, -0.011068197898566723, -0.006488016806542873, -0.013168882578611374, -0.007259415462613106, 0.009236125275492668, -0.018720200285315514, 0.02939581125974655, -0.008251214399933815, -0.004528525285422802, 0.018623776733875275, -0.0186513252556324, -0.023334819823503494, -0.0005897414521314204, 0.0005609000800177455, -0.0037605701945722103, 0.0018406820017844439, 0.004597400315105915, -0.027591289952397346, -0.004063619766384363, -0.0068048411048948765, -0.00807213969528675, -0.017618201673030853, -0.0008570621721446514, 0.01495963055640459, 0.029065212234854698, 0.02603471651673317, -0.00964248739182949, -0.025538817048072815, -0.03677920252084732, -0.012066883966326714, -0.022852694615721703, 0.005003762431442738, -0.015992755070328712, 0.02708161436021328, 0.013141333125531673, -0.03487825393676758, -0.029946811497211456, -0.0146428057923913, -0.01739780232310295, -0.01220463402569294, 0.0011519328691065311, -0.0022315471433103085, 0.017824826762080193, 0.03724755346775055, 0.020827773958444595, 0.035759855061769485, 0.0119429100304842, 0.005344693083316088, -0.014821880497038364, 0.0036434826906770468, -0.006291723344475031, -0.015042280778288841, 0.005792379844933748, 0.00984911248087883, -0.027425989508628845, -0.014119356870651245, 0.0026861215010285378, 0.00610231701284647, 0.02373429387807846, 0.01687435247004032, -0.014821880497038364, 0.0038879886269569397, 0.014257106930017471, 0.02545616775751114, 0.00036137804272584617, -0.011447010561823845, -0.01681925356388092, -0.020607372745871544, -0.005234493408352137, -0.00019618587975855917, -0.000527754018548876, -0.024271517992019653, 0.013451269827783108, -0.01917477510869503, 0.011880922131240368, -0.015180030837655067, -0.0013387560611590743, 0.014353531412780285, -0.020979298278689384, -0.005926685873419046, 0.010172824375331402, 0.008120352402329445, -0.0023503564298152924, 0.034602753818035126, -0.006178079638630152, -0.01954670064151287, -0.019505374133586884, 0.0238858200609684, -0.002190222265198827, -0.02347256988286972, -0.0017537274397909641, -0.02724691480398178, -0.00817545223981142, -0.007782864850014448, -0.019946174696087837, 0.018362050876021385, -0.014146906323730946, -0.030415160581469536, 0.0034471892286092043, -0.007066566031426191, 0.003075264859944582, 0.005206943489611149, -0.012555896304547787, -0.007348952814936638, -0.010172824375331402, -0.013148220255970955, -0.013382394798099995, -0.0034437456633895636, 0.002734333975240588, -0.019656900316476822, -0.012597220949828625, -0.025580141693353653, -0.006550004240125418, -0.03598025441169739, 0.0006203046650625765, 0.008864200673997402, -0.016309579834342003, 0.006842722650617361, 0.20915932953357697, 0.006298610474914312, -0.003023608587682247, 0.022701170295476913, 0.016378453001379967, -0.012301059439778328, 0.022880245000123978, 0.027370890602469444, -0.001202728133648634, -0.0008643801556900144, 0.008416513912379742, -0.004270244389772415, 0.011894697323441505, -0.001923331874422729, 0.029037661850452423, -0.02854176238179207, -0.01743912696838379, -0.009518512524664402, 0.0012870999053120613, -0.01349259540438652, 0.02854176238179207, 0.009552950039505959, -0.021020622923970222, -0.04165554791688919, 0.01228728424757719, 0.014284656383097172, 0.0021678379271179438, 0.00885042641311884, 0.006508679129183292, 0.0076588899828493595, -0.026268891990184784, 0.014215781353414059, 0.010703161358833313, -0.006188410799950361, 0.014009157195687294, -2.0595212845364586e-05, 0.009732024744153023, -0.010303686372935772, 0.03440990671515465, 0.007300740573555231, -0.003898320021107793, -0.024216419085860252, 0.0016202823026105762, 0.0028324807062745094, 0.004611175507307053, -0.016998328268527985, -0.010703161358833313, 0.027963213622570038, 0.010889123193919659, 0.025332191959023476, -0.02332104556262493, -0.010510311461985111, -0.0006323577836155891, 0.03581495210528374, -0.019078349694609642, -0.004573293961584568, -0.0009685534168966115, 0.00963560026139021, 0.0146428057923913, 0.02064869925379753, -0.007459152955561876, 0.01906457543373108, -0.007652002852410078, -0.0014971683267503977, -0.034492556005716324, 0.0019956505857408047, -0.013706107623875141, 0.0016220042016357183, 0.009683812037110329, -0.01655752770602703, 0.023872043937444687, -0.027963213622570038, -0.014780555851757526, -0.005816486198455095, -0.03102125972509384, -0.037633251398801804, 0.018100326880812645, -0.006240067072212696, 0.002799765206873417, 0.032453857362270355, 0.00019747728947550058, -0.007865514606237411, -0.022935345768928528, -0.01618560403585434, 0.014215781353414059, -0.009057050570845604, 0.02095174789428711, -0.02713671512901783, -0.012900270521640778, -0.008258101530373096, -0.011991121806204319, -0.0026964526623487473, -0.009077712893486023, -0.0294509120285511, -0.01891305111348629, -0.010696273297071457, 0.023073095828294754, 0.019574249163269997, -0.006336492020636797, -0.001916444394737482, -0.03038761019706726, 0.05498972907662392, -0.01635090447962284, 0.006267616990953684, -0.012252846732735634, 0.010517198592424393, -0.008795326575636864, -0.0011553766671568155, -0.02137877233326435, 0.011825822293758392, 0.004432100336998701, -0.0449339933693409, 0.016433553770184517, -0.006295166909694672, -0.005017537157982588, 0.03328035771846771, 0.024230193346738815, -0.0321783572435379, 0.015951428562402725, 0.009621825069189072, -0.005458336789160967, 0.01084779854863882, -0.00021501886658370495, -0.0010253752116113901, 0.004232363309711218, -0.01254212111234665, 0.008141014724969864, -0.0033094394020736217, -0.021075723692774773, -0.006515566725283861, -0.004580181557685137, 0.004280575551092625, 0.01152277272194624, -0.0011226610513404012, -0.004232363309711218, -0.02095174789428711, -0.003571164095774293, -0.005861254874616861, -0.014780555851757526, -0.009966199286282063, 0.006009336095303297, 0.0006095429416745901, -0.013706107623875141, -0.012535233981907368, 0.007355840411037207, -0.0033146049827337265, 0.01928497478365898, 0.015152480453252792, -0.014821880497038364, -0.018871726468205452, -0.0119704594835639, 0.011440122500061989, -0.001033123699016869, -0.017466677352786064, 0.01516625564545393, 0.01155032217502594, -0.02997436188161373, -0.016833027824759483, -0.0001805814099498093, -0.015042280778288841, -0.04380444437265396, 0.0007072592270560563, 0.0099524250254035, 0.0028221495449543, -0.016791703179478645, -0.01790747605264187, -0.17764216661453247, 0.011729397810995579, 0.006722191348671913, -0.015648379921913147, 0.0030494367238134146, 0.02310064435005188, 0.004049844574183226, 0.007314515765756369, -0.007879289798438549, -0.005689067766070366, -0.009814674966037273, 0.015097380615770817, -0.033555857837200165, -0.02859686315059662, -0.007645115256309509, 0.00588191719725728, -0.016571303829550743, 0.016171829774975777, 0.03278445824980736, 0.009752687066793442, 0.03333545848727226, -0.017425352707505226, -0.009938649833202362, -0.00563396792858839, 0.019353849813342094, 0.008478501811623573, -0.009966199286282063, 0.022618521004915237, -0.001003851881250739, -0.017149852588772774, -0.02283892035484314, 0.00027420822880230844, 0.01133680995553732, 0.0014429293805733323, 0.007342065684497356, 0.009835337288677692, 0.004890118725597858, 0.0017270384123548865, -0.01698455400764942, 0.02592451684176922, 0.009008838795125484, 0.02777036465704441, 0.014002269133925438, 0.012135758996009827, 0.0241062194108963, 0.006646428722888231, 0.029202962294220924, -0.01891305111348629, 0.02221904695034027, -0.024822518229484558, -0.010227924212813377, -0.009435863234102726, 0.007528027985244989, -0.00775531493127346, 0.009546062909066677, 0.018403375521302223, 0.001640944741666317, 0.010489649139344692, -0.008630026131868362, -0.02173692174255848, -0.00877466332167387, -0.02556636743247509, 0.007596902549266815, -0.021915996447205544, -0.010331236757338047, -0.018293175846338272, -0.0159238800406456, 0.020731348544359207, -0.004070507362484932, 0.007348952814936638, -0.01352014485746622, 0.021516522392630577, -0.027536189183592796, 0.0017192899249494076, 0.0012208077823743224, -0.0008596449624747038, -0.011936021968722343, 0.04496154189109802, 0.019794650375843048, -0.019147224724292755, -0.010813361033797264, 0.016901902854442596, 0.01902325078845024, 0.000297023041639477, 0.004903893452137709, 0.017480451613664627, 0.023183295503258705, -0.001329285791143775, -0.02237057127058506, 0.0004799720481969416, 0.03374870494008064, -0.014312206767499447, -0.00025203911354765296, 0.009986862540245056, 0.025332191959023476, 0.0027274463791400194, 0.00400507589802146, 0.01139191072434187, 0.020524723455309868, 0.010668723843991756, 0.018720200285315514, -0.0017588930204510689, 0.004029182251542807, 0.007810414768755436, 0.027673939242959023, 0.02812851406633854, -0.00909148808568716, -0.022618521004915237, 0.03766079992055893, -0.015152480453252792, -0.030663110315799713, -0.019780874252319336, 0.028321363031864166, 0.03171001002192497, -0.00048772047739475965, 0.03176511079072952, 0.023541444912552834, 0.0019904850050807, 0.009973087348043919, 0.013354845345020294, 0.04336364567279816, -0.014160681515932083, -0.0017545883310958743, 0.02383071929216385, -0.01152277272194624, -0.013561470434069633, -0.10298176854848862, -0.02294912002980709, -0.0012302780523896217, -0.00013688889157492667, 0.0013800810556858778, 0.013823194429278374, -0.01698455400764942, 0.00898817554116249, 0.010269248858094215, 0.016171829774975777, -0.021544072777032852, -0.0476614385843277, -0.02593829110264778, 0.017370251938700676, 0.010462098754942417, 0.005496217869222164, -0.017466677352786064, -0.008953738026320934, -0.0006960670580156147, 0.007782864850014448, -0.0039809700101614, 0.022177722305059433, 0.0031734115909785032, -0.01097177341580391, 0.0037812325172126293, 0.016102954745292664, -0.03757815062999725, -0.002806652570143342, 0.026751015335321426, 0.0010408720700070262, 0.01215642224997282, -0.017852377146482468, -0.0007912005530670285, -0.02588319219648838, -0.0159238800406456, -0.0073076281696558, -0.017576877027750015, -0.010207261890172958, 0.008974401280283928, -0.0347956046462059, -0.0021420097909867764, -0.01670905388891697, -0.002033531665802002, -0.016681503504514694, -0.004442431963980198, -0.02436794340610504, -0.0016469713300466537, 0.04441054165363312, -0.003364539472386241, 0.0031010927632451057, -0.03132430836558342, 0.018155425786972046, 0.008299427106976509, -0.005964567419141531, 0.022866470739245415, 0.021337447687983513, -0.01188780926167965, -0.02069002389907837, -0.026158690452575684, -0.011378135532140732, -0.008244327269494534, -0.007528027985244989, -0.03793630003929138, 0.011385022662580013, -0.011922246776521206, -0.006129866931587458, -0.03394155576825142, 0.0033524862956255674, 0.008457839488983154, 0.00885731354355812, -0.016102954745292664, 0.004042957443743944, -0.009897325187921524, 0.005709730088710785, -0.017287602648139, -0.0042909071780741215, -0.02640664018690586, -0.014367306604981422, 0.02887236326932907, -0.0034850705415010452, 0.003140695858746767, -0.015896329656243324, -0.011812047101557255, -0.041049446910619736, 0.007466040551662445, 0.011991121806204319, -0.005368799436837435, 0.012487021274864674, 0.012280396185815334, -0.015965204685926437, 0.012280396185815334, 0.008299427106976509, 0.018265627324581146, -0.007810414768755436, 0.0005561648868024349, 0.0037777889519929886, -0.030883509665727615, 0.0016245869919657707, 0.004315013065934181, -0.0018733976176008582, -0.039782147854566574, -0.0036882515996694565, -0.05410813167691231, 0.0015496854903176427, 0.01484943088144064, -0.0021867784671485424, 0.0027911558281630278, -0.01891305111348629, 0.002191944047808647, 0.0005763969384133816, 0.021833347156643867, -0.0008458700031042099, -0.025800541043281555, 0.008698901161551476, -0.01760442741215229, -0.009862887673079967, -0.013823194429278374, 0.0009057050920091569, 0.026902539655566216, 0.001730482093989849, 0.046752288937568665, 2.0998777472414076e-05, -0.0017433961620554328, 0.0014472340699285269, 0.004645612556487322, 0.04096679762005806, -0.006887491326779127, 0.009249900467693806, -0.015689704567193985, 0.026916315779089928, -0.019257424399256706, 0.0023658531717956066, -2.1456149625009857e-05, -0.010255474597215652, -0.014215781353414059, -0.005110518541187048, 0.008347638882696629, -0.01618560403585434, -0.0012896826956421137, 0.02022167481482029, 0.02267361991107464, -0.001952603692188859, -0.010751374065876007, 0.008712676353752613, 0.009855999611318111, -0.0008781551150605083, -0.01844470202922821, -0.010827136225998402, 0.006498347967863083, 0.006673979107290506, 0.019822198897600174, 0.014629031531512737, 0.05132558196783066, 0.0030494367238134146, -0.019946174696087837, -0.02483629249036312, -0.018610000610351562, -0.0346854068338871, 0.008519826456904411, -0.0002798043133225292, 0.015634605661034584, 0.00017563102301210165, 0.030552910640835762, 0.008251214399933815, 0.010586073622107506, -0.0008755723247304559, -0.005286149214953184, -0.0009332550689578056, -0.01252145878970623, 0.010172824375331402, -0.019188549369573593, -0.009484075009822845, -0.03330790624022484, -0.01665395312011242, -0.0058337049558758736, 0.007975714281201363, 0.009435863234102726, -0.01980842463672161, 0.004500975366681814, 0.005237936973571777, -0.011488335207104683, 0.02829381264746189, 0.01661262847483158, -0.0020972411148250103, -0.006288279313594103, 0.015689704567193985, 0.02599339187145233, 0.021778246387839317, 0.007376503199338913, 0.01613050326704979, 0.004480313044041395, 0.007252528332173824, -0.0074522653594613075, -0.0007158686057664454, -0.006870272569358349, -0.004697268828749657, 0.007893064990639687, 0.01385763194411993, -0.0076382276602089405, 0.005220718216150999, 0.03986479714512825, 0.02577299252152443, 0.01932629942893982, 0.012590333819389343, -0.00974579993635416, -0.015910103917121887, -0.018596226349473, 0.009573612362146378, -0.02304554544389248, -0.01708097755908966, -0.0009547784575261176, -0.00522416178137064, 0.02498781867325306, -0.02089664712548256, 0.01902325078845024, -0.0072663030587136745, -0.009222351014614105, 0.0038432199507951736, -0.004163488280028105, -0.010827136225998402, -0.01776972785592079, 0.01771462708711624, 0.028899911791086197, 0.0071767657063901424, 0.018375826999545097, 0.0038948762230575085, 0.03542925417423248, 0.0006504374323412776, -0.011302373372018337, -0.01071693655103445, 0.012397483922541142, -0.007328290492296219, 0.00921546295285225, 0.005478999111801386, -0.0005901718977838755, 0.004790250211954117, -0.025635242462158203, 0.025593917816877365, 0.0026534060016274452, 0.04914913699030876, -0.004680050071328878, 0.0793989971280098, 0.008650688454508781, -0.017108527943491936, 0.0070114657282829285, -0.023169519379734993, 0.010090174153447151, 0.028486663475632668, 0.008471613749861717, -0.01943649910390377, 0.004132494796067476, 0.04011274874210358, 0.011750060133635998, 0.011619197204709053, -0.026310216635465622, -0.0009323941194452345, 0.01249390933662653, -0.005651186686009169, 0.032756906002759933, -0.04507174342870712, 0.003929313737899065, 0.02242567017674446, -0.008698901161551476, 0.013120670802891254, -0.02730201557278633, -0.01827940158545971, -0.003133808495476842, 0.01932629942893982, 0.0067635164596140385, -0.015262680128216743, -0.029726412147283554, 0.030249860137701035, -0.011357473209500313, -0.010041962377727032, -0.012452583760023117, -0.023224620148539543, 0.027894338592886925, -0.006512122694402933, -0.025235766544938087, 0.026020942255854607, 0.011557210236787796, -0.009084600955247879, 0.0056442990899086, -0.023252170532941818, -0.0343548059463501, 0.002190222265198827, -0.0023348594550043344, 0.009718249551951885, -0.021254798397421837, -0.012810733169317245]} +{"id": "test:10000045", "text": "\"As I have been discussing the magic of wordpress themes before. WordPress has brough a revolution as it\u2019s ideal for less tech savy people to manage their website. Now almost 20% of the websites over world wide web are built using wordpress. Lots of other cms are available offering ecommerce solutions.\"", "vector_field": [0.009988240897655487, -0.0015517446445301175, 0.002266811905428767, -0.021649837493896484, 0.006051317788660526, -0.0016538971103727818, -0.03637276589870453, -0.025476500391960144, 0.0007872225833125412, -0.033960022032260895, 0.0016620043898001313, 0.013866792432963848, -0.011966431513428688, 0.0007612790795974433, 0.000794519204646349, 0.03551663085818291, 0.03357086703181267, 0.008613236248493195, -0.04259920120239258, -0.018251240253448486, -0.0116551099345088, -0.008347315713763237, -0.003888279665261507, 0.0068101645447313786, -0.037851542234420776, 0.011719969101250172, 0.00714094378054142, -0.012589075602591038, -0.017434021458029747, 0.002112772548571229, 0.002154930727556348, -0.02014511451125145, -0.009008875116705894, -0.0013263606233522296, -0.0344010591506958, 0.006612345576286316, 0.02489277347922325, -0.015579062514007092, 0.016084959730505943, -0.009521258994936943, 0.014022452756762505, 0.009942839853465557, -0.0014268916565924883, -0.005431917030364275, -0.030976518988609314, 0.012005346827208996, -0.00427743187174201, -0.024088524281978607, -0.001801450620405376, 0.004890346899628639, 0.0015866061439737678, 0.018056664615869522, -0.03676191717386246, -0.005714052356779575, -0.0097871795296669, -0.008742953650653362, -0.031469445675611496, 0.014035425148904324, 0.03759210929274559, 0.004540109541267157, -0.0028602690435945988, 0.021442290395498276, -0.004089341498911381, -0.003038630587980151, -0.01987270824611187, -0.0041736578568816185, -0.014489435590803623, 0.021688751876354218, 0.021053137257695198, 0.019224122166633606, 0.030924633145332336, 0.01987270824611187, -0.006307509727776051, -0.015125051140785217, 0.018147466704249382, 0.020832616835832596, 0.0037293757777661085, 0.007231746334582567, -0.0008480276446789503, -0.000993959722109139, -0.0032056416384875774, -0.0006076450226828456, 0.000598321610596031, 0.017122698947787285, -0.0016620043898001313, 0.004510923288762569, 0.012388013303279877, 0.015553118661046028, -0.021053137257695198, -0.008879157714545727, -0.011259471997618675, 0.003109975252300501, 0.0016652473714202642, 0.0200283695012331, -0.0009963919874280691, 0.016733547672629356, -0.016007129102945328, 0.006479385308921337, 0.007796016987413168, 0.004601725377142429, 0.019561387598514557, 0.008840242400765419, -0.009482343681156635, -0.020521296188235283, -0.03497181460261345, 0.0016603829571977258, 0.00917750783264637, 0.008548378013074398, 0.026202918961644173, 0.016214678063988686, -0.020702900364995003, 0.030587367713451385, -0.009728806093335152, -0.007971135899424553, 0.0019133319146931171, 0.02406258136034012, -0.006673961412161589, -0.007037170231342316, 0.014463492669165134, 0.01419108547270298, 0.01410028338432312, 0.029186420142650604, 0.044596847146749496, -0.0160719882696867, 0.020132143050432205, -0.00014127037138678133, -0.035179365426301956, 0.009806636720895767, 0.027370374649763107, -0.021468233317136765, 0.017667513340711594, 0.006077261175960302, 0.009865010157227516, 0.007497666869312525, 0.009080219082534313, 0.02391989156603813, 0.002377071650698781, 0.018095580860972404, -0.014022452756762505, -0.016149818897247314, 0.02698122337460518, 0.01214155089110136, 0.008671609684824944, -0.010370907373726368, 0.001546880230307579, 0.004543352406471968, 0.020197002217173576, -0.01147350575774908, 0.005795125849545002, 0.004699013661593199, 0.016785433515906334, 0.006476141978055239, 0.008275971747934818, 0.006275080144405365, 0.024257156997919083, 0.022817295044660568, 0.014696983620524406, 0.02075478807091713, -0.0031569977290928364, -0.025761879980564117, -0.0013596006901934743, 0.007808988448232412, 0.004880617838352919, 0.010435766540467739, 0.03048359416425228, 0.024334987625479698, 0.04270297288894653, 0.018640393391251564, -0.005973487161099911, -0.00691393855959177, 0.0022051960695534945, 0.009949326515197754, -0.021584978327155113, 0.017667513340711594, 0.007037170231342316, 0.007543067913502455, -0.021105024963617325, 0.025813765823841095, -0.016188733279705048, -0.014165142551064491, -0.0029024272225797176, 0.007569011300802231, 0.005535691045224667, 0.012057234533131123, -0.016720576211810112, 0.006965825334191322, 0.03346709534525871, -0.011797799728810787, 0.017252417281270027, -0.028641605749726295, 0.017667513340711594, 0.023958807811141014, 0.004144471604377031, -0.007257689721882343, -0.6637380719184875, -0.0015736344503238797, -0.0031894270796328783, 0.010182817466557026, 0.0013758153654634953, -0.008989416994154453, 0.011454048566520214, 0.0027662240900099277, -0.013607357628643513, -0.00691393855959177, -0.006706390529870987, -0.02139040268957615, -0.004118528217077255, -0.021870356053113937, -0.020184030756354332, -0.015034249052405357, 0.004916290286928415, -0.036632198840379715, -0.00972232036292553, 0.0067388201132416725, -0.004815759137272835, -0.00016630179015919566, -0.04838459938764572, -0.007089057005941868, -0.007815474644303322, 0.006219950504601002, -0.009385054931044579, 0.020261861383914948, -0.009819609113037586, 0.03419351205229759, -0.016214678063988686, 0.036243047565221786, -0.029056701809167862, 0.018705252557992935, 0.044596847146749496, 0.01311443094164133, -0.015643920749425888, 0.01781020127236843, 0.007179859094321728, 0.032533127814531326, -0.022441113367676735, -0.027007166296243668, 0.009890953078866005, 0.014242972247302532, 0.009164535440504551, 0.019457612186670303, 0.030379820615053177, -0.0008431632304564118, 0.02084559015929699, -0.006044831592589617, 0.001331225037574768, -0.004478493705391884, -0.0008561349823139608, -0.013957594521343708, 0.00987798161804676, 0.024412818253040314, 0.02853783220052719, 0.0008131660870276392, 0.0032688789069652557, 0.028849154710769653, -0.00387855083681643, -0.0025359755381941795, -0.011389189399778843, -0.03691757842898369, -0.014567266218364239, 0.011596737429499626, -0.00860026478767395, 0.0029835007153451443, 0.0204823799431324, 0.0002916615048889071, -0.0006137255695648491, 0.032014258205890656, -0.02661801502108574, -0.023375079035758972, -0.003113218117505312, 0.013283063657581806, 0.022389227524399757, -0.008801327086985111, 0.00046779343392699957, 0.02412744052708149, -0.012796623632311821, -0.01971704699099064, -0.00027747367857955396, -0.031158125028014183, 0.03357086703181267, 0.007679271046072245, 0.006268594413995743, -0.03811097890138626, 0.01701892539858818, 0.012290725484490395, -0.006609102711081505, 0.021234741434454918, -0.010948150418698788, -0.028641605749726295, -0.03471238166093826, 0.010072557255625725, 0.0027094725519418716, 0.010370907373726368, 0.013237662613391876, -0.004264459945261478, -0.017278360202908516, -0.01113624032586813, 0.03188454359769821, 0.0071020289324223995, 0.006673961412161589, 0.006385339889675379, 0.003226720727980137, 0.004731442779302597, 0.046386949717998505, -0.018095580860972404, 0.017239445820450783, -0.03017227165400982, -0.006508571561425924, 0.0015387728344649076, 0.010338478721678257, -0.023997722193598747, 0.023893948644399643, -0.006200492847710848, -0.010857348330318928, -0.00935911200940609, 0.014956418424844742, -0.0040342118591070175, 0.0009242366068065166, -0.009151563979685307, -0.013334950432181358, 0.02419229969382286, -0.0027678455226123333, -0.01156430784612894, -0.003058088244870305, -0.002730551641434431, 0.015319627709686756, 0.01481372956186533, 0.0034180539660155773, -0.007316062692552805, 0.006985282991081476, 0.014684012159705162, 0.03694352135062218, -0.02679961919784546, -0.005808097310364246, -0.03956381231546402, 0.01175888441503048, -0.004822244867682457, -0.004786572884768248, -0.0387076772749424, -0.016448168084025383, -0.02980906330049038, -0.02166280895471573, 0.002459766576066613, 0.003885036800056696, -0.015592033974826336, 0.002030077623203397, -0.014580237679183483, -0.01330900751054287, 0.026643957942724228, 0.018588505685329437, 1.4225845916371327e-05, -0.022311396896839142, -0.02297295443713665, 0.0006514246924780309, -0.005247069522738457, 0.0004098259669262916, 0.02485385723412037, -0.009456399828195572, -0.025839710608124733, -0.005084923002868891, -0.006142119877040386, -0.029835006222128868, 0.01905548945069313, -0.011687539517879486, -0.03206614777445793, 0.01339980959892273, 0.022531915456056595, 0.004209330305457115, 0.008775383234024048, -0.013451696373522282, 0.019496528431773186, 0.0012193437432870269, -0.025450557470321655, 0.007718186359852552, 0.0014139198465272784, -0.006391826085746288, 0.010072557255625725, -0.007367949467152357, -0.009702863171696663, -0.0036580313462764025, 0.0031667265575379133, -0.009398027323186398, 0.009942839853465557, -0.03175482526421547, -0.013711131177842617, 0.0055454196408391, 0.020015398040413857, 0.025294896215200424, 0.0007247960893437266, 0.020339692011475563, 0.004144471604377031, -0.006018888205289841, 0.01643519662320614, -0.0021954672411084175, 0.02236328274011612, -0.00489358976483345, 0.02962745912373066, -0.013185775838792324, -0.022233566269278526, 0.0056491936556994915, -0.030120383948087692, -0.009203450754284859, -0.004209330305457115, 0.011479991488158703, 0.018160438165068626, -0.00013772341480944306, -0.025813765823841095, 0.005597306881099939, -0.014165142551064491, 0.015189910307526588, 0.016668688505887985, -0.00030443057767115533, -0.007770073600113392, -0.005954029504209757, -0.0011139482958242297, 0.012362070381641388, -0.014645096845924854, 0.003862336277961731, -0.015345570631325245, -0.0018954958068206906, 0.015812553465366364, 0.002236003987491131, 0.02200007438659668, -0.016214678063988686, -0.03333737701177597, -0.019185205921530724, 0.012848510406911373, 0.021338514983654022, 0.0033758957870304585, 0.007374435197561979, 0.01808260753750801, 0.031313784420490265, -0.030587367713451385, 0.02929019369184971, -0.0008334344020113349, 0.01382787711918354, 0.041068535298109055, 0.0020187273621559143, -0.033985964953899384, 0.030068498104810715, 0.019730020314455032, 0.015656892210245132, -0.009261824190616608, -0.011732940562069416, 0.004394177813082933, -0.00930073857307434, -0.012705821543931961, -0.002038185019046068, 0.020456437021493912, 0.016629774123430252, -0.005133566912263632, 0.018640393391251564, -0.005068708211183548, 0.02111799642443657, 0.02166280895471573, -0.006226436235010624, -0.006609102711081505, 0.014580237679183483, -0.00032936062780208886, -8.867401629686356e-05, -0.0017609139904379845, 0.00508168013766408, 0.0032850936986505985, -0.00021099348668940365, -0.038396354764699936, -0.020962335169315338, 0.00394989550113678, 0.048540256917476654, -0.018588505685329437, 0.005642707925289869, 0.008185168728232384, -0.008029508404433727, -0.01620170660316944, -0.006628560367971659, 0.044960059225559235, -0.03686568886041641, -0.023050785064697266, 0.007257689721882343, -0.004666584078222513, 0.01166808232665062, -0.0071474299766123295, -0.018640393391251564, 0.01095463614910841, 0.0056945946998894215, 0.029886893928050995, -8.690053800819442e-05, 0.031313784420490265, 0.013373865745961666, -0.010182817466557026, 0.0061972495168447495, -0.004309861455112696, 0.03333737701177597, -0.004569296259433031, 0.007082571275532246, -0.02401069365441799, 0.02622886188328266, -0.00981312245130539, -0.016123875975608826, -0.01670760288834572, 0.02582673914730549, 0.005937814712524414, -0.014398633502423763, 0.0032721220050007105, -0.012070205993950367, -0.026514241471886635, 0.0008447847212664783, -0.029835006222128868, 0.010987065732479095, 0.008068423718214035, 0.002929992275312543, 0.01713567040860653, 0.02507437765598297, -0.015215853229165077, 0.012128578498959541, 0.0044655222445726395, 0.016668688505887985, -0.009670433588325977, -0.007400378584861755, 0.003456969279795885, 0.09401918947696686, -0.010896263644099236, 0.013542498461902142, 0.021507147699594498, -0.004494708497077227, 0.001267177052795887, -0.01595524325966835, -0.00990392453968525, 0.02641046606004238, 0.02600834332406521, 0.016123875975608826, -0.014204057864844799, 0.018886856734752655, -0.019457612186670303, 0.010734116658568382, -0.005431917030364275, -0.013049572706222534, -0.015280712395906448, -0.0341675691306591, -0.035542573779821396, 0.004257974214851856, 0.0051465388387441635, -0.005337871611118317, 0.01133730262517929, 0.016396282240748405, -0.002231139689683914, 0.017732370644807816, 0.04428552836179733, -0.03408973664045334, -0.006699904799461365, -0.00202521332539618, 0.003309415653347969, 0.0013506825780496001, 0.01996351033449173, -0.014554294757544994, 0.01880902610719204, -0.0068296222016215324, 0.024075552821159363, 0.011771855875849724, -0.022129792720079422, -0.014502407982945442, 0.022843237966299057, -0.0007154726190492511, 0.007056627422571182, -0.006203735712915659, -0.03406379371881485, -0.0022927552927285433, 0.043040238320827484, -0.009235880337655544, -0.0384741872549057, 0.02251894399523735, 0.010662771761417389, -0.024879802018404007, 0.0037650479935109615, -0.006745305843651295, 0.018043693155050278, 0.023349136114120483, 0.021468233317136765, -0.029653402045369148, -0.014437548816204071, -0.012725278735160828, -0.00940451305359602, -0.015306655317544937, 0.0030759242363274097, -0.01273176446557045, -0.020767759531736374, -0.021066108718514442, -0.003933680709451437, -0.03266284614801407, -0.004361748229712248, -0.004163929261267185, -0.050460077822208405, -0.02351776883006096, -0.004728199914097786, 0.010740602388978004, -0.02890104055404663, 0.009436942636966705, 0.0026608286425471306, 0.009988240897655487, 0.015462316572666168, 0.00010630748147377744, -0.003898008493706584, -0.0009566659573465586, -0.05583037808537483, -0.013373865745961666, -0.0028926983941346407, -0.03460860624909401, -0.009158049710094929, -0.02489277347922325, 0.020897476002573967, 0.0026186704635620117, -0.006881508976221085, 0.029731232672929764, -0.014541322365403175, 0.004903318360447884, 0.02211681939661503, 0.007160401437431574, 0.006725848186761141, 0.012219380587339401, -0.01524179708212614, 0.022778378799557686, -0.00013498718908522278, -0.026397494599223137, -0.0002148444764316082, 0.02750009298324585, 0.00246949540451169, 0.00015859981067478657, 0.004481737036257982, 0.021403374150395393, -0.008023022674024105, -0.004987634718418121, -0.0018452303484082222, 0.025904567912220955, -0.008567835204303265, -0.003051602281630039, 0.03551663085818291, 0.003901251358911395, 0.02679961919784546, 0.036787860095500946, -0.004575781989842653, -0.007361463736742735, -0.0416392907500267, 0.01325063407421112, 0.01312740333378315, -0.005153024569153786, 0.008587293326854706, 0.004206087440252304, 0.01808260753750801, 0.0020787217654287815, -0.010546025820076466, 0.001932789571583271, 0.003518585115671158, 0.0030062012374401093, -0.009028332307934761, -0.017862088978290558, -0.032221805304288864, 0.019859736785292625, 0.004157443530857563, -0.019859736785292625, -0.021143939346075058, -0.0003887469065375626, -0.0011755641316995025, 0.0016222784761339426, -0.007497666869312525, -0.005730267148464918, -0.024088524281978607, 0.009365597739815712, -0.01180428545922041, -0.00432931911200285, 0.0193538386374712, -0.005201668478548527, -0.005315171089023352, -0.03157322108745575, -0.00032611770438961685, 0.025294896215200424, -0.05894359573721886, -0.00474765757098794, -0.01001418475061655, 0.020832616835832596, 0.03315577283501625, 0.02944585494697094, -0.0044914656318724155, 0.0016814620466902852, -0.012816080823540688, 0.017667513340711594, -0.02148120477795601, -0.011739426292479038, 0.0041023134253919125, -0.007504152599722147, 0.017875060439109802, 0.002615427365526557, 0.016642745584249496, -0.008788355626165867, 0.009138592518866062, 0.02564513310790062, 0.020261861383914948, -0.0033758957870304585, -0.0011942109558731318, -0.01747293584048748, -0.023634513840079308, 0.004261217080056667, -0.02220762148499489, -0.007646841928362846, 0.0067842211574316025, -0.04342939332127571, 0.004416878335177898, -0.004303375259041786, 0.007731158286333084, 0.03868173435330391, -0.0002351128205191344, 0.041405800729990005, -0.009203450754284859, -0.009028332307934761, -0.008678095415234566, 0.0024808456655591726, -0.024542536586523056, 0.003904494224116206, -0.03245529904961586, -0.013957594521343708, 0.02312861569225788, 0.030457649379968643, 0.0018792811315506697, -0.010630342178046703, -0.0003429404168855399, 0.0037942344788461924, 0.028641605749726295, -0.0271628275513649, -0.013983537442982197, -0.0059702442958951, -0.010079043917357922, 0.003135918639600277, -0.020586155354976654, -0.00808139517903328, -0.014541322365403175, -0.014567266218364239, -0.007964649237692356, -0.02419229969382286, 0.01829015649855137, -0.004043940454721451, -0.017550766468048096, 0.02449064888060093, -0.008451090194284916, 0.03650248050689697, 0.0037034323904663324, 0.02340102195739746, 0.016331423074007034, 0.004254731349647045, -0.014113254845142365, 0.005269770044833422, -0.017148643732070923, -0.02631966397166252, 0.023777203634381294, 0.025567304342985153, 0.01643519662320614, -0.012854996137320995, 0.0013814904959872365, -0.02254488691687584, -0.015125051140785217, -0.03232558071613312, -0.010520082898437977, 0.0025975913740694523, 0.010883291251957417, -0.02419229969382286, 0.02799301967024803, -0.008204626850783825, 0.03951192647218704, -0.005107623524963856, -0.0012696092016994953, 0.0016328180208802223, -0.013334950432181358, -0.001138270366936922, 0.023076729848980904, -0.002281405031681061, -0.002200331771746278, -0.02254488691687584, -0.017421049997210503, 0.004261217080056667, -0.03694352135062218, -0.010046614333987236, 0.0026608286425471306, -0.028408115729689598, 0.0003319955139886588, -0.02081964537501335, -0.0032575286459177732, 0.03984919190406799, 0.03071708418428898, -0.012517730705440044, 0.0049941204488277435, -0.029653402045369148, 0.017823172733187675, 0.02066398411989212, 0.0198078490793705, 0.013944623060524464, 0.01179131306707859, -0.008969959802925587, -0.018095580860972404, -0.007705214899033308, -0.029653402045369148, -0.016824349761009216, -0.007808988448232412, 0.009527744725346565, 0.0049681770615279675, -0.011908059008419514, 0.008502976968884468, -0.005438402760773897, -0.01207669172435999, 0.001926303724758327, -0.014878587797284126, 0.030639255419373512, -0.01847176067531109, -0.012102635577321053, -0.0193538386374712, -0.02752603590488434, 0.0021695238538086414, -0.003842878621071577, 0.008652151562273502, 0.00036523560993373394, 0.040679384022951126, -0.002615427365526557, -0.0041055562905967236, 0.024957630783319473, -0.012070205993950367, -0.006524786353111267, -0.005370301194489002, -0.002855404745787382, -0.036787860095500946, 0.010688715614378452, -0.0016190354945138097, -0.013724102638661861, 0.002714337082579732, 0.020158087834715843, -0.007789531257003546, 0.0019830551464110613, 0.014658068306744099, -0.011823742650449276, 0.03351898118853569, -0.005548662506043911, 0.011084353551268578, -0.03694352135062218, 0.007607926614582539, -0.003218613564968109, -0.000668855442199856, -0.0034731838386505842, -0.005749724805355072, -0.010033642873167992, 0.0037747768219560385, 0.002073857234790921, 0.022830266505479813, -0.04244353994727135, 0.001177185564301908, -0.027396319434046745, 0.020313747227191925, -0.008944015949964523, -0.013957594521343708, -0.010377394035458565, -0.009469371289014816, -0.008477033115923405, 0.0176545400172472, -0.004728199914097786, -0.022194650024175644, -0.006177792325615883, -0.001340953866019845, 0.012744736857712269, -0.0014244593912735581, -4.079004793311469e-05, -0.009657462127506733, -0.014346746727824211, -0.02011917158961296, -0.010370907373726368, -0.004682798869907856, -0.04786572605371475, 0.03907088562846184, 0.014619152992963791, -0.02695528045296669, -0.0060837469063699245, 0.0031407829374074936, -0.0574907585978508, -9.105554636335e-05, 0.013211718760430813, 0.00803599413484335, 0.016992982476949692, 0.016642745584249496, 0.014619152992963791, -0.006362639367580414, -0.014826701022684574, -0.0008366773836314678, -0.01926303654909134, 0.01183671411126852, -0.015410429798066616, -0.022285452112555504, -0.0009428834891878068, -0.014450520277023315, -0.016266563907265663, 0.023180503398180008, 0.021714696660637856, -0.006038345862179995, -0.008853213861584663, 0.001344196731224656, -0.013257120735943317, 0.014917503111064434, 0.013633300550282001, 0.0004084071842953563, 0.017965862527489662, 0.009274795651435852, 0.03017227165400982, -0.04703553766012192, -0.0024857099633663893, 0.00256029749289155, -0.019276008009910583, -0.007063113618642092, -0.002258704509586096, -0.023777203634381294, 0.004170414991676807, -0.014696983620524406, 0.013698159717023373, -0.0233102198690176, -0.0076727853156626225, -0.0021306085400283337, 0.02516517974436283, -0.014230000786483288, -0.020612098276615143, 0.022324368357658386, -0.0015784988645464182, -0.005814583506435156, 0.007465237285941839, 0.02980906330049038, 0.005512990523129702, 0.010714658536016941, 0.008652151562273502, -0.024814942851662636, -0.01480075716972351, -0.004355262499302626, 0.02349182404577732, 0.007439293898642063, 0.01595524325966835, -0.007893305271863937, 0.008632694371044636, 0.007912762463092804, -0.0015168830286711454, 0.02494465932250023, 0.008969959802925587, 0.0019036030862480402, -0.009566660039126873, 0.011240014806389809, 0.002406258136034012, -0.029342079535126686, 0.009158049710094929, 0.012498273514211178, -0.03582794964313507, 0.016214678063988686, 0.0024451734498143196, -0.013049572706222534, 0.01156430784612894, 0.00850946269929409, -0.012355584651231766, 0.021169882267713547, 0.19696292281150818, -0.013905707746744156, -0.024659281596541405, 0.041613347828388214, 0.03341520577669144, 0.025995370000600815, -0.004919533152133226, 0.0066934190690517426, 0.001984676579013467, 0.0033272518776357174, -0.011142726056277752, 0.0010085529647767544, -0.03616521507501602, -0.005973487161099911, 0.010461709462106228, -0.03336331993341446, -0.022129792720079422, -0.007231746334582567, -0.007679271046072245, 0.021169882267713547, 0.018303127959370613, -0.0025959699414670467, -0.03323360159993172, -0.00378774874843657, 0.0041023134253919125, 0.0037909916136413813, -0.019898653030395508, 0.012984713539481163, 0.022350311279296875, 0.008392716757953167, 0.007354977540671825, -0.0056945946998894215, -0.008937530219554901, 0.010584941133856773, -0.0066934190690517426, 0.0075819832272827625, 0.027811415493488312, -0.009203450754284859, 0.00969637744128704, 0.017978833988308907, -0.035023704171180725, 0.014593210071325302, -0.0015347192529588938, 0.004874132107943296, 0.013516555540263653, 0.013347922824323177, 0.013056058436632156, 0.004695770796388388, 0.005033035762608051, 0.014645096845924854, -0.018406901508569717, -0.007439293898642063, 0.021675780415534973, 0.01282256655395031, -0.014048396609723568, 0.0023462637327611446, 0.0013141996460035443, 0.012057234533131123, -0.003810449270531535, 0.00810085330158472, -0.013594385236501694, 0.02534678392112255, -0.024594422429800034, 0.01535854209214449, 0.009209936484694481, 0.017784258350729942, -0.007588468957692385, -0.00516599602997303, 0.0010174709605053067, -0.005461103282868862, 0.002711093984544277, 0.007406864780932665, -0.012770679779350758, 0.0005614332039840519, -0.014476464129984379, -0.02118285372853279, 0.011570793576538563, 0.00714094378054142, 0.005428674165159464, 0.01661680079996586, -0.028304340317845345, -0.01589038409292698, -0.013801933266222477, -0.018705252557992935, 0.032714731991291046, -0.029238305985927582, 0.004640640690922737, -0.008379745297133923, -0.011908059008419514, -0.0019214393105357885, -0.017589682713150978, -0.006553972605615854, -0.016759490594267845, -0.02100124955177307, 0.004112042021006346, 0.016033073887228966, 0.013594385236501694, 0.02188332937657833, -0.009702863171696663, 0.0067842211574316025, -0.020093228667974472, 0.03813692182302475, 0.014217029325664043, -0.010630342178046703, -0.004718471318483353, 0.005444888491183519, -0.01704486832022667, 0.01016336027532816, -0.016357365995645523, 0.007594955153763294, 0.004695770796388388, -0.005448131822049618, 0.012887425720691681, 0.002691636560484767, 8.370826981263235e-05, 0.019276008009910583, -0.0014009481528773904, -0.02729254588484764, -0.0139186792075634, -0.007633870001882315, -0.012057234533131123, -0.008593779057264328, 0.004047183319926262, -0.0033531952649354935, 0.0032153704669326544, 0.016188733279705048, -0.030794914811849594, 0.012258295901119709, -0.030950576066970825, -0.031080294400453568, 0.014294859953224659, -0.016059016808867455, -0.005587577819824219, -0.008275971747934818, -0.02661801502108574, -0.03250718489289284, 0.003097003325819969, -0.016487084329128265, 0.006978797260671854, 0.018938742578029633, -0.008723496459424496, 0.009793665260076523, -0.02014511451125145, -0.014839672483503819, 0.002949449932202697, -0.015125051140785217, 0.0072641754522919655, 0.017070813104510307, -0.0005946732708252966, -0.008230570703744888, -0.015877412632107735, 0.02729254588484764, -0.005058979149907827, -0.014891560189425945, 0.004754143301397562, 0.012122092768549919, -0.03250718489289284, -0.022376254200935364, -0.004841702524572611, 0.0047379289753735065, -0.02679961919784546, 0.008386231027543545, 0.04988931864500046, 0.0016620043898001313, 0.0010766545310616493, -0.016331423074007034, -0.16354772448539734, 0.02136445976793766, 0.007257689721882343, -0.01472292747348547, 0.03375247120857239, 0.010383879765868187, 0.016889208927750587, 0.00488386070355773, -0.0023041055537760258, -0.029523685574531555, -0.0037228900473564863, -0.005448131822049618, -0.0262548066675663, -0.013996509835124016, 0.007893305271863937, -0.003936923574656248, -0.016603829339146614, 0.027059054002165794, 0.024542536586523056, 0.004416878335177898, 0.031676992774009705, 0.010085529647767544, 0.023362107574939728, -0.014735898934304714, 0.021597949787974358, 0.029653402045369148, -0.009534230455756187, 0.032040201127529144, 0.014891560189425945, -0.03032793290913105, -0.016020100563764572, 0.001923060743138194, 0.029757175594568253, 0.004060155246406794, -0.01677246205508709, -0.0048773749731481075, 0.010221732780337334, 0.006346424575895071, 0.0011634031543508172, 0.019522471353411674, 0.023297248408198357, -0.013198747299611568, -0.002991607878357172, 0.007465237285941839, 0.007329034153372049, 0.005393001716583967, 0.030094441026449203, -0.003885036800056696, -0.012109121307730675, 0.010656286031007767, 0.007439293898642063, 0.0076273842714726925, 0.02570999227464199, -0.004640640690922737, 0.00592484325170517, 0.013477640226483345, 0.00331265851855278, 0.005188697017729282, -0.007653327658772469, -0.001778750098310411, -0.007199316751211882, -0.020832616835832596, -0.0036256019957363605, 0.0011188127100467682, -0.007445780094712973, -0.02109205164015293, -0.009378569200634956, 0.0071733733639121056, -0.0077441297471523285, 0.018834969028830528, -0.002612184500321746, 0.004014754202216864, -0.017239445820450783, -0.015475288033485413, 0.019496528431773186, -0.006849079858511686, -0.035386912524700165, 0.0065604583360254765, -0.002740280469879508, 0.002630020724609494, -0.009073733352124691, -0.010027156211435795, -0.004728199914097786, 0.013412781059741974, -0.016889208927750587, 0.012173979543149471, 0.005136809777468443, -0.004523895215243101, -0.021351486444473267, 0.0125047592446208, 0.014956418424844742, -0.03525719419121742, -0.033778417855501175, -0.007108514662832022, -0.008522434160113335, 0.017965862527489662, 0.007056627422571182, 0.010260648094117641, -0.015721751376986504, -0.026514241471886635, 0.01034496445208788, 0.0011025980347767472, 0.006142119877040386, -0.0017868574941530824, 0.02215573564171791, -0.0075560398399829865, 0.007517124526202679, 0.014320802874863148, 0.03831852599978447, 0.0217276681214571, -0.011648624204099178, 0.03035387583076954, 0.031080294400453568, 0.01570877991616726, -0.016733547672629356, 0.007569011300802231, 0.03209209069609642, -0.015371514484286308, -0.0015485016629099846, 0.017511852085590363, 0.054896410554647446, -0.020443465560674667, -0.009034818038344383, -0.008840242400765419, 0.004377963021397591, -0.03904494270682335, -0.10133524984121323, -0.020858561620116234, 0.014048396609723568, 0.012375041842460632, -0.008392716757953167, 0.024166354909539223, -0.020832616835832596, 0.0029672859236598015, -0.02591754123568535, 0.03188454359769821, -0.0125047592446208, -0.03922654688358307, -0.014463492669165134, 0.0011974539374932647, 0.012842024676501751, -0.006362639367580414, -0.0030564668122678995, -0.0071020289324223995, -0.017498880624771118, -0.015073164366185665, -0.02075478807091713, -0.01579958200454712, -0.011544850654900074, -0.0057172952219843864, -0.008431632071733475, -0.007919248193502426, -0.0073095764964818954, 0.01862742193043232, 0.0056718941777944565, -0.004429849795997143, 0.004423364065587521, -0.000506303331349045, 0.002043049316853285, -0.00944991409778595, 0.02179252728819847, -0.009826094843447208, -0.0019133319146931171, 0.009663947857916355, -0.0010685472516342998, -0.019094403833150864, 0.0048514315858483315, 0.0059702442958951, 0.007115000393241644, -0.03741050511598587, -0.0102411899715662, -0.025450557470321655, -0.016175761818885803, 0.008937530219554901, 0.010837890207767487, -0.018432844430208206, -0.031080294400453568, 0.0024808456655591726, -0.005597306881099939, -0.03453077748417854, 0.01914629153907299, -0.020080257207155228, 0.010974093340337276, 0.00022619475203100592, 0.002639749553054571, 0.01241395715624094, 0.007997078821063042, -0.028615662828087807, -0.03460860624909401, 0.025904567912220955, 0.02491871640086174, -0.004160686396062374, 0.032377466559410095, 0.0067388201132416725, 0.009352626278996468, -0.04955205321311951, -0.004112042021006346, 0.026903392747044563, -0.018977658823132515, 0.0033921105787158012, -0.02679961919784546, -0.00399529654532671, -0.029212363064289093, -0.042106274515390396, 0.024659281596541405, 0.013146860525012016, -0.009683405049145222, -0.013438724912703037, -0.007017712574452162, -0.018393930047750473, 0.023089701309800148, 0.01762859709560871, 0.011869143694639206, 0.019548414275050163, -0.0016214677598327398, -0.029030758887529373, -0.003904494224116206, -0.0076273842714726925, 0.03497181460261345, 0.004510923288762569, 0.017421049997210503, 0.01921115070581436, -0.0162925086915493, -0.0051692393608391285, 0.00016011993284337223, 0.01277716550976038, -0.036268990486860275, -0.006907452829182148, -0.05121243745088577, 0.00963151827454567, -0.02494465932250023, -0.021442290395498276, 0.02093639224767685, -0.015306655317544937, -0.014152170158922672, -0.0042417594231665134, 0.015617976896464825, 0.0037585622631013393, -0.027266601100564003, 0.015306655317544937, -0.014489435590803623, -0.0053346287459135056, -0.028797267004847527, 0.01914629153907299, 0.016370337456464767, 0.01028659101575613, -0.01756373792886734, 0.010137416422367096, -0.019768934696912766, -0.03050953708589077, 0.005415702238678932, 0.02428310178220272, -0.014074340462684631, 0.01701892539858818, -0.020041340962052345, 0.0024678739719092846, 0.002482467098161578, -0.0005804854445159435, 0.0030548451468348503, -0.016980011016130447, 0.023543711751699448, 0.0002446389407850802, 0.004264459945261478, -0.0278373584151268, -0.03188454359769821, 0.02731848880648613, -0.00020602774748113006, 0.03600955754518509, -0.027629811316728592, -0.02890104055404663, -5.538325785892084e-05, -0.019159262999892235, -0.014294859953224659, 0.00039928642218001187, -0.014658068306744099, -0.00935911200940609, 0.025333812460303307, 0.0198078490793705, 0.03995296359062195, 0.00812679622322321, -0.012070205993950367, -0.02552838809788227, -0.0024257157929241657, -0.022402198985219002, -0.0017933433409780264, 0.0003413189551793039, 0.009683405049145222, 0.0017187558114528656, 0.02996472455561161, 0.0009169399854727089, 0.004348776303231716, -0.024477677419781685, -0.007796016987413168, -0.01472292747348547, -0.010961121879518032, 0.020702900364995003, 0.008593779057264328, -0.00812679622322321, 0.008619722910225391, -0.005840526893734932, 0.012783652171492577, 0.02170172519981861, 0.012699335813522339, -0.012426928617060184, 0.0007008794345892966, 0.021130967885255814, 0.003200777340680361, 0.02163686603307724, -0.015877412632107735, 0.01259556133300066, -0.02609914541244507, 0.04291052371263504, 0.025139236822724342, -0.002602455671876669, -0.019042517989873886, -0.02014511451125145, -0.017485907301306725, -0.0028213539626449347, -0.001306092250160873, -0.010597913525998592, -0.0019765691831707954, 0.016785433515906334, 0.01589038409292698, -0.009307225234806538, 0.0034504833165556192, 0.01917223446071148, 0.009702863171696663, 0.014956418424844742, 0.010396851226687431, -0.004569296259433031, -0.012284239754080772, -0.005052493419498205, -0.007912762463092804, -0.004627668764442205, -0.0020916934590786695, -0.021105024963617325, 0.011583765968680382, 0.010695201344788074, -0.012264781631529331, -0.0188998281955719, 0.015163966454565525, -0.016992982476949692, -0.024075552821159363, -0.0020171059295535088, 0.003011065535247326, 0.008723496459424496, -0.013451696373522282, 0.029653402045369148, 0.005097894463688135, -0.0053346287459135056, 0.03458266332745552, -0.011544850654900074, 0.019379783421754837, 0.0007357409922406077, 0.012835538946092129, -0.01015038788318634, 0.019457612186670303, -0.01993756741285324, 0.010584941133856773, 0.011577279306948185, 0.00046657733037136495, -0.003797477576881647, -0.0066934190690517426, 0.0032980653923004866, -0.0072187744081020355, 0.03087274543941021, 0.0037780199199914932, 0.05640113353729248, 0.013516555540263653, -0.008515948429703712, 0.023595599457621574, -0.007089057005941868, 0.023647485300898552, 0.005623250268399715, -0.004896832630038261, 0.0033531952649354935, -0.01214155089110136, 0.03458266332745552, 0.00992338266223669, 0.0021306085400283337, -0.0061518484726548195, -0.021922243759036064, -0.010137416422367096, -0.021948186680674553, 0.018834969028830528, -0.006972311530262232, 0.02306375652551651, -0.006362639367580414, -0.013775990344583988, 0.006622074171900749, -0.016824349761009216, -0.002991607878357172, -0.015501231886446476, 0.03312982991337776, 0.012109121307730675, -0.01941869780421257, -0.02618994750082493, 0.0014690498355776072, -0.022934040054678917, -0.026643957942724228, 0.0017057841178029776, -0.012284239754080772, -0.005859984550625086, 0.007906276732683182, -0.013503583148121834, -1.1027247637684923e-05, 0.019833793863654137, -0.015125051140785217, 0.017161615192890167, -0.009748264215886593, -0.01960030198097229, 0.01419108547270298, -0.019859736785292625, -0.001572823734022677, -0.016889208927750587, -0.03453077748417854]} +{"id": "test:10000046", "text": "\"Farmington, CT, August 30, 2016 -- Many Americans realize they need long-term care insurance, but balk at the premiums. \\\"Now there's an easy way to pay,\\\" says Larry Golfin, who represents ACSIA Partners LLC in CT. \\\"Just use some of the money that's already in your Health Savings Account.\\\"\\n\\\"It's a shame so few people know about this,\\\" says Golfin. \\\"We're spreading the word through our business partners and to the public directly.\\\"\\n\\\"In the years ahead, HSAs promise to be an increasingly important tool for the health and wellbeing of our longer-living population,\\\" says Golfin.\\nACSIA Partners has agents in all parts of the country who are glad to answer questions. \\\"Depending on your circumstances, long-term care insurance may or may not be the best solution for you,\\\" says Golfin. \\\"And the possible role of an HSA depends on your circumstances too. For example, some people may choose to pay some of their care costs directly from their HSA, and the balance from an LTC policy paid for with HSA dollars.\\\"\\nFor information, contact Golfin at larry.golfin@acsiapartners.com, http://www.larrygolfin.com or 860-677-4075.\"", "vector_field": [0.022273670881986618, 0.024775277823209763, 0.03157108649611473, -0.054018665105104446, -0.03408607468008995, 0.0317048653960228, -0.02397262305021286, 0.005009903572499752, -0.012608368881046772, 0.012086642906069756, 0.025818727910518646, 0.037323448807001114, -0.022728508338332176, 0.002856113249436021, 0.01464844960719347, 0.004441356286406517, 0.038340143859386444, -0.005427952855825424, 0.02905610390007496, -0.006802499294281006, -0.013464533723890781, 0.022126516327261925, -0.01379228476434946, 0.00985258724540472, -0.0021838899701833725, 0.007585087791085243, 0.01365181989967823, -0.006524914409965277, -0.0035417142789810896, -0.004909571725875139, 0.02020014449954033, 0.007845950312912464, -0.03373825550079346, -0.010093383491039276, 0.0019982759840786457, -0.004217281937599182, 2.2705306037096307e-05, 0.002150445943698287, 0.027852121740579605, -0.0213372390717268, 0.02137737162411213, -0.005511562805622816, 0.011016436852514744, -0.0014623366296291351, -0.02474852278828621, -0.00325576844625175, -0.029510941356420517, -0.017604894936084747, -0.037323448807001114, 0.025203360244631767, -0.00647140434011817, -0.0025534455198794603, -0.01934397965669632, 0.006207196973264217, 0.008842580020427704, -0.035985689610242844, 0.005534973461180925, 0.026848802343010902, 0.021965986117720604, -0.008661982603371143, -0.001315183355472982, -0.013283936306834221, -0.011283988133072853, 0.019959349185228348, -0.006732266861945391, -0.008233900181949139, -0.00790614914149046, -0.005846002139151096, -0.014247122220695019, 0.015705278143286705, 0.0227151308208704, 0.00822721142321825, -0.006722233723849058, 0.0009991379920393229, 0.03063465654850006, -0.0008670343668200076, -0.025577932596206665, -0.024695012718439102, 0.00605335459113121, 0.017564762383699417, 0.004213937558233738, -0.022032873705029488, -0.010494710877537727, 0.010681997053325176, 0.01690926030278206, 0.019758684560656548, -0.013330758549273014, 0.01970517449080944, -0.001082747825421393, 0.008120191283524036, 0.04363766312599182, 0.0033076065592467785, 0.001754135126248002, 0.022701753303408623, 0.013604998588562012, 0.022487711161375046, -0.007150316145271063, -0.007384424097836018, -0.03100922890007496, -0.018139997497200966, -0.005327621009200811, 0.020039614289999008, -0.010126828216016293, -0.0048460280522704124, -0.029350409284234047, -0.019183449447155, 0.0168289951980114, -0.004538343753665686, 0.01143783051520586, -0.0004937162739224732, -0.033952295780181885, 0.01982557401061058, 0.01597283035516739, -0.004739007446914911, 0.018340662121772766, -0.019932594150304794, -0.0015501270536333323, -0.0023912424221634865, 0.015063154511153698, -0.03221321105957031, 0.0028343747835606337, 0.026099657639861107, 0.0272233746945858, 0.026915689930319786, -0.01098968181759119, -0.0032658015843480825, -0.03946385905146599, -0.0015166831435635686, -0.01492937933653593, -0.016561442986130714, 0.003916286397725344, 0.0021186741068959236, -0.0053777871653437614, 0.010059939697384834, -0.013912682421505451, -0.005551695358008146, -0.010702063329517841, -0.014113346114754677, -0.026594627648591995, -0.01856807991862297, 0.0223405584692955, 0.005929612088948488, 0.010126828216016293, -0.002252449980005622, -0.014394275844097137, 0.019397489726543427, 0.026835424825549126, -0.008615161292254925, -0.012340817600488663, 0.004575132392346859, -0.002354454016312957, -0.027330394834280014, 0.0031771750655025244, -0.006444648839533329, -0.010902727022767067, 0.018541324883699417, -0.007237270474433899, 0.009585035964846611, -0.026929067447781563, -0.030581146478652954, -0.015183553099632263, 0.010962926782667637, 0.0313570462167263, 0.01202644407749176, -0.01583905518054962, 0.01438089832663536, -0.003879497991874814, 0.009511458687484264, 0.005187156610190868, 0.01220704149454832, 0.009257284924387932, 0.027450794354081154, 0.0012600007466971874, 0.006752333138138056, -0.009745567105710506, 0.004029995761811733, -0.02149777114391327, 0.010608420707285404, -0.010855905711650848, -0.041256457567214966, 0.033952295780181885, 0.013076583854854107, 0.010561599396169186, 0.006029943935573101, -0.01715005747973919, 0.001983226276934147, 0.01593269780278206, 0.01888914220035076, -0.011872601695358753, -0.004892849829047918, -0.0015484548639506102, 0.02969822660088539, 0.009036554954946041, -0.022487711161375046, -0.642123818397522, -0.019959349185228348, 0.0020969356410205364, -0.02084226906299591, 0.02779860980808735, 0.025858862325549126, -0.00885595753788948, 0.0033527559135109186, -0.029403919354081154, 0.007310847286134958, -0.015330706723034382, 0.022153271362185478, -0.011391009204089642, -0.009845898486673832, -0.012554858811199665, -0.009765633381903172, 0.015384216792881489, -0.018956029787659645, -0.008528207428753376, 0.021872343495488167, -0.005411230958998203, 0.006187130697071552, -0.010956238023936749, -0.006571736186742783, 0.025912372395396233, 0.02620667777955532, 0.006317561957985163, -0.034353625029325485, -0.0016847389051690698, 0.017484497278928757, -0.02311645820736885, 0.019999481737613678, -0.020507829263806343, 0.007083428557962179, 0.05153043568134308, -8.632719254819676e-05, 0.005749014671891928, -0.0007704650051891804, 0.00922384113073349, 0.0238923579454422, -0.01954464428126812, -0.011651871725916862, 0.013357513584196568, 0.007177071645855904, -0.002361142775043845, 0.003956419415771961, -0.02556455507874489, -0.006026599556207657, -0.004150393884629011, 0.005715570878237486, 0.03202592581510544, 0.019718552008271217, -0.009906097315251827, -0.02555117756128311, -9.991379920393229e-05, 0.0014907640870660543, 0.00395307457074523, -0.03226672112941742, 0.017859069630503654, 0.028173182159662247, 0.006387794390320778, -0.008795758709311485, -0.021845588460564613, -0.010173649527132511, -0.03603919968008995, 0.028842061758041382, -0.005929612088948488, 0.007170382887125015, 0.03280182555317879, 0.00175246293656528, 0.0036420461256057024, 0.03087545372545719, -0.015905942767858505, -0.003933008294552565, 0.010588354431092739, 0.009377683512866497, -0.005150367971509695, 0.012768900021910667, -0.0026002670638263226, 0.017484497278928757, 0.01657482050359249, -0.023745203390717506, 0.018541324883699417, -0.00904324371367693, 0.02462812326848507, -0.0010710424976423383, -0.007698797155171633, -0.0035149590112268925, -0.0013712019426748157, -0.009317484684288502, 0.0018210230628028512, 0.026715027168393135, 0.013511355035007, -0.02494918555021286, 0.006782433018088341, -0.00025793645181693137, -0.010046562179923058, -0.00029179846751503646, 0.01904967427253723, -0.03836689889431, -0.005749014671891928, 0.00558848399668932, -0.005551695358008146, 0.010956238023936749, 0.013952815905213356, 0.02201949618756771, -0.020614849403500557, 0.0035985689610242844, 0.019999481737613678, -0.018996162340044975, 0.010187027044594288, -0.01649455539882183, -0.0016111622098833323, 0.004317613784223795, 0.003899564500898123, -0.009116820991039276, 0.0006563374772667885, 0.0008302460191771388, 0.0085148299112916, -0.026929067447781563, 0.02022690139710903, 0.01343108993023634, -0.0020835581235587597, -0.01456818450242281, 0.002401275560259819, 0.02555117756128311, 0.015611635521054268, 0.0070031629875302315, 0.014474540948867798, 0.020467696711421013, 0.00254508457146585, -0.022728508338332176, 0.0005171270458959043, -0.018434304744005203, 0.004384501837193966, -0.0054513635113835335, 0.025618065148591995, 0.0026638105046004057, 0.022581353783607483, -0.027049466967582703, -0.016227005049586296, -0.0062941513024270535, 0.0048460280522704124, -0.009284039959311485, 0.004240692593157291, -0.0325075201690197, -0.014554806984961033, -0.002361142775043845, -0.00726402597501874, -0.00492629362270236, -0.003655423643067479, -0.031196516007184982, -0.02173856645822525, 0.009350928477942944, -0.025444157421588898, -0.011892667971551418, 0.013966193422675133, -0.01994597166776657, -0.01657482050359249, -0.02247433364391327, 0.0014364175731316209, 0.019598154351115227, -0.019290469586849213, 0.027932386845350266, -0.014514673501253128, 0.017043037340044975, 0.02758456952869892, 0.0407213531434536, -0.001209834823384881, -0.021885721012949944, 0.0018477782141417265, -0.028494244441390038, -2.211089167758473e-06, 0.025577932596206665, 0.023303743451833725, 0.006508192513138056, 0.0022875661961734295, 0.037002384662628174, -0.007290781009942293, 0.0026370554696768522, 0.0008820841321721673, 0.011558229103684425, 0.006819221191108227, -0.0095917247235775, 0.01990583911538124, -0.0136116873472929, -0.009478014893829823, 0.007946282625198364, -0.007692108396440744, 0.009946230798959732, -0.00518046785145998, 0.014581562019884586, -0.010842528194189072, 0.011551540344953537, -0.01375215221196413, -0.0003114467835985124, -0.02040080912411213, 0.012969563715159893, 0.024614745751023293, 0.004280825611203909, 0.030099553987383842, -0.004090195056051016, 0.001192276831716299, 0.017096547409892082, 0.019183449447155, -0.018233641982078552, 0.0004144959384575486, -0.0033293452579528093, 0.028494244441390038, 0.0031989137642085552, -0.016467800363898277, -0.025096340104937553, 0.031918905675411224, -0.012835787609219551, 0.013176916167140007, 0.01438089832663536, 0.00333770620636642, -0.004140360746532679, 0.023664938285946846, -0.0017658405704423785, 0.0152771957218647, 0.008447941392660141, 0.010066628456115723, -0.04117618873715401, -0.004832650534808636, 0.009297417476773262, 0.012039821594953537, 0.0073442910797894, -0.0248689204454422, -0.0033661334309726954, 0.01176558155566454, 0.0073643578216433525, 0.010280669666826725, 0.03499574959278107, 0.030179819092154503, -0.0018461060244590044, 0.011197034269571304, -0.0025701674167066813, 0.01452805195003748, -0.017939334735274315, 0.009611790999770164, -0.008682048879563808, 0.013584932312369347, -0.0037055895663797855, 0.010240537114441395, 0.00818707887083292, 0.028146427124738693, 0.030286841094493866, -0.007518199738115072, 0.02064160630106926, -0.01796608977019787, 0.018340662121772766, 0.0009824159787967801, -0.0011655216803774238, 0.03221321105957031, -0.010742196813225746, 0.005528284702450037, 0.029350409284234047, 0.02620667777955532, 0.0309289637953043, 0.007029918022453785, 0.025858862325549126, -0.012140153907239437, -0.01904967427253723, -0.010608420707285404, -0.0034079384058713913, 0.0026621383149176836, -0.005484807770699263, -0.028066162019968033, -0.013604998588562012, 0.003402921836823225, -0.007952971383929253, 0.009284039959311485, -0.03071492351591587, 0.02311645820736885, 0.008280721493065357, -0.00726402597501874, -0.010160272009670734, 0.009197086095809937, -0.016882505267858505, -0.012895986437797546, -0.04831981658935547, 0.03737695887684822, -0.007845950312912464, -0.015451104380190372, -0.010434512048959732, 0.005949678365141153, 0.0036654567811638117, -0.010287358425557613, 0.010996370576322079, -0.0021972674876451492, -0.014474540948867798, 0.008193767629563808, -0.000719462928827852, 0.002122018486261368, -0.01986570656299591, 0.01824701949954033, 0.014661827124655247, 0.006849320605397224, -0.005367754027247429, -0.0013854155549779534, -0.025203360244631767, -0.0024380639661103487, -0.00963185727596283, 0.03676158934831619, 0.010240537114441395, -0.04184506833553314, -0.021404126659035683, -0.024440838024020195, -0.042192887514829636, 0.014046458527445793, -0.021270351484417915, -0.010956238023936749, 0.02263486571609974, 0.0333101749420166, 0.005458052735775709, -0.017457742244005203, -0.012113397940993309, 0.03737695887684822, -0.010802395641803741, 0.002503279596567154, -0.034433890134096146, -0.034433890134096146, -0.001316855545155704, 0.10364948958158493, 0.0048594060353934765, -0.022567976266145706, 0.01739085279405117, -0.0013753824168816209, -0.021471016108989716, 0.0016747057670727372, -0.024106398224830627, 0.04355739802122116, 0.029510941356420517, -0.0045182774774730206, 0.01710992492735386, -0.007799129001796246, -0.003172158496454358, 0.018835632130503654, 0.03619972988963127, -0.009966297075152397, -0.02441408298909664, -0.00914357602596283, -0.026460852473974228, 0.019063051789999008, -0.015451104380190372, 0.014902623370289803, 0.060680702328681946, 0.005926267709583044, -0.022808773443102837, 0.0395708791911602, 0.007758995983749628, 0.0024280305951833725, -0.0221398938447237, 0.0168289951980114, 0.008059991523623466, 0.013223737478256226, 0.010093383491039276, -0.014942756853997707, -0.012160220183432102, 0.003742377972230315, 0.012220419012010098, 0.019451001659035683, -0.008949601091444492, 0.029403919354081154, 0.009063309989869595, 0.012046510353684425, -0.01117027923464775, 0.018073109909892082, -0.030286841094493866, -0.00012959529703948647, 0.0340593196451664, 0.011116769164800644, 0.018193507567048073, 0.02730363979935646, -0.011190345510840416, -0.002068508183583617, -0.002648760797455907, 0.016628332436084747, -0.018862387165427208, -0.011966245248913765, -0.027932386845350266, 0.0006266559939831495, -0.012334128841757774, -0.0029313622508198023, -0.01589256525039673, -0.0013385940110310912, -0.006648656912147999, -0.00014140519488137215, -0.0356111153960228, -0.020507829263806343, 0.004718941170722246, -0.005471430253237486, -0.017257077619433403, 0.01884900964796543, -0.026795292273163795, -0.02766483463346958, 0.014822358265519142, 0.012374261394143105, -0.0063978275284171104, 0.018180130049586296, 0.0075717102736234665, -0.006100176367908716, 0.01974530704319477, 0.007451311685144901, -0.01625376008450985, -0.02381209097802639, -0.02247433364391327, 0.011277299374341965, -0.013424401171505451, -0.00041219667764380574, -0.0146885821595788, 0.0031771750655025244, 0.043503887951374054, 0.009919475764036179, -0.0068325987085700035, 0.015558125451207161, -0.004805895499885082, -0.0084144975990057, 0.00045149330981075764, 0.018461059778928757, 0.013725397177040577, -0.015437726862728596, -0.009478014893829823, 0.011223789304494858, -0.01669522002339363, -0.01767178252339363, -0.0293771643191576, 0.018782122060656548, -0.009859276004135609, -0.0008845924749039114, -0.0034748262260109186, -0.005444674752652645, -0.02345089800655842, 0.021350616589188576, -0.014447785913944244, -0.00830747652798891, -0.008468007668852806, -0.00873555988073349, 0.02173856645822525, -0.007150316145271063, -0.004213937558233738, -0.016039717942476273, -0.02255459874868393, 0.004528310615569353, -0.016561442986130714, 0.022688375785946846, -0.0034982371143996716, -0.02104293368756771, 0.003090220969170332, 0.03657430410385132, 0.01868847943842411, 0.0025584620889276266, -0.012267240323126316, -0.0005672929692082107, 9.265018888982013e-05, -0.010902727022767067, -0.029724981635808945, -0.027236752212047577, 0.000327959714923054, -0.021136576309800148, 0.0012064904440194368, -0.001943093491718173, -0.013384268619120121, 0.015825677663087845, -0.00342298811301589, -0.0016320646973326802, -0.005401197820901871, 0.024106398224830627, -0.02668827213346958, 0.0006847648764960468, 0.002779192291200161, 0.018434304744005203, 0.01946437917649746, -0.005601861514151096, -0.015986207872629166, -0.019611531868577003, 0.022246915847063065, 0.002904606983065605, -0.008869335055351257, 0.01365181989967823, 0.006284118164330721, 0.034915484488010406, 0.01641429029405117, 0.02494918555021286, -0.007116872351616621, -0.0034112827852368355, 0.009932853281497955, 0.015063154511153698, 0.012588302604854107, -0.011999689042568207, 0.0004015364102087915, 0.00029723308398388326, -0.0012583285570144653, 0.0063309394754469395, 0.023103080689907074, 0.017778802663087845, -0.004996526055037975, 0.0021437571849673986, 0.009886031039059162, -0.007310847286134958, -0.04591185227036476, -0.01908980682492256, -0.04040028899908066, -0.01673535257577896, -0.021136576309800148, -0.008849268779158592, -0.003284195903688669, -0.023210100829601288, -0.005250699818134308, 0.02235393598675728, -0.01316353864967823, 0.006153686437755823, -0.006320906337350607, 0.026233434677124023, -0.01379228476434946, 0.027022710070014, 0.011872601695358753, -0.018554702401161194, 0.005160401575267315, -0.02889557182788849, -0.025832105427980423, 0.007692108396440744, 0.014742093160748482, 0.017845692113041878, -0.0012992974370718002, 0.031999170780181885, -0.042192887514829636, 0.004698874894529581, 0.016240382567048073, -0.034273359924554825, -0.013110027648508549, -0.015905942767858505, -0.012414393946528435, 0.008240588940680027, -0.021992741152644157, -0.02572508528828621, -0.012594991363584995, 0.006157030817121267, -0.01353142224252224, -0.00836767628788948, 0.023905735462903976, -0.0024531136732548475, -0.016267137601971626, -0.008401120081543922, -0.0064847818575799465, 0.010608420707285404, -0.02006636932492256, 0.020213522017002106, 0.0033460671547800303, -0.003795888274908066, -0.0293771643191576, 0.0095917247235775, 0.031303536146879196, -0.004876127932220697, -0.002526690252125263, 0.037002384662628174, -0.03582515940070152, -0.009959608316421509, -0.012180286459624767, 0.029724981635808945, -0.0188088770955801, -0.049309760332107544, 0.004143705125898123, -0.0029129679314792156, 0.011364254169166088, -0.014768848195672035, -0.02478865534067154, -0.0014163512969389558, 0.021925853565335274, 0.029965778812766075, -0.003386199939996004, 0.00424738135188818, -0.006672068033367395, -0.002397931180894375, -0.006076765712350607, -0.007217204198241234, 0.022246915847063065, 0.006310873199254274, -0.003989863209426403, -0.016962770372629166, -0.012347506359219551, 0.004458078648895025, 0.00763859786093235, -0.005846002139151096, 0.048988696187734604, 0.003565124934539199, 0.03986518830060959, 0.0018109898082911968, 0.0024798689410090446, -0.003993207588791847, -0.004468111786991358, 0.003735689213499427, 0.0221398938447237, -0.018835632130503654, -0.019277092069387436, 0.003715622704476118, 0.0009431193466298282, -0.017417607828974724, -0.009471326135098934, -0.0100800059735775, 0.02006636932492256, -0.01625376008450985, -0.013845794834196568, 0.03652079403400421, -0.006327595096081495, -0.013003007508814335, -0.022608108818531036, -0.009397749789059162, 0.003902908880263567, 0.0060700769536197186, 0.006254018284380436, 0.019397489726543427, -0.002886212896555662, -0.002897918224334717, -0.00034468169906176627, -0.007063362281769514, -0.0026420720387250185, -0.0017206912161782384, -0.007183760404586792, -0.02945743128657341, 0.020414186641573906, -0.02401275560259819, 0.03705589473247528, -0.01657482050359249, 0.012949497438967228, -0.010554910637438297, 0.040213003754615784, -0.00018059732974506915, -0.015825677663087845, 0.009585035964846611, 0.019250337034463882, -0.013130094856023788, -0.014407653361558914, 0.016066472977399826, 0.0019246992887929082, 0.01755138486623764, 0.0030952375382184982, -0.0051035466603934765, 0.0186082124710083, 0.009524837136268616, -0.0022440890315920115, -0.029510941356420517, 0.013899304904043674, -0.012347506359219551, -0.0074379341676831245, 0.008468007668852806, -0.025952504947781563, 0.02108306623995304, -0.035049259662628174, 0.034514155238866806, 0.00010169050801778212, -0.001274214475415647, -0.012454526498913765, -0.012494659051299095, 0.007859327830374241, -0.0004036266473121941, -0.008153635077178478, -0.01727045513689518, 0.005799180828034878, -0.015798920765519142, 0.011564917862415314, 0.006153686437755823, -0.013023073785007, 0.0005505710141733289, 0.01202644407749176, 0.025256870314478874, -0.001712330151349306, -0.010253914631903172, 0.010334180667996407, -0.026139790192246437, -0.0065382919274270535, -0.015076532028615475, 0.01261505763977766, -0.009538214653730392, 0.053135745227336884, -0.0009105115314014256, -0.03432686999440193, -0.010996370576322079, 0.009578347206115723, -0.05318925529718399, -0.009872653521597385, 0.008648605085909367, 0.011872601695358753, 0.01743098720908165, 0.0434771329164505, -0.00786601658910513, 0.01686912775039673, 0.014875868335366249, 0.01892927475273609, -0.007672042120248079, 0.008769003674387932, 0.01820688508450985, -0.011197034269571304, 0.0033477393444627523, -0.0012190319830551744, -0.02303619310259819, 0.011183656752109528, 0.006956341210752726, 0.0019280436681583524, -0.0047423518262803555, -0.0016111622098833323, -0.01922358199954033, -0.0038326766807585955, 0.03438038006424904, 0.0027323707472532988, -0.021511148661375046, -0.004270792473107576, -0.0031470756512135267, -0.01563839055597782, 0.018755367025732994, 0.007732240948826075, -0.016855750232934952, -0.02654111757874489, -0.001354479929432273, 0.00020818859047722071, -0.01649455539882183, 0.007999792695045471, 0.011404386721551418, 0.005575106479227543, 0.0023310433607548475, -0.030367106199264526, 0.03133029118180275, 0.01117696799337864, -0.0022156615741550922, 0.015531370416283607, 0.006233952008187771, -0.007397801615297794, 0.012909363955259323, -0.006949652452021837, 0.0004464767116587609, -0.027370527386665344, -0.006595146842300892, -0.01828715205192566, 0.02811967208981514, -0.011832469142973423, 0.0018093176186084747, 0.01751125231385231, -0.02547091245651245, -0.02125697396695614, 0.005822591483592987, -0.016842372715473175, 0.03314964100718498, -0.008153635077178478, -0.013645131140947342, 0.022889038547873497, -0.012969563715159893, -0.016186870634555817, -0.002966478234156966, -0.02686217986047268, -0.01904967427253723, -0.008789069950580597, -0.01117696799337864, 0.011056569404900074, 0.00967198982834816, -0.011812402866780758, -0.014166857115924358, 0.010367624461650848, -0.0008348445990122855, 0.0034146271646022797, 0.2059077024459839, -0.007478067185729742, -0.015651768073439598, 0.01950451172888279, -0.01216690894216299, 0.019879084080457687, 0.017992844805121422, 0.01735072024166584, -0.01129736565053463, 0.022982681170105934, -0.026888934895396233, 0.026848802343010902, -0.011424452997744083, -0.0046018874272704124, -0.0077121746726334095, -0.024400705471634865, -0.004973115399479866, -0.008140257559716702, -0.02104293368756771, -0.012467904016375542, -0.002770831109955907, 0.011631805449724197, 0.008682048879563808, -0.007431245408952236, 0.008769003674387932, -0.008100124076008797, 0.01739085279405117, 0.0013076583854854107, 0.016106605529785156, 7.635932888661046e-06, -0.010688685812056065, 0.002208972815424204, 0.007892771624028683, 0.0028544410597532988, -0.012956186197698116, -0.008033236488699913, -0.01567852310836315, -0.008314166218042374, 0.026300322264432907, 0.007845950312912464, 0.0019263714784756303, 0.01088934950530529, -0.025992637500166893, -0.002692237962037325, 0.019437624141573906, 0.015825677663087845, 0.017257077619433403, -0.01876874454319477, -0.02080213651061058, -0.001568521256558597, -0.032561030238866806, -0.02076200395822525, -0.01550461445003748, 0.02454785816371441, -0.0008661982719786465, 0.01397957094013691, 0.011799025349318981, 0.006635279394686222, -0.03355097025632858, 0.04885492101311684, 0.0152771957218647, 0.03282858058810234, -0.0031755028758198023, 0.007852639071643353, 0.0013486272655427456, 0.021925853565335274, 0.01848781481385231, -0.006922897417098284, 0.014447785913944244, -0.008360987529158592, 0.005919578950852156, -0.004029995761811733, -0.018233641982078552, -0.020654983818531036, -0.025671575218439102, -0.019036296755075455, 0.010688685812056065, 0.046339936554431915, -0.007625220343470573, 0.018782122060656548, -0.015384216792881489, 0.0043577468022704124, -0.00522060040384531, 0.021310484036803246, 0.013992948457598686, -0.028253449127078056, 0.003070154460147023, 0.0013845794601365924, -0.016066472977399826, -0.00840780884027481, 0.00973218958824873, 0.005197189748287201, -0.005805869586765766, -0.003008283209055662, 0.0009707106510177255, -0.01062848698347807, 0.01852794736623764, 0.01492937933653593, -0.014273877255618572, 0.0015308967558667064, -0.013009696267545223, 0.0555972196161747, 0.0010409429669380188, 0.014635072089731693, 0.004545032512396574, -0.0044146012514829636, -0.01188597921282053, 0.018541324883699417, 0.0005325949168764055, -0.00786601658910513, 0.016441045328974724, -0.007150316145271063, -0.0016429340466856956, -0.007999792695045471, 0.0005994827952235937, 0.012280617840588093, -0.016842372715473175, -0.03478170558810234, 0.029002591967582703, -0.009692056104540825, 0.00954490341246128, -0.033711500465869904, -0.009906097315251827, 0.012681945227086544, -0.004417945630848408, -0.01787244714796543, -0.04615265130996704, -0.027771854773163795, -0.028413979336619377, -0.03039386123418808, -0.005210567265748978, -0.005431297235190868, 0.008581717498600483, -0.031517576426267624, -0.027852121740579605, 0.012648501433432102, 0.004013273864984512, -0.009016488678753376, -0.015745410695672035, 0.013270558789372444, -0.01567852310836315, -0.0019497822504490614, -0.015437726862728596, -0.011263921856880188, 0.021404126659035683, -0.01601296290755272, 0.009605102241039276, 0.018594834953546524, -0.012795655056834221, -0.02291579358279705, -0.006050010211765766, -0.0035383698996156454, -0.0007453820435330272, 0.008909467607736588, 0.03015306405723095, -0.012287306599318981, -0.03242725133895874, -0.011263921856880188, 0.0321061909198761, 0.008140257559716702, -0.024601368233561516, 0.01832728460431099, 0.03470144048333168, 0.007805817760527134, -0.0007303322199732065, -0.0010551565792411566, -0.16962771117687225, 0.011692004278302193, -0.022447578608989716, -0.003272490343078971, 0.011778959073126316, 0.012447837740182877, -0.0022925827652215958, -0.006618557497859001, -0.01942424476146698, 0.016240382567048073, -0.0003956837172154337, -0.01054153311997652, -0.02275526337325573, -0.021644923835992813, 0.010942860506474972, 0.025069585070014, -0.026313699781894684, 0.03751073405146599, 0.003668801160529256, -0.0019029607065021992, 0.02121684141457081, -0.021136576309800148, 0.026260189712047577, -0.008193767629563808, 0.017685160040855408, 0.018180130049586296, -0.0018611557316035032, -0.0011730465339496732, -0.04141698777675629, -0.013223737478256226, -0.014447785913944244, 0.025056207552552223, -0.0023243543691933155, -0.006274085026234388, -0.0050901691429317, -0.009250596165657043, -0.018313907086849213, -0.017016280442476273, -0.01816675253212452, 0.04259421303868294, 0.007578399032354355, -0.003275834722444415, -0.003381183370947838, 0.01306320633739233, -0.015384216792881489, 0.03186539560556412, 0.025791972875595093, -0.025537800043821335, 0.026661517098546028, -0.005003214813768864, 0.0005614403053186834, 0.0024480971042066813, 0.023999378085136414, 0.0336579903960228, -0.0068626985885202885, 0.0101803382858634, -0.006237296387553215, 0.009665301069617271, 0.0009188724798150361, 0.022447578608989716, -0.013658508658409119, -0.011912734247744083, 0.006136964540928602, -0.00818707887083292, 0.0015919320285320282, -0.05083480104804039, -0.009391061030328274, -0.015705278143286705, -0.00922384113073349, 0.025430778041481972, 0.001754135126248002, -0.015357461757957935, 0.004404568113386631, -0.009785699658095837, 0.029029347002506256, -0.007324224803596735, -0.023986000567674637, 0.009177019819617271, -0.00023348057584371418, 0.00010247435420751572, -0.013551488518714905, 0.023384008556604385, -0.03397905081510544, -0.03828663378953934, -0.016762107610702515, -0.00808674655854702, 0.015558125451207161, 0.0053978534415364265, 0.025323757901787758, 0.0024815411306917667, 0.010514778085052967, -0.021604791283607483, -0.0178323145955801, -0.013404334895312786, 0.005290832836180925, 0.02343752048909664, 0.0030417272355407476, -0.01113014668226242, -0.0039129420183598995, -0.015544747933745384, 0.0035417142789810896, -0.022808773443102837, -0.016441045328974724, 0.009685367345809937, 0.01170538179576397, 0.0013829072704538703, -0.005524940323084593, -0.003973141312599182, -0.0011036503128707409, -0.0333101749420166, -0.011778959073126316, 0.038902003318071365, 0.035530850291252136, 0.016762107610702515, 0.0008871007594279945, 0.006270740646868944, -0.035049259662628174, -0.04216613247990608, -0.015437726862728596, -0.026380587369203568, 0.017404230311512947, -0.007524888496845961, -0.00891615729779005, 0.0009857603581622243, 0.0058794463984668255, 0.008929534815251827, -0.10290034115314484, -0.0221398938447237, 0.01515679806470871, 0.01824701949954033, 0.004374468699097633, 0.018220262601971626, -0.007023229263722897, 0.0152771957218647, -0.029109613969922066, 0.020052991807460785, -0.010528155602514744, -0.027049466967582703, -0.004866094794124365, -0.015879187732934952, 0.044761382043361664, -0.021096443757414818, -0.002165495650842786, -0.041336722671985626, 0.01275552250444889, 0.007618531584739685, 0.007397801615297794, -0.0073442910797894, 0.003106942865997553, -0.03234698623418808, -0.006825909949839115, -0.035290054976940155, -0.035290054976940155, 0.02450772561132908, 0.01438089832663536, 0.025363890454173088, -0.008387742564082146, -0.007123561110347509, 0.009056621231138706, -0.005411230958998203, 0.010494710877537727, -0.029805246740579605, -0.013364202342927456, -0.006869387347251177, 0.05177123472094536, -0.02999253384768963, -0.0018628279212862253, 0.008180390112102032, -0.005023281089961529, -0.02458799071609974, 0.016066472977399826, -0.026380587369203568, -0.006929586175829172, -0.008715493604540825, 0.010454578325152397, -0.0180062223225832, -0.011411075480282307, -0.03298911079764366, -0.03173162043094635, 0.0016822306206449866, 0.0005727276438847184, -0.00363201298750937, -0.004351057577878237, 0.05864730849862099, -0.006377761252224445, -0.016427667811512947, 5.89514911553124e-07, -0.011625116690993309, -0.017578139901161194, -0.004307580646127462, 0.04213937744498253, -0.002190578728914261, -0.014140102081000805, -0.016400912776589394, 0.010414445772767067, -0.007725552190095186, -0.021885721012949944, 0.02230042591691017, -0.025163227692246437, 0.0025768561754375696, -0.051824744790792465, 0.01379228476434946, 0.007651975378394127, -0.014621694572269917, 0.0007896952447481453, -0.006106865126639605, -0.019036296755075455, -0.025082962587475777, 0.0004147049621678889, -0.03933008387684822, -0.010307424701750278, -0.016681842505931854, 0.030527636408805847, 0.001479894737713039, 0.005585139617323875, -0.02315659075975418, 0.03847391903400421, 0.028012651950120926, -0.0106686195358634, 0.010554910637438297, -0.0021119853481650352, -0.0010192043846473098, -0.01491600088775158, -0.0015818987740203738, 0.016307270154356956, 0.012340817600488663, -0.03108949586749077, -0.0009665301186032593, -0.06453344225883484, 0.009397749789059162, 0.015665145590901375, 0.005638649687170982, 0.0052573890425264835, -0.014166857115924358, 0.00418383814394474, -0.008474696427583694, -0.008026547729969025, -0.004344368819147348, -0.028949081897735596, 0.013444467447698116, 0.005631960928440094, 0.020213522017002106, -0.017725292593240738, -0.025136472657322884, 0.023598050698637962, 0.002817652653902769, 0.04401223734021187, 0.027196619659662247, 0.002257466549053788, -0.00991278700530529, -0.008528207428753376, 0.0009966297075152397, 0.007136938627809286, 0.0293771643191576, 0.0019531266298145056, 0.02510971762239933, -0.0018862386932596564, 0.00605335459113121, 0.004919604863971472, -0.0026855492033064365, -0.0008937895181588829, 0.02040080912411213, -0.009350928477942944, -2.265305010951124e-05, 0.008922846056520939, -0.004815928637981415, 0.014273877255618572, -0.011163590475916862, -0.02409302070736885, 0.009451259858906269, 0.0009916131384670734, -0.02458799071609974, -0.02640734240412712, -0.016976147890090942, -0.017859069630503654, 0.023477653041481972, 0.022741885855793953, -0.007892771624028683, 0.0423266626894474, 0.016387535259127617, -0.0442797876894474, -0.027932386845350266, -0.02758456952869892, -0.013176916167140007, 0.004213937558233738, 0.003030021907761693, 0.0016788862412795424, -0.017016280442476273, 0.023303743451833725, 0.018273774534463882, 0.023664938285946846, 0.0012625090312212706, 0.012307372875511646, 0.0016337368870154023, -0.02197936363518238, -0.010307424701750278, -0.005815902724862099, -0.02881530672311783, -0.03015306405723095, 0.012949497438967228, 0.0003850234788842499, 0.017925957217812538, 0.01795271225273609, 0.005407886579632759, 0.0073643578216433525, -0.014635072089731693, -0.012742144986987114, 0.0013285608729347587, 0.0026621383149176836, 0.0007111019804142416, -0.00991278700530529, 0.027116354554891586, 0.009090065024793148, 0.01966504193842411, -0.0007704650051891804, 0.01828715205192566, -0.01072881929576397, -0.007203826680779457, 0.002006636932492256, 0.006401171907782555, -0.010875971987843513, -0.0005677110166288912, 0.0233171209692955, 0.01339095737785101, 0.0029999222606420517, 0.0043778130784630775, -0.016976147890090942, 0.00932417344301939, 0.04655397683382034, 0.0046386756002902985, -0.012300684116780758, 0.006648656912147999, -0.01165856048464775, 0.009919475764036179, -0.008989733643829823, -0.03023332916200161, -0.004678808618336916, 0.021404126659035683, 0.010802395641803741, 0.0018477782141417265, 0.027317017316818237, 0.006009877659380436, -0.03454091027379036, 0.02185896597802639, 0.011150212958455086, -0.02327698841691017, -0.007243959233164787, 0.007538266014307737, 0.008501451462507248, 0.005324276629835367, 0.020347299054265022, 0.00012719152437057346, 0.008936223573982716, -0.006357694510370493, 0.025417400524020195, -0.0037055895663797855, -0.000855328980833292, -0.0020384087692946196, -0.0031587809789925814, -0.010936171747744083, -0.06078772246837616, -0.008668671362102032, -0.025256870314478874, -0.006377761252224445, 0.01625376008450985, 0.005702193360775709, -0.024521103128790855, 0.06485450267791748, -0.0030768432188779116, -0.001354479929432273, -0.0036487348843365908, 0.00454168813303113, 0.015370839275419712, -0.014942756853997707, 0.007725552190095186, 0.0014013014733791351, -0.01251472532749176, -0.0011638493742793798, -0.008501451462507248, -0.001943093491718173, -0.023905735462903976, -0.0031604531686753035, 0.006280773784965277, 0.021765321493148804, 0.014875868335366249, 0.007952971383929253, 0.0046386756002902985, 0.005143679212778807, 0.007036606781184673, -3.851279689115472e-05, -0.023424141108989716, -0.012360883876681328, -0.0101803382858634, 0.004384501837193966, -0.008869335055351257, 0.007330913562327623, -0.03563787043094635, 0.0025985948741436005, -0.0033109509386122227, -0.0090031111612916, 0.004588509909808636, 0.013451156206429005, 0.013083272613584995, -0.014996266923844814, -0.009069998748600483, 0.009758944623172283, 0.028574509546160698, -0.020039614289999008, 0.044467076659202576, -0.009839209727942944, -0.03039386123418808, -0.009725500829517841, 0.028735041618347168, -0.013417712412774563, 0.007110183592885733, -0.01832728460431099]} +{"id": "test:10000047", "text": "\"So many things happening this weekend!\\nThe highly anticipated PBS Kids in the Park festival is tomorrow, Saturday, June 21. But there are lot of other events for families to enjoy this weekend as well.\\nAlso on Saturday, check out the free Summer Solstice Celebration at the Indianapolis Museum of Art\u2019s 100 Acre Woods. The Windsor Park Neighborhood Association will also celebrate the solstice with a festival in Fletcher Park from 12 p.m.-11 p.m.\\nSaturday and Sunday, you can experience Native American culture through myriad experiences on offer at the Eiteljorg Indian Market and Festival.\\nSaturday, the Irvington Folk Festival features its Children\u2019s Day from 11 a.m.-2 p.m. at Ellenberger Park.\"", "vector_field": [0.006860240362584591, 0.01155694480985403, -0.008396418765187263, -0.010634559206664562, -0.03594589605927467, 0.019125929102301598, -0.019220881164073944, 0.00879657082259655, -0.005083292257040739, 0.004893389530479908, 0.0005900553078390658, 0.003223261795938015, -0.0037031054962426424, 0.0011996094835922122, -0.01642659679055214, 0.005683520808815956, 0.01900384947657585, -0.03304309397935867, 0.013205030001699924, -0.011367041617631912, 0.008647361770272255, 0.02348013035953045, 0.0161281768232584, -0.011021147482097149, -0.0019007240189239383, 0.016847096383571625, 0.011258525773882866, -0.01783730275928974, 0.0008558345143683255, -0.001576871844008565, 0.010153019800782204, 0.0007121356902644038, -0.002538254950195551, -0.016942046582698822, -0.008118346333503723, -0.02194734290242195, -0.006978929508477449, -0.020129701122641563, 0.012065612711012363, -0.007602896075695753, 0.024226177483797073, 0.00017400695651303977, -0.013564488850533962, -0.01755244843661785, -0.027400268241763115, 0.005639436189085245, 0.0060226330533623695, 0.017810175195336342, 0.005639436189085245, 0.01667075790464878, 0.018230672925710678, -0.004611926153302193, -0.03016742318868637, -0.003920137416571379, 0.01908523589372635, -0.007643589749932289, 0.012099524028599262, 0.030357327312231064, -0.004767918027937412, -0.024402515962719917, -0.01496163196861744, -0.011855363845825195, -0.008769442327320576, 0.01698273979127407, -0.02786146104335785, -0.020373862236738205, -0.01965494453907013, 0.003906572703272104, -0.006070109084248543, -0.014107069000601768, 0.04951038584113121, 0.017783045768737793, -0.0019397219875827432, -0.016494419425725937, 0.01008519809693098, -0.015449953265488148, -0.02168961800634861, 0.014608955010771751, 0.025935303419828415, -0.008036959916353226, -0.00777245219796896, -0.032934579998254776, -0.057296402752399445, 0.02547411061823368, 0.02168961800634861, 0.019709203392267227, -0.002558601787313819, 0.04034079238772392, 0.0018939417786896229, 0.00894578080624342, 0.004567841533571482, 0.010153019800782204, -0.0056123072281479836, -0.01378830336034298, -0.004500019364058971, 0.025881044566631317, -0.007921661250293255, 0.020306039601564407, -0.018922463059425354, 0.0069280629977583885, -0.0016760622384026647, 0.007087445817887783, 0.022394971922039986, -0.017660964280366898, -0.024388952180743217, 0.005459706764668226, 0.0021601449698209763, -0.0015183750074356794, 0.017742352560162544, 0.028783846646547318, -0.028865233063697815, 0.005079901311546564, 0.02255774475634098, -0.05249457433819771, -0.019559992477297783, -0.018027206882834435, 0.003192741656675935, 0.007067098747938871, -0.011923185549676418, -0.025582626461982727, 0.03106267936527729, 0.026776300743222237, 0.010932978242635727, -0.0350506417453289, 0.017511755228042603, 0.005666565150022507, -0.001371708931401372, -0.012716708704829216, -0.014622519724071026, -0.01724046654999256, -0.003570851869881153, -0.011896057054400444, 0.006626253016293049, -0.023547952994704247, -0.025867480784654617, 0.009264546446502209, -0.0021262336522340775, 0.006978929508477449, -0.025121433660387993, -0.0019159840885549784, 0.02140476368367672, 0.01700986921787262, -0.02727818861603737, -0.01871899515390396, 0.028268394991755486, 0.03659699112176895, 0.006165060214698315, -0.002974014263600111, 0.013211812824010849, -0.018868204206228256, 0.027061155065894127, 0.016643628478050232, -0.003781101433560252, 0.003950657323002815, 0.016331644728779793, 0.010200495831668377, -0.0029146696906536818, 0.012370814569294453, 0.0062396652065217495, -0.013903601095080376, 0.008138693869113922, -0.021553972736001015, -0.002251705154776573, 0.022381406277418137, 0.012601410038769245, 0.012526805512607098, -0.0009588398388586938, 0.02021108940243721, -0.0061989715322852135, -0.0117197185754776, -0.010207277722656727, -0.009732521139085293, -0.02578609436750412, 0.005649609956890345, 0.009223852306604385, 0.02905513532459736, 0.009271328337490559, 0.01637233793735504, -0.02726462297141552, -0.019248010590672493, -0.010437874123454094, 0.012981216423213482, 0.025664012879133224, 0.02192021533846855, -0.02520282007753849, -0.02075366862118244, 0.024104097858071327, -5.2429928473429754e-05, 0.014269842766225338, 0.010932978242635727, 0.012119870632886887, 0.006087064743041992, 0.024429645389318466, -0.010030940175056458, -0.6454526782035828, -0.003330082166939974, 0.0032639552373439074, -0.03496925160288811, 0.022313585504889488, 0.034426674246788025, 0.010234407149255276, 0.006168451625853777, -0.014039246365427971, 0.012628539465367794, 0.0019024196080863476, 0.02311388961970806, -0.0037031054962426424, -0.009590093977749348, -0.011502686887979507, -0.015178663656115532, 0.01420202013105154, -0.00939340889453888, 0.007874186150729656, -0.00014783174265176058, -0.014134197495877743, 0.023588646203279495, -0.00019159840303473175, -0.017660964280366898, 0.008905087597668171, 0.014893809333443642, 0.006253229454159737, -0.011041494086384773, -0.03315161168575287, -0.010186931118369102, -0.02166248857975006, 0.03301596641540527, 0.004126995801925659, 0.01188249234110117, 0.03713957220315933, 0.01642659679055214, -0.012214822694659233, 0.03301596641540527, 0.01451400388032198, 0.05504469573497772, -0.036515604704618454, 0.003189350478351116, 0.030574358999729156, 0.02407696843147278, 0.02845829911530018, 0.01274383720010519, 0.013449191115796566, -0.007555420510470867, 0.0027959803119301796, -0.02867533080279827, 0.017186207696795464, 0.017647400498390198, -0.0029604497831314802, 0.022462794557213783, 0.004628881812095642, 0.0004912888398393989, 0.0021855782251805067, 0.004917127545922995, 0.0011894361814484, 0.009257763624191284, -0.011333130300045013, 0.02075366862118244, -0.027630863711237907, 0.012974433600902557, -0.0019566775299608707, 0.021336941048502922, 0.005405448842793703, -0.0011385693214833736, 0.009135683067142963, -0.005069728009402752, 0.012574281543493271, 0.032066453248262405, -0.015938274562358856, -0.006358354352414608, 0.019844846799969673, -0.01698273979127407, 0.005768299102783203, 0.0054664891213178635, -0.02255774475634098, 0.027332445606589317, 0.013232159428298473, -0.023954888805747032, -0.027061155065894127, -0.0042321207001805305, 0.02927216701209545, 0.006493999157100916, -0.006951800547540188, -0.00862023327499628, 0.03426390141248703, -0.0054257954470813274, 0.027888590469956398, -0.0033368642907589674, 0.013489884324371815, -0.046743229031562805, -0.004303334280848503, 0.0035030292347073555, -0.027183236554265022, 0.0190309789031744, 0.0004007882671430707, -0.025311335921287537, 0.007630025036633015, -0.012859135866165161, 0.004350809846073389, -0.008396418765187263, 0.008091217838227749, 0.004791655577719212, -0.0018057726556435227, -0.002163535915315151, 0.03456231951713562, -0.02727818861603737, -0.025433417409658432, -0.005161288194358349, -0.022449228912591934, 0.0022008384112268686, -0.01363231148570776, -0.02546054683625698, 0.014324100688099861, -0.0046458374708890915, 0.009088207967579365, 0.0036929321940988302, 0.007229872513562441, -0.009006820619106293, -0.014798857271671295, -0.011156792752444744, 0.011740065179765224, 0.0060226330533623695, 0.013537360355257988, -0.009949552826583385, -0.001748123555444181, -0.013984988443553448, -0.03830611705780029, 0.009495142847299576, 0.029977520927786827, -0.005764907691627741, 0.015056583099067211, -0.0021381026599556208, 0.009556182660162449, -0.009094989858567715, -0.00010581361129879951, -0.01844770461320877, -0.01724046654999256, 0.009339150972664356, -0.005924290511757135, -0.025989560410380363, 0.002997752046212554, -0.04066633805632591, -0.022788342088460922, 0.006375310011208057, -0.04413884878158569, -0.002031282288953662, -0.03079139068722725, -0.0039540487341582775, -0.009935988113284111, -0.010675253346562386, -0.002090626861900091, 0.0036556299310177565, 0.018000077456235886, -0.022693390026688576, -0.02639649622142315, -0.01965494453907013, 0.014025681652128696, 0.02110634557902813, -0.019858412444591522, -0.0190445426851511, -0.0003802295832429081, -0.0038455326575785875, -0.0027332445606589317, 0.006405829917639494, -0.0053342352621257305, -0.01584332250058651, -0.005958201829344034, -0.008823700249195099, -0.006636426318436861, 0.00659234169870615, 0.016819966956973076, -0.00020611664513126016, -0.01422914955765009, -0.009284893050789833, 0.0032249572686851025, 0.024266870692372322, -0.00923741701990366, -0.0068873693235218525, -0.008674491196870804, 0.0035911984741687775, 0.008464241400361061, 0.011828234419226646, 0.012547152116894722, 0.003974395338445902, -0.0007155268103815615, 0.005988722201436758, 0.02463311143219471, 0.023846371099352837, 0.0060226330533623695, -0.0013886645901948214, -0.0015268528368324041, -0.0008134454838000238, -0.035620350390672684, -0.009827472269535065, 0.020034750923514366, 0.029651973396539688, 0.03052010014653206, 0.01850196346640587, 0.0409918874502182, 0.0039574396796524525, 0.005242675077170134, -0.03049297258257866, -0.00311135477386415, -0.029787618666887283, 2.1830350306117907e-05, 0.01695561222732067, 0.028024233877658844, -0.013930730521678925, -0.020862184464931488, 0.009718956425786018, 0.0027281579095870256, -0.005476662423461676, -0.012696362100541592, 7.041877688607201e-05, -0.013957859016954899, -0.0031825683545321226, 0.0045135836116969585, -0.022964680567383766, 0.02076723240315914, -0.0004993427428416908, -0.0380619578063488, 0.005890379659831524, 0.009746085852384567, 0.04538678005337715, 0.028322653844952583, 0.0001438047765986994, 0.0017803391674533486, 0.015531339682638645, 0.013550925068557262, 0.01961425133049488, 0.008172605186700821, -0.00018396838277112693, 0.02079436182975769, -0.01696917600929737, 0.027725815773010254, 0.011394171044230461, 0.03247338533401489, 0.018542656674981117, 0.030927035957574844, 0.001993980025872588, 0.009142465889453888, -0.007243437226861715, 0.00805730652064085, 0.005853076931089163, -0.010905849747359753, 0.005123985931277275, -0.039743952453136444, 0.015083711594343185, -0.027115413919091225, 0.023086760193109512, 0.021743876859545708, -0.019275140017271042, 0.001337797730229795, -0.028838103637099266, 0.01115000993013382, 0.022734083235263824, 0.01961425133049488, -0.0010605735005810857, 0.027454525232315063, 0.0036013717763125896, -0.0190581064671278, 0.0024738237261772156, 0.012852353043854237, -0.027617299929261208, 0.011095752008259296, -0.0047780913300812244, 0.004015089012682438, -0.02582678757607937, -0.010465003550052643, -0.021852392703294754, 0.02021108940243721, -0.016304515302181244, -0.019207317382097244, 0.0109940180554986, 0.005819166079163551, 0.018542656674981117, -0.0007905553793534636, -0.007921661250293255, -0.010234407149255276, 0.025284208357334137, -0.013903601095080376, -0.002412783447653055, 0.006212536245584488, -0.004554277285933495, -0.0009749476448632777, -0.009624005295336246, 0.008803353644907475, 0.0409918874502182, -0.005754734389483929, -0.006327833980321884, -0.007697847671806812, 0.00689415168017149, 0.01873255893588066, 0.010559954680502415, 0.012947305105626583, -0.0014853115426376462, 0.0013589923037216067, 0.0047577447257936, -0.0002846423303708434, -0.019410783424973488, 0.01968207396566868, 0.012418289668858051, -0.005547876004129648, -0.02402270957827568, -0.004876433871686459, 0.0008448133594356477, 0.01203848421573639, -0.008233644999563694, -0.025433417409658432, -0.0012301296228542924, -0.010003810748457909, 0.0016582588432356715, 0.007786016911268234, 0.011360259726643562, 0.015355002135038376, -0.020984264090657234, -0.01639946736395359, -0.03637995943427086, -0.015192227438092232, -0.00040100020123645663, 0.12403368949890137, -0.027020461857318878, 0.02170318178832531, 0.03448093309998512, 0.007874186150729656, -0.0002801914815790951, -0.005161288194358349, -0.010465003550052643, 0.008816918358206749, 0.005239284131675959, 0.01273027341812849, -0.02170318178832531, -0.00020918984955642372, -0.0005362211959436536, 0.0007744475733488798, 0.005442751105874777, -0.021757440641522408, -0.0012250429717823863, -0.01962781511247158, 0.0031028769444674253, 0.003291084198281169, -0.017430368810892105, -0.013076167553663254, 0.027942847460508347, -0.0007333302055485547, -0.0219609085470438, 0.028187008574604988, 0.023303791880607605, 0.004174471367150545, -0.03168664500117302, -0.016752144321799278, -0.005758125800639391, 0.018339188769459724, 0.010614212602376938, -0.01962781511247158, 0.005361364223062992, 0.008545627817511559, 0.011204267852008343, 0.033856965601444244, -0.0034674224443733692, 0.004527148324996233, 0.013367803767323494, -0.0032588685862720013, -0.02932642586529255, 0.004808611236512661, 0.0014276624424383044, -0.002818022621795535, 0.027034027501940727, -0.00023356355086434633, 0.009468013420701027, 0.032934579998254776, 0.010885502211749554, 0.005646218545734882, -0.011638332158327103, 0.013801868073642254, -0.002558601787313819, 0.016453726217150688, 0.005107030272483826, -0.014636083506047726, -0.02399558201432228, -0.002943494124338031, -0.005147723481059074, 0.004079520236700773, -0.004211774095892906, -0.008715184405446053, 0.013842561282217503, 0.011156792752444744, -0.006429567933082581, -0.004289769567549229, -0.004296551924198866, -0.013143990188837051, -0.005045989993959665, -0.013449191115796566, 0.0017235379200428724, 0.032391998916864395, 0.002460259245708585, 0.03157813102006912, -3.825609746854752e-05, 0.005880205892026424, -0.0007278196280822158, -0.011896057054400444, -0.03071000427007675, 0.0017006478738039732, -0.024741629138588905, -0.006134540308266878, 0.0053749289363622665, 0.027440961450338364, 0.015951840206980705, -0.011129663325846195, 0.011753629893064499, 0.005195199511945248, 0.009827472269535065, 0.03448093309998512, 0.004425414372235537, -0.003971004392951727, 0.0037539724726229906, 0.01334745716303587, 0.03369419276714325, -0.000254546117503196, 0.011468775570392609, 0.00980712566524744, -0.014378358609974384, -0.007182396948337555, -0.005673347506672144, -0.0072502195835113525, 0.013835779391229153, -0.005107030272483826, 0.010363269597291946, -0.022951114922761917, 0.00965113379061222, 0.01521935686469078, -0.029489198699593544, 0.006334616336971521, 0.009006820619106293, 0.005235892720520496, 0.0054664891213178635, -0.017688093706965446, 0.008694837801158428, 0.005951419472694397, -0.003706496674567461, -0.015449953265488148, -0.019220881164073944, 0.008030178025364876, 0.0028095447923988104, -0.0014446181012317538, -0.025853917002677917, -0.007474033627659082, -0.0011267004301771522, -0.001399685745127499, 0.0030452278442680836, -0.026152335107326508, 0.018854640424251556, 0.026735607534646988, 0.008172605186700821, -0.010871938429772854, 0.01481242198497057, 0.009156030602753162, 0.01996692828834057, -0.016331644728779793, -0.011780759319663048, -0.023561518639326096, -0.013544142246246338, 0.012248734012246132, -0.018339188769459724, 0.01582975871860981, -0.020658716559410095, -0.005992113146930933, -0.02577252872288227, -0.008416765369474888, 0.010885502211749554, -0.024212613701820374, -0.007148485630750656, 0.00895256269723177, 0.01642659679055214, -0.001836292678490281, -0.014731035567820072, -0.03250051662325859, -0.013137207366526127, 0.02311388961970806, 0.013686569407582283, 0.02430756390094757, -0.0131846833974123, 0.01509727630764246, -0.004445761442184448, 0.021282684057950974, -0.016480853781104088, -0.008599885739386082, -0.018624043092131615, -0.01696917600929737, 0.0068873693235218525, 0.013171118684113026, 0.0057886457070708275, 0.0008808440179564059, -0.01155016291886568, 0.011929968371987343, 0.034426674246788025, 0.014893809333443642, -0.026532141491770744, -0.012174129486083984, -0.028593942523002625, 0.0015861974097788334, 0.012547152116894722, -0.012547152116894722, 0.017348982393741608, 0.01635877415537834, 0.004781482275575399, 0.018122157081961632, -0.0015175272710621357, 0.022354278713464737, -0.016006097197532654, 0.013971423730254173, -0.01730828732252121, 0.033287256956100464, -0.013578053563833237, -0.007989483885467052, 0.023208841681480408, -0.005171461496502161, -0.04251110926270485, -0.004971385467797518, 0.013245724141597748, 0.02986900508403778, 0.008694837801158428, -0.03439954295754433, -0.000729939085431397, -0.005218937061727047, 0.02843116968870163, -0.004506801720708609, -0.015192227438092232, 0.004157515708357096, -0.021241990849375725, 0.002948580775409937, 0.0006786483572795987, -0.005581787321716547, -0.04332497715950012, 0.022381406277418137, -0.01155694480985403, -0.02289685793220997, -0.010966889560222626, -0.002577252918854356, -0.005212155170738697, 0.006493999157100916, -0.03534905984997749, 0.01814928650856018, 0.008884740062057972, 0.03157813102006912, 0.0072502195835113525, 0.008755877614021301, -0.019180187955498695, 0.0205095075070858, 0.0044118501245975494, 0.01436479389667511, 0.018352754414081573, 0.011977444402873516, 0.013822214677929878, -0.021282684057950974, 0.00790131464600563, 0.009128901176154613, -0.034426674246788025, -0.039418406784534454, 0.01993979886174202, -0.012248734012246132, -0.012140218168497086, -0.010580301284790039, 0.007975920103490353, 3.894491965183988e-05, 0.017064128071069717, 0.0014556392561644316, -0.00033444943255744874, -0.02371072769165039, -0.0248908381909132, -0.0032809108961373568, 0.0006006525363773108, -0.02339874394237995, 0.017226900905370712, 0.013903601095080376, -0.030682874843478203, -0.021581102162599564, -0.006775462534278631, 0.003085921285673976, 0.003345342120155692, 0.01787799596786499, -0.004954429809004068, -0.011828234419226646, 0.0020228044595569372, 0.024158354848623276, 0.012404724955558777, 0.004788264632225037, 0.008036959916353226, -0.003550505032762885, 0.030574358999729156, -0.00021745571575593203, -0.006978929508477449, -0.02170318178832531, 0.009854601696133614, -0.012777748517692089, -0.0008342160726897418, -0.013767956756055355, -0.024429645389318466, -0.009000038728117943, 0.013238941319286823, 0.01877325214445591, 0.017701657488942146, 0.021784570068120956, 0.00300283869728446, -0.00267559546045959, -0.009447666816413403, 0.002109277993440628, -0.017348982393741608, 0.012954086996614933, -0.022205069661140442, -0.04343349486589432, -0.008315031416714191, -0.018054334446787834, -0.006765288766473532, 0.00936627946794033, 0.004934082739055157, -0.004889998119324446, 0.015978967770934105, -0.01935652643442154, 0.00964435189962387, -0.014758164063096046, 0.017986511811614037, -0.017796609550714493, -0.024199048057198524, -0.0043643745593726635, -0.026776300743222237, 0.024144791066646576, 0.005581787321716547, -0.025894610211253166, 0.005625871941447258, 0.02870245836675167, -0.00980034377425909, -0.005269804038107395, -0.018379881978034973, -0.0038150125183165073, 0.0033368642907589674, -0.023832807317376137, 0.008742312900722027, -0.052575960755348206, 0.024266870692372322, -0.016101049259305, -0.014459745027124882, 0.01585688814520836, -0.002055020071566105, 0.03288032114505768, -0.011868927627801895, 0.025365594774484634, 0.009413755498826504, -0.004377938807010651, -0.0017498191446065903, -0.009773214347660542, 0.007406210992485285, -0.023344485089182854, -0.012153781950473785, -0.01850196346640587, -0.0069721476174890995, -0.015978967770934105, 0.028214138001203537, -0.02193377912044525, 0.0012479330180212855, 0.01053282618522644, 0.04044930636882782, 0.014758164063096046, -0.0016506287502124906, -0.004476281348615885, -0.015151534229516983, -0.006741551216691732, -0.04196852818131447, -0.010437874123454094, 0.008898304775357246, -0.01130600180476904, 0.05219615623354912, 0.009142465889453888, -0.02992326393723488, -0.026762736961245537, -0.01989910565316677, -0.025012917816638947, -0.01871899515390396, -0.001257258583791554, -0.00541223119944334, 0.03605441376566887, -0.00047433323925361037, 0.009189940989017487, 0.034697964787483215, 0.003540331730619073, 0.04457291215658188, -0.02136407047510147, 0.008308249525725842, -0.001919375266879797, -0.016467289999127388, 0.010058068670332432, 0.007311259862035513, -0.01638590358197689, -0.003169003874063492, 0.03727521747350693, 0.001013097818940878, 0.007765669841319323, 0.028214138001203537, -0.013890037313103676, 0.018922463059425354, -0.019871976226568222, 0.0025467327795922756, 0.009061078540980816, 0.0023703945335000753, 0.007243437226861715, -0.023683598265051842, -0.0023229187354445457, -0.008138693869113922, -0.013510230928659439, -0.013150772079825401, -0.00891186948865652, -0.001113983686082065, -0.0032351305708289146, 0.013903601095080376, 0.0020380644127726555, -0.02200160175561905, -0.0205095075070858, -0.012689579278230667, 0.024755192920565605, -0.02722392976284027, -0.028593942523002625, 0.01672501489520073, 0.0003901910095009953, -0.02022465318441391, -0.00935949757695198, -0.007758887950330973, -0.03174090385437012, -0.018637608736753464, 0.022639133036136627, -0.0005099400295875967, 0.02223219722509384, -0.01991266943514347, 0.002436521463096142, 0.0018023814773187041, -0.031523872166872025, -0.021486150100827217, -0.006378700956702232, -0.037085313349962234, 0.01960068754851818, -0.004266032017767429, -0.01877325214445591, -0.01203170232474804, 0.00027319727814756334, -0.010559954680502415, -0.012472547590732574, -0.018230672925710678, -0.015002325177192688, -0.013774738647043705, -0.0004972232854925096, 0.020373862236738205, -0.014432616531848907, -0.010458220727741718, -0.018108593299984932, 0.010492132045328617, 0.00014412269229069352, 0.02026534639298916, 0.20162256062030792, 0.008471023291349411, 0.0059785484336316586, 0.027969976887106895, -0.01615530624985695, 0.04023227468132973, 0.023547952994704247, 0.0010173367336392403, -0.013123643584549427, 0.007453687023371458, -0.027942847460508347, -0.0018718994688242674, -0.016182435676455498, 0.001530243898741901, 0.013306763954460621, -0.016318080946803093, -0.017484625801444054, -0.020889313891530037, -0.054122310131788254, -0.057296402752399445, 0.0006031958619132638, -0.014893809333443642, 0.005062945652753115, 0.011631549336016178, 0.008871176280081272, 0.0067992000840604305, -0.005839512683451176, 0.02055020071566105, 0.01346275582909584, 0.0019108974374830723, -0.02053663693368435, 0.009569747373461723, -0.00014083755377214402, -0.005520747043192387, -0.025107869878411293, -0.03271754831075668, 0.009135683067142963, -0.006470261141657829, 0.0321478396654129, 0.010030940175056458, -0.0074875978752970695, -0.010776986368000507, -0.014948067255318165, -0.01873255893588066, -0.008118346333503723, -0.0034894647542387247, -8.854008046910167e-05, -0.026423625648021698, 0.00630748737603426, 0.011048275977373123, -0.0248908381909132, -0.020360298454761505, 0.012872700579464436, 0.019248010590672493, 0.011102534830570221, -0.0017413413152098656, 0.0016565632540732622, 0.012194476090371609, -0.013944295234978199, -0.0013522100634872913, 0.0073248241096735, 0.02924503944814205, -0.025094304233789444, 0.005473271477967501, -0.010431092232465744, 0.009820690378546715, -0.006317660678178072, 0.029733359813690186, 0.012682797387242317, -0.003930310718715191, 0.006019242107868195, -0.032364871352910995, -0.013001563027501106, -0.020414555445313454, 0.0021262336522340775, -0.010810897685587406, 0.038794439285993576, 0.022991809993982315, 0.011075405403971672, 0.020007621496915817, 0.009929206222295761, 0.015192227438092232, 0.0011987617472186685, -0.0002589969662949443, 0.020848620682954788, -0.023235969245433807, 0.022150810807943344, -0.006538083776831627, -0.00821329839527607, 0.007969137281179428, 0.0010554868495091796, 0.0009079729788936675, -0.005178243853151798, -0.03407399728894234, 0.01606035605072975, 0.0014403791865333915, -0.01669788546860218, 0.01728115975856781, -0.028566814959049225, 0.0031079635955393314, 0.0069721476174890995, 0.009508706629276276, 0.006822938099503517, 0.010641342028975487, -0.02369716204702854, 0.010214060544967651, 0.035620350390672684, -0.00013670462067238986, 0.011529815383255482, -0.006663555279374123, 0.025975996628403664, -0.029380682855844498, 0.006578777451068163, -0.008009830489754677, -0.016548676416277885, 0.00033487332984805107, -0.02957058697938919, -0.002797675784677267, -0.007514726836234331, -0.002075366908684373, -0.007684282958507538, -0.02436182275414467, 0.000568860792554915, 0.01009197998791933, -0.03133397176861763, -0.03141535818576813, -0.00490356283262372, 0.015626290813088417, -0.004672966431826353, -0.03567460551857948, 0.0014251191169023514, -0.02078079804778099, 0.02144545689225197, 0.011353477835655212, -0.00983425509184599, -0.023846371099352837, -0.022489923983812332, -0.008355725556612015, -0.003818403696641326, 0.012628539465367794, 0.009861383587121964, -0.0024941705632954836, -0.017348982393741608, -0.014310535974800587, 0.015911145135760307, -0.014595390297472477, 0.03019455261528492, -0.019790589809417725, -0.0027179846074432135, -0.020319605246186256, -0.0014929416356608272, 0.018596915528178215, -0.005737778730690479, 0.0005972614162601531, 0.012201257981359959, -0.01635877415537834, -0.02554193325340748, -0.01615530624985695, -0.018936026841402054, 0.01493450254201889, -0.008850828744471073, 0.005819166079163551, 0.017213337123394012, -0.0007443513604812324, -0.022367842495441437, 0.02022465318441391, -0.17373397946357727, 0.000698571209795773, 0.014215584844350815, -0.02259843982756138, 0.007731758989393711, 0.005354582332074642, 0.04275526851415634, -0.023575082421302795, -0.0030706613324582577, -0.013042256236076355, 0.016182435676455498, -0.005239284131675959, -0.03434528782963753, -0.003933701664209366, 0.021309811621904373, 0.0007189179304987192, -0.005368146579712629, 0.03643421828746796, 0.010268318466842175, -0.0008511716732755303, 0.016657192260026932, -0.0037437989376485348, 0.016480853781104088, -0.018922463059425354, 0.003496247110888362, 0.01420202013105154, -0.009657916612923145, -0.008810135535895824, -0.01111609861254692, -0.00038192514330148697, 0.00910177268087864, 0.022313585504889488, 0.011604420840740204, 0.00526641309261322, -0.0011394170578569174, -0.01960068754851818, -0.010241189040243626, -0.001997370971366763, 0.013293199241161346, 0.030899906530976295, -0.005893770605325699, 0.007162050344049931, 7.91615093476139e-05, 0.008640579879283905, -0.012228387407958508, -0.013130425475537777, 0.00793522596359253, 0.006897542625665665, -0.003048619022592902, -0.01639946736395359, 0.007365517783910036, -0.013591618277132511, 0.011183921247720718, -0.014120632782578468, -0.010044503957033157, 0.03258190304040909, -0.03567460551857948, -0.019424349069595337, 0.01962781511247158, -0.014324100688099861, -0.0014276624424383044, -0.018922463059425354, -0.008559192530810833, -0.015707679092884064, -0.018569786101579666, -0.005602133926004171, -0.01317790150642395, 0.014649648219347, -0.02962484396994114, 0.02406340464949608, 0.0025908173993229866, -0.03136109933257103, 0.009128901176154613, -0.022177940234541893, 0.00490356283262372, -0.03933702036738396, -0.01334745716303587, -0.015205792151391506, -0.011916403658688068, -0.011773976497352123, -0.007372299674898386, 0.05192486569285393, -0.040313661098480225, 0.016521546989679337, -0.007691065315157175, 0.005168070551007986, 0.011950314976274967, 0.0076164607889950275, -0.0027315490879118443, 0.0064770434983074665, 0.026165898889303207, -0.02463311143219471, 0.005910726264119148, 0.013266070745885372, 0.02664065733551979, 0.029814748093485832, 0.008077653124928474, 0.01155016291886568, 0.018298495560884476, -0.01608748361468315, -0.008091217838227749, 0.008586321957409382, 0.016046790406107903, 0.01989910565316677, 0.02725105918943882, 0.015585598535835743, 0.019465042278170586, 0.006880586966872215, 0.016914917156100273, 0.008640579879283905, -0.01606035605072975, 0.011048275977373123, -0.007568984758108854, -0.004934082739055157, 0.01407993957400322, 0.03022168204188347, 0.032337743788957596, -0.0030994857661426067, 0.021228425204753876, -0.02171674743294716, 0.0249179657548666, -0.0025569063145667315, -0.021608231589198112, 0.014785293489694595, 0.004445761442184448, 0.005239284131675959, -0.09028524160385132, -0.007134921383112669, 0.009420537389814854, 0.04419310390949249, 0.0021889694035053253, 0.02810562215745449, -0.00996311753988266, 0.0064736525528132915, 0.013361021876335144, 0.029760489240288734, -0.015192227438092232, 0.0016972566954791546, 0.021296247839927673, 0.02055020071566105, 0.03665124997496605, -0.020129701122641563, -0.00638548331335187, -0.03285319358110428, -0.014012116938829422, 0.01301512774080038, 0.0003259716322645545, 0.014880244620144367, -0.006999276578426361, -0.005870032589882612, -0.011414517648518085, -0.017389675602316856, -0.042945172637701035, 0.039418406784534454, 0.009115336462855339, 0.02753591351211071, -0.011421299539506435, 0.0010707469191402197, 0.00862701516598463, -0.02485014498233795, -0.014025681652128696, 0.006178624927997589, -0.04519687965512276, -0.012587846256792545, -0.008715184405446053, -0.017484625801444054, 0.016630062833428383, -0.01375439204275608, 0.04620065167546272, -0.02992326393723488, 0.01111609861254692, -0.027644429355859756, 0.0022093162406235933, -0.03635283187031746, 0.03171377629041672, -0.026003126055002213, -0.02897374890744686, -0.012255515903234482, -0.029434941709041595, -0.002500952687114477, 0.0023415698669850826, 0.009115336462855339, -0.007501162588596344, 0.021513279527425766, -0.002431434579193592, 0.004527148324996233, -0.012465765699744225, -0.02254418097436428, -0.0001778219739207998, 0.005622480995953083, -0.005269804038107395, 0.007195961661636829, -0.01085837371647358, -0.017742352560162544, -0.019153058528900146, -0.01422914955765009, -0.03963543847203255, 0.008294684812426567, 0.00019551940204109997, 0.03019455261528492, -0.008966127410531044, 0.003192741656675935, -0.003842141479253769, -0.007942008785903454, 0.0109804542735219, 0.0038828349206596613, -0.01615530624985695, -0.021024959161877632, 0.005903943907469511, -0.01273027341812849, 0.035267673432826996, 0.0035911984741687775, 0.02550124004483223, 0.008884740062057972, -0.0013259288389235735, -0.006663555279374123, -0.004154124762862921, 0.0041473424062132835, 0.01638590358197689, -0.02754947729408741, -0.012967651709914207, 0.012235169298946857, 0.0009164508082903922, -0.01755244843661785, 0.0003098637971561402, 0.018027206882834435, -0.018122157081961632, 0.0014505526050925255, -0.07655797898769379, 0.0124589828774333, 0.009542617946863174, -0.0038997905794531107, -0.006341398693621159, 0.009122119285166264, 0.0060633267275989056, 0.006009068805724382, 0.02315458282828331, 0.0016624977579340339, -0.0440032035112381, -0.007080663461238146, 0.014907374046742916, 0.001748123555444181, 0.0014683560002595186, 0.0011385693214833736, 0.03743799030780792, 0.010214060544967651, -0.0035437229089438915, -0.014880244620144367, -0.00968504510819912, 0.008728749118745327, -6.051033778931014e-05, 0.005663174204528332, -0.0234530009329319, 0.0008757573086768389, -0.030682874843478203, 0.006354963406920433, -0.014853115193545818, -0.0015438083792105317, 0.04251110926270485, -0.04723155125975609, -0.018312061205506325, 0.01083124428987503, -0.00402526231482625, -0.0307642612606287, 0.0003179177292622626, 0.03108980879187584, -0.0009172986028715968, 0.007358735427260399, -0.012825224548578262, -0.015463517978787422, 0.0204959437251091, -0.010078415274620056, -0.007202743552625179, -0.012879482470452785, 0.018976720049977303, 0.025270642712712288, 0.008287902921438217, -0.009603658691048622, 0.023222405463457108, 0.017932254821062088, -0.01818997971713543, 0.006829720456153154, 0.00409308448433876, -0.02400914579629898, -0.010749857872724533, -0.009203505702316761, 0.024511031806468964, -0.012350467033684254, 0.03551183268427849, 0.023792114108800888, 0.04454578086733818, 0.016887789592146873, 0.01289304718375206, -0.013279634527862072, -0.013171118684113026, 0.006090455688536167, 0.018393447622656822, -0.013035474345088005, -0.02164892479777336, 0.003540331730619073, 0.014419051818549633, 0.032337743788957596, 0.025324901565909386, -0.011624767445027828, 0.014608955010771751, -0.029760489240288734, -0.010064850561320782, 0.023819243535399437, 0.016874223947525024, -0.007528291549533606, -0.03494212403893471, 0.011584073305130005, 0.015572033822536469, 0.0080505246296525, -0.024470338597893715, 0.009549400769174099, -0.012648886069655418, 0.01818997971713543, -0.0234258733689785, -0.006046371068805456, 0.0004101138620171696, -0.0017201467417180538, 0.017362546175718307, 0.010627777315676212, -0.011231397278606892, -0.007589331828057766, -0.005469880066812038, 0.04272814095020294, 0.003016403177753091, 0.009766432456672192, -0.00923063512891531, 0.0006515193963423371, -0.016345210373401642, -0.004761135671287775, -0.02550124004483223, -0.022408535704016685, -0.0035810251720249653, -0.016765708103775978, -0.015734806656837463, 0.03730234503746033, -0.011394171044230461, -0.00023144409351516515, -0.0102547537535429, 0.0190445426851511, 0.009881730191409588, 0.016168871894478798, -0.013869689777493477, 0.026477882638573647, 0.012391161173582077, 0.0033182131592184305, 0.005402057897299528, 0.012255515903234482, 0.018298495560884476, -0.007304477505385876, -0.0018854639492928982, 0.0009410364436917007, -0.005819166079163551, 0.02407696843147278, 0.0019855021964758635, -0.0009274719632230699, -0.03868592157959938, 0.01159763801842928, 0.007874186150729656, 0.017633836716413498, -0.006351571995764971, 0.025935303419828415, -0.00038870738353580236, 0.05827304720878601, -0.012533588334918022, 0.00038298487197607756, 0.02406340464949608, -0.01907167211174965, 0.012384378351271152, -0.003202914958819747, 0.006012459751218557, -0.017972948029637337, -0.013510230928659439, 0.032093580812215805, -0.007426557596772909, 0.006612688302993774, -0.038224730640649796, -0.027969976887106895, -0.032337743788957596, -0.01277096662670374, 0.01762027107179165, -0.016630062833428383, -0.00528336875140667, 0.03480647876858711, -0.0032944753766059875, 0.04023227468132973, 0.007107792422175407, -0.015612727031111717, -0.02700689807534218, 0.025311335921287537, -0.025257078930735588, -0.00402526231482625, -0.06326477974653244, -0.0003929462982341647, 0.003221566090360284, -0.014893809333443642, 0.001860030577518046, 0.02136407047510147, -0.031225454062223434, -0.028241265565156937, -0.02221863344311714, 0.018352754414081573, 0.009732521139085293, -0.0028536294121295214, -0.011400952935218811, -0.009332368150353432, -0.011706153862178326, 0.01578906551003456, 0.00467974878847599, 0.0017955992370843887, -0.022923987358808517, -0.04769274219870567]} +{"id": "test:10000048", "text": "\"Deportivo is finally responding the questions regarding the polemic case with Vecindario; the lawyers of the club aren\u2019t only saying that the suspension of Oltra\u2019s license is unfair, but also accused the RFEF of been attacking Depor.\\nDeportivo is now open to negotiate with UD Vecindario in order to solve once and for all the conflict with the Canarian club. Vecindario is asking for \u20ac700,000 as part of a polemic collaboration deal that was cancelled on the past year.\\nThe conflict between Deportivo and Vecindario is now threatening Deportivo. A debt of \u20ac100,000 with Vecindario has turned into a loss of \u20ac700,000. For this conflict Oltra wasn\u2019t on the bench for the Copa clash.\\nOne of the few positive news after the relegation to Segunda is the surprising reaction of Depor\u2019s fans; a clear example is the number of socios, which is about to reach the mark of 20,000, the highest since the years of the Champions League.\\nDeportivo can\u2019t inscribe players due to a legal demand of UD Vecindario, while Adrian is doing everything in order to join Atletico Madrid. Those are the two current problems that are focusing the attention of president Lendoiro.\\nLendoiro travelled to Madrid in order to solve several pending issues and to participate in the draw of the liga calendar; before travelling he talked to reporters.\\nThe season 2011/12 will bring some changes to the structure of base football at Deportivo; the most notorious modification is the appearance of the Benjamin squad, the ninth team at the youth system composed by 10-year-old kids.\\nLendoiro is on war against the clubs that haven\u2019t paid their obligations to Deportivo; there\u2019s a well-known battle against Real Zaragoza, and since Friday the war was extended to Racing Santander; all for the transfer of Sergio Canales.\\nDepor\u2019s president continues his battle against the bankruptcy law and Zaragoza, while he announced that the club will make six or seven signings ahead of next season.\"", "vector_field": [-0.009287448599934578, -8.914135105442256e-05, 0.020972568541765213, -0.016092602163553238, -0.0444556400179863, 0.018532585352659225, -0.013871231116354465, 0.010006750002503395, -0.02451265975832939, -0.04657123237848282, 0.02925158478319645, 0.025880740955471992, -0.0021296958439052105, 0.01250314898788929, -0.005747358314692974, -0.007189486641436815, 0.0107331033796072, -0.017559412866830826, 0.030295277014374733, -0.006223366595804691, -0.024625491350889206, 0.006632380653172731, -0.03610610216856003, 0.0014350765850394964, -0.015965666621923447, 0.012249277904629707, 0.01983014866709709, 0.006043541245162487, 0.006212788634002209, 0.01296857837587595, 0.01193193905055523, 0.0033972880337387323, 0.013610308058559895, -0.003843325423076749, -0.017023462802171707, -0.03743187338113785, -0.0016492801951244473, -0.008145028725266457, 0.014153310097754002, -0.003790435614064336, 0.024808842688798904, 0.000682719168253243, -0.026529522612690926, -0.03621893376111984, -0.018617209047079086, 0.010154841467738152, 0.005479383282363415, -0.02757321484386921, -0.02571149542927742, 0.023398447781801224, 0.022425275295972824, 0.04651481658220291, -0.00045088553451932967, 0.003949104808270931, 0.010669635608792305, -0.03317248076200485, 0.01272176019847393, 0.0026744608767330647, 0.01117737777531147, -0.014153310097754002, 0.0011186192277818918, -0.019548069685697556, 0.013151929713785648, 0.008490575477480888, -0.021395687013864517, 0.035372696816921234, 0.019519861787557602, -0.002963591832667589, -0.014456545002758503, 0.0015038333367556334, 0.051676858216524124, 0.01785559579730034, 0.0022196085192263126, -0.020253267139196396, 0.027643734589219093, -0.014654000289738178, 0.014809143729507923, -0.012834591791033745, 0.0019727894105017185, 0.005906027741730213, 0.013476320542395115, -0.020478930324316025, -0.02765783853828907, 0.00808861292898655, 0.027361655607819557, -0.004499158822000027, 0.005091524682939053, 0.022199612110853195, -0.015556653030216694, 0.013123721815645695, -0.002873679157346487, 0.03424438089132309, 0.017460685223340988, -0.0038680073339492083, -0.03480853885412216, -0.007214168552309275, 0.007623183075338602, 0.036388181149959564, -0.0030006146989762783, 0.007051973138004541, 0.002612756099551916, 0.003790435614064336, -0.03968850523233414, -0.016163121908903122, -0.027361655607819557, -0.012171706184744835, 0.023440759629011154, 0.002822552341967821, -0.0011195007245987654, -0.003185729030519724, -0.011847315356135368, 0.02796812541782856, -0.016769591718912125, -0.023031745105981827, -0.008257860317826271, -0.016826007515192032, 0.04236825555562973, 0.0052819279953837395, -0.007672546897083521, -0.022100884467363358, 0.013328229077160358, 0.021550830453634262, 0.013532736338675022, -0.03274936228990555, 0.02314457669854164, -0.0019551594741642475, 0.010817727074027061, 0.001051625469699502, -0.016290057450532913, 0.01688242331147194, -0.012242225930094719, 0.011896679177880287, -0.005715624429285526, 0.0027238246984779835, 0.00494695920497179, 0.01223517395555973, -0.01841975376009941, 0.017079878598451614, -0.012199914082884789, -0.03207237273454666, 0.009795190766453743, -0.001844090991653502, -0.036162517964839935, -0.028433555737137794, -0.0030711344443261623, 0.013927646912634373, -0.0048623355105519295, -0.002753795590251684, 0.022961225360631943, -0.013814815320074558, 0.02925158478319645, -0.025344792753458023, 0.005768514238297939, 0.0036053212825208902, 0.00782063789665699, 0.0011741535272449255, -0.00772191071882844, -0.010726051405072212, -0.01544382143765688, -0.005708572454750538, -0.004760081879794598, -0.007242376450449228, -0.002789055462926626, 0.02562687173485756, 0.022143196314573288, 0.03018244542181492, 0.00856814719736576, -0.005891923792660236, -0.0037128638941794634, -0.03074660338461399, 0.011163273826241493, 0.007898209616541862, -0.05376424267888069, 0.03602147847414017, 0.02730523981153965, -0.015077118761837482, -0.0023906189016997814, 0.016374681144952774, -0.035147033631801605, 0.0034889637026935816, -0.004266443662345409, 0.00685804383829236, 0.007284688297659159, 0.05867241695523262, -0.012778175994753838, -0.007905261591076851, 0.013222449459135532, 0.013017942197620869, -0.010860038921236992, -0.022016260772943497, 0.010486284270882607, 0.007531507406383753, 0.01820819452404976, 0.004104248248040676, -0.6219276189804077, -0.0386166051030159, -0.002942435909062624, 0.0007849727990105748, 0.019294198602437973, 0.015429717488586903, 0.014146258123219013, -0.0008383032982237637, -0.021579038351774216, -0.025471728295087814, -0.012298641726374626, -0.01576821133494377, -0.011367781087756157, -0.0014350765850394964, -0.021494414657354355, -0.02461138740181923, -0.0005288979737088084, 0.0031063943170011044, 0.020732801407575607, 0.0050210049375891685, -0.016952943056821823, 0.001828224048949778, -0.010535648092627525, 0.014724520035088062, 0.009216928854584694, -0.009774034842848778, 0.011191481724381447, -0.04493517428636551, -0.006812206003814936, 0.014110998250544071, -0.025471728295087814, -0.010775415226817131, -0.03274936228990555, -0.019759628921747208, 0.04916635900735855, -0.0003640581271611154, 0.012799331918358803, 0.02208678051829338, 0.008631614968180656, 0.019759628921747208, -0.01478093583136797, -0.0006765486905351281, 0.02664235420525074, -0.011431248858571053, -0.0044780028983950615, 0.02040841057896614, 0.02301764115691185, 0.013843023218214512, -0.0003607525140978396, 0.0007475091842934489, 0.026712873950600624, 0.014738623984158039, -0.02637437917292118, -0.0014227356296032667, 0.00663943262770772, -0.0004308314819354564, 0.014964287169277668, -0.030238861218094826, -0.006473711226135492, 0.007919365540146828, -0.018786456435918808, 0.0015479081775993109, 0.008363639935851097, -0.014921975322067738, 0.011205585673451424, -0.011120961979031563, -0.014273193664848804, 0.035372696816921234, 0.013532736338675022, 0.01820819452404976, 0.017926115542650223, 0.021677765995264053, -0.020789217203855515, 0.0026321490295231342, 0.019519861787557602, 2.5797549824346788e-05, 0.02045072242617607, 0.01166396401822567, -0.0056662606075406075, 0.02840534783899784, -0.0034325479064136744, -0.028786154463887215, -0.032326243817806244, 0.005595740862190723, 0.0189133919775486, 0.00627273041754961, -0.027023160830140114, 0.028334828093647957, 0.012594824656844139, 0.015697691589593887, 0.01490787137299776, 0.040365494787693024, 0.01887108013033867, 0.024978090077638626, 0.017785076051950455, 0.005927183665335178, -0.024949882179498672, 0.014809143729507923, 0.01380776334553957, -0.034441836178302765, 0.030633771792054176, -0.015274574048817158, 0.008032197132706642, -0.014527064748108387, 0.01947754994034767, 0.023158680647611618, -0.03255190700292587, 0.00039204565109685063, 0.028772050514817238, -0.022213716059923172, -0.025640975683927536, 0.0019022697815671563, -0.00455204863101244, -0.013046150095760822, 0.005595740862190723, -0.021536726504564285, -0.008060405030846596, -0.013603256084024906, 0.02513323351740837, -0.011198533698916435, 0.007390467915683985, -0.0008784114033915102, 0.01152997650206089, 0.004696614108979702, -0.010387556627392769, 0.0067205303348600864, -0.008018093183636665, -0.016290057450532913, -0.013624412007629871, -0.03308785706758499, 0.01223517395555973, 0.008582251146435738, 0.04090144485235214, -0.012467889115214348, 0.03229803591966629, 0.000700349104590714, 0.04767134040594101, 0.02301764115691185, -0.024456243962049484, -0.01126905344426632, -0.011120961979031563, -0.014132154174149036, -0.004019624553620815, -0.0061916327103972435, -0.004086618311703205, -0.04651481658220291, 0.00887843407690525, -0.002089146990329027, -0.009463747963309288, 0.008744446560740471, -0.015147638507187366, -0.02090204879641533, -0.02597946859896183, 0.0015091223176568747, -0.0026885648258030415, -0.00490464735776186, 0.007362260017544031, -0.018983911722898483, -0.014837351627647877, -0.0057896701619029045, 0.005645104683935642, 0.005722676403820515, -0.024978090077638626, 0.012841643765568733, 0.008518783375620842, -0.01557075697928667, 0.009915074333548546, 0.001480914419516921, -0.006713478360325098, -0.0053348178043961525, 0.021931637078523636, 0.015373301692306995, 0.00640671793371439, 0.029082337394356728, -0.014484752900898457, 0.004463898949325085, -0.02664235420525074, 0.021226439625024796, -0.0009705278207547963, 0.0148232476785779, 9.927856444846839e-05, -0.0015972719993442297, -0.01205887459218502, -0.005504065193235874, 0.007757170591503382, 0.014096894301474094, -0.0013178375083953142, 0.04501979798078537, 0.0167836956679821, 0.022636834532022476, -0.014766831882297993, 0.009118201211094856, 0.008095664903521538, 0.012030666694045067, -0.009075889363884926, 0.015881042927503586, -0.004386327229440212, 0.021240543574094772, 0.012333901599049568, 0.02293301746249199, 0.008822018280625343, -0.021593142300844193, 0.021889325231313705, -0.011826159432530403, -0.0038539033848792315, -0.01323655340820551, 0.013955854810774326, -0.004587308503687382, 0.03579581528902054, 0.024540867656469345, 0.014978391118347645, -0.010126633569598198, -0.011981302872300148, -0.00567683856934309, -0.020775113254785538, 0.045837827026844025, 0.0027961074374616146, 0.002767899539321661, -0.004633146338164806, 0.007118966896086931, -0.009125253185629845, -0.02473832294344902, 0.03672667592763901, 0.006801628042012453, -0.02181880548596382, -0.031874917447566986, -0.014125102199614048, 0.01665676012635231, 0.008807914331555367, -0.019280094653367996, -0.009689411148428917, -0.00030984607292339206, -0.01517584640532732, 0.0067381602711975574, 0.008187340572476387, 0.007750118616968393, 0.02487936243414879, -0.012115290388464928, 0.028038645163178444, -0.025260169059038162, 0.007341104093939066, 0.019505757838487625, 0.01281343586742878, -0.007270584348589182, 0.00963299535214901, -0.009795190766453743, 0.029308000579476357, 0.026219235733151436, -0.0025175544433295727, 0.011360729113221169, -0.020351994782686234, 0.0011274341959506273, -0.00047821193584240973, -0.015020702965557575, -0.005017478950321674, -0.027756566181778908, -0.0015893385279923677, -0.010599115863442421, 0.035936854779720306, 0.013441060669720173, 0.024357516318559647, 0.008060405030846596, 0.02633206732571125, 0.005274876020848751, 0.02624744363129139, 0.009026525542140007, 0.0009423199226148427, 0.003205121960490942, -0.0021949266083538532, -0.013300021179020405, -0.005511117167770863, -0.009774034842848778, 0.0005368314450606704, 0.016318265348672867, 0.001586694037541747, 0.01621953770518303, -0.032410867512226105, -0.020295578986406326, 0.021282855421304703, 0.01887108013033867, -0.007637287024408579, -0.019237782806158066, 0.023031745105981827, 0.001479151425883174, -0.001234095310792327, -0.021621350198984146, -0.01657213643193245, 0.024498555809259415, 0.0009934466797858477, 0.024329308420419693, -0.009132305160164833, -0.00410777423530817, -0.03762932866811752, 0.011297261342406273, -0.004865861497819424, -0.0038010135758668184, 0.01367377582937479, -0.005525221116840839, -0.0020961989648640156, -0.020478930324316025, 0.004139508120715618, 0.014089842326939106, -0.006198684684932232, -0.01837744191288948, 0.05066137760877609, 0.030238861218094826, 0.0235959030687809, -0.024681907147169113, -0.0038010135758668184, -0.013095513917505741, 0.013659671880304813, 0.004569678567349911, -0.020168643444776535, 0.010669635608792305, -0.01634647324681282, 0.003289745422080159, -0.017601724714040756, 0.0015011888463050127, 0.027827085927128792, -0.007044921163469553, -0.001184731489047408, 0.0007655798690393567, -0.008060405030846596, -0.0169670470058918, 0.10346654802560806, 0.029392624273896217, -0.020648177713155746, 0.015740003436803818, -0.008603407070040703, 0.011008130386471748, -0.025217857211828232, -0.03004140593111515, 0.011537028476595879, 0.0035013046581298113, -0.01789790764451027, -0.023468967527151108, 0.00914640910923481, 0.011544080451130867, 0.023821566253900528, 0.010845934972167015, 0.00932270847260952, -0.006399665959179401, 0.016586240381002426, -0.022721458226442337, -0.024710115045309067, -0.009294500574469566, 0.009900970384478569, 0.03698054701089859, -0.00887843407690525, -0.0011071597691625357, 0.030013198032975197, 0.046937935054302216, 0.007090758997946978, 0.012517252936959267, -0.007679598871618509, 0.002699142787605524, -0.00435106735676527, 0.03176208585500717, -0.0226227305829525, -0.029590079560875893, 0.006131690926849842, -0.02305995300412178, 0.020535346120595932, 0.02345486357808113, -0.003443125868216157, 0.012778175994753838, -0.001983367372304201, -0.006410243920981884, 0.009111149236559868, -0.0251755453646183, -0.01692473515868187, -0.002281313296407461, -0.0022301864810287952, -0.024710115045309067, 0.004188871942460537, -0.026896225288510323, -0.019519861787557602, -0.00776422256603837, -0.011149169877171516, 0.0017770972335711122, -0.01258777268230915, -0.028969505801796913, -0.014611688442528248, 0.024089541286230087, -0.025246065109968185, -0.03305964916944504, 0.018969807773828506, 0.011304313316941261, 0.002417063806205988, -0.03226982802152634, -0.019378822296857834, -0.013173085637390614, -0.008166184648871422, -0.005761462263762951, -0.01305320207029581, -0.013525684364140034, -0.041465602815151215, 0.01719271019101143, 0.012799331918358803, 0.013617360033094883, 0.014040478505194187, -0.001035758526995778, 0.024272892624139786, 0.021762389689683914, -0.010937610641121864, -0.009245136752724648, -0.005937761627137661, 0.0044427430257201195, 0.0020768060348927975, -0.003460755804553628, -0.012615980580449104, -0.00015117667498998344, -0.005200830288231373, -0.005514643155038357, 0.008180288597941399, -0.013046150095760822, 0.0016554506728425622, -0.012686500325798988, -0.00045220777974464, 0.012996786274015903, 0.02386387810111046, -0.002908939030021429, 0.004834127612411976, -0.0011459456291049719, 0.011276105418801308, -0.0031751510687172413, -0.00616342481225729, -0.014696312136948109, -0.0010190100874751806, 0.016670864075422287, 0.004615516401827335, -0.018010739237070084, -0.002260157372802496, 0.013469268567860126, 0.015514341183006763, -0.0006285071140155196, 0.0029829847626388073, -0.0009052970563061535, -0.03133896738290787, -0.0052290381863713264, 0.007129544857889414, 0.00602943729609251, 0.01590925082564354, -0.001479151425883174, 0.0015170557890087366, -0.005243142135441303, 0.034131549298763275, 0.000865188951138407, 0.006353828124701977, 0.008638666942715645, 0.025161441415548325, -0.028997713699936867, -0.02314457669854164, -0.0009661203366704285, -0.005338343791663647, 0.005578110925853252, -0.010465128347277641, 0.0077007547952234745, -0.04002700001001358, -0.009922126308083534, 0.012573668733239174, -0.009082941338419914, -0.01402637455612421, -0.004033728502690792, -0.010697843506932259, -0.015937458723783493, -0.01904032751917839, -0.02447034791111946, 0.014625792391598225, -0.02032378688454628, -0.01486555952578783, 0.012573668733239174, 0.00048129717470146716, 0.005186726339161396, -0.014167414046823978, 0.00663943262770772, -0.004851757548749447, -0.002450560685247183, 0.0027573215775191784, -0.009865710511803627, -0.01398406270891428, -0.03895509988069534, 0.008032197132706642, 0.021085400134325027, 0.04922277480363846, -0.0008074509096331894, -0.008398899808526039, -0.014837351627647877, 0.013786607421934605, 0.01250314898788929, 0.002487583551555872, -0.0007739540887996554, -0.013969958759844303, 0.009386176243424416, 0.0366702601313591, 0.03486495465040207, -0.0031134462915360928, -0.03503420203924179, -0.025161441415548325, 0.034526459872722626, -0.01228453777730465, 0.013511580415070057, -0.01856079325079918, -0.016445200890302658, 0.01806715503334999, -0.009216928854584694, -0.013920594938099384, 0.0011591680813580751, -0.028913090005517006, 0.0044780028983950615, 0.032015956938266754, -0.007468039635568857, -0.0037481237668544054, -0.013871231116354465, 0.01016189344227314, -0.02544352039694786, 0.025020401924848557, -0.005549903027713299, -0.0121928621083498, 0.01851848140358925, -0.016247745603322983, -0.01289805956184864, -0.019914772361516953, 0.0020380201749503613, -0.004978693090379238, -0.006276256404817104, 0.030013198032975197, 0.002893072087317705, -0.010585011914372444, 0.02297532930970192, -0.01109275408089161, -0.005109154619276524, 0.005408863537013531, -0.017206814140081406, -0.020845633000135422, -0.02283428981900215, -0.002734402660280466, -0.02809506095945835, -0.005313661880791187, 0.014223829843103886, -0.02014043554663658, 0.0023359660990536213, -0.0041818199679255486, -0.030803019180893898, 0.006995557341724634, -0.003920896910130978, 0.017023462802171707, -0.011741535738110542, 0.011304313316941261, 0.02655773051083088, -0.01576821133494377, -0.026896225288510323, -0.010944662615656853, 0.006367932073771954, 0.007870001718401909, 0.008250808343291283, -0.0024241157807409763, -0.025880740955471992, -0.01152997650206089, 0.03418796509504318, 0.002783766482025385, -0.037572912871837616, -0.03204416483640671, 0.03853198140859604, 0.0017312593990936875, 0.0386166051030159, -0.0023888559080660343, -0.007189486641436815, -0.013370540924370289, 0.02386387810111046, -0.03255190700292587, -0.0013169560115784407, -0.0020909099839627743, -0.013666723854839802, -0.011953094974160194, 0.02668466605246067, -0.03286219388246536, 0.03207237273454666, 0.012573668733239174, -0.022157300263643265, -0.02045072242617607, -0.013264761306345463, 0.019942980259656906, 0.037262625992298126, -0.01978783681988716, 0.017206814140081406, -0.004016098566353321, 0.008328380063176155, 0.0004145237908232957, 0.029082337394356728, -0.015077118761837482, -0.0023095211945474148, -0.003030585590749979, 0.027827085927128792, -0.008871382102370262, 5.326856808096636e-06, 0.017333749681711197, 0.020704593509435654, -0.009513111785054207, 0.00042422025580890477, -0.014950183220207691, 4.013564466731623e-05, -0.017178606241941452, -0.01607849821448326, 0.008074508979916573, 0.001643991214223206, -0.027277031913399696, 0.011452404782176018, 0.008814966306090355, -0.011981302872300148, -0.02925158478319645, -0.016938839107751846, 0.007087233010679483, 0.03035169281065464, -0.008060405030846596, -0.003942052833735943, 0.005937761627137661, 0.01926599070429802, -0.01038050465285778, 0.0028119743801653385, -0.0053665516898036, -0.007327000144869089, -0.05263592675328255, 0.015429717488586903, -0.01590925082564354, 0.014435389079153538, -0.04465309530496597, 0.013067306019365788, -0.021043088287115097, -0.011727431789040565, 0.02492167428135872, 8.208937651943415e-05, -0.020027603954076767, 0.009823398664593697, 0.0009608313557691872, -0.010352296754717827, -0.01179795153439045, -0.02500629797577858, 0.02757321484386921, -0.021367479115724564, 0.019689109176397324, -0.024230580776929855, -0.047248221933841705, 0.03387767821550369, -0.008102716878056526, -0.006445503793656826, 0.012489045038819313, -0.039773128926754, 0.008356587961316109, 0.01612081006169319, -0.005994177423417568, -0.010711947456002235, -0.018349234014749527, -0.0035030676517635584, 0.017869699746370316, 0.004054884426295757, -0.03635997325181961, -0.012552512809634209, -0.03410334140062332, -0.02258041873574257, 0.0010454549919813871, 0.009428488090634346, 0.0035189345944672823, -0.009781086817383766, 0.018363337963819504, 0.03579581528902054, -0.00448505487293005, 0.004421587102115154, 0.012912162579596043, -0.01961858943104744, 0.003600032301619649, -0.01113506592810154, -0.006089379079639912, -0.01613491401076317, -0.028645114973187447, 0.02465369924902916, 0.006050593219697475, -0.019985292106866837, 0.006413769908249378, 0.009788138791918755, -0.02647310681641102, 0.020577657967805862, -0.024597283452749252, 0.03525986522436142, 0.04527366906404495, -0.025288376957178116, 0.0005165570182725787, 0.020337890833616257, -0.0030200076289474964, 0.03328531235456467, 0.001679251086898148, -0.03494957834482193, -0.01025356911122799, -0.01895570382475853, 0.015556653030216694, 0.007531507406383753, -0.036247141659259796, -0.0040231505408883095, 0.017122190445661545, 0.021127711981534958, 0.017418373376131058, 0.0358804389834404, -0.029477247968316078, -0.004851757548749447, 0.004470950923860073, -0.0053665516898036, -0.015189950354397297, -0.00554285105317831, 0.011381885036826134, -0.011466508731245995, -0.0235959030687809, 0.0046190423890948296, 0.0035612464416772127, 0.010549752041697502, -0.008991265669465065, -0.006812206003814936, -0.018715936690568924, 0.03215699642896652, -0.0148232476785779, -0.004153612069785595, -0.0029865107499063015, -0.026712873950600624, 0.024442140012979507, 0.02707957662642002, 0.013652619905769825, 0.02633206732571125, 0.02142389491200447, -0.000796432257629931, -0.007728962693363428, -0.005493487231433392, -0.013701983727514744, 0.004361645318567753, 0.003081712406128645, -0.002572207245975733, 0.013927646912634373, -0.029590079560875893, 0.008490575477480888, -0.0394064262509346, -0.001163575565442443, -0.02028147503733635, 0.02487936243414879, -0.009759930893778801, 0.019153159111738205, 0.015246366150677204, -0.03396230190992355, -0.017079878598451614, -0.019872460514307022, 0.022650938481092453, -0.02283428981900215, -0.021268751472234726, -0.007418675813823938, -0.008109768852591515, -0.04507621377706528, -0.019336510449647903, -0.017912011593580246, 0.0065301270224153996, 0.020154539495706558, -0.03658563643693924, -0.013046150095760822, 0.013857127167284489, 0.19689109921455383, 0.012954474426805973, 0.010063165798783302, 0.03207237273454666, 0.0005936879897490144, 0.03331352025270462, 0.028221996501088142, -0.007376363966614008, 0.0005055383080616593, -0.020041707903146744, -0.0031257872469723225, -0.014301401562988758, -0.032636530697345734, -0.008666874840855598, 0.024286996573209763, 0.007221220526844263, -0.036388181149959564, -0.04152201861143112, -0.019773732870817184, -0.010881194844841957, 0.0021120659075677395, 0.00245232367888093, 0.010521544143557549, -0.01736195757985115, 0.01455527264624834, 0.010965818539261818, 0.005645104683935642, -0.003638818161562085, 0.017305541783571243, 0.02624744363129139, -0.008638666942715645, -0.00688272574916482, 0.0012305693235248327, 0.010817727074027061, -0.004569678567349911, 0.00918872095644474, -0.022467587143182755, 0.0026021781377494335, 0.019646797329187393, 0.013469268567860126, 0.03887047618627548, -0.0057367803528904915, -0.010281777009367943, -0.02699495293200016, -0.0036846559960395098, 0.030690187588334084, 0.00776422256603837, 0.013271813280880451, 0.001783267711289227, -0.01051449216902256, -0.028250204399228096, 0.014992495067417622, 0.0068615698255598545, 0.02063407376408577, 0.013032046146690845, -0.010493336245417595, -0.0012649476993829012, 0.008518783375620842, -0.012778175994753838, 0.004961063154041767, -0.031395383179187775, 0.017220918089151382, 0.018363337963819504, 0.03573939949274063, -0.024710115045309067, 0.020337890833616257, 0.015204054303467274, 0.005109154619276524, 0.02854638732969761, -0.012305693700909615, 0.01544382143765688, -0.018038947135210037, -0.01402637455612421, 0.00772191071882844, -0.020789217203855515, -0.028221996501088142, 0.014527064748108387, -0.0006051474483683705, 0.01572589948773384, 0.006783998105674982, 0.013518632389605045, 0.019872460514307022, -0.028391243889927864, 0.006568912882357836, -0.03243907541036606, -0.03675488382577896, 0.015316885896027088, -0.004171242006123066, -0.007926417514681816, 0.0044780028983950615, 0.016995254904031754, -0.020577657967805862, -0.011367781087756157, -0.00923103280365467, 0.019463445991277695, 0.016684968024492264, 0.030295277014374733, 0.03754470497369766, -0.019646797329187393, 0.0009634758462198079, -0.02301764115691185, 0.061211127787828445, 0.02585253305733204, 0.004234709776937962, -0.011790899559855461, 0.029054129496216774, 0.009252188727259636, 0.01947754994034767, -0.015316885896027088, -0.014992495067417622, 0.022284235805273056, -0.023751046508550644, 0.006064697168767452, 0.001765637774951756, 0.007926417514681816, -0.010465128347277641, 0.0028507602401077747, -0.009294500574469566, 0.01966090127825737, 0.0037833836395293474, 0.0112196896225214, -0.021494414657354355, 0.019999396055936813, -0.009527215734124184, 0.007118966896086931, -0.011318417266011238, -0.02372283861041069, 0.008772654458880424, -0.001102752285078168, -0.014033426530659199, 0.0017488893354311585, -0.03243907541036606, 0.012672396376729012, -0.014654000289738178, 0.009400280192494392, 0.0013804237823933363, 0.01926599070429802, 0.01210823841392994, -0.013194241560995579, 0.0026850388385355473, 0.0005465279100462794, -0.012862799689173698, 0.015415613539516926, 0.014287297613918781, -0.007975781336426735, 0.01038050465285778, 0.015020702965557575, -0.01590925082564354, -0.007799482438713312, -0.013292969204485416, -0.011233793571591377, 0.00356300943531096, -0.009922126308083534, -0.02045072242617607, -0.004749503917992115, -0.012644188478589058, -0.053566787391901016, -0.035682983696460724, 0.005468805320560932, 0.0030411635525524616, -0.0592929907143116, 0.03336993604898453, -0.0002787733101285994, -0.015147638507187366, -0.013469268567860126, -0.01841975376009941, -0.18143317103385925, 0.024329308420419693, 0.005253720097243786, -0.0035083566326647997, 0.027093680575489998, 0.00905473344027996, 0.009555423632264137, -0.004534418694674969, -0.02310226485133171, -0.011551132425665855, 0.020873840898275375, 0.008723290637135506, -0.018123570829629898, -0.008779706433415413, 0.0053665516898036, 0.0027626105584204197, -0.00923103280365467, 0.00479534175246954, 0.007418675813823938, 0.023736942559480667, 0.024371620267629623, -0.027545006945729256, -0.0007408979581668973, -0.03901151567697525, 0.01478093583136797, -0.030718395486474037, -0.0009149935212917626, 0.022199612110853195, 0.0037692796904593706, -0.02248169109225273, -0.009731722995638847, 0.007312896195799112, 0.022467587143182755, 0.01864541694521904, 0.008159132674336433, 0.010951714590191841, 0.017742764204740524, 0.009167565032839775, -0.007933469489216805, 0.0245690755546093, -0.020958464592695236, 0.008321328088641167, -0.011741535738110542, 0.015345093794167042, -0.021311063319444656, 0.0448787584900856, -0.006939141545444727, 0.0009123490308411419, -0.009597735479474068, 0.005927183665335178, 0.003924422897398472, 0.0057367803528904915, 0.013843023218214512, 0.019054431468248367, 0.005398285575211048, 0.009731722995638847, -0.007658442948013544, -0.008420055732131004, -0.0006540705217048526, -0.008370691910386086, -0.012827539816498756, -0.01139598898589611, 0.00856814719736576, -0.012214018031954765, 0.01802484318614006, 0.004019624553620815, 0.009125253185629845, -0.001722444430924952, -0.031310759484767914, 0.0035594834480434656, 0.0048799654468894005, -0.00681573199108243, 0.007214168552309275, 4.192618507659063e-05, 0.02784118987619877, 0.013779555447399616, -0.01621953770518303, -0.013130773790180683, -0.0020115752704441547, -0.03573939949274063, -0.028207892552018166, 0.0319877490401268, -0.007207116577774286, 0.005712098442018032, -0.011226741597056389, 0.024681907147169113, 0.010845934972167015, 0.00578614417463541, 0.0064948671497404575, 6.71039306325838e-05, 0.013462216593325138, -0.012122342362999916, 0.012827539816498756, 0.002700905781239271, -0.0389833077788353, 0.020041707903146744, 0.018532585352659225, 0.016981150954961777, -0.041860513389110565, -0.005338343791663647, 0.02248169109225273, -0.018546689301729202, -0.025288376957178116, 0.02152262255549431, 0.02009812369942665, 0.011713327839970589, -0.008737394586205482, 0.021000776439905167, -0.00016902698553167284, -0.032636530697345734, -0.008032197132706642, 0.03012602962553501, 0.008229652419686317, 0.022763770073652267, -0.0052290381863713264, 0.011783847585320473, -0.00437574926763773, -0.0006448148051276803, 0.01258777268230915, -0.022524002939462662, 0.06262151896953583, -0.013758399523794651, 0.006660588551312685, -0.004037254489958286, -0.012178758159279823, 0.0017277334118261933, -0.11102626472711563, 0.009583631530404091, 0.015895146876573563, 0.020436618477106094, -0.009879814460873604, 0.02349717542529106, -0.013109617866575718, 0.05032288283109665, 0.00914640910923481, 0.0065301270224153996, -0.028264308348298073, -0.01109275408089161, -0.0026321490295231342, 0.01833513006567955, 0.022425275295972824, -0.020930256694555283, 0.004559100605547428, -0.008652770891785622, -0.02191753312945366, 0.033595599234104156, -0.013638515956699848, -0.0008497627568431199, -0.024371620267629623, -0.004781237803399563, -0.018137674778699875, -0.006092905066907406, -0.005151466466486454, 0.027587318792939186, 0.0015258707571774721, -7.839810859877616e-05, 0.010676687583327293, 0.013116669841110706, 0.03221341222524643, -0.0018546689534559846, -0.018476169556379318, -0.02761552669107914, -0.018899288028478622, -0.019632693380117416, 0.006593594793230295, -0.034300796687603, 0.02227013185620308, 0.0064948671497404575, -0.0018141200998798013, -0.023694630712270737, -0.02142389491200447, -0.005151466466486454, -0.008229652419686317, 0.02416006103157997, 0.03923717886209488, -0.01709398254752159, -0.025683287531137466, -0.016022082418203354, 0.011607548221945763, -0.021494414657354355, 0.029843950644135475, 0.006064697168767452, 0.018306922167539597, 0.010902350768446922, -0.04315807670354843, -0.00985160656273365, -0.006124638952314854, 0.03133896738290787, 0.009118201211094856, -0.00870213471353054, 0.020309682935476303, -0.03114151395857334, -0.024174164980649948, -0.021804701536893845, 0.012658292427659035, -0.015895146876573563, -0.022100884467363358, -0.009435540065169334, -0.00857519917190075, -0.003688181983307004, -0.0035965063143521547, 0.020845633000135422, -0.015415613539516926, -0.02558455988764763, 0.020351994782686234, -0.0037304938305169344, 0.007552663329988718, -0.017023462802171707, 0.021677765995264053, -0.014611688442528248, 0.017023462802171707, 0.03449825197458267, 0.019054431468248367, 0.0013583863619714975, 0.010077269747853279, -0.020845633000135422, -0.019280094653367996, 0.01922367885708809, 0.024089541286230087, -0.02660004235804081, -0.009647099301218987, 0.025965364649891853, 0.018546689301729202, -0.004386327229440212, -0.005641578696668148, 0.016558032482862473, -0.006819257978349924, -0.015345093794167042, -0.051451198756694794, 0.015246366150677204, -0.01816588267683983, 0.009343864396214485, -0.03328531235456467, -0.00949900783598423, -0.012658292427659035, 0.005465279333293438, -0.007658442948013544, 0.00839184783399105, -0.014625792391598225, -0.0032104109413921833, 0.0030235336162149906, -0.02850407548248768, -0.02916696108877659, -0.009428488090634346, 0.018673624843358994, -0.01152997650206089, 0.034441836178302765, 0.005553429014980793, 0.0006765486905351281, 0.029843950644135475, 0.003236855613067746, 0.023877982050180435, -0.03195954114198685, -0.0040760403499007225, -0.005426493473351002, 0.03274936228990555, 0.02523196116089821, -0.014364869333803654, 0.003353213192895055, -0.03562656790018082, -0.03125434368848801, 0.0038221694994717836, -0.0034131549764424562, -0.003635292174294591, -0.02283428981900215, 0.010006750002503395, -0.0008250808459706604, 0.017376061528921127, -0.026402587071061134, -0.03966029733419418, -0.00011823073873529211, -0.0208597369492054, -0.005119732581079006, -0.0004918751074001193, -0.035457320511341095, -0.003231566632166505, 0.013730191625654697, 0.010704895481467247, -0.004731873981654644, 0.010761311277747154, -0.014308453537523746, -0.03142359107732773, 2.4847187887644395e-05, -0.01750299707055092, 0.0065124970860779285, 0.0062445225194096565, 0.021790597587823868, -0.01771455630660057, 0.003009429667145014, 0.004869387485086918, 0.010408712550997734, -0.011078650131821632, 0.010105477645993233, -0.0018141200998798013, -0.03455466777086258, -0.0018546689534559846, 0.030408108606934547, 0.0036599740851670504, -0.030605563893914223, 0.02801043726503849, 0.00011349269334459677, -0.003621188225224614, -0.011099806055426598, 0.02345486357808113, 0.011226741597056389, 0.0026515419594943523, -0.007616131100803614, 0.005514643155038357, -0.01700935885310173, -0.02813737280666828, 0.004534418694674969, 0.008187340572476387, 0.02981574274599552, 0.014146258123219013, -0.02956187166273594, 0.012432629242539406, 0.0012755256611853838, 0.0012481992598623037, -0.009329760447144508, 0.02527427300810814, -0.02142389491200447, -0.008589303120970726, -0.012214018031954765, -0.011050442233681679, 0.011022234335541725, -0.00356300943531096, -0.0015708270948380232, 0.0049504851922392845, 0.012757020071148872, 0.021593142300844193, -0.014541168697178364, -0.007090758997946978, -0.011635756120085716, 0.0057720402255654335, -0.01424498576670885, -0.001391001744195819, 0.008920745924115181, 0.01947754994034767, 0.009216928854584694, -0.026303859427571297, -0.0028648641891777515, 0.016896527260541916, 0.01389238703995943, 0.00395615678280592, 0.008201444521546364, -0.02032378688454628, -0.01983014866709709, 0.00932270847260952, 0.01771455630660057, 0.02098667249083519, -0.004686036147177219, 0.013828919269144535, -0.0017312593990936875, 0.014639896340668201, 0.021804701536893845, -0.01864541694521904, 0.005817878060042858, 0.005066842772066593, 0.0052466681227087975, -0.008504679426550865, -0.014752727933228016, -0.015345093794167042, -0.0030940533615648746, -0.0031257872469723225, 0.002380040939897299, -0.002730876673012972, -0.015330989845097065, 0.06758610904216766, 0.045048005878925323, -0.014625792391598225, 0.00879381038248539, -0.004358119331300259, 0.009555423632264137, -0.0026215710677206516, -0.003342635231092572, -0.002505213487893343, -0.014484752900898457, 0.012749968096613884, -0.001695118029601872, 0.006480763200670481, -0.009872762486338615, -0.025161441415548325, 0.008511731401085854, -0.014484752900898457, 0.015782315284013748, -0.044117145240306854, -0.005482909269630909, 0.03562656790018082, 0.011896679177880287, 0.013709035702049732, 0.002732639666646719, -0.019421134144067764, 0.00015977126895450056, 0.006551282946020365, 0.007552663329988718, -0.016600344330072403, -0.026092300191521645, -0.006195158697664738, 0.032100580632686615, -0.0036088472697883844, -0.03057735599577427, -0.005831982009112835, -0.009442592039704323, -0.007884105667471886, 0.006953245494514704, 0.0005703283241018653, 0.009315656498074532, -0.004767133854329586, 0.032015956938266754, -0.02372283861041069, -0.019280094653367996, -0.008779706433415413, -0.0005183200119063258, -0.008222600445151329, -0.014992495067417622, -0.009287448599934578]} +{"id": "test:10000049", "text": "\"It is possible to erase up to 4 HDD / SSD at the same time. IDE HDD connection is also possible with dedicated adapter.\\nHDD / SSD is exchanged from the order in which erasing is completed, and asynchronous erase function which can erase newly is installed.Five types of erase algorithms are installed. The erasing method can be selected according to the application.\\nHDD copy function installed. It is possible to copy one HDD data to up to three HDDs simultaneously.\\nData delete contents can be printed with the attached dedicated printer . It is also possible to output text data of work log to USB memory.\\nDedicated carrying case with waterproof / dustproof specification is included, which can contain the main body and all accessories.\\nThis product is compatible with SATA 6 Gbps HDD / SSD, but the internal transfer speed is up to 130 MB / sec.\\n2.5 \u201cHDD and 3.5\u201d HDD can not be connected at the same time.\"", "vector_field": [-0.012598935514688492, 0.0321325808763504, -0.0033035811502486467, -0.01995713822543621, -0.03509703651070595, 0.025462554767727852, -0.025846345350146294, -0.02044680155813694, -0.008588979952037334, -0.03615576773881912, 0.0036625582724809647, 0.03615576773881912, -0.027209466323256493, 1.804190469556488e-05, -0.00996533501893282, 0.01511342916637659, 0.008688236586749554, 0.013856182806193829, 0.005531885661184788, -0.00451947096735239, 0.00015343373524956405, -0.000947070715483278, 0.0022332672961056232, 0.004138988442718983, -0.026521289721131325, -0.007867718115448952, 0.015206068754196167, 0.0008726284722797573, -0.003320124000310898, 0.014769340865314007, 0.023596536368131638, -0.028003517538309097, -0.01801171340048313, -0.02274954877793789, -0.027632959187030792, -0.00907202810049057, -0.0020430260337889194, -0.01017708145081997, -0.0004208054451737553, -0.006051327101886272, 0.006613779813051224, -0.01177841704338789, 0.007000879384577274, 0.017283834517002106, -0.022537801414728165, 0.01999684050679207, -0.016291271895170212, -0.015695733949542046, -0.0034872053656727076, 0.02064531482756138, -0.005263893399387598, 0.01124243251979351, -0.007325116544961929, -0.0029876152984797955, -0.015510454773902893, 0.014359081164002419, -0.024165606126189232, 0.016119226813316345, 0.0063424790278077126, -0.00357984472066164, 0.0019470781553536654, 0.0002863958361558616, -0.022140776738524437, 0.010388828814029694, -0.00972050242125988, -0.00797359086573124, -0.0164897833019495, -0.042296428233385086, 0.01119611319154501, 0.0037849743384867907, 0.031709086149930954, 0.006580694112926722, 0.004268021788448095, 0.008330914191901684, 0.0006302776164375246, -0.015285473316907883, 0.014901682734489441, 0.007808164227753878, 0.03586461767554283, 0.02122761867940426, 0.016952980309724808, -0.03147087246179581, 0.01834256760776043, 0.0013523673405870795, 0.03080916218459606, 0.03020039014518261, 0.006934708449989557, 0.03006804920732975, 0.007331733591854572, -0.0075368634425103664, 0.0031398083083331585, 0.013128302991390228, -0.011222581379115582, -0.01090496126562357, 0.022828953340649605, 0.017932308837771416, -0.004274638835340738, 0.0258860494941473, -0.003490513889119029, 0.002418545773252845, 0.0076030343770980835, 0.02274954877793789, -0.018541080877184868, -0.012724660336971283, -0.036923352628946304, -0.013187856413424015, 0.003229138907045126, -0.010249869897961617, 0.00863530021160841, 0.01928219571709633, -0.011163027957081795, 0.037796806544065475, 0.022405460476875305, -0.014306144788861275, 0.02177022024989128, -0.0016898388275876641, 0.019983606413006783, -0.0025608132127672434, -0.02170404978096485, -0.00994548387825489, 0.022656910121440887, 0.01540458109229803, 0.03295309841632843, -0.017574986442923546, -0.013392986729741096, 0.007993442006409168, -0.025740472599864006, -0.004585641901940107, 0.023742111399769783, 0.0033945662435144186, 0.02601839043200016, 0.006676641758531332, 0.009124964475631714, 0.014279676601290703, -0.015642795711755753, 0.000904059677850455, -0.01711178943514824, 0.02333185262978077, -0.019427770748734474, -0.007847866974771023, 0.039808403700590134, -0.0015293745091184974, 0.0014971161726862192, -0.03139146789908409, -0.01453112531453371, 0.04449329897761345, 0.0049429647624492645, 0.021664345636963844, 0.0013391332468017936, 0.0073714363388717175, -0.015364878810942173, -0.00178992236033082, 0.008575745858252048, -0.003275458700954914, 0.015986884012818336, -0.020909998565912247, 0.0037551976274698973, -0.00224815565161407, -0.03033273108303547, -0.008516192436218262, -0.007060432806611061, 0.007358201779425144, -0.007417755667120218, 0.016344208270311356, 0.02199520170688629, 0.014795809052884579, -0.006074486766010523, -0.009376414120197296, 0.011189496144652367, -0.035176441073417664, -0.04205821081995964, 0.023596536368131638, -0.033323656767606735, 0.005121626425534487, 0.02304070070385933, 0.0007113369647413492, 0.0211879163980484, 0.02274954877793789, -0.005568279884755611, -0.00022022495977580547, 0.005118317902088165, 0.007225860375910997, 0.02646835334599018, 0.012843768112361431, -0.04102594777941704, -0.010891727171838284, -0.00884704664349556, -0.0025905899237841368, 0.029671022668480873, -0.0045591737143695354, 0.0037353462539613247, 0.015034024603664875, -0.021082043647766113, -0.00074938521720469, -0.6318525075912476, -0.009224221110343933, -0.009369797073304653, -0.018898403272032738, 0.010335891507565975, 0.0340118333697319, 0.01717795990407467, 0.008218423463404179, 0.0059586879797279835, -0.001028957194648683, 0.0013341703452169895, 0.02747415006160736, -0.00459225894883275, -0.0105939581990242, -0.033455997705459595, -0.01192399300634861, -0.014689935371279716, 0.00045823334949091077, -0.006491363514214754, -0.0168206375092268, 0.00034781068097800016, 0.003897464834153652, -0.020393865182995796, -0.002461556810885668, 0.02863875776529312, 0.0105939581990242, -0.013081983663141727, -0.018130822107195854, -0.02478761225938797, 0.011493882164359093, -0.015034024603664875, 0.007073667366057634, -0.0022498099133372307, 0.007397904526442289, 0.041105352342128754, -0.037637997418642044, 0.015139897353947163, 0.04004661738872528, 0.020036542788147926, 0.039093755185604095, 0.002264698501676321, -0.008774259127676487, 0.028691694140434265, 0.011884290724992752, 0.031709086149930954, -0.010620426386594772, -0.006706418935209513, -0.015073726885020733, -0.005607982166111469, 0.006051327101886272, -0.015523688867688179, -0.01684710569679737, 0.012883470393717289, -0.001528547378256917, -0.004605493508279324, -0.005244042258709669, 0.03626164421439171, -0.0045327055267989635, -0.00788756925612688, -0.0023589918855577707, 0.021042339503765106, 0.011818119324743748, -0.058442123234272, -0.026296308264136314, -0.039967212826013565, 0.016608892008662224, -0.03162968158721924, -0.029300466179847717, 0.024218542501330376, -0.013790011405944824, -0.008655151352286339, 0.014187037013471127, 0.01736323907971382, -0.014041461050510406, 0.004403672181069851, 0.020685017108917236, 0.027262402698397636, -0.0020794200245290995, -0.012122505344450474, 0.008086081594228745, -0.00228454964235425, -0.012512913905084133, 0.013247410766780376, -0.022789251059293747, 0.010071207769215107, 0.01742940954864025, 0.0020364089868962765, -0.0006443389574997127, -0.007080284412950277, 0.011520350351929665, 0.03512350469827652, 0.00027130061062052846, -0.00206618569791317, -0.026322776451706886, -0.030385669320821762, 0.02834760583937168, -0.005942145362496376, -0.013042280450463295, -0.0062134456820786, 0.007741993293166161, -0.009680800139904022, 0.0008999239653348923, 0.01729706861078739, 0.009872695431113243, 0.03152380883693695, 0.012493062764406204, -0.037796806544065475, -0.0012564195785671473, 0.023795049637556076, -0.023675940930843353, -0.0014888448640704155, 0.010382210835814476, -0.017085321247577667, 0.014279676601290703, -0.02863875776529312, -0.015735436230897903, 0.01749558188021183, 0.00795373972505331, 0.012579084374010563, -0.03268841654062271, 0.024893485009670258, 0.000595124380197376, -0.005535194184631109, 0.010289572179317474, -0.018157290294766426, 0.01594718173146248, 0.006100955419242382, -0.018832232803106308, -0.005899134092032909, -0.0018064650939777493, -0.015536922961473465, -0.018474910408258438, 0.0015302016399800777, -0.008291210979223251, -0.011818119324743748, 0.03218551725149155, 0.01600011996924877, -0.004608802031725645, 0.010038122534751892, -0.030544478446245193, -0.013816479593515396, 0.0034872053656727076, 0.008284593932330608, -0.0063623301684856415, -0.006921474356204271, -0.015933947637677193, -0.005647684913128614, -0.017879372462630272, -0.004291181452572346, -0.02787117473781109, 0.006603853777050972, -0.002370571717619896, -0.015867777168750763, 0.024800846353173256, -0.009627862833440304, -0.022206947207450867, -0.013935587368905544, -0.020407099276781082, -0.024152372032403946, -0.010137379169464111, -0.006458277814090252, 0.0434080995619297, -0.0037287292070686817, 0.011718862690031528, -0.02478761225938797, -0.002540961839258671, 0.0015657684998586774, -0.005270510911941528, -0.0193086639046669, -0.015550157055258751, 0.000558730389457196, -0.0018924871692433953, 0.018421974033117294, 0.0065873111598193645, -0.0018213535659015179, -0.014703169465065002, -0.020658548921346664, -0.008774259127676487, 0.018514612689614296, 0.00017793763254303485, 0.020711485296487808, -0.000715472677256912, 0.008933069184422493, 0.01772056147456169, 0.032794289290905, 0.02990923821926117, 0.009614628739655018, -0.016158929094672203, -0.00868161953985691, -0.007947122678160667, 0.01908368244767189, 0.03327072039246559, -0.010752768255770206, -0.025131700560450554, 0.000499590125400573, 0.006928091403096914, 0.005164637230336666, -0.015470752492547035, 0.007702290546149015, 0.016503017395734787, 0.06325936317443848, -0.014359081164002419, -0.007252328563481569, -0.009455818682909012, 0.00965433195233345, 0.002325906418263912, 0.017958777025341988, -0.013492242433130741, 0.03607636317610741, -0.0028536191675812006, -0.012731277383863926, -0.040549516677856445, -0.008403701707720757, -0.014981087297201157, -0.018024947494268417, 0.013359900563955307, -0.02310687117278576, 0.015722202137112617, -0.0022316130343824625, 0.0030653660651296377, 0.002137319417670369, -0.002155516529455781, 0.0021058882120996714, -0.006272999569773674, -0.0026650321669876575, 0.025608131662011147, 0.015073726885020733, -0.004019881132990122, 0.029432807117700577, -0.04224349185824394, 0.022723080590367317, 0.013988524675369263, -0.008238274604082108, -0.003318469738587737, 0.039543718099594116, 0.05812450125813484, 0.019679220393300056, -0.003075291635468602, 0.029300466179847717, -0.018382271751761436, 0.004486385732889175, 0.016542719677090645, 0.009422733448445797, -0.02391415648162365, 0.003788282861933112, 0.02731533907353878, 0.019838029518723488, -0.0030438604298979044, -0.005568279884755611, 0.0163971446454525, -0.00677258986979723, 0.003897464834153652, -0.03729390725493431, -0.011943844147026539, 0.02903578244149685, -0.02060561254620552, 0.011665926314890385, -0.008562511764466763, 0.005548428278416395, 0.02154523879289627, 0.023821517825126648, 0.016317740082740784, 0.0003480174927972257, -0.018580783158540726, -0.02219371311366558, -0.005789951886981726, -0.012545999139547348, -0.00686853751540184, -0.006848686374723911, -0.020340928807854652, 0.013293730095028877, 0.0003598041657824069, -0.006468203850090504, -0.022220181301236153, 0.005081923678517342, 0.004238245077431202, 0.01762792281806469, 0.002076111501082778, 0.003933859057724476, 0.003202670719474554, -0.0357058085501194, -0.0358910858631134, -0.0022696612868458033, 0.01717795990407467, 0.0006753565394319594, -0.0014086125884205103, -0.015444283373653889, 0.006729578599333763, 0.01197031233459711, 0.03252960368990898, 0.011454179883003235, 0.020658548921346664, 0.012982727028429508, 0.015298707410693169, -0.019401302561163902, -0.012168824672698975, 0.040417175740003586, 4.244241790729575e-05, 0.033588338643312454, -0.008105932734906673, -0.007404521573334932, 0.00023656089615542442, -0.01594718173146248, -0.04671664163470268, 0.02413913793861866, -0.007894186303019524, -0.004820548929274082, 0.003679100889712572, 0.03726743906736374, -0.023225979879498482, 0.020063010975718498, -0.006378872785717249, -0.03181495890021324, 0.008721321821212769, -0.03644692152738571, 0.012479828670620918, -0.02458909898996353, 0.01409439742565155, 0.03700275719165802, 0.004536014050245285, -0.0013622930273413658, -0.007880952209234238, -0.013439306057989597, -0.00853604357689619, 0.02420530840754509, 0.014081163331866264, -0.004549248144030571, 0.02653452381491661, -0.00993224885314703, -0.01740294136106968, -0.02760649099946022, -0.03398536518216133, 0.018514612689614296, -0.006934708449989557, 0.0015905825421214104, 0.02977689728140831, 0.023742111399769783, -0.024086201563477516, -0.02728887088596821, 0.009369797073304653, 0.01704561896622181, 0.0027560172602534294, -0.008866897784173489, -0.02228635363280773, 0.009065411053597927, -0.02035416290163994, 0.006226679775863886, 0.030756225809454918, 0.0386173278093338, -0.004929730668663979, 0.03470000997185707, 0.008787493221461773, 0.02053944207727909, -0.012440125457942486, -0.01417380291968584, 0.020565910264849663, 0.01791907474398613, -0.003659249749034643, 0.004794080276042223, 0.012248230166733265, 0.006895005702972412, -0.029088718816637993, 0.005498800426721573, -0.00669980188831687, 0.0031580054201185703, 0.01821022666990757, 0.02333185262978077, 0.003440885804593563, 0.018316099420189857, 0.016185397282242775, -0.005240733735263348, 0.034752946346998215, -0.008615449070930481, -0.011692394502460957, 0.012479828670620918, -0.0033002726268023252, -0.017389707267284393, 0.009561692364513874, -0.00993224885314703, -0.0045393225736916065, -0.00168074038811028, -0.03480588272213936, -0.0064417351968586445, -0.02932693436741829, 0.007192774675786495, 0.01036897674202919, 0.005478948820382357, -0.008675002492964268, 0.006861920468509197, -0.02310687117278576, -0.024099435657262802, -0.00921760406345129, -0.0281887948513031, 0.016503017395734787, -0.008000059984624386, -0.008258125744760036, -0.008125783875584602, 0.019758624956011772, -0.01109685655683279, 0.0210952777415514, 0.004976050462573767, -0.0066667161881923676, -0.0069413254968822, 0.03933197259902954, -0.009938866831362247, -0.013988524675369263, -0.0008238274604082108, -0.0063689472153782845, -0.0015550156822428107, -0.007404521573334932, 0.00028598227072507143, 0.0011215964332222939, 0.01025648694485426, 0.007033964619040489, 0.020949700847268105, 0.0017022459069266915, 0.01652948558330536, 0.014689935371279716, -0.005111700389534235, 0.02008947916328907, 0.015311941504478455, 0.00544255506247282, 0.030438605695962906, -0.017958777025341988, -0.012486445717513561, 0.007258945610374212, -0.008039762265980244, -0.019480707123875618, 0.018938105553388596, 0.005644376389682293, -0.00933009386062622, -0.010421914048492908, 0.021439366042613983, 0.006107572466135025, -0.0025508874095976353, -0.008516192436218262, -0.01453112531453371, -0.00878087617456913, 0.002668340690433979, 0.045181479305028915, 0.014359081164002419, 0.010124145075678825, -0.015960415825247765, 0.008019911125302315, -0.004919805098325014, -0.04454623907804489, 0.015219302847981453, 0.01023663580417633, -0.012029866687953472, 0.02787117473781109, 1.3505580682249274e-05, -0.024509694427251816, -0.0024582482874393463, 0.01646331511437893, -0.020301226526498795, 0.03207964450120926, -0.0002539307461120188, -0.023014232516288757, -0.026521289721131325, -0.02067178301513195, 0.011705628596246243, 0.03724097087979317, -0.0029826525133103132, -0.01197031233459711, -0.012182059697806835, 0.008741172961890697, -0.007708907593041658, 0.025489022955298424, -0.015629561617970467, -0.018276397138834, -0.016741232946515083, 0.013306964188814163, 0.0031778565607964993, 0.042693451046943665, 0.028691694140434265, -0.014769340865314007, -0.014769340865314007, 0.00490987952798605, -0.006306084804236889, 0.0005938836839050055, -0.014107631519436836, -0.014134100638329983, 0.05028986930847168, 0.025091998279094696, 0.004985976032912731, -0.00515140313655138, 0.02096293494105339, 0.02165111154317856, 0.006064561195671558, -0.000701824901625514, -0.012956258840858936, -0.02031446062028408, -0.0046915155835449696, 0.022299587726593018, 0.025356682017445564, 0.0013258990366011858, 0.014253207482397556, -0.0012688266579061747, 0.002989269560202956, 0.032291390001773834, 0.02326568216085434, -0.014994321390986443, -0.0076096514239907265, -0.048040058463811874, 0.015391346998512745, -0.032026708126068115, 0.0041489144787192345, 0.008840429596602917, -0.033164843916893005, -0.01082555577158928, -0.01296287588775158, -0.018170524388551712, 0.02404649741947651, 0.01950717531144619, 0.024814080446958542, 0.005349915940314531, 0.01570896804332733, 0.0319472998380661, -0.002704734681174159, -0.001578175462782383, 0.0008056304650381207, -0.02628307417035103, 0.006828834768384695, 0.028506414964795113, 0.003980178385972977, -0.0006902450113557279, -0.00195700372569263, 0.0003149320255033672, -0.0012440126156434417, -0.001614569453522563, -0.027553554624319077, -0.0016956288600340486, -0.009727119468152523, -0.032741352915763855, 0.008820578455924988, -0.029432807117700577, 0.0113218380138278, -0.011235815472900867, -0.011189496144652367, 0.018885169178247452, -0.0026021699886769056, 0.018977809697389603, -0.013333432376384735, -0.028400542214512825, -0.021015871316194534, -0.0075964173302054405, 0.06045371666550636, 0.004575716331601143, 0.01901751197874546, 0.02816232666373253, 0.011963695287704468, -0.025621365755796432, -0.031841427087783813, 0.005058764014393091, -0.004661738406866789, 0.027712365612387657, 0.015986884012818336, -0.022656910121440887, 0.0014747835230082273, -0.005105083342641592, -0.007821397855877876, 0.005581513978540897, -0.047907717525959015, 0.0018031565705314279, 0.014769340865314007, -0.0032804214861243963, 0.005852814298123121, 0.015920713543891907, -0.01928219571709633, -0.001421846798621118, -0.02585958130657673, 0.02167758159339428, -0.009852844290435314, 0.013895885087549686, -0.03131205961108208, 0.0201556496322155, -0.029829833656549454, 0.0025740473065525293, 0.006292850710451603, -0.012380572035908699, -0.014835511334240437, -0.02834760583937168, -0.01111670769751072, -0.005217574071139097, 0.009548458270728588, 0.03380008786916733, -0.018713125959038734, -0.00936318002641201, 0.027421213686466217, 0.006848686374723911, -0.008774259127676487, -0.029856301844120026, -0.010309423319995403, 0.015722202137112617, -0.023093637079000473, -0.026812441647052765, -0.0038015171885490417, 0.013095217756927013, 0.014107631519436836, -0.016225099563598633, -0.015510454773902893, -0.023887688294053078, -0.00855589471757412, -0.006547608878463507, 0.008436786942183971, 0.004191925283521414, 0.008615449070930481, -0.043566908687353134, -0.012638638727366924, -0.027685897424817085, -0.014994321390986443, -0.00473783491179347, 0.029723959043622017, -0.027341807261109352, -0.0077684614807367325, -0.0048635597340762615, -0.005521960090845823, -0.007199391722679138, 0.012989344075322151, 0.01155343558639288, -0.0007779214065521955, 0.03581168130040169, -0.021439366042613983, -0.018752828240394592, 0.029115187004208565, 0.01525900512933731, -0.026415415108203888, 0.008125783875584602, -0.0007229168550111353, -0.01236072089523077, 0.009151432663202286, -2.0187288100714795e-05, -0.016939746215939522, -0.01901751197874546, 0.005780026316642761, 0.0006141485064290464, 0.005214265547692776, -0.00040467630606144667, 0.014636998996138573, -0.018104353919625282, 0.006484746467322111, 7.877178177295718e-06, -0.04234936460852623, -0.004225010983645916, -0.034752946346998215, -0.024125903844833374, 0.024575864896178246, 0.010282955132424831, -0.01577513851225376, -0.01082555577158928, 0.0058792829513549805, -0.007338350638747215, -0.025422852486371994, -0.019652752205729485, 0.001614569453522563, -0.015497220680117607, -0.013895885087549686, 0.009879312478005886, 0.008430169895291328, -0.004939656239002943, -0.01082555577158928, -0.018766062334179878, 0.0003465699846856296, -0.00592229375615716, 0.007119986694306135, 9.434520325157791e-05, 0.005786643363535404, 0.014001758769154549, 0.0032671871595084667, -0.015801606699824333, 0.01090496126562357, -0.023318618535995483, -0.03282075747847557, -0.0011348306434229016, -0.010057973675429821, 0.029247529804706573, -0.00178992236033082, -0.008105932734906673, -0.03451473265886307, 0.005786643363535404, -0.008277976885437965, -0.021373195573687553, -0.021624643355607986, 0.0035335251595824957, 0.025290511548519135, 0.030412137508392334, 0.0031100313644856215, 0.014014992862939835, 0.025330213829874992, 0.029591618105769157, -0.0266403965651989, -0.019758624956011772, -0.02702418714761734, 0.0017005916452035308, -0.005862739868462086, 0.01323417667299509, -0.043593376874923706, -0.005237425211817026, -0.003240718971937895, 0.0019024128559976816, -0.01581484079360962, 0.01700591668486595, 0.0038478367496281862, -0.01380324549973011, -0.0014971161726862192, 0.007880952209234238, 0.024443523958325386, -0.005396235268563032, 0.008787493221461773, -0.024549396708607674, -0.003457428654655814, 0.013260644860565662, -0.002277932595461607, -0.020486505702137947, -2.2616999558522366e-05, 0.0024499769788235426, -0.0011885944986715913, 0.0038081342354416847, -0.029432807117700577, -0.02255103550851345, 0.003609621664509177, -0.017932308837771416, 0.008906600996851921, 0.004701441153883934, -0.012665106914937496, 0.013763543218374252, 0.012545999139547348, -0.01749558188021183, 0.016926512122154236, 0.0017436026828363538, -0.021148214116692543, 0.00967418309301138, 0.013909119181334972, -0.03335012495517731, 0.02073795348405838, 0.0029015932232141495, 0.007225860375910997, -0.006352404598146677, 0.00041729010990820825, -0.011705628596246243, -0.006021550390869379, -0.03610283136367798, 0.01807788386940956, 0.021201150491833687, -0.0032721501775085926, -0.012519530951976776, 0.0113218380138278, -0.020142415538430214, 0.01928219571709633, -0.03080916218459606, 0.01236072089523077, -0.036632198840379715, -0.014491423033177853, -0.008741172961890697, -0.0075302463956177235, -0.00832429714500904, 0.0042051593773067, 0.003038897644728422, 0.0016939745983108878, 0.002491333754733205, 0.19978313148021698, -0.009978569112718105, 0.004575716331601143, 0.02433764934539795, 0.0016120880609378219, 0.006292850710451603, -0.002954529831185937, -0.0020182118751108646, 0.022378992289304733, 0.012241613119840622, -0.039067286998033524, -0.002234921557828784, -0.014200271107256413, -0.01322094164788723, 0.011771799996495247, -0.025700770318508148, -0.03724097087979317, -0.017799967899918556, 0.007642736658453941, 0.015073726885020733, -0.0020976169034838676, 0.005320138763636351, -0.01840873993933201, -0.021624643355607986, 0.007728758733719587, 0.0034673542249947786, -0.012539382092654705, 0.009111730381846428, 0.010316040366888046, -0.0037717402447015047, -0.024840548634529114, 0.002473136642947793, 0.019335132092237473, 0.00541939539834857, -0.019136618822813034, 0.001010760199278593, -0.004575716331601143, -0.01729706861078739, -0.01623833365738392, -0.004767612088471651, 0.01140785962343216, 0.02990923821926117, -0.007192774675786495, 0.003191090654581785, 0.004377203993499279, 0.02177022024989128, -0.0014433523174375296, 0.0211879163980484, -0.013790011405944824, -0.025581663474440575, -0.025489022955298424, -0.04118475690484047, -0.004175382666289806, 0.03586461767554283, -0.016701530665159225, -0.010726300068199635, 0.027262402698397636, 0.013670903630554676, -0.005154711659997702, 0.028718162328004837, -0.01863371953368187, 0.03380008786916733, -0.026719801127910614, 0.015695733949542046, -0.004363969434052706, 0.03141793608665466, -0.0266403965651989, 0.016635360196232796, 0.04923113435506821, 0.01373707503080368, 0.016966214403510094, -0.01697944849729538, -0.025197871029376984, 0.016860339790582657, -0.008006677031517029, -0.015920713543891907, 0.03120618686079979, 0.0037221121601760387, 0.016595657914876938, 0.03941137716174126, -0.010137379169464111, -0.00459225894883275, -0.0029082102701067924, 0.004536014050245285, 0.00715968944132328, -0.03136499598622322, 0.016053056344389915, -0.020949700847268105, 0.008602214977145195, -0.003679100889712572, -0.000992563203908503, -0.015060492791235447, 0.019454238936305046, -0.025039061903953552, 0.0249464213848114, 0.013869416899979115, 0.022431928664445877, 0.015470752492547035, -0.005657610483467579, -0.0035103652626276016, 0.008132400922477245, 0.07691702991724014, -0.00490987952798605, -0.023318618535995483, -0.01409439742565155, 0.009641097858548164, -0.017482345923781395, 0.010812321677803993, 0.01959981583058834, -0.01937483437359333, 0.01590747945010662, -0.042111147195100784, 0.0077486103400588036, -0.0038776136934757233, 0.01344592310488224, 0.028135858476161957, -0.00016728826449252665, -0.03335012495517731, -0.00483709154650569, -0.003222521860152483, 0.028691694140434265, -0.009389648213982582, 0.00022105210518930107, 0.024536162614822388, 0.006127423606812954, 0.006147274747490883, -0.042852263897657394, 0.005813112016767263, -0.016304505988955498, -0.03716156631708145, 0.009667566046118736, 0.008370616473257542, 0.017230898141860962, 0.006223371252417564, -0.00036393984919413924, -0.0076030343770980835, -0.0013780086301267147, -0.01124243251979351, -0.0017204429022967815, -0.008483107201755047, 0.005680770147591829, 0.014994321390986443, -0.026587460190057755, 0.011454179883003235, -0.008165487088263035, -0.018421974033117294, 0.02284218743443489, -0.002334177726879716, -0.015629561617970467, -0.0210952777415514, -0.04290520027279854, 0.013723840937018394, 0.014425252564251423, -0.030597414821386337, -0.00635902164503932, -0.0012274698819965124, -0.03586461767554283, -0.01717795990407467, -0.011685777455568314, -0.002327560679987073, -0.02747415006160736, -0.012148973532021046, 0.002741128671914339, -0.0032936555799096823, -0.021426131948828697, -0.021955497562885284, -0.1662212610244751, 0.012420274317264557, 0.016211865469813347, -0.0062829251401126385, 0.026336010545492172, 0.0006960349855944514, 0.03051801025867462, 0.002873470541089773, -0.01834256760776043, -0.009310242719948292, 0.002952875569462776, 0.01197031233459711, -0.02705065719783306, -0.021042339503765106, 0.004985976032912731, 0.011930610053241253, -0.016807403415441513, 0.025581663474440575, 0.026402181014418602, 0.0032010164577513933, 0.021174682304263115, -0.03912022337317467, 0.007801546715199947, -0.011374774388968945, 0.021783454343676567, -0.033006034791469574, 0.0049363477155566216, 0.010342508554458618, 0.011871055699884892, -0.01921602338552475, -0.009548458270728588, -0.013055514544248581, 0.01525900512933731, 0.027341807261109352, 0.014306144788861275, -0.007781695574522018, -0.0034739712718874216, -0.0064549692906439304, 0.004430140368640423, 0.018316099420189857, 0.018514612689614296, 0.025422852486371994, -0.008661768399178982, -0.002425162820145488, 0.020433567464351654, 0.02566106803715229, -0.015086960978806019, -0.0163971446454525, 0.005730398464947939, -0.014769340865314007, 0.018091119825839996, -0.046451959758996964, 0.0019007585942745209, 0.006216754205524921, 0.01300919521600008, 0.004751069471240044, 0.005373075604438782, 0.011864438652992249, 0.007655970752239227, 0.010521169751882553, -0.018673421815037727, -0.025091998279094696, 0.02167758159339428, 0.02002330869436264, -0.0014160568825900555, -0.033588338643312454, -0.00797359086573124, 0.014663467183709145, 0.0033714063465595245, -0.0075898002833127975, -0.01801171340048313, -0.03020039014518261, -0.019176321104168892, -0.012638638727366924, 0.02199520170688629, -0.018196992576122284, -0.04166118800640106, -0.0011439290829002857, 0.018474910408258438, -0.004562482237815857, -0.002795719774439931, 0.025462554767727852, -0.0005074479267932475, -0.014213505201041698, -0.006802366580814123, -0.0004071576986461878, 0.010421914048492908, 0.0008275496074929833, -0.007940505631268024, 0.017416175454854965, 0.03062388300895691, -0.03522937744855881, 0.016503017395734787, -0.008833812549710274, -0.0023060552775859833, 0.003970252815634012, 0.00025517147150821984, -0.006908239796757698, 0.005664227530360222, 0.00243012560531497, 0.020711485296487808, -0.015603093430399895, -0.010249869897961617, 0.0022448471281677485, 0.0131084518507123, 0.014557593502104282, -0.007497160695493221, 0.00834414828568697, 0.05169269070029259, 0.002109196735545993, -0.01642361283302307, 0.0010099330684170127, 0.013909119181334972, 0.033455997705459595, 0.011315220966935158, 0.008800727315247059, 0.00630939332768321, -0.031867895275354385, 0.0043937466107308865, -0.009323476813733578, 0.051666222512722015, -0.044599175453186035, -0.026031624525785446, -0.0011803230736404657, 0.002966109663248062, -0.036632198840379715, -0.10746151208877563, -0.02183639071881771, 0.013988524675369263, 0.04179352894425392, -0.0019503866787999868, 0.02167758159339428, 0.008357382379472256, 0.02158494107425213, -0.018554314970970154, 0.020751187577843666, -0.033455997705459595, -0.0248537827283144, -0.0065277572721242905, -0.003081908682361245, 0.0036427071318030357, 0.0028188794385641813, -0.012552616186439991, 0.009032324887812138, -0.03583814948797226, 0.02270984649658203, 0.006028167437762022, -0.0036427071318030357, 0.00435073534026742, -0.032609011977910995, -0.01438554935157299, -0.014650233089923859, -0.018580783158540726, -0.008377233520150185, 0.008436786942183971, 0.030041581019759178, 0.011282135732471943, -0.015272239223122597, -0.0098660783842206, -0.009952100925147533, -0.010342508554458618, 0.016807403415441513, 0.006696493364870548, 0.010918195359408855, -0.0022498099133372307, -0.007920654490590096, -0.0020413717720657587, -0.0012572467094287276, 0.004258096218109131, -0.031603213399648666, -0.008893366903066635, -0.008794110268354416, -0.01733677089214325, 0.00614065770059824, -0.021359959617257118, -0.0008850355516187847, -0.04277285933494568, -0.03078269399702549, -0.014067929238080978, 0.010309423319995403, 0.014663467183709145, 0.005283745005726814, 0.016158929094672203, 0.014451720751821995, -0.02112174592912197, 0.018871935084462166, -0.0077618444338440895, -0.012876853346824646, -0.017601454630494118, -0.005710546858608723, 0.018157290294766426, -0.023887688294053078, 0.0006435118266381323, 0.012155590578913689, 0.0029727269429713488, 0.013763543218374252, -0.022378992289304733, 0.013829714618623257, -0.010858641937375069, 0.016026588156819344, 0.0044797686859965324, 0.02261720784008503, -0.013644435442984104, -0.021756986156105995, 0.012545999139547348, 0.0036757923662662506, -0.010931429453194141, -0.015695733949542046, -0.011884290724992752, -0.02064531482756138, 0.022987764328718185, 0.0035401422064751387, -0.0044599175453186035, 0.0050951577723026276, 0.00449631130322814, -0.02456263080239296, 0.0030835631769150496, 0.05343960225582123, 0.0041555315256118774, 0.01232763472944498, -0.010679979808628559, 0.02465526945888996, -0.021307023242115974, 0.02398032695055008, 0.013551796786487103, -0.0015971997054293752, -0.018355801701545715, -0.0196395181119442, -0.06082427501678467, 0.015298707410693169, -0.01095128059387207, -0.012804065831005573, 0.003970252815634012, 0.027685897424817085, -0.009442584589123726, -0.019811561331152916, 2.4594372007413767e-05, 0.0020843828096985817, -0.007411138620227575, -0.005366458557546139, -0.02116144821047783, -0.01866018772125244, -0.011156410910189152, -0.01250629685819149, 0.02002330869436264, -0.013346666470170021, 0.020843828096985817, 0.005842888727784157, -0.01613246090710163, 0.00018279705545865, 0.006921474356204271, -0.009230838157236576, -0.013479008339345455, 0.005839580204337835, -0.019229257479310036, 0.001728714327327907, -0.005065381061285734, -0.014610530808568, 0.010997600853443146, 0.00043217858183197677, -0.01662212610244751, -0.007636119611561298, -0.012407040223479271, -0.011652692221105099, -0.002454939763993025, 0.01496785320341587, 0.018580783158540726, 0.01124243251979351, 0.005353224463760853, -0.015285473316907883, 0.006881771609187126, -0.011202730238437653, -0.0027030804194509983, 0.015311941504478455, 0.013670903630554676, -0.005469023250043392, -2.5447356165386736e-05, 0.03790267929434776, 0.02951221354305744, 0.002678266493603587, 0.02073795348405838, -0.03311190754175186, -0.0013813171535730362, -0.028850505128502846, 0.02488025091588497, -0.011725479736924171, 0.008377233520150185, -0.010329274460673332, 0.027394745498895645, 0.003411108860746026, 0.022895125672221184, 0.004370586480945349, 0.010441765189170837, 0.0012961220927536488, -0.003124919952824712, -0.016886809840798378, -0.00795373972505331, -0.02874463051557541, -0.02601839043200016, -0.0027262403164058924, 0.005124934948980808, 0.02064531482756138, 0.01511342916637659, -0.008675002492964268, -0.01769409328699112, 0.0201556496322155, -0.030570946633815765, -0.0040165726095438, -0.013604733161628246, -0.005313521716743708, -0.005402852315455675, -0.013194473460316658, 0.026706567034125328, 0.00679574953392148, 0.01733677089214325, 0.021955497562885284, -0.014703169465065002, 0.016886809840798378, -0.005379692651331425, 0.032873693853616714, -0.008363999426364899, 0.00013099766511004418, -0.006378872785717249, 0.018977809697389603, -0.005809803493320942, -0.0009007510961964726, 0.026269840076565742, 0.015179600566625595, -0.0022382300812751055, 0.010871876031160355, -0.022339290007948875, 0.009230838157236576, -0.03628811240196228, 0.026852143928408623, -0.009766821749508381, -0.015060492791235447, -0.0029379872139543295, 0.03205317631363869, -0.01959981583058834, 0.012519530951976776, 0.02129378914833069, 0.017773497849702835, -0.02293482795357704, 0.013710606843233109, -0.030888566747307777, -0.008483107201755047, -0.02196873351931572, 0.020208587870001793, -0.00339787476696074, 0.007741993293166161, 0.010818938724696636, -0.0015690770233049989, 0.03205317631363869, 0.00471467524766922, 0.017680859193205833, -0.012539382092654705, 0.051375072449445724, 0.008708087727427483, 0.021108511835336685, 0.010799087584018707, 0.014557593502104282, -0.032900162041187286, -0.006216754205524921, -0.018474910408258438, -0.0210952777415514, 0.03078269399702549, 0.011520350351929665, 0.09703297913074493, 0.021214384585618973, -0.01337975263595581, 0.00884704664349556, -0.0011687432415783405, 0.00160795240662992, 0.050528086721897125, -0.010620426386594772, 0.0038346026558429003, -0.006590619683265686, 0.054683614522218704, -0.014716403558850288, -0.015007555484771729, -0.00010256486712023616, -0.00824489165097475, 0.01177841704338789, -0.019586581736803055, 0.027010953053832054, -0.023093637079000473, 0.012784214690327644, 0.03559993579983711, -0.012876853346824646, 0.013684138655662537, -0.011057154275476933, -0.01588101126253605, -0.006048018578439951, 0.026005156338214874, 0.011215964332222939, -0.023953858762979507, -0.028268201276659966, -0.012645255774259567, 0.001973546575754881, -0.018329333513975143, -0.02300099842250347, -0.021333491429686546, 0.015642795711755753, -0.014848745428025723, -0.00043838209239766, -0.0033879491966217756, 0.016635360196232796, 0.014557593502104282, -0.004377203993499279, 0.0012076186249032617, -0.007325116544961929, 0.0028106081299483776, -0.01776026375591755, -0.007662588264793158, 0.007986824959516525, -0.01069321483373642]} +{"id": "test:10000050", "text": "\"\\\"Whoever gets him, they'll be getting a good one,\\\" David Montgomery said.\\nINDIANAPOLIS \u2014 Hakeem Butler has been surrounded by some of the best wide receivers on the planet this week at the NFL Scouting Combine.\\nIt\u2019s an experience that might humble some. But for Butler, it has only enhanced his confidence.\\nAs it stands, 22-year-old Butler is not regarded as the best wide receiver in this year\u2019s NFL Draft. He\u2019s projected by some experts to go as late as the third round. But when wide receivers were measured Thursday, Butler gained some attention: He led all receivers in height (6-foot-5 3/8), arm length (35 1/4 inches) and wingspan (83 7/8 inches).\\nOn Thursday, running back David Montgomery, who played with Butler at Iowa State, captured the general vibe surrounding Butler here.\\nButler says he\u2019s met with every NFL team on an informal basis. He had \u201cnine or 10\u201d formal meetings set up for Friday night, but didn\u2019t divulge which teams he\u2019d be sitting down with.\\nThere is clear interest in Butler, who declared for the draft after his junior season in which he had 60 receptions for 1,318 yards and nine touchdowns.\\nBut in his mind, the hype machine hasn\u2019t been turned up high enough \u2014 yet.\\nButler, of course, is talking about Saturday\u2019s wide receiver workouts. If he crushes the drills, he could vault up the draft board \u2014 perhaps into the first round. And he feels well-prepared because he\u2019s spent the past few months working out with some legendary NFL receivers in Calvin Johnson and Anquan Boldin.\\nButler met Boldin at the South Florida gym where he trains. He was connected to Johnson through his agent.\\nJohnson and Boldin are known for being precise, tactical receivers. But Butler says the most valuable lesson of working with them has been learning about how they think.\\nButler admitted the first time he worked with Johnson, who went to six Pro Bowls with the Detroit Lions from 2010-15, that he was a little starstruck.\\nButler is hoping that one day he can leave a legacy like Boldin or Johnson. But for now, his goal is to prove what he already believes is true: that he is the top receiver in this draft.\"", "vector_field": [-0.04548555985093117, -0.00824442133307457, 0.010622493922710419, 0.001998697407543659, -0.01994427479803562, 0.042857855558395386, -0.032583530992269516, -0.005948465317487717, -0.04096590727567673, -0.01782897301018238, 0.01583191752433777, 0.01281005796045065, -0.009978706017136574, -0.011975761502981186, 0.0021826366428285837, 0.0048086983151733875, 0.031926605850458145, 0.008080190047621727, 0.01379544660449028, -0.020128212869167328, -0.004490089602768421, 0.004720013588666916, -0.02601427026093006, -0.002310737268999219, -0.005626571364700794, 0.00013035876327194273, 0.02552814595401287, -0.026684334501624107, -0.008494053967297077, -0.0012547287624329329, 0.019734058529138565, 0.015634840354323387, -0.020929662510752678, -0.020259598270058632, -0.031190847977995872, -0.021310679614543915, 0.007410125806927681, -0.01425529457628727, 0.015069883316755295, 0.01104949600994587, 0.0288521908223629, 0.006457583047449589, -0.0013122097589075565, 0.0009303714614361525, -0.024950049817562103, -0.00425031129270792, -0.010537093505263329, -0.027380676940083504, -0.012856042012572289, 0.007817420177161694, 0.020167628303170204, -0.009059010073542595, -0.003954694606363773, -0.008099897764623165, -0.006832030601799488, 0.02343912050127983, 0.011568467132747173, 0.0208902470767498, 0.004220749717205763, 0.017671309411525726, 0.019681503996253014, 0.0031269679311662912, -0.001120880013331771, 0.011752407066524029, 0.0022893871646374464, -0.01026118453592062, -0.0044736661948263645, -0.004867821931838989, -0.027170460671186447, 0.002660550409927964, 0.02472669631242752, 0.018656698986887932, -0.004345565568655729, -0.0005251302500255406, 0.0288259144872427, 0.014872805215418339, -0.012764072977006435, -0.012934873811900616, -0.006608675699681044, -0.023820137605071068, 0.023334013298153877, -0.025830332189798355, -0.03247842192649841, 0.02052236907184124, -0.007167063187807798, 0.005337523762136698, -0.020456677302718163, 0.00030300713842734694, -0.00963053572922945, -0.0022959564812481403, -0.007646618876606226, 0.030717860907316208, 0.0069240001030266285, 0.0036919242702424526, -0.029272623360157013, -0.004023672081530094, -0.021087326109409332, 0.021481480449438095, -0.008835654705762863, -0.030402537435293198, 0.0047988444566726685, -0.01027432270348072, -0.019970551133155823, -0.011706422083079815, 0.003204156644642353, -0.006963416002690792, -0.01797349564731121, -0.0020496093202382326, 0.026369011029601097, -0.021310679614543915, -0.0006561048794537783, 0.0159764401614666, -0.002734454581514001, -0.04685196280479431, 0.0015281741507351398, 0.0050616152584552765, -0.01914282515645027, -0.005912334192544222, 0.013913692906498909, -0.016725337132811546, 0.02195446752011776, -0.0022811756934970617, -0.003084267722442746, -0.02664491906762123, -0.01275093387812376, 0.001075716339983046, -0.03137478604912758, -0.019182240590453148, 0.03171638771891594, 0.011141465045511723, 0.02552814595401287, -0.022532563656568527, 0.03820681944489479, -0.00139186205342412, -0.005445916671305895, 0.005380223970860243, -0.038679804652929306, -0.010780155658721924, 0.003217295277863741, -0.023964662104845047, -0.0008441499667242169, 0.0010083814850077033, -0.0014977913815528154, -0.018078604713082314, -0.03529006615281105, 0.003338826587423682, 0.02455589547753334, 0.015818778425455093, -0.036814134567976, -0.010655339807271957, 0.021534034982323647, 0.014347264543175697, -0.0024536186829209328, -0.006007588468492031, 0.007994789630174637, 0.00938747264444828, -0.009978706017136574, 0.028326651081442833, -0.002023332053795457, -0.012829765677452087, -0.00983418244868517, -0.03142734244465828, -0.008382376283407211, -0.017303431406617165, 0.0025636537466198206, 0.014544342644512653, -0.00752837210893631, -0.013545814901590347, -0.0026162078138440847, 0.005084607284516096, 0.0027131044771522284, 0.028431758284568787, -0.008986747823655605, 0.04748261347413063, 0.010340015403926373, 0.011877222917973995, -0.006487144622951746, 0.01104949600994587, -0.024608448147773743, -0.019471287727355957, 0.0002520953712519258, -0.025462452322244644, 0.027223015204072, 0.012698380276560783, -0.02521282061934471, -0.004966360982507467, 0.0108064329251647, -0.013191075064241886, -0.0035868161357939243, 0.0066875070333480835, 0.031532447785139084, 0.03077041544020176, -0.010707894340157509, -0.0030629176180809736, -0.6323307156562805, 0.003517838893458247, 0.01281005796045065, 0.010110091418027878, 0.00795537419617176, 0.013039981946349144, 0.008953901939094067, 0.0014871163293719292, 0.019957412034273148, 0.001219418947584927, -0.006086419802159071, 0.014649450778961182, 0.009926152415573597, -0.014872805215418339, -0.03460686281323433, -0.002642484847456217, 0.03613093122839928, -0.011699852533638477, -0.01096409559249878, -0.024858081713318825, -0.02547559142112732, 0.008395514450967312, -0.015766223892569542, -0.03804915398359299, 0.015017328783869743, -0.008769962005317211, -0.01244874857366085, -0.005278400611132383, 0.00021555385319516063, -0.010642201639711857, -0.034475479274988174, 0.032609809190034866, -0.0009788197930902243, 0.0036360856611281633, 0.04879646375775337, 0.0012867538025602698, -0.01143708173185587, 0.0062539358623325825, 0.019826026633381844, -0.0019379317527636886, 0.005314531736075878, -0.014662588946521282, 0.03013976663351059, 0.007922528311610222, -0.006030580960214138, 0.007633480243384838, -0.0021465057507157326, -0.007127647288143635, -0.006611960474401712, -0.008421791717410088, 0.016357457265257835, 0.017934080213308334, 0.02378072217106819, -0.03245214745402336, -0.017474232241511345, -0.008086759597063065, 0.023425981402397156, -0.0015413126675412059, -0.002545588416978717, -0.024082908406853676, -0.007574357092380524, 0.02487121894955635, -0.01493849791586399, 0.010661909356713295, 0.004256880842149258, 0.02487121894955635, -0.00022684477153234184, -0.0026983236894011497, 0.016383735463023186, 0.007929096929728985, 0.017106354236602783, 0.03804915398359299, -0.011115188710391521, 0.02343912050127983, 0.017093215137720108, -0.010326877236366272, 0.016239210963249207, -0.012061161920428276, -0.005321100819855928, 0.041806772351264954, -0.005534601863473654, -0.014991051517426968, -0.017106354236602783, 0.0006811501807533205, 0.007206478621810675, 0.00415834179148078, -0.0069240001030266285, 0.009906444698572159, 0.014899082481861115, -0.03247842192649841, 0.020509229972958565, -0.006349190138280392, -0.004946652799844742, -0.008796239271759987, 0.004674028605222702, 0.004805414006114006, -0.00794223602861166, -0.006766337901353836, 0.018223127350211143, -0.03502729535102844, -0.01809174194931984, -0.004949937574565411, 0.021704835817217827, -0.013072827830910683, 0.0152144068852067, -0.020535508170723915, 0.0007640870753675699, 0.0029101823456585407, 0.02698652073740959, -0.02521282061934471, 0.004394835326820612, -0.012429040856659412, -0.025738362222909927, 0.019812889397144318, 0.02112674154341221, -0.026342732831835747, 0.0030267867259681225, -0.002158001996576786, 0.005636425223201513, -0.013887416571378708, 0.025882884860038757, 0.007252463139593601, 0.03345067426562309, -0.006378751713782549, -0.027801109477877617, 0.02401721477508545, 0.0030202174093574286, -0.032898854464292526, -0.013020274229347706, 0.002780439332127571, -0.023662475869059563, -0.021770529448986053, 0.028142711147665977, -0.00706852413713932, 0.023018687963485718, -0.02787994034588337, -0.004155057016760111, 0.001506002969108522, -0.010471400804817677, -0.04945339262485504, -0.008973609656095505, 0.004102502949535847, 0.006168535444885492, -0.00020949781173840165, -0.01539834588766098, -0.05428836867213249, -0.029114961624145508, 0.007081662770360708, -0.015582285821437836, -0.02307124249637127, 0.031900327652692795, -0.0159764401614666, -0.009814474731683731, 0.006585683207958937, 0.007048816420137882, -0.005288254469633102, 0.007377279456704855, -0.014465510845184326, -0.014045078307390213, -0.021915052086114883, -0.00582364946603775, 0.04351478070020676, -0.03431781753897667, 0.006684222258627415, 0.03013976663351059, -0.03137478604912758, 0.02049609273672104, 0.005055045709013939, 0.011069203726947308, 0.006050288677215576, 0.009328349493443966, -0.0056988331489264965, -0.014268433675169945, 0.023623060435056686, 0.014018801040947437, -0.006001019384711981, -0.0025357345584779978, 0.004677313379943371, 0.0013516253093257546, 0.01848589815199375, 0.00786997377872467, 0.000997706432826817, -0.026093101128935814, -0.02890474535524845, 0.025541283190250397, 0.0005000849487259984, -0.008835654705762863, 0.03182149678468704, -0.011095480993390083, 0.008592592552304268, -0.00573824904859066, -0.011686714366078377, -0.014005662873387337, 0.01177868340164423, -0.019760334864258766, -0.005002491641789675, 0.014307849109172821, -0.013243628665804863, 0.013381583616137505, 0.030192319303750992, 0.04096590727567673, 0.01505674421787262, 0.010018122382462025, -0.0011824668617919087, -0.00867142342031002, -0.028063880279660225, 0.008855362422764301, -0.005626571364700794, 0.0061980970203876495, -0.03397621586918831, 0.014899082481861115, -0.04038781300187111, 0.015766223892569542, 0.020758861675858498, 0.0060897041112184525, 0.02753833867609501, -0.012724657543003559, 0.022230377420783043, -0.0024700418580323458, -0.004424396902322769, 0.05386793240904808, -0.0024848226457834244, 0.003583531593903899, 0.014991051517426968, 0.0028214973863214254, 0.017868388444185257, 0.019221656024456024, 0.024450786411762238, 0.008526899851858616, 0.0071867709048092365, 0.011594744399189949, -0.0035638236440718174, 0.03292513266205788, 0.012080869637429714, 0.010451693087816238, -0.0304550901055336, -0.010537093505263329, -0.005120738409459591, 0.05586498975753784, 0.017421677708625793, 0.024516480043530464, 0.006690791808068752, 0.01505674421787262, -0.007305017206817865, 0.028063880279660225, -0.016751613467931747, 0.022571979090571404, 0.013939970172941685, 0.0016226072330027819, -0.0044408198446035385, -0.008533469401299953, 0.035710498690605164, -0.009137840941548347, 0.014320987276732922, -7.487930270144716e-05, -0.031611278653144836, -0.012599840760231018, -0.0184333436191082, 0.012501302175223827, 0.006316343788057566, 0.005097745917737484, 0.012711518444120884, 0.009755351580679417, 0.006845169235020876, 0.028142711147665977, 0.004178049508482218, 0.02258511632680893, 0.006845169235020876, -0.020220182836055756, -0.000977998599410057, 0.006441159639507532, -0.02258511632680893, -0.00911813322454691, -0.017132630571722984, -0.009242949075996876, -0.006345905363559723, -0.021599728614091873, -0.01989172026515007, 0.014123909175395966, 0.02309751883149147, -0.01719832420349121, -0.021731112152338028, 0.006112696602940559, 0.02132381871342659, -0.020456677302718163, -0.030244873836636543, -0.009321779944002628, -0.0011529051698744297, -0.02353109046816826, 0.0010297315893694758, -0.002092309296131134, 0.02275591716170311, 0.025173405185341835, 0.0031548873521387577, 0.020653754472732544, 0.026263901963829994, 0.014150186441838741, 0.0022466869559139013, -0.010615924373269081, -0.021967606619000435, 0.017737003043293953, -0.0007234397926367819, -0.009597688913345337, -0.025265375152230263, 0.020115075632929802, 0.013664061203598976, -0.007291879039257765, -0.014005662873387337, -0.00963053572922945, -0.016699058935046196, 0.011752407066524029, -0.003514554351568222, -0.011601313948631287, 0.0353688970208168, -0.001656274776905775, 0.009991845116019249, -0.024752972647547722, 0.008835654705762863, 0.00485796807333827, -0.006858307868242264, -0.019760334864258766, -0.00026810794952325523, -0.05194970965385437, 0.0007571072201244533, 0.13569463789463043, 0.008993317373096943, -0.011456790380179882, 0.01731657050549984, 0.013269905932247639, 0.011712990701198578, -0.01461003441363573, -0.024109184741973877, 0.038653526455163956, -0.004033525940030813, 0.0041057877242565155, 0.00030588117078877985, 0.004634613171219826, -0.007449541240930557, -0.016015857458114624, -0.008257560431957245, -0.005324385594576597, -0.011299127712845802, 0.006161966361105442, -0.02370189130306244, -0.004736436530947685, 0.009893305599689484, -0.01488594338297844, 0.021941328421235085, 0.0023255180567502975, -0.014793974347412586, 0.021363234147429466, 0.022217238321900368, 0.028694529086351395, -0.013716615736484528, 0.015083021484315395, 0.00930207222700119, 0.016304904595017433, 0.007686034310609102, -0.0032846301328390837, 0.028484312817454338, -0.014636311680078506, -0.012888888828456402, 0.028694529086351395, -0.004979499150067568, 0.006674368400126696, 0.0008761750650592148, 0.012172839604318142, -0.008605730719864368, 0.008737116120755672, -0.021862497553229332, 0.004401404410600662, 0.04270019009709358, -0.007265601772814989, -0.021507758647203445, 0.030901800841093063, 0.006280212663114071, -0.00917068775743246, -0.017040660604834557, -0.022966133430600166, -0.004641182254999876, -0.006421451922506094, 0.0008466134313493967, -0.009026163257658482, -0.009124702773988247, -0.017395401373505592, 0.0016570958541706204, 0.03592071682214737, 0.0007776361890137196, -0.01719832420349121, -0.008139313198626041, -0.015043606050312519, -0.012245100922882557, -0.0015306376153603196, 0.009860459715127945, 0.006657945457845926, 0.01219254732131958, 0.00628349743783474, 0.0007612130139023066, 0.02887846902012825, -0.0037411937955766916, 0.0192610714584589, 0.005636425223201513, 0.00042618074803613126, -0.00025866462965495884, 0.016843583434820175, -0.0008433287730440497, 0.01955011859536171, -0.011903499253094196, -0.010622493922710419, 0.027643447741866112, -0.00016648968448862433, -0.0002814517356455326, -0.015766223892569542, 0.02307124249637127, -0.006316343788057566, 0.009617396630346775, 0.022020161151885986, -0.010123230516910553, 0.011377958580851555, 0.004155057016760111, 0.010313739068806171, -0.02180994488298893, 0.01811802014708519, -0.016068410128355026, -0.003833163296803832, -0.0016349246725440025, 0.0029003284871578217, 0.02516026608645916, 0.015503454022109509, -0.0022368330974131823, 0.010911541059613228, -0.01018892228603363, -0.00698312371969223, -0.005439347587525845, -0.00930207222700119, -0.03397621586918831, 0.002135009504854679, -0.015083021484315395, -0.011594744399189949, 0.015293237753212452, 0.012593272142112255, 0.010727602057158947, -0.007679465226829052, -0.0074889566749334335, -0.005478763021528721, -0.04267391562461853, 0.01957639493048191, 0.016501981765031815, -0.0030070787761360407, 0.028090156614780426, 0.01408449374139309, -0.018906330689787865, -0.009945860132575035, 0.015661116689443588, 0.014150186441838741, 0.005987880751490593, 0.003918563947081566, -0.035079848021268845, -0.030402537435293198, -0.019182240590453148, 0.006746630184352398, 0.008336391299962997, 0.0016053629806265235, 0.0012276305351406336, -0.009512288495898247, -0.023793861269950867, 0.004851398523896933, 0.002141578821465373, -0.009926152415573597, -0.023347150534391403, -0.02787994034588337, 0.03726084530353546, 0.007935666479170322, 0.02392524667084217, 0.016449427232146263, 0.0022302637808024883, -0.02209899201989174, -0.0014690508833155036, 0.015188129618763924, 0.0008236209978349507, 0.005311246961355209, -0.013263336382806301, 0.02827409654855728, 0.01113489642739296, 0.022190961986780167, -0.006030580960214138, -0.010353154502809048, 0.01702752336859703, 0.014386679977178574, 0.0035736775025725365, -0.0062539358623325825, -0.01147649809718132, -0.010149506852030754, 0.02656608819961548, -0.005964888259768486, 0.0240434929728508, -0.03150617331266403, 0.04149144887924194, -0.008362668566405773, 0.01697496883571148, 0.004788990598171949, -0.01688299886882305, -0.035815607756376266, -0.03268864005804062, 0.0017195038963109255, -0.03295141085982323, -0.0014723354252055287, -0.008283836767077446, -0.026750028133392334, -0.013066258281469345, 0.04325200989842415, -0.0036590779200196266, 0.008625438436865807, 0.018709253519773483, 0.028300372883677483, -0.02610624022781849, 0.007285309489816427, -0.02615879476070404, 0.003402876900509, 0.002180994488298893, -0.009669951163232327, -0.020246461033821106, 0.002949597779661417, 0.027485784143209457, 0.013026842847466469, -0.0037149167619645596, -0.013381583616137505, 0.00867142342031002, -0.0033585343044251204, 0.007672896143049002, -0.03174266591668129, 0.024109184741973877, -0.0025997846387326717, -0.02647411823272705, 0.0017014384502545, -0.04093962907791138, -0.018827499821782112, -0.03342439606785774, -0.012599840760231018, 0.0073904176242649555, -0.006838600151240826, 0.00938747264444828, 0.00896047055721283, -0.011450220830738544, 0.01811802014708519, -0.003256710711866617, 0.0304288137704134, 0.01960267312824726, 0.015608562156558037, 0.032268207520246506, 0.007981651462614536, -0.04072941467165947, 0.026145655661821365, 0.004989353008568287, -0.0031318948604166508, 0.01671219803392887, -0.0009591120178811252, -0.012948011979460716, 0.012317363172769547, 0.01345384493470192, -0.015634840354323387, -0.03694551810622215, -0.047640275210142136, 0.01646256633102894, 0.027643447741866112, -0.021402649581432343, -0.00947287306189537, -0.004969645291566849, -0.007035677786916494, 0.006050288677215576, -0.009998413734138012, 0.01346698310226202, 0.0003795800730586052, -0.04932200536131859, -0.02887846902012825, 0.015490315854549408, -0.019103409722447395, 0.003902140539139509, 0.009111563675105572, -0.004670743830502033, -0.03247842192649841, -0.018591007217764854, 0.007541510742157698, 0.004611620679497719, -0.009104995056986809, 0.001721146167255938, -0.023715030401945114, -0.005179861560463905, 0.004480235278606415, 0.006519990973174572, -0.012028315104544163, 0.003882432822138071, 0.000803502625785768, 0.01860414445400238, -0.013979385606944561, 0.00252259592525661, -0.001698153791949153, 0.0038068862631917, 0.007915958762168884, 0.017093215137720108, -0.007738588377833366, -0.02753833867609501, 0.019826026633381844, 0.008691131137311459, 0.010254614986479282, 0.00980133656412363, 0.029903272166848183, 0.031190847977995872, 0.007318155840039253, 0.004168195649981499, -0.012389624491333961, 0.025830332189798355, 0.004926945082843304, -0.030323704704642296, -0.017697587609291077, -0.029614225029945374, -0.006487144622951746, 0.02212526835501194, 0.006651375908404589, -0.004059802740812302, -0.014675727114081383, -0.0029249631334096193, -0.005560878664255142, 0.005810510832816362, -0.035736776888370514, 0.0031828065402805805, -0.001936289481818676, -0.010904972441494465, 0.004003964364528656, -0.012212255038321018, 0.026684334501624107, 0.011916638351976871, 0.010609355755150318, -0.00565941771492362, -0.009729074314236641, 0.002933174604550004, -0.00983418244868517, -0.015464038588106632, 0.011883791536092758, 0.04553811252117157, 0.01097723376005888, -0.016357457265257835, -0.024069769307971, -0.0019445010693743825, 0.025515006855130196, -0.011660437099635601, 0.03983599320054054, 0.005091176833957434, -0.012271378189325333, -0.0048973835073411465, 0.015135576017200947, -0.01290859654545784, -0.017566202208399773, -0.002473326399922371, -0.02922006882727146, 0.011870653368532658, -0.003524408210068941, 0.01488594338297844, -0.01792094111442566, 0.013177935965359211, -0.007364140823483467, 0.04201699048280716, -0.015372068621218204, 0.00573824904859066, 0.014846527948975563, 0.015464038588106632, 0.0003621304640546441, 0.01476769708096981, -0.00769260386005044, -0.021770529448986053, -0.012422471307218075, -0.029246347025036812, -0.02793249487876892, -0.0047988444566726685, 0.00256858067587018, 0.013394721783697605, 0.01345384493470192, -0.027039075270295143, -0.02506829798221588, 0.0002137062547262758, -0.031006908044219017, -0.035710498690605164, -0.0030941215809434652, 0.005432778038084507, 0.0044966586865484715, 0.0021399364341050386, 0.02418801560997963, 0.020614339038729668, 0.005183146335184574, -0.017697587609291077, -0.02496318891644478, -0.019734058529138565, 0.017855249345302582, -0.004352135118097067, 0.017080076038837433, -0.0032829877454787493, -0.04672057926654816, 0.024661002680659294, 0.016580812633037567, -0.014268433675169945, 0.0024913919623941183, 0.02501574344933033, -0.00778457336127758, -0.014123909175395966, -0.00023279816377907991, -0.002667119726538658, -0.010320307686924934, -0.003233718452975154, -0.0032517837826162577, -0.028983576223254204, 0.005094461143016815, 0.01458375807851553, -0.01729029230773449, -0.012711518444120884, 0.0053408085368573666, 0.006066712085157633, -0.010898402892053127, 0.021192433312535286, 0.015004190616309643, -0.021455204114317894, 0.016869859769940376, 0.002913466887548566, 0.023741306737065315, -0.012185977771878242, -0.008178728632628918, 0.02212526835501194, 0.019628949463367462, -0.017474232241511345, -0.001180003397166729, 0.0168304443359375, 0.023465396836400032, 0.006694076117128134, 0.016646506264805794, -0.021599728614091873, 0.0067860460840165615, -0.01702752336859703, 0.0027886510360985994, -0.0104451235383749, -0.0160815492272377, 0.015306376852095127, -0.005166723392903805, -0.027801109477877617, 0.03860097378492355, -0.000796933367382735, -0.030402537435293198, 0.021047910675406456, 0.0014107486931607127, 0.002793577965348959, -0.0159764401614666, -0.013756031170487404, 0.0013401290634647012, -0.019655227661132812, -0.026658058166503906, 0.010898402892053127, -0.016042133793234825, -0.0031696681398898363, -0.0074232639744877815, -0.007416694890707731, -0.004358704201877117, -0.01000498328357935, 0.24174878001213074, -0.0061652506701648235, -0.018275681883096695, 0.016699058935046196, -0.016042133793234825, 0.017579341307282448, 0.031900327652692795, 0.011069203726947308, 0.020193906500935555, -0.013598368503153324, -0.0071210782043635845, 0.01583191752433777, -0.037050627171993256, -0.004818552639335394, 0.02370189130306244, 0.03276747092604637, -0.009985275566577911, -0.03179521858692169, -0.014294710010290146, -0.0012949653901159763, 0.0015142144402489066, 0.0004462991200853139, -0.02074572443962097, -0.013145090080797672, 0.0353688970208168, 0.014242156408727169, 0.022532563656568527, 0.02006252110004425, -0.0017720579635351896, 0.03334556519985199, -0.014570618979632854, -0.021100463345646858, -0.00010613461199682206, 0.0272492915391922, -0.015319515019655228, -0.006703929975628853, -0.006421451922506094, -0.012980857864022255, 0.018682975322008133, 0.018788084387779236, 0.01651512086391449, -0.021967606619000435, 0.008822516538202763, -0.007245894055813551, -0.007081662770360708, 0.014702004380524158, -0.00469373632222414, 0.0006889511714689434, -0.023176349699497223, -0.005229131318628788, -0.018354512751102448, -0.015871332958340645, 0.02824781835079193, 0.013834862038493156, 0.01106263417750597, -0.015529731288552284, 0.002542303642258048, -0.004999206867069006, -0.00068607711000368, 0.012645825743675232, -0.015595423988997936, 0.01874866895377636, -0.006910861935466528, 0.022480009123682976, -0.034133877605199814, 0.03397621586918831, -0.0030727714765816927, 0.017106354236602783, 0.00012676620099227875, -0.003908710088580847, -0.009768489748239517, -0.011699852533638477, -0.03197915852069855, -0.016449427232146263, -0.024030353873968124, -0.018420206382870674, 0.014741419814527035, 0.0014747988898307085, 0.0016341034788638353, 0.00874368567019701, 0.03657764196395874, 0.0007587495492771268, -0.02192819118499756, 0.0010748952627182007, -0.015608562156558037, -0.02146834321320057, 0.01309253554791212, -0.027012798935174942, -0.026539811864495277, -0.017553063109517097, 0.014202740974724293, -0.0001637182867852971, 0.004700305871665478, -0.015818778425455093, 0.003151602577418089, -0.022887302562594414, 0.02137637324631214, 0.016764752566814423, -0.019707780331373215, -0.010477970354259014, -0.012100577354431152, 0.019799750298261642, -0.0052422694861888885, -0.008224713616073132, -0.017093215137720108, 0.020246461033821106, 0.017815833911299706, 0.007075093220919371, 0.0046378979459404945, -0.01186408381909132, -0.010707894340157509, -0.035447727888822556, 0.008901347406208515, 0.014163325540721416, 0.014123909175395966, 0.013000566512346268, 0.013072827830910683, -0.023399705067276955, 0.020706309005618095, 0.004667459521442652, 0.033844828605651855, -0.008927624672651291, 0.028773359954357147, 0.026434702798724174, 0.0011808245908468962, -0.008165590465068817, -0.016410011798143387, 0.010136368684470654, 0.009203533641994, -0.01923479326069355, 0.0052751158364117146, -0.03113829344511032, 0.0026392003055661917, -0.01413704827427864, -0.0037083474453538656, -0.007640049792826176, -0.013053120113909245, -0.01746109314262867, -0.005728395190089941, 9.847474757407326e-06, 0.008625438436865807, -0.03268864005804062, 0.004867821931838989, -0.016528258100152016, 0.005370370112359524, -0.02387269213795662, 0.01185094565153122, 0.008513761684298515, 0.005120738409459591, -0.02054864540696144, 0.021731112152338028, 0.009742213413119316, -0.011299127712845802, -0.0256726685911417, -0.004427681211382151, 0.009965567849576473, 0.0009426888427697122, -0.03276747092604637, -0.03712945803999901, -0.003171310294419527, -0.014491788111627102, 0.01777641847729683, 0.00106257782317698, -0.032110545784235, -0.008559745736420155, -0.013572092168033123, -0.16575556993484497, 0.0336083360016346, 0.03854842111468315, -0.030980631709098816, 0.029929550364613533, 0.008001359179615974, 0.025278514251112938, 0.00037075261934660375, -0.022966133430600166, -0.01461003441363573, -0.01685672253370285, 0.011988899670541286, -0.01726401597261429, -0.0208376944065094, -2.4390945327468216e-05, -0.00397768709808588, -0.01425529457628727, 0.019471287727355957, 0.011102049611508846, 0.0015749800950288773, 0.043724995106458664, -0.01823626644909382, -0.007075093220919371, -0.014636311680078506, 0.009623966179788113, 0.006125835236161947, 0.001147157046943903, 0.007672896143049002, 0.004736436530947685, -0.007015970069915056, 0.018183711916208267, -0.015818778425455093, -0.0008141776779666543, -0.02178366668522358, 0.012435609474778175, -0.006027296185493469, -0.001873881439678371, 0.005163438618183136, 0.002489749575033784, 0.029666779562830925, 0.023623060435056686, 0.006326197646558285, -0.00690429238602519, 0.015450900420546532, 0.007429833523929119, 0.01253414899110794, 0.01734284684062004, 0.01760561764240265, 0.0032665645703673363, -0.023136934265494347, 0.02353109046816826, -0.006247366312891245, 0.019392456859350204, 0.018656698986887932, -0.0015725166304036975, -0.0152144068852067, -0.023964662104845047, -0.008139313198626041, -0.004411258269101381, -0.04364616423845291, -0.022992411628365517, -0.012435609474778175, -0.002640842692926526, -0.008165590465068817, -0.005294823553413153, -0.014977913349866867, -0.028090156614780426, -0.021599728614091873, -0.015897609293460846, 0.0008244421915151179, -0.009420319460332394, -0.02950911782681942, -0.014347264543175697, -0.010826140642166138, 0.022545700892806053, 0.014005662873387337, -0.01914282515645027, 0.007429833523929119, -0.007850266061723232, -0.02117929421365261, -0.0010321950539946556, 0.012527579441666603, -0.021507758647203445, -0.009761921130120754, -0.008868501521646976, -0.027748554944992065, -0.010550231672823429, 0.030323704704642296, 0.00823128316551447, -0.01777641847729683, 0.00252259592525661, -0.01729029230773449, -0.0009730716701596975, -0.020798277109861374, -0.006224374286830425, 0.010793294757604599, 0.012323932722210884, -0.002668761881068349, 0.0009665024117566645, -0.019077131524682045, 0.004887529648840427, 0.004848114214837551, -0.005741533357650042, 0.01726401597261429, -0.008855362422764301, 0.016383735463023186, 0.010523955337703228, 0.01077358704060316, 0.018617283552885056, 8.206443453673273e-05, -0.005068184342235327, 0.010727602057158947, 0.02207271382212639, 0.0052422694861888885, 0.0029906558338552713, 0.012304224073886871, -0.009775059297680855, -0.0320579893887043, 0.04551183432340622, -0.031874049454927444, 0.05749416723847389, -9.154623512586113e-06, -0.031322233378887177, -0.0014526277082040906, 0.008835654705762863, -0.025633253157138824, -0.07856835424900055, -0.01700124517083168, 0.012875749729573727, 0.021862497553229332, 0.027827385812997818, 0.036709025502204895, -0.014780835248529911, 0.00938747264444828, -0.03757616877555847, 0.030849246308207512, -0.017934080213308334, -0.02787994034588337, -0.007469248957931995, 0.017395401373505592, 0.01139109767973423, -0.013480122201144695, 0.025055158883333206, -0.011023218743503094, -0.0015996148576959968, 0.037392228841781616, -0.013072827830910683, -0.009893305599689484, -0.014991051517426968, -0.022164683789014816, 0.005376939661800861, -0.019182240590453148, -0.03237331286072731, 0.004069656599313021, 0.011640729382634163, -0.014780835248529911, -0.0005415533669292927, -0.0011028145672753453, 0.025738362222909927, -0.01299399696290493, 0.01940559409558773, -0.0063524749130010605, -0.04099218547344208, 0.011509343981742859, 0.0020413976162672043, -0.013118812814354897, -0.010412277653813362, 0.01729029230773449, -0.006385320797562599, -0.03400249034166336, 0.007640049792826176, -0.007265601772814989, -0.025580698624253273, 0.012902026996016502, 0.012002038769423962, 0.0006577472086064517, -0.026198210194706917, -0.014478649944067001, -0.02824781835079193, 0.009775059297680855, 0.004076226148754358, 0.0037313399370759726, 0.006076565943658352, 0.0070225391536951065, -0.0009763563284650445, 0.027328122407197952, -0.01622607372701168, 0.013611507602036, -0.0036262315697968006, 0.0030432099010795355, 0.0037247706204652786, -0.02272964082658291, -0.010793294757604599, 0.000151195636135526, 0.0004454779555089772, -0.038364481180906296, 0.007712311577051878, 0.02263767085969448, -0.011588174849748611, 0.03471197187900543, -0.013939970172941685, 0.014058217406272888, -0.03237331286072731, -0.00015273530152626336, -7.190260657807812e-05, -0.013177935965359211, -0.00654626777395606, -0.010970664210617542, 0.01408449374139309, -0.02853686735033989, 0.015647977590560913, 0.012856042012572289, -0.012461886741220951, -0.0075218030251562595, 0.012724657543003559, -0.023662475869059563, 0.008796239271759987, 0.00777800427749753, 0.017093215137720108, -0.024542756378650665, -0.00964367389678955, 0.001421423745341599, 0.021560311317443848, -0.015674255788326263, 0.002479895716533065, -0.006845169235020876, -0.025882884860038757, -0.009939290583133698, -0.059175897389650345, 0.032583530992269516, -0.02450334094464779, -0.006963416002690792, -0.004871106706559658, -0.026079963892698288, -0.015240684151649475, -0.006125835236161947, -0.007462679874151945, 0.03626231849193573, -0.021560311317443848, 0.000715638801921159, -0.023176349699497223, -0.010471400804817677, -0.022256653755903244, -0.0039415559731423855, 0.001926435623317957, -0.014110771007835865, -0.007357571739703417, -0.000295822013868019, 0.007449541240930557, 0.021901912987232208, 0.02258511632680893, 0.009663381613790989, -0.009420319460332394, 0.011056064628064632, -0.009880167432129383, 0.013979385606944561, 0.000847434566821903, -0.012041454203426838, 0.027958771213889122, -0.009479442611336708, -0.02793249487876892, 0.022309208288788795, 0.020456677302718163, -0.045722052454948425, -0.0033503228332847357, 0.021192433312535286, 0.008454637601971626, 0.00841522216796875, -0.03728711977601051, -0.008901347406208515, 0.0022466869559139013, -0.01442609541118145, -0.011732698418200016, 0.0017868387512862682, -0.01104949600994587, 0.00012060751032549888, 0.00760720344260335, 0.006851738318800926, 0.012008607387542725, 0.026526672765612602, 0.0037510476540774107, -0.044697247445583344, -0.016068410128355026, 0.0017162192380055785, -0.022598255425691605, 0.010372862219810486, 0.010024691000580788, -0.02372816763818264, 0.028983576223254204, -0.0029446708504110575, -0.00068607711000368, 0.02372816763818264, 0.020141351968050003, 0.010826140642166138, -0.0026786159723997116, 0.01960267312824726, 0.028116434812545776, -0.011739267967641354, -0.02384641394019127, 8.006286407180596e-06, 0.004220749717205763, 0.0041287802159786224, 0.014058217406272888, 0.020482953637838364, -0.007732019294053316, 0.009761921130120754, 0.0027426660526543856, 0.029325177893042564, 0.02756461687386036, -0.023544229567050934, 0.0027952203527092934, 0.03079669177532196, 0.027328122407197952, 0.013230490498244762, -0.011942915618419647, 0.0015749800950288773, 0.008106467314064503, 0.025515006855130196, 0.030586475506424904, 0.01571367122232914, -0.025725223124027252, 0.0028576282784342766, 0.007278740406036377, 0.0136246457695961, -0.014110771007835865, -0.012245100922882557, 0.030691584572196007, 0.02598799392580986, -0.00857288483530283, 0.007403556257486343, 0.007666326593607664, 0.008191867731511593, -0.007377279456704855, 0.011331973597407341, -0.016134103760123253, -0.031611278653144836, 0.011798391118645668, -0.015332653187215328, 0.009104995056986809, 0.009341487661004066, 0.026658058166503906, 0.012087439186871052, -0.019799750298261642, 0.010642201639711857, 0.003399592125788331, -0.00012994818098377436, -0.02081141620874405, 0.02180994488298893, 0.020338429138064384, 0.008211575448513031, -0.013125382363796234, 0.008264129050076008, 0.001421423745341599, 0.03197915852069855, -0.0011060992255806923, -0.03310907259583473, 0.010530523955821991, -0.0011923207202926278, 0.016186656430363655, 0.004043379798531532, -0.0021202287171036005, -0.0013795446138828993, -0.0013861139304935932, 0.006066712085157633, 0.009380904026329517, -0.00046559632755815983, -0.0005904122954234481, 0.05026797950267792, 0.00904587097465992, -0.003439007792621851, 0.012882319279015064, -0.020798277109861374, 0.018012911081314087, 0.01034658495336771, 0.0031499601900577545, -0.008513761684298515, 0.002598142484202981, 0.013926832005381584, -0.007048816420137882, 0.00331254955381155, -0.03368716686964035, -0.022138407453894615, 0.015306376852095127, 0.03276747092604637, -0.010530523955821991, 0.008921055123209953, 0.013112243264913559, 0.029430286958813667, 0.007298448123037815, 0.024569032713770866, 0.013223920948803425, 0.0024256992619484663, -0.017737003043293953, 0.013085966929793358, -0.006194812245666981, 0.017093215137720108, -0.03967833146452904, 0.008632007986307144, -0.005094461143016815, -0.03657764196395874, -0.012691810727119446, 0.023163212463259697, -0.03279374912381172, -0.0031466756481677294, -0.020023105666041374, 0.0029167516622692347, 0.016475705429911613, 0.029903272166848183, 0.015161852352321148, -0.011542189866304398, -0.023623060435056686, 0.03079669177532196, 0.0030448520556092262, -0.012080869637429714, -0.012133424170315266, 0.012488164007663727]} +{"id": "test:10000051", "text": "\"Building a successful software platform is done by building software with a purpose by solving a problem that people already have. The problem may not be obvious, and it may not even be a problem that people are aware of, but the problem exists. Using Instagram as a case study, I think there are four keys a platforms success.\\nMost people aren't professional photographers. They don't have access to the tools professional photographers use nor the time or money to buy the tools and learn them. But they do want professional looking photos. Instagram built a platform that had intrinsic value on it's own by providing people the ability to easily create professional looking photographs.\\nOne of the keys that made Instagram successful was that they took something people were already doing and made it easier and better. They removed the need to download photos from a mobile device and then upload them from a desktop or laptop. They simplified the process and allowed users to share exiting photos right from their phones or take a new photo to share.\\nThey also simplified an otherwise complex photo editing process by providing a set of templates that people could apply without having to understand image editing. This enabled a regular person to have professional looking photos without any work.\\nOne thing that Instagram did very well is that it built it's software to work with exiting services like Facebook and Twitter. Services that already had a huge community of users. They didn't try to re-create Twitter or Facebook. Instead, they allowed Twitter and Facebook to get better by making their service interoperable.\\nInstagram didn't force people to ONLY use their service. Instead, it allowed people to continue to use their existing social media services and augment a specific workflow to be better if they wanted to. Users don't want to be asked to chose between this OR that. They want to be able to use this AND that. Will your software provide additional choices or limit choices?\"", "vector_field": [-0.005251719616353512, -0.005338085815310478, 0.005610470660030842, -0.023837028071284294, -0.009327532723546028, -0.0027255129534751177, -0.02456781640648842, -0.02734481729567051, 0.022136280313134193, -0.03034769743680954, -0.0014441398670896888, -0.0032254394609481096, -0.0067564817145466805, -0.008277853019535542, -0.009672996588051319, -0.0033583103213459253, 0.03606114536523819, -0.0013968045823276043, -0.025657357648015022, -0.0009508568909950554, -0.020940443500876427, 0.00451096473261714, -0.013645834289491177, 0.005331442225724459, -0.030108530074357986, 0.019465576857328415, 0.014549355953931808, -0.024514667689800262, 0.0031905609648674726, -0.006743194535374641, 0.011433535255491734, 0.0030809424351900816, -0.03853254020214081, 0.002836792264133692, -0.02548462525010109, -0.013845141045749187, 0.005587218329310417, -0.005763272289186716, -0.009646422229707241, 0.016064083203673363, 0.005298224277794361, 0.009467046707868576, -0.01323393452912569, -0.01886765845119953, -0.011592980474233627, 0.017538949847221375, 0.0025893202982842922, 0.0035709035582840443, -0.011666059494018555, 0.0032586571760475636, -0.009958668611943722, 0.009327532723546028, -0.026016108691692352, -0.0036938090343028307, -0.0029214974492788315, -0.012131107039749622, -0.021936973556876183, 0.023478275164961815, 0.03760244697332382, -0.017352931201457977, -0.013340231962502003, 0.032420482486486435, -0.018415898084640503, -0.008490446023643017, -0.021711094304919243, -0.004085778258740902, -0.016662001609802246, 0.02319924719631672, 0.01223740354180336, 0.004597330931574106, 0.025670643895864487, 0.02620212733745575, 0.0020993591751903296, 0.010549943894147873, 0.014429772272706032, 0.00233354396186769, -0.011579693295061588, 0.0006996479933150113, 0.0026856516487896442, 0.01388500165194273, -0.0007860140176489949, -0.027424540370702744, -0.02404962107539177, -0.0025859985034912825, -0.010437004268169403, 0.013466458767652512, -0.0006224167882464826, 0.01355946809053421, -0.0009707875433377922, 0.005029160995036364, -0.007580280769616365, 0.014190604910254478, -0.006175172049552202, 0.01517384871840477, -0.02463425137102604, 0.021352343261241913, -0.007015579845756292, 0.021551648154854774, -0.005484243389219046, 0.01801728457212448, 0.0001671265927143395, -0.0025012933183461428, -0.019080251455307007, -0.014416485093533993, -0.009327532723546028, 0.005132135935127735, -0.004148891661316156, -0.014150743372738361, 0.05041119456291199, -0.011891939677298069, -0.01254964992403984, 0.059632427990436554, -0.02678675949573517, -0.01625010184943676, -0.006706655025482178, -0.009805867448449135, 0.005726732779294252, -0.024727262556552887, -0.006467487663030624, -0.006347903981804848, 0.01925298385322094, -0.003969516139477491, 0.03215474262833595, -0.00807854626327753, -0.006062231492251158, -0.0039994120597839355, -0.03263307735323906, -0.009035216644406319, -0.009281028062105179, -0.020475395023822784, 0.029816213995218277, -0.0030012198258191347, 0.020515255630016327, -0.0026574167422950268, 0.0012365292059257627, 0.009526838548481464, -0.003766888054087758, -0.011387030594050884, -0.023810453712940216, -0.023292256519198418, 0.01752566359937191, 0.006072197109460831, -0.022109705954790115, -0.008570169098675251, 0.01308777742087841, 0.0058363513089716434, 0.004600652493536472, 0.0024647540412843227, -0.0035410076379776, 0.018429184332489967, -0.0063047208823263645, -0.02698606625199318, -0.010835615918040276, -0.00338156265206635, 0.0054676346480846405, 0.018588628619909286, 0.014509494416415691, 0.032553352415561676, -0.0014582574367523193, 0.011858722195029259, 0.014323475770652294, 0.032978542149066925, -0.021259333938360214, 0.008151625283062458, 0.022069845348596573, 0.018003998324275017, 0.031144922599196434, 0.013061203062534332, 0.008563525043427944, -0.004999265074729919, -0.014482920989394188, 0.02678675949573517, -0.028088893741369247, 0.014921394176781178, 0.017605384811758995, -0.0026192162185907364, -0.006597036961466074, 0.01537315547466278, 0.0049162209033966064, -0.022269152104854584, 0.025311892852187157, 0.018880944699048996, 0.011692632921040058, 0.03178270161151886, -0.015997648239135742, -0.005640367045998573, 0.0019698101095855236, -0.0069491444155573845, 0.004544182680547237, 0.006457522511482239, -0.003064333461225033, 0.010642953217029572, -0.0008977085817605257, -0.00414557009935379, -0.6373547911643982, -0.011081427335739136, 0.025046151131391525, -0.013140925206243992, -0.0022604649420827627, 0.020090069621801376, 0.01801728457212448, 0.010456934571266174, -0.016821447759866714, 0.01066952757537365, -0.02685319446027279, -0.0006510670646093786, 0.017538949847221375, -0.006660150364041328, -0.014908106997609138, -0.013924863189458847, 0.034413546323776245, -0.038851432502269745, -0.022867070510983467, 0.006155241280794144, -0.016064083203673363, 8.003391849342734e-05, -0.01951872557401657, 0.0005123831215314567, -0.01788441464304924, -0.0033732582814991474, -0.009161443449556828, -0.018841084092855453, -0.01264265924692154, 0.05166018009185791, -0.01124087255448103, 0.049720264971256256, 0.01417731773108244, 0.007998824119567871, 0.04836498200893402, -0.008802692405879498, -0.033722616732120514, 0.025896525010466576, 0.026972778141498566, 0.04363477975130081, -0.015878064557909966, -0.04108365997672081, 0.022986654192209244, -0.001772164716385305, 0.0016725115710869431, 0.002742121694609523, 0.03404150903224945, -0.006357869133353233, -0.01124087255448103, 0.002120950724929571, 0.020276088267564774, -0.02891269326210022, 0.002479702001437545, -0.006025692448019981, 0.012649303302168846, -0.018841084092855453, 0.006570462603121996, -0.005367981735616922, -0.00021103625476825982, -0.01687459647655487, 0.01474866271018982, -0.008722970262169838, -0.026162266731262207, -0.021564936265349388, -0.0522182360291481, 0.011101357638835907, 0.004613939672708511, -0.0033915280364453793, 0.0015695367474108934, -0.004182109609246254, 0.0029862718656659126, -0.0031141601502895355, 0.00011231737880734727, -0.012423423118889332, -0.024713974446058273, 0.03749614953994751, 0.029842788353562355, 0.0057433415204286575, -0.004401346202939749, 0.0065073492005467415, 0.0010430359980091453, -0.015917925164103508, -0.006969075184315443, -0.034280676394701004, 0.019691457971930504, 0.010217767208814621, -0.028142042458057404, -0.019917337223887444, 0.0148018104955554, -0.002466414822265506, 0.025086013600230217, 0.014310188591480255, -0.014987830072641373, -0.020541829988360405, -0.006543888710439205, 0.04509636014699936, -0.026746898889541626, -0.010430360212922096, 0.014150743372738361, -0.019797753542661667, -0.02353142388164997, -0.01323393452912569, 0.009639779105782509, 0.04036615788936615, 0.04453830420970917, 0.006394408643245697, 0.00551413930952549, 0.025989534333348274, 0.021192897111177444, -0.006726585794240236, 0.02019636705517769, -0.012337056919932365, -0.004846463445574045, 0.015133988112211227, 0.02734481729567051, -0.014881533570587635, 0.010237697511911392, -0.029258158057928085, 0.02046210877597332, -0.018389323726296425, 0.008722970262169838, -0.027424540370702744, 0.005809776950627565, 0.003590834327042103, -0.010350638069212437, 0.0486307255923748, -0.0069491444155573845, -0.0034380326978862286, -0.00455082580447197, 0.010470221750438213, -0.007361044175922871, -0.017113763839006424, 0.01615709252655506, -0.010111470706760883, 0.00434155436232686, 0.02042224630713463, 0.005128813907504082, -0.006517314352095127, 0.01510741375386715, -0.024806983768939972, 0.00796560663729906, -0.023677581921219826, 0.013127638027071953, 0.01509412657469511, -0.004826533142477274, -0.0312512181699276, -0.034413546323776245, -0.016263389959931374, -0.008576812222599983, -0.0032586571760475636, -0.0066867247223854065, -0.032978542149066925, -0.019199835136532784, 0.01687459647655487, -0.0007237308309413493, 0.002491328166797757, -0.004720236174762249, -0.022216003388166428, -0.005069022066891193, -0.02936445362865925, -0.022920219227671623, 0.01622352935373783, -0.030985478311777115, -0.026614027097821236, -0.024860132485628128, -0.017631959170103073, -0.006068875081837177, 0.026055969297885895, 0.002539493842050433, -0.015147275291383266, -0.002527867676690221, 0.003736992133781314, -0.01187865249812603, 0.020834146067500114, -0.016914457082748413, 0.013021341525018215, -0.010696101933717728, 8.36671024444513e-05, 0.011380386538803577, -0.008331000804901123, 0.012277265079319477, -0.004079134669154882, -0.006633576471358538, 0.0009342480334453285, -0.002076106844469905, -0.0027271737344563007, 0.016954317688941956, 0.023252395913004875, -0.017273208126425743, 0.014389910735189915, 0.0036306953988969326, 0.018987242132425308, 0.014668939635157585, 0.023730730637907982, -0.001626006793230772, 0.02183067798614502, 0.0226146150380373, 0.017180198803544044, -0.010138044133782387, 0.003444676287472248, 0.00866317842155695, 0.0025046151131391525, 0.004447851330041885, -0.02846093289554119, 0.00020646881603170186, -0.017871126532554626, -0.0011808895505964756, -0.0173263568431139, 0.022667763754725456, -0.005763272289186716, 0.0036904874723404646, -0.0198774766176939, 0.004371450282633305, -0.035715680569410324, 0.010204480029642582, 0.020674701780080795, 0.006789699662476778, 0.021285906434059143, -0.006334616802632809, 0.018322886899113655, -0.0009217914193868637, -0.019213123247027397, 0.029098711907863617, -0.0338023379445076, 0.005823064129799604, 0.02411605603992939, 0.004610617645084858, 0.011008348315954208, 0.010403785854578018, -0.04833840951323509, 0.006301399320363998, 0.010051678866147995, 0.015811629593372345, 0.003929655067622662, 0.027956023812294006, -0.004138926509767771, 0.014270327053964138, -0.03401493281126022, 0.03765559196472168, -0.023225821554660797, 0.01755223609507084, 0.005686871707439423, 0.008523663505911827, -0.011048209853470325, 0.02653430588543415, 0.020581692457199097, 0.027530835941433907, -0.008716326206922531, -0.014868246391415596, 0.0038831501733511686, -0.00894220732152462, -0.010616379790008068, -0.014310188591480255, 0.0017821299843490124, -0.0035775471478700638, -0.005464313086122274, 0.004976012744009495, 0.01615709252655506, 0.03308483585715294, 0.011021635495126247, 0.0009525177883915603, 0.006793021224439144, 0.0004978504148311913, 0.008244635537266731, 0.003604121273383498, -0.003041081130504608, 0.01094855647534132, -0.0173263568431139, -0.009633135050535202, -0.023278970271348953, -0.0061386325396597385, -0.00013868392852600664, 0.023292256519198418, 0.013054559007287025, -0.0033417013473808765, 0.008085190318524837, 0.0022737521212548018, 0.005447704344987869, 0.006215033121407032, 0.006829560734331608, -0.020714562386274338, -0.009493621066212654, -0.003650626167654991, -0.0022870393004268408, 0.013978011906147003, -0.013911576010286808, -0.048019517213106155, -0.0005929360631853342, 0.0016750029753893614, 0.00450432114303112, -0.024302074685692787, 0.016343113034963608, -0.033430300652980804, 0.02160479687154293, 0.0005356355686672032, 0.019731318578124046, 0.03207501769065857, -0.010403785854578018, -0.005979187320917845, -0.01582491584122181, 0.012715738266706467, -0.0001164695859188214, -0.006424304563552141, -0.03188899904489517, 0.042997002601623535, -0.007148450706154108, -0.013991299085319042, -0.018708212301135063, -0.0010106487898156047, -0.015519313514232635, 0.005162031855434179, -0.004467781633138657, -0.008350932039320469, 0.006005761679261923, 0.013924863189458847, 0.0015637235483154655, -0.005348050966858864, -0.009945381432771683, 0.02012993022799492, 0.015320006757974625, -0.01631653867661953, -0.025338467210531235, -0.00812505092471838, 0.019983774051070213, 0.10278887301683426, 0.013632547110319138, -0.017220059409737587, 0.037443000823259354, -0.0160507969558239, 0.018841084092855453, -0.02604268305003643, -0.028673525899648666, 0.01716691069304943, 0.016887882724404335, 0.009526838548481464, 0.004647157154977322, 0.016662001609802246, -0.002851740224286914, 0.011480039916932583, 0.005125492345541716, 0.015904638916254044, -0.01545287761837244, -0.017140338197350502, -0.013041271828114986, 0.013174142688512802, 0.005042448174208403, 0.011367099359631538, 0.02137891761958599, 0.02904556319117546, 0.012230760417878628, 0.010775824077427387, 0.026879768818616867, -0.0217642430216074, -0.015413016080856323, 0.0014931359328329563, 0.036140866577625275, -6.78367869113572e-05, 0.0016957640182226896, -0.03133094310760498, 0.021511787548661232, -0.0021292550954967737, 0.016170380637049675, 0.0018053824314847589, 0.009912163950502872, 0.02059497870504856, 0.023876888677477837, 0.01156640611588955, -0.01345981564372778, -0.003252013586461544, 0.0010530013823881745, -0.02806231938302517, 0.028381209820508957, -0.004351519979536533, -0.00862996093928814, -0.014615791849792004, -0.004414633382111788, -0.018296314403414726, -0.00973943155258894, 0.005225145258009434, -0.011287377215921879, 0.00414557009935379, -0.020794285461306572, -0.030135104432702065, -0.024594390764832497, -0.0099852429702878, -0.03279251977801323, 0.00034214238985441625, -0.02382373996078968, 0.018003998324275017, -0.018455758690834045, -0.030374271795153618, -0.027238519862294197, -0.025338467210531235, 0.004610617645084858, 0.006520635914057493, -0.04033958539366722, -0.031995296478271484, 0.009400611743330956, 0.011493327096104622, -0.004786671604961157, 0.010795755311846733, -0.004447851330041885, -0.011201011016964912, 0.02528531849384308, -0.004846463445574045, -0.03438697010278702, 0.006019048858433962, -0.04913563281297684, -0.017910989001393318, 0.00019889102259185165, -0.03749614953994751, -0.017419366165995598, -0.025577634572982788, 0.014496208168566227, 0.011998236179351807, 0.0033417013473808765, 0.0013411649269983172, -0.009832441806793213, 0.004806602373719215, 0.01194508746266365, -0.0008067750604823232, 0.017445940524339676, 0.015851490199565887, -0.02420906536281109, 0.007327826227992773, -0.011167793534696102, -0.01644940860569477, -0.0407913438975811, -0.01801728457212448, 0.009287671186029911, 0.0008786083781160414, -0.015917925164103508, -0.018415898084640503, -0.01694103144109249, 0.009805867448449135, -0.013831853866577148, -0.012443353421986103, 0.011725851334631443, -0.005029160995036364, 0.011254159733653069, -0.010862190276384354, 0.01837603561580181, 0.031144922599196434, 0.015320006757974625, -0.0011244193883612752, -0.03903745114803314, 0.0325799286365509, 0.015040977858006954, -0.024009758606553078, -0.016821447759866714, 0.015997648239135742, -0.020847434177994728, -0.03335057944059372, 0.005852960050106049, 0.0015637235483154655, 0.009699570946395397, -0.004783350042998791, 0.011759068816900253, -0.027716856449842453, -0.019731318578124046, 0.004982656333595514, -0.0036306953988969326, 0.012662590481340885, -0.010363925248384476, 0.018283026292920113, -0.001255629351362586, 0.01781797781586647, -0.038984302431344986, 0.004613939672708511, -0.03425410017371178, -0.004603974521160126, 0.03090575523674488, -0.005052413325756788, 0.017060615122318268, -0.0074274796061217785, 0.0015014404198154807, -0.02792944945394993, 0.0004289236676413566, 0.0002767450350802392, -0.0353170670568943, -0.004441207740455866, -0.01710047572851181, 0.0406053252518177, 0.0530686117708683, 0.0023584573063999414, -0.0029214974492788315, 0.01625010184943676, -0.020183078944683075, 0.013532894663512707, -0.01958516053855419, -0.0007581941899843514, -0.005716767627745867, -0.010078252293169498, 0.01184543501585722, 0.028726674616336823, 0.03622058779001236, 0.010350638069212437, -0.00704215420410037, 0.010802398435771465, 0.026148980483412743, -0.014150743372738361, -0.013406666927039623, -0.017671819776296616, -0.023863600566983223, -0.0012365292059257627, 0.001963166519999504, -0.019332706928253174, 0.023225821554660797, -0.03303168714046478, 0.01546616479754448, -0.011367099359631538, -0.021285906434059143, 0.005142101086676121, 0.006922570522874594, 0.015147275291383266, -0.018721500411629677, -0.01039049867540598, 0.0010878799948841333, 0.0066036805510520935, -0.02404962107539177, 0.004039273131638765, -0.05075665935873985, 0.001118606305681169, 0.024687400087714195, 0.02059497870504856, 0.018429184332489967, -0.0010280880378559232, -0.00941389799118042, -0.013752130791544914, 0.019651595503091812, -0.014363337308168411, -0.026507731527090073, 0.010782468132674694, -0.0127954613417387, 0.004424598533660173, -0.004235257860273123, 4.3650143197737634e-05, -0.0010123096872121096, -0.011181080713868141, 0.015200423076748848, 0.015280146151781082, -0.0025079369079321623, -0.01579834148287773, -0.018495619297027588, 0.005431095138192177, 0.0019282879075035453, 0.029258158057928085, 0.02290693111717701, 0.04695655032992363, 0.01745922677218914, 0.0021358986850827932, -0.023146098479628563, -0.002110985340550542, 0.016861308366060257, -0.003952907398343086, 0.017937561497092247, 0.028859544545412064, 0.007487271446734667, -0.019983774051070213, 0.006799664814025164, -0.011141219176352024, -0.02137891761958599, -0.0148948198184371, 0.017020754516124725, 0.0045740786008536816, -0.020209653303027153, -0.006278146989643574, -0.0006265690317377448, -0.008284496143460274, 0.0036107648629695177, -0.010450290516018867, 0.022295726463198662, -0.0269594918936491, -0.012476570904254913, -0.009599917568266392, -0.000832518795505166, 0.0008686430519446731, 0.02803574502468109, 0.004976012744009495, -0.022641189396381378, -0.010855547152459621, -0.009360750205814838, 0.022720912471413612, -0.005460991058498621, -0.009028572589159012, 0.0019997060298919678, -0.026148980483412743, 0.014243753626942635, 0.02963019534945488, 0.0300288088619709, 4.9748708988772705e-05, -0.014602504670619965, -0.0005543205188587308, 0.02718537300825119, 0.002431536326184869, -0.02196354791522026, -0.016701864078640938, 0.01755223609507084, -0.006862778682261705, -0.029417602345347404, -0.008430654183030128, -0.03473243489861488, 0.0024381796829402447, -0.015971073880791664, 0.03207501769065857, 0.006849491503089666, -0.012144394218921661, 0.019040390849113464, 0.0036439825780689716, -0.0037768534384667873, 0.033137984573841095, 0.0035642599686980247, -0.011745781637728214, -0.021166322752833366, 0.0010363925248384476, -0.030427420511841774, -0.020501969382166862, 0.003909724298864603, -0.010490152053534985, -0.0024979717563837767, 0.0018336174543946981, 0.022999940440058708, -0.02284049615263939, 0.013738843612372875, -0.005686871707439423, -0.012895114719867706, -0.019133400171995163, 0.02626856416463852, -0.023837028071284294, -0.02355799823999405, 0.03377576544880867, -0.01345981564372778, -0.026507731527090073, -0.01217096857726574, 0.003926333039999008, 0.014589217491447926, -0.0033699364867061377, 0.0018552090041339397, 0.005451025906950235, 0.011201011016964912, -0.0013228951720520854, -0.020674701780080795, -0.032420482486486435, -0.010769180953502655, 0.005706802010536194, 0.011254159733653069, 0.029391027987003326, 0.004783350042998791, -0.013579399324953556, -0.01896066777408123, 0.01602422259747982, -0.00047044578241184354, -0.03119807131588459, -0.011785643175244331, -0.0084505844861269, 0.020501969382166862, 0.006308042909950018, -0.007208242546766996, -0.01344652846455574, -0.024793697521090508, -0.02284049615263939, 0.0002657416625879705, -0.007015579845756292, -0.019399141892790794, 0.012675877660512924, 0.007287965156137943, 0.013951437547802925, -0.017140338197350502, -0.014336762949824333, -0.026707036420702934, -0.02539161592721939, 0.0004322454333305359, -0.017924275249242783, -0.006122023798525333, -0.006597036961466074, 0.042864128947257996, 0.00020408129785209894, 0.006364512722939253, -0.009633135050535202, -0.005969222169369459, -0.05075665935873985, -0.00704879779368639, -0.026348285377025604, -0.014469633810222149, 0.005544035695493221, -0.0003319694660604, 0.005185284186154604, 0.03143724054098129, -0.022176142781972885, -0.00010053547157440335, -0.012788817286491394, -0.022999940440058708, -0.00338156265206635, 0.01510741375386715, 0.010124756954610348, -0.022601328790187836, 0.002386692212894559, 0.008510377258062363, 0.006205067969858646, -0.02864695154130459, -0.0038698629941791296, -0.005610470660030842, -0.00643759174272418, 0.020860720425844193, 0.028567228466272354, 0.013486389070749283, 0.022720912471413612, 0.0026623993180692196, 0.03377576544880867, -0.02173766866326332, -0.013273796066641808, 0.022960079833865166, -0.014031159691512585, -0.0037602444645017385, -0.005351372994482517, -0.012941619381308556, -0.008457228541374207, -0.00511552719399333, -0.011466752737760544, -0.010290846228599548, -0.018708212301135063, 0.023438414558768272, 0.03906402364373207, -0.009307601489126682, 0.002350152935832739, 0.013054559007287025, -0.005142101086676121, -0.008018754422664642, -0.0018601916963234544, 0.02575036697089672, -0.008032041601836681, 0.008570169098675251, -0.0023202570155262947, -0.031277794390916824, -0.007095302455127239, -0.002627520589157939, 0.027105649933218956, -0.018362749367952347, -0.014469633810222149, 0.0113538121804595, 0.0009749397286213934, 0.005680228117853403, 0.004567435011267662, 0.006052266340702772, 0.011101357638835907, 0.005457669496536255, -0.0005343898665159941, -0.011905226856470108, -0.010702745988965034, -0.008171556517481804, 0.02333211898803711, 0.0025079369079321623, 0.0004986808635294437, 0.00877611804753542, 0.006902639754116535, 0.0013860089238733053, 0.005045769736170769, 0.03752272203564644, -0.01658228039741516, -0.010835615918040276, 0.21663260459899902, 0.02483355812728405, -0.0016069066477939487, 0.03746957331895828, -0.0020046886056661606, 0.01195837464183569, 0.0212460458278656, 0.012881827540695667, -0.001681646448560059, 0.007905814796686172, -0.029231583699584007, 0.004680375102907419, -0.01651584357023239, -0.007181668654084206, -0.0016866291407495737, -0.017897700890898705, 0.0022471779957413673, -0.012370274402201176, 0.005989152938127518, 0.01509412657469511, -0.003351666731759906, -0.012974836863577366, -0.019691457971930504, -0.005926039069890976, 0.028115468099713326, 0.037283554673194885, -0.021910401061177254, 0.02137891761958599, 0.03351002559065819, 0.006690046284347773, -0.010596448555588722, 0.00519524933770299, 0.017206773161888123, 0.024806983768939972, 0.004045916721224785, -0.011971661821007729, 0.022933505475521088, -0.01445634663105011, 0.01345981564372778, -0.0065804277546703815, 0.007527132518589497, -0.011818860657513142, 0.01761867292225361, -0.017990710213780403, 0.012815391644835472, -0.014057734049856663, -0.023438414558768272, -0.016369687393307686, 0.014150743372738361, 0.027663707733154297, 0.0036971308290958405, -0.01380527950823307, 0.014031159691512585, 0.020342525094747543, -0.01725992187857628, 0.0023634398821741343, -0.01628996431827545, 0.008311070501804352, 0.011898582801222801, 0.022787347435951233, -0.02238873578608036, 0.03008195571601391, 0.004394702613353729, 0.013871714472770691, -0.022083131596446037, -0.010682814754545689, -0.010709389112889767, 0.020741136744618416, -0.019864188507199287, 0.013160855509340763, 0.0016475983429700136, -0.0069624315947294235, -0.02499300427734852, 0.024102769792079926, -0.007420836016535759, -0.04751460999250412, 0.017352931201457977, -0.002016314771026373, 0.03906402364373207, 0.049613967537879944, -0.017153624445199966, -0.002944749779999256, -0.023278970271348953, -0.01772496849298477, -0.011672702617943287, -0.019266270101070404, 0.04855100065469742, -0.03226103633642197, -0.0355030857026577, -0.021790817379951477, -0.02411605603992939, 0.0037967839743942022, -0.017538949847221375, -0.02711893618106842, 0.004394702613353729, -0.011559762060642242, 0.0021425422746688128, 0.014389910735189915, -0.00995202548801899, -0.0023069698363542557, -0.02030266262590885, 0.04974684119224548, 0.022960079833865166, 0.02745111472904682, 0.0028567228000611067, 0.0014051090693101287, -0.012456640601158142, 0.012655946426093578, 0.009991887025535107, -0.012356987223029137, 0.024195779114961624, -0.02102016471326351, -0.0007802009349688888, -0.001431683194823563, 0.0064940620213747025, 0.026720324531197548, 0.016622141003608704, -0.0327659472823143, 0.007175025064498186, -0.0009209609706886113, 0.006573784630745649, 0.00959327444434166, 0.007261390797793865, 0.026587452739477158, 0.004786671604961157, -0.01102827861905098, -0.008636604063212872, 0.02862037718296051, -0.014974542893469334, -0.017140338197350502, 0.015040977858006954, 0.011858722195029259, 0.002572711557149887, 0.005517461337149143, -0.020488683134317398, -0.02068798802793026, 0.013552824966609478, -0.016529131680727005, 0.0024730584118515253, 0.017445940524339676, -0.018455758690834045, 0.021657945588231087, 0.002783644013106823, 0.01886765845119953, 0.015599035657942295, -0.019664883613586426, -0.01876136101782322, 0.012044740840792656, -0.01120765507221222, -0.014708801172673702, 0.0015695367474108934, 0.033988360315561295, -0.008815979585051537, -0.03061343915760517, 0.0033931888174265623, 0.012018166482448578, -0.018216591328382492, -0.01372555736452341, 0.023278970271348953, -0.014509494416415691, -0.0020844112150371075, 0.021910401061177254, 0.02862037718296051, -0.014150743372738361, -0.0172333475202322, -0.027079075574874878, -0.1674172431230545, 0.01031077653169632, 0.008749544620513916, -0.002720530377700925, 0.01710047572851181, -0.017220059409737587, 0.023146098479628563, 0.008111764676868916, -0.0198774766176939, 0.0008399927755817771, 0.004231935832649469, -0.00046920013846829534, -0.01765853352844715, -0.0003784742730204016, -0.0006390256457962096, 0.007327826227992773, 0.013227291405200958, 0.02920500934123993, 0.05490222945809364, 0.0072348169051110744, 0.05724075436592102, 0.0005165353650227189, 0.006474131252616644, 0.012476570904254913, 0.0012846948811784387, 0.01880122348666191, -0.008264565840363503, 0.013307013548910618, -0.009766005910933018, -0.02640143409371376, 9.960122406482697e-05, 0.008656534366309643, 0.030666587874293327, -0.004301693290472031, -0.018774649128317833, -0.012111176736652851, 0.010762537829577923, 0.0022471779957413673, -0.002544476417824626, 0.027796577662229538, 0.017538949847221375, 0.025803515687584877, 0.010157975368201733, 0.013273796066641808, 0.008982067927718163, 0.0006680911756120622, 0.017047327011823654, -0.004398024640977383, 0.015226997435092926, -0.015771767124533653, 0.028832970187067986, -0.024753836914896965, -0.01905367709696293, 0.029683344066143036, -0.004301693290472031, -0.010005173273384571, 0.0013752131490036845, 0.028540654107928276, 0.00201465399004519, -0.015001117251813412, 0.014363337308168411, -0.017910989001393318, 0.014124169014394283, -0.031277794390916824, -0.024527955800294876, -0.02523217163980007, -0.013061203062534332, 0.013107707723975182, -0.011692632921040058, 0.019492151215672493, -0.003457963466644287, -0.009865659289062023, 0.014868246391415596, -0.0010571535676717758, 0.004132282920181751, 0.005690193269401789, 0.012184255756437778, 0.022734198719263077, 0.0135262506082654, 0.027690282091498375, 0.0256839320063591, 0.01517384871840477, 0.008689752779901028, 0.011101357638835907, -0.017990710213780403, -0.021644659340381622, 0.016010934486985207, 0.02734481729567051, -0.03792133554816246, -0.007168381474912167, 0.01824316568672657, -0.04039273411035538, -0.023504849523305893, -0.00046546311932615936, 0.020183078944683075, 0.01130730751901865, 0.02917843498289585, -0.005112205166369677, -0.0032736051362007856, -0.004255188629031181, 0.008928920142352581, 0.007387618068605661, -0.008118407800793648, -0.0021790815517306328, 0.02196354791522026, 0.011134576052427292, 0.0020462109241634607, 0.002783644013106823, 0.02241531014442444, 0.010257627815008163, -0.03119807131588459, 0.016396259889006615, 0.01981104165315628, 0.02806231938302517, -0.014031159691512585, 0.03329743072390556, 0.008822623640298843, -0.013566112145781517, -0.009433829225599766, -0.010370568372309208, 0.055327415466308594, 0.005105561576783657, -0.03853254020214081, 0.0072481040842831135, -0.0053845904767513275, -0.03351002559065819, -0.11543817818164825, -0.045176081359386444, 0.009526838548481464, 0.01158633641898632, -0.01259615458548069, 0.027743428945541382, -0.001835278351791203, 0.013273796066641808, -0.015187135897576809, 0.04642506688833237, -0.013453171588480473, -0.0063180080614984035, -0.01801728457212448, -0.010809042491018772, 0.015213710255920887, -0.009234522469341755, -0.00868310872465372, -0.004241901449859142, -0.01284196600317955, 0.021498501300811768, 0.008065259084105492, -0.022601328790187836, 0.024089481681585312, -0.01094855647534132, 0.0014042785624042153, -0.0021674553863704205, -0.021724380552768707, 0.004334910772740841, 0.010417073033750057, -0.0038598976098001003, 0.0007590246386826038, -0.0011335542658343911, 0.006052266340702772, -0.02349156327545643, 0.017738256603479385, 0.007347756996750832, -0.00486971577629447, -0.009433829225599766, 0.04121653363108635, -0.0355030857026577, 0.0009408915648236871, -0.015585748478770256, 0.011699276976287365, -0.027690282091498375, 0.011805573478341103, 0.005823064129799604, -0.010377212427556515, 0.015067552216351032, 0.01357275526970625, -0.005487565416842699, -0.04209347814321518, 0.014947968535125256, -0.025272032245993614, -0.019532011821866035, 0.020209653303027153, -0.014203892089426517, 0.008257921785116196, -0.006989005953073502, 0.017419366165995598, -0.010217767208814621, -0.001038883812725544, -0.012277265079319477, 0.0008154947427101433, 0.017738256603479385, 0.017073901370167732, -0.008244635537266731, -0.0059193954803049564, -0.006470809690654278, 0.023318830877542496, -0.01573190651834011, 0.01473537553101778, 0.004278440959751606, -0.01509412657469511, 0.019890762865543365, -0.012941619381308556, -0.0019432358676567674, -0.03505132719874382, 0.004849785473197699, 0.006929214112460613, -0.011891939677298069, -0.008437298238277435, -0.017220059409737587, 0.018774649128317833, -0.016715150326490402, 0.013738843612372875, -0.012689164839684963, 0.0019847580697387457, 0.008430654183030128, -0.0033500059507787228, -0.010862190276384354, 0.007035510614514351, -0.004105708561837673, 0.010011817328631878, -0.017871126532554626, 0.015917925164103508, -0.014057734049856663, -0.010755893774330616, -0.00897542480379343, -0.0028417748399078846, 0.013685695827007294, -0.015306719578802586, 0.004049238748848438, -0.02127262018620968, 0.021352343261241913, 0.018070433288812637, -0.0038897935301065445, -0.004896290134638548, -0.01664871536195278, -0.02947075106203556, -0.009865659289062023, -0.0022305690217763186, -0.012556293979287148, -0.02326568216085434, 0.04751460999250412, 0.002579354913905263, -0.009553412906825542, -0.021259333938360214, -0.0025943031068891287, 0.008669821545481682, 0.006364512722939253, 0.002625859808176756, 0.009453759528696537, -0.02317267283797264, -0.014549355953931808, 0.013532894663512707, -0.0067697688937187195, -0.003280248725786805, 0.019797753542661667, -0.023092949762940407, 0.038160502910614014, 0.002944749779999256, -0.029550472274422646, 0.01820330321788788, -0.006523957941681147, -0.012058028019964695, 0.009327532723546028, -0.013081133365631104, -0.024076195433735847, -0.013672408647835255, 0.036884941160678864, -0.01452278159558773, 0.015785055235028267, -0.002627520589157939, -0.009885589592158794, 0.005112205166369677, -0.022402022033929825, -0.00575995072722435, -0.029125286266207695, 0.0036672349087893963, -0.0017190163489431143, 0.0076201423071324825, -0.0004916220786981285, 0.014336762949824333, 0.015572461299598217, -0.04010041803121567, -0.030507143586874008, 0.02163137122988701, -0.027716856449842453, 0.012828678824007511, 0.008297783322632313, 0.014389910735189915, -0.010383855551481247, 0.015413016080856323, -0.03207501769065857, -0.02355799823999405, -0.02427550032734871, 0.025564348325133324, -0.013140925206243992, 0.008184843696653843, 0.033137984573841095, 0.004404668230563402, -0.007660003378987312, -0.0021956905256956816, -0.008410723879933357, 0.005783203057944775, 0.023026514798402786, 0.013406666927039623, 0.015346581116318703, -0.0005522443680092692, 0.0006722433608956635, -0.02180410362780094, 0.02610911801457405, -0.006467487663030624, 0.0038897935301065445, -0.0016941031208261847, 0.010563231073319912, 0.037416424602270126, -0.008271208964288235, -0.014974542893469334, 0.019691457971930504, -0.011320594698190689, 0.010616379790008068, -0.021006878465414047, 0.016369687393307686, -0.029231583699584007, -0.014389910735189915, -0.013978011906147003, -0.0035044681280851364, -0.007779587060213089, 0.012928332202136517, 0.015280146151781082, 0.012476570904254913, -0.0055739316157996655, -0.012788817286491394, 0.001359434681944549, -0.0172333475202322, -0.013067846186459064, 0.029311304911971092, 0.008556881919503212, -0.017924275249242783, -0.016794873401522636, 0.02082085981965065, -0.0030809424351900816, 0.012948262505233288, 0.002413266571238637, -0.015027690678834915, -0.028939267620444298, 0.008842553943395615, 0.033430300652980804, -0.029444176703691483, -0.01026427187025547, 0.015333293937146664, 0.014296901412308216, 0.0038565758150070906, 0.010941913351416588, -0.03377576544880867, 0.021060027182102203, 0.006264859810471535, 0.011460109613835812, -0.015426303260028362, -0.006793021224439144, -0.022561468183994293, 0.010377212427556515, -0.013074490241706371, 0.014509494416415691, -0.012064671143889427, 0.001976453699171543, -0.01582491584122181, 0.008397436700761318, 0.012503145262598991, 0.008218061178922653, 0.058888353407382965, 0.01409759558737278, -0.009001999162137508, 0.009646422229707241, -0.018136868253350258, 0.01130730751901865, 0.0039728377014398575, -0.0005144592723809183, -0.004710271023213863, -0.012018166482448578, 0.02438179776072502, 0.008390792645514011, 0.012941619381308556, -0.015971073880791664, -0.0049062552861869335, 0.023278970271348953, -0.021126462146639824, 0.039861250668764114, -0.03364289551973343, -0.0016567332204431295, 0.008224704302847385, 0.0030975511763244867, 0.019784467294812202, 0.0032154740765690804, -0.004780028015375137, -0.020408960059285164, 0.00708865886554122, 0.000459234812296927, -0.009872302412986755, -0.032686226069927216, 0.008802692405879498, 0.012463283725082874, -0.023212533444166183, -0.018628491088747978, 0.01958516053855419, 0.027397966012358665, 0.006793021224439144, -0.010138044133782387, 0.0029513933695852757, 0.008085190318524837, 0.0001514519826741889, -0.0148948198184371, -0.029391027987003326, -0.0418277382850647, -0.008882415480911732, 0.016077371314167976, -0.010662884451448917, -0.021192897111177444, -0.01538644265383482]} +{"id": "test:10000052", "text": "\"In a small bowl, combine fennel, capers, lemon juice, parsley and olive oil and set aside.\\nPreheat oven to 400\u00b0F. Lay trout on a greased baking sheet. Stuff each fish with slices of red onion, lemon, whole sprigs of parsley and oregano. Drizzle with olive oil and season with salt and pepper.\\nBake 15 to 18 minutes or until fish is firm and flakey. Serve with fennel salad.\"", "vector_field": [0.017553316429257393, 0.0029271708335727453, -0.008375365287065506, -0.01599992625415325, -0.008718405850231647, -0.01225236989557743, -0.019443275406956673, -0.03383802995085716, -0.00900319404900074, -0.04437519982457161, 0.01761804148554802, 0.02731378935277462, -0.033139005303382874, -0.016245879232883453, 0.0017572733340784907, 0.029462646692991257, 0.01101612951606512, 0.0007677957764826715, 0.02733967825770378, -0.019456220790743828, -0.029126077890396118, 0.009436849504709244, 0.0032783017959445715, 0.020388254895806313, 0.006310650147497654, -0.01609054021537304, 0.0016731313662603498, -0.009792834520339966, -0.00862779188901186, -0.014226471073925495, 0.029825104400515556, 0.019456220790743828, -0.009145588614046574, -0.02634291909635067, -0.0060129170306026936, -0.016608336940407753, -0.0054854112677276134, 0.0017006393754854798, 0.01609054021537304, 0.0030048403423279524, 0.006262106820940971, -0.022084040567278862, -0.00827827863395214, 0.002169892890378833, -0.00568282138556242, 0.004417455289512873, -0.002542059402912855, -0.013397996313869953, 0.00336891645565629, 0.01820056326687336, 0.03404514864087105, -0.009268565103411674, -0.008563066832721233, -0.011236193589866161, 0.011857549659907818, 0.006546895019710064, 0.015093781054019928, 0.008912580087780952, 0.01722969301044941, -0.0002560667635407299, -0.014821937307715416, -0.010796066373586655, -0.008097049780189991, 0.011132634244859219, -0.011495091952383518, 0.007074400782585144, 0.026770101860165596, 0.009398014284670353, -0.025708617642521858, 0.014109966345131397, 0.022472387179732323, -0.002085750922560692, 0.030834807083010674, 0.0020776602905243635, 0.02840116247534752, -0.017514482140541077, -0.036996591836214066, -0.002925552660599351, 0.007993490435183048, -0.003145616501569748, 0.024944867938756943, -0.007611615117639303, -0.03761794790625572, 0.027753915637731552, 0.012575993314385414, 0.013631004840135574, -0.0187830850481987, 0.005643986631184816, 0.0013632623013108969, -0.004711952060461044, -0.008802548050880432, 0.030575908720493317, -0.0007839769241400063, -0.009054973721504211, -0.022938404232263565, 0.0047669680789113045, 0.004359202925115824, 0.034252267330884933, -0.01616820879280567, -0.04882825165987015, 0.026498258113861084, -0.005339780822396278, -0.02159213274717331, -0.01183813251554966, -0.00910028163343668, -0.016582448035478592, 0.021889865398406982, -0.00413913931697607, 0.011423895135521889, -0.025708617642521858, -0.005821979604661465, 0.019417384639382362, 0.011637486517429352, -0.03782506659626961, -0.027986925095319748, 0.014899606816470623, 0.009443322196602821, -0.025708617642521858, -0.017605096101760864, -0.008051742799580097, 0.01706140860915184, 0.00284141069278121, -0.027986925095319748, -0.028219932690262794, 0.014394755475223064, 0.017294418066740036, 0.016957849264144897, 0.00046884894254617393, 0.008569539524614811, 0.015417403541505337, 0.0518573634326458, -0.011715156026184559, 0.026718322187662125, -0.004585739225149155, -0.0208413265645504, 0.018912533298134804, -0.004323604516685009, -0.015598633326590061, -0.02423289604485035, -0.01011645793914795, 0.014860772527754307, 0.02326202765107155, -0.01875719428062439, 0.010084095411002636, 0.0017135842936113477, -0.013229711912572384, -0.006802557501941919, 0.0244788508862257, -0.01594814658164978, -0.014886662364006042, 0.0053268359042704105, -0.0025663310661911964, 0.04354672133922577, -0.02428467757999897, -0.013423886150121689, 0.005546899978071451, -0.009650440886616707, -0.02031058445572853, -0.014563038945198059, -0.004168265499174595, 0.008718405850231647, 0.0008074396173469722, 0.022847790271043777, 0.027831586077809334, 0.006297705229371786, 0.01533973403275013, 0.03725548833608627, 0.00020570291962940246, 0.010977295227348804, -0.013281491585075855, 0.000536405248567462, 0.010621310211718082, -0.007656922098249197, 0.03495129197835922, -0.004423927515745163, 0.01282194722443819, -0.012854308821260929, -0.008174719288945198, 0.01917143166065216, -0.0034951292909681797, -0.01182518806308508, -0.013236184604465961, 0.035132523626089096, 0.011559817008674145, -0.011721628718078136, 0.00013824773486703634, 0.008343003690242767, 0.012517740949988365, -0.012071141041815281, -0.008129412308335304, -0.005663404241204262, 0.03445938602089882, -0.0017847813433036208, 0.0018300885567441583, -0.6341977119445801, -0.028918959200382233, -0.00917147845029831, 0.0010550112929195166, 0.03254353627562523, 0.028918959200382233, -0.004016162361949682, -0.026614762842655182, -0.005970845930278301, -0.025307325646281242, -9.506428614258766e-05, 0.011533927172422409, 0.0037507915403693914, -0.009333290159702301, -0.00019912933930754662, -0.011572761461138725, 0.017851049080491066, -0.013734564185142517, -0.02195459045469761, 0.019818678498268127, -0.018912533298134804, 0.005711947567760944, -0.004802566487342119, 0.008012907579541206, 0.02317141368985176, -0.0016148792346939445, 0.006129421293735504, -0.002166656544432044, 0.015935201197862625, 0.011533927172422409, 0.006757250055670738, -0.003244321560487151, -0.00691906176507473, 0.009430376812815666, 0.0523492693901062, -0.010504805482923985, -0.024271732196211815, 0.0204788688570261, 0.024582410231232643, 0.05680232495069504, -0.008666626177728176, -0.010342993773519993, 0.03495129197835922, -0.028349382802844048, -0.03360502049326897, -0.009190895594656467, 0.011611596681177616, -0.00313267158344388, 0.01839473657310009, 0.0024611535482108593, 0.018135838210582733, -0.012194118462502956, 0.01082195620983839, -0.014084076508879662, 0.004579266533255577, -0.018097003921866417, 0.019080817699432373, -0.00797407329082489, -0.0007791225798428059, 0.00827827863395214, -0.00450159702450037, 0.00951451901346445, 0.005042047705501318, -0.026071075350046158, -0.00413913931697607, -0.01786399446427822, -0.03479595482349396, 0.0010186036815866828, 0.04258880019187927, -0.02990277297794819, 0.011359170079231262, 0.03298366442322731, -0.0262911394238472, -0.0022249086759984493, 0.010550112463533878, 0.012575993314385414, -0.01101612951606512, 0.0019886638037860394, 0.015883421525359154, -0.009333290159702301, 0.012634245678782463, -0.001215204712934792, -0.008802548050880432, -0.00768928462639451, 0.025630949065089226, -0.005236221477389336, 0.007553362753242254, 0.008012907579541206, -0.004805802833288908, -0.012789584696292877, 0.0029417339246720076, 0.04271824657917023, 0.0029757143929600716, -0.013760454021394253, 0.016194099560379982, -0.016232933849096298, -0.011410949751734734, -0.014174691401422024, 0.00958571583032608, -0.024349400773644447, 0.004728133324533701, -0.030601799488067627, 0.03953379765152931, -0.006158547475934029, 0.0018851044587790966, 0.019067872315645218, -0.021553298458456993, -0.0006614046869799495, 0.021126115694642067, 0.008090577088296413, -0.002703870879486203, -0.03184451162815094, -0.04326193407177925, 0.001275883987545967, 0.001437695580534637, -0.015689248219132423, -0.006375374738126993, -0.001622969750314951, -0.00875076837837696, -0.02889307029545307, 0.028634170070290565, -0.0065857297740876675, 0.011831659823656082, 0.0019805734045803547, -0.01667306199669838, 0.022847790271043777, 0.0016415781574323773, -0.005527482368052006, -0.003375388914719224, -0.0034498220775276423, -0.005498356185853481, 0.0197539534419775, 0.006110004149377346, -0.005786380730569363, -0.032439976930618286, 0.014485369436442852, 0.011197359301149845, -0.009022612124681473, 0.04318426549434662, -0.0212037842720747, 0.008964359760284424, 0.010556585155427456, -0.027003111317753792, -0.030627688392996788, 0.008109994232654572, -0.01997401751577854, -0.005915829911828041, -0.017164967954158783, -0.031740952283144, -0.008770185522735119, -0.001660186448134482, -0.0034336410462856293, -0.00797407329082489, 0.010297686792910099, 0.005818743258714676, -0.021320289000868797, -0.005802561994642019, -0.04284769669175148, -0.0147831030189991, -0.034304048866033554, 0.0069773136638104916, 0.02801281400024891, -0.020569482818245888, 0.004446581471711397, -0.012427126988768578, -0.009520991705358028, 0.02081543765962124, 0.01403229683637619, -0.0049708508886396885, -0.012608355842530727, 0.037307269871234894, -0.008614846505224705, -0.004465998616069555, 0.013993462547659874, -0.021139061078429222, 0.0195080004632473, -0.03298366442322731, -0.006129421293735504, -0.015495073981583118, -0.006187673658132553, -0.0022653616033494473, 0.003195778001099825, -0.011915802024304867, -0.027857474982738495, 0.007300937082618475, 0.008511287160217762, -0.0004110013251192868, 0.04186388477683067, -0.0348995141685009, 0.034304048866033554, -0.003027494065463543, 0.008355948142707348, 0.02100961096584797, 0.013203822076320648, 0.028582390397787094, 0.004724896978586912, 0.033941589295864105, -0.03114548698067665, -0.0031375258695334196, 0.018213506788015366, 0.02537205070257187, -0.01655655726790428, 0.018796028569340706, -0.0018381791887804866, 0.029048409312963486, 0.0021407667081803083, 0.002373775467276573, 0.005928774829953909, 0.021721580997109413, 0.019715119153261185, 0.02539793960750103, -0.015857530757784843, -0.04025871306657791, 0.0014563038712367415, 0.01421352569013834, 0.020375309512019157, 0.03407103940844536, 0.01650477759540081, 0.02345620095729828, 0.023417366668581963, 0.03873120993375778, -0.024828363209962845, 0.021527407690882683, 0.016232933849096298, -0.020582428202033043, 0.032051630318164825, 0.027469128370285034, -0.0018139074090868235, 0.032077521085739136, -0.022821899503469467, 0.012886671349406242, 0.00911322608590126, 0.011391532607376575, 0.02726200968027115, 0.018122892826795578, -0.011922274716198444, -0.006394792348146439, -0.02692544087767601, -0.0025501500349491835, 0.010109985247254372, 0.04352083429694176, 0.006252398248761892, 0.022834844887256622, -0.01839473657310009, 0.024465905502438545, 0.005035575479269028, 0.023404421284794807, 0.010770176537334919, 0.009553353302180767, 0.009702220559120178, -0.028685951605439186, 0.01461481861770153, -0.006472461856901646, 0.002611638279631734, 0.016776621341705322, 0.0035016017500311136, -0.0054951198399066925, -0.02534615993499756, 0.01694490574300289, 0.03720371052622795, 0.03497718274593353, 0.013566279783844948, 0.0052491663955152035, -0.004326840862631798, 0.015262064523994923, -0.02336558699607849, -0.03482184559106827, -0.014873716980218887, -0.010278269648551941, -0.016608336940407753, -0.015676302835345268, -0.02064715325832367, 0.016349438577890396, -0.004511306062340736, -0.001964392140507698, -0.0065857297740876675, 0.004576030652970076, 0.03474417328834534, -0.0047669680789113045, 0.0005784762906841934, -0.02425878681242466, -0.010478915646672249, 0.040025703608989716, -0.007896403782069683, -0.007844624109566212, -0.034640613943338394, -0.025113152340054512, 0.016569502651691437, -0.015469184145331383, 0.011300918646156788, 0.019106706604361534, 0.004145612008869648, -0.0024287912528961897, 0.03067946806550026, 0.013876957818865776, -0.008220026269555092, 0.034640613943338394, -0.026640653610229492, -0.0065533677116036415, -0.009087336249649525, -0.006906116846948862, -0.013902847655117512, -0.010796066373586655, -0.025100206956267357, 0.036245785653591156, -0.012815474532544613, -0.03528786078095436, -0.028194043785333633, 0.01820056326687336, -0.03748849779367447, 0.004288006108254194, -0.012478906661272049, 0.00574430963024497, -0.016219988465309143, 0.004818747751414776, 0.009689275175333023, -0.0005675539723597467, -0.0034206961281597614, -0.0033090461511164904, -0.010543640702962875, -0.006673108320683241, -0.03593510761857033, -0.031352605670690536, 0.0067896125838160515, 0.06441394239664078, 0.03197396174073219, -0.008187663741409779, 0.024634189903736115, 0.0027394695207476616, 0.010155292227864265, -0.0039126030169427395, 0.0011399623472243547, 0.002700634766370058, -0.019223211333155632, 0.024970756843686104, -0.002477334812283516, 0.020828381180763245, 0.009508046321570873, -0.010207071900367737, -0.003702248213812709, 0.0011051729088649154, -0.043443161994218826, 0.019986962899565697, -0.01348861027508974, 0.006394792348146439, -0.000722488563042134, -0.007760481908917427, 0.032957773655653, 0.041475534439086914, 0.004061469808220863, 0.03590921685099602, 0.02478952892124653, 0.010323576629161835, 0.012077613733708858, -0.012931978330016136, -0.0009498337749391794, 0.0027216700837016106, -0.008064687252044678, 0.01135269831866026, -0.010381828993558884, 0.004812275525182486, -0.009559825994074345, 0.016181154176592827, 0.0073397718369960785, -0.001846269704401493, -0.010576002299785614, -0.018964312970638275, -0.005135898478329182, -0.015961090102791786, -0.013100262731313705, -0.022550057619810104, 0.004870527423918247, 0.010388300754129887, -0.028556501492857933, 0.016388272866606712, 0.011592179536819458, 0.0020841327495872974, 0.019430330023169518, 0.0006909352960065007, -0.008744295686483383, -0.0003689303121063858, -0.01711318828165531, 0.0060355705209076405, -0.010368883609771729, 0.0015711900778114796, 0.019572723656892776, 0.012517740949988365, -0.007430386263877153, -0.004592211451381445, 0.0037993350997567177, 0.0003116894804406911, 0.0074174413457512856, -0.039300788193941116, 0.004220045171678066, 0.0069578965194523335, -0.017553316429257393, -0.027831586077809334, 0.00012803338177036494, 0.016906069591641426, 0.012983758002519608, 0.019262045621871948, -0.00862779188901186, 0.007009676191955805, 0.0120581965893507, -0.006320358719676733, -0.013449775986373425, -0.00539156049489975, -0.024349400773644447, 0.004142375662922859, 0.01747564598917961, 0.004359202925115824, -0.01681545563042164, -0.002042061649262905, 0.004349494352936745, 8.849069126881659e-05, 0.012362401932477951, -0.0008665008354000747, -0.021708637475967407, -0.012647190131247044, 0.010834900662302971, -0.007022621110081673, 0.004553376697003841, -0.0013236184604465961, -0.005572789814323187, -0.008770185522735119, -0.021022556349635124, -0.024271732196211815, -0.030058111995458603, -0.016737787052989006, -0.0024514449760317802, 0.007825206033885479, -0.00674430513754487, -0.01403229683637619, -0.013190876692533493, 0.026718322187662125, -0.009042029269039631, -0.008537176996469498, 0.024776583537459373, 0.006006444338709116, 0.01933971606194973, -0.018640689551830292, 0.03332023322582245, -0.016763675957918167, 0.027029000222682953, -0.0008632646058686078, -0.022990183904767036, 0.04207100346684456, -0.0015978890005499125, -0.005226512905210257, -0.006307413801550865, 0.013876957818865776, -0.009999953210353851, -0.00768928462639451, 0.005650459323078394, 0.022252323105931282, 0.02598046138882637, -0.014990221709012985, -0.021462682634592056, -0.03971502557396889, -0.00281390268355608, -0.018291177228093147, -0.004854346159845591, -0.04059528186917305, -0.010504805482923985, -0.028323492035269737, 0.010983767919242382, -0.023210247978568077, 0.011715156026184559, 0.015818696469068527, -0.010912570171058178, -0.0069578965194523335, -0.02067304216325283, 0.0052103316411376, 0.013152042403817177, 0.011870495043694973, -0.023986943066120148, -0.04812922701239586, -0.014498314820230007, 0.011851077899336815, -0.016465943306684494, 0.02803870476782322, -0.012394764460623264, -0.0002789226418826729, -0.008200609125196934, 0.017579205334186554, -0.008847855031490326, -0.0012597028398886323, -0.03948201611638069, -0.001164234010502696, 0.006796084810048342, 0.011753990314900875, -0.034718286246061325, -0.010375356301665306, 0.0016512868460267782, 0.02498370222747326, 0.015585687942802906, 0.01023943442851305, -0.004019398707896471, -0.01589636690914631, 0.027831586077809334, -0.008970832452178001, -0.023973997682332993, -0.017540371045470238, -0.014886662364006042, 0.005067937541753054, -0.007967600598931313, 0.00021804105199407786, 0.008815493434667587, 0.031223155558109283, 0.009080863557755947, 0.05732012167572975, 0.018291177228093147, 0.011941691860556602, 0.008252388797700405, 0.030886588618159294, -0.02925552800297737, 0.01589636690914631, -0.04163087531924248, 0.004546904470771551, 0.012446544133126736, 0.010744286701083183, -0.015624523162841797, -0.021669801324605942, 0.0007633459754288197, 0.008291224017739296, -0.011009657755494118, -0.006297705229371786, 0.01118441391736269, -0.02576039731502533, 0.018796028569340706, 0.004155320581048727, 0.010355939157307148, -0.016375327482819557, -0.010090568102896214, 0.0031245809514075518, -0.018407681956887245, -0.01953388936817646, 0.014744267798960209, -0.008647209033370018, -0.010498332791030407, -0.007728119380772114, 0.025669783353805542, -0.00911322608590126, -0.017294418066740036, 0.013566279783844948, -0.018265286460518837, 0.034278158098459244, -0.0060032084584236145, 0.047611430287361145, 0.006886699236929417, 0.00502263056114316, -0.0029627694748342037, 0.004355966579169035, 0.007747536990791559, -0.01165690366178751, 0.014524204656481743, 0.006297705229371786, 0.001420705346390605, -0.02122967503964901, -0.028944849967956543, 0.021682746708393097, -0.018653634935617447, -0.02251122146844864, 0.041889771819114685, -0.0025016064755618572, -0.001927175442688167, -0.02772802673280239, 0.0020873688627034426, -0.011669849045574665, 0.040750619024038315, -0.022045204415917397, 0.01570219174027443, -0.01496433187276125, 0.006932006683200598, -0.024245841428637505, 0.02692544087767601, -0.004119721706956625, 0.040310490876436234, 0.010465971194207668, 0.014666598290205002, 0.001589798484928906, -0.015909310430288315, -0.026459423825144768, -0.007469221018254757, 0.009042029269039631, 0.02498370222747326, 0.000321398169035092, -0.003058238187804818, 0.009249147959053516, 0.015029055997729301, -0.011126161552965641, -0.02314552292227745, 0.008297695778310299, 0.027158450335264206, -0.01460187416523695, -0.012498323805630207, -0.0010639108950272202, 0.01053069531917572, -0.00284141069278121, -0.02287367917597294, -0.007540417835116386, 0.0015986980870366096, 0.00034931066329590976, -0.0007111617596819997, 0.026640653610229492, -0.016349438577890396, -0.02845294214785099, -0.003145616501569748, -0.004407746717333794, -0.009902866557240486, -0.009889921173453331, 0.002137530595064163, -0.012757222168147564, -0.033915698528289795, -0.0013276637764647603, 0.001734619727358222, -0.012569520622491837, 0.003475711913779378, 0.00928151048719883, -0.002936879638582468, 0.018550075590610504, -0.015469184145331383, -0.04018104448914528, 0.009165005758404732, -0.025747453793883324, 0.03264709562063217, -0.014770157635211945, 0.009035556577146053, -0.016232933849096298, -0.025902792811393738, 0.02564389444887638, -0.002381865866482258, -0.006860809400677681, -0.014511259272694588, 0.0001245948951691389, -0.003273447509855032, 0.004977323114871979, -0.005420686677098274, 0.009236202575266361, -0.0010914189042523503, 0.016232933849096298, -0.010472442954778671, -0.022407662123441696, 0.017941664904356003, -0.006097059231251478, -0.003802571212872863, 0.02311963401734829, -0.008563066832721233, -0.012886671349406242, 0.02033647522330284, 0.02612285502254963, 0.03686714172363281, -0.009922283701598644, 0.017656875774264336, -0.012621300294995308, 0.024711858481168747, -0.013695728965103626, -0.016297658905386925, 0.004990268032997847, -0.007779899053275585, -0.0019126124680042267, 0.013864013366401196, -0.02639469876885414, -0.015909310430288315, -0.007805788889527321, 0.03497718274593353, 0.02142384834587574, -0.011974054388701916, 0.00526534765958786, 0.0006326831644400954, -0.02350798062980175, -0.04012926295399666, -0.01555979810655117, -0.030213451012969017, -0.006048515439033508, 0.014498314820230007, 0.030109891667962074, -0.02847883105278015, 0.0005351916770450771, -0.01271191518753767, -0.002980568679049611, -0.016595391556620598, -0.012679552659392357, 0.002339794998988509, 0.004546904470771551, 0.006165019702166319, 0.003627814818173647, 0.017928719520568848, -0.02478952892124653, 0.013462720438838005, -0.01836884580552578, 0.01271191518753767, -0.006543658673763275, -0.01040771882981062, 0.001451449585147202, -0.008129412308335304, -0.03445938602089882, -0.0037993350997567177, 0.02687366120517254, -0.015961090102791786, -0.0065177688375115395, -0.010912570171058178, -0.0032685932237654924, -0.012414181604981422, -0.012459488585591316, -0.005660167895257473, 0.02280895598232746, -0.006938478909432888, -0.013967572711408138, -0.013915793038904667, 0.00544010428711772, 0.011197359301149845, 0.022278213873505592, -0.02639469876885414, 0.00336891645565629, -0.021695692092180252, 0.0023980471305549145, 0.00826533418148756, -0.018407681956887245, 0.005818743258714676, -0.010478915646672249, -0.025074316188693047, 0.014278250746428967, -0.010938460007309914, -0.012575993314385414, 0.00045388139551505446, 0.01614231988787651, -0.03303544595837593, -0.02300312928855419, -0.01196110900491476, -0.04220044985413551, -0.009792834520339966, 0.005313890986144543, -0.009372124448418617, 0.006268579047173262, -0.007326826918870211, 0.013462720438838005, -0.01744975708425045, -0.004705479834228754, -0.026472369208931923, 0.007333299145102501, -0.0047766766510903835, 0.027909254655241966, 0.02765035629272461, -0.007313882000744343, -0.017630986869335175, 0.01861480064690113, 0.007527472916990519, -0.0035727987997233868, -0.006592202465981245, 0.015171450562775135, -0.03228463977575302, 0.013320326805114746, 0.0028446470387279987, -0.009236202575266361, -0.03826519474387169, -0.004304186906665564, -0.023404421284794807, 0.006886699236929417, 0.01589636690914631, 0.19334538280963898, -0.0018397972453385592, -0.009255620650947094, 0.03114548698067665, -0.0012807383900508285, 0.019961072131991386, 0.030161671340465546, 0.007456276100128889, -0.003961146809160709, 0.00431389594450593, 0.012103503569960594, -0.019249102100729942, -0.0295662060379982, 0.00028256341465748847, 0.02295134961605072, -0.03596099838614464, -0.016129374504089355, -0.013113207183778286, -0.01703551970422268, -0.027469128370285034, 0.025294380262494087, -0.0129060884937644, -0.004113249480724335, -0.016802510246634483, 0.024440016597509384, 0.008783130906522274, -0.013307381421327591, -0.0067896125838160515, 0.02195459045469761, -0.005977318622171879, -0.0038252249360084534, 0.006165019702166319, 0.006398028694093227, 0.011618069373071194, -0.01460187416523695, 0.007870513945817947, 0.025553278625011444, 0.005614860448986292, 0.013527445495128632, 0.0023025781847536564, 0.02881539985537529, 0.010355939157307148, 0.009398014284670353, 0.0051391348242759705, -0.01928793638944626, 0.029850993305444717, 0.009508046321570873, -0.018330011516809464, -7.888313120929524e-05, 0.01900314725935459, -0.030498240143060684, -0.0029676237609237432, -0.001471676048822701, 0.009119698777794838, 0.009527463465929031, 0.006579257547855377, 0.036608245223760605, 0.012045251205563545, -0.006245925556868315, 0.004508069716393948, -0.015508018434047699, 0.04460820555686951, -0.01700962893664837, 0.013423886150121689, -0.04256290942430496, 0.03691892325878143, -0.004941724706441164, -0.009889921173453331, 0.014006407000124454, 0.014679543673992157, 0.014860772527754307, -0.029643874615430832, -0.00917147845029831, 0.01725558377802372, -0.025527389720082283, -0.002088987035676837, -0.000541664136108011, -0.010187654756009579, 0.02481541782617569, 0.03552087023854256, -0.0060129170306026936, 0.015313844196498394, -0.009637495502829552, 0.024970756843686104, 0.014135856181383133, -0.04626515507698059, 0.009974063374102116, 0.010537168011069298, -0.007786371745169163, -0.03036879003047943, 0.005220040213316679, -0.0038834770675748587, -0.016038760542869568, 0.011993471533060074, 0.025100206956267357, 0.00597408227622509, -0.015365623869001865, 0.006932006683200598, -0.01645299792289734, 0.01329443696886301, -0.035417310893535614, -0.038187526166439056, -0.0026844535022974014, 0.0017605095636099577, -0.006232980638742447, -0.0009441703441552818, -0.02183808572590351, 0.024219952523708344, 0.01761804148554802, -0.02651120349764824, 0.016737787052989006, -0.01609054021537304, -0.002076042117550969, 0.008427144959568977, 0.011398005299270153, -0.006132657639682293, -0.013346216641366482, -0.02806459367275238, 0.0030517657287418842, 0.007980545051395893, 0.016737787052989006, -0.027805695310235023, -0.0011763699585571885, 0.011359170079231262, -0.03212929889559746, -0.04051760956645012, -0.0193914957344532, 0.015495073981583118, -0.009728110395371914, -0.05035575106739998, -0.001737855956889689, -0.004537195898592472, -0.02425878681242466, 0.012071141041815281, 0.0005198196158744395, -0.01875719428062439, -0.013631004840135574, 0.013100262731313705, -0.01366983912885189, -0.0262911394238472, 0.015287954360246658, -0.023352641612291336, 0.0026585636660456657, 0.0237539354711771, -0.006320358719676733, -0.0019287936156615615, -0.0033494990784674883, -0.0261746346950531, -0.023831604048609734, 0.008737822994589806, -0.020944885909557343, -0.020569482818245888, -0.007249157410115004, -0.004012926481664181, 0.012278259731829166, -0.02733967825770378, -0.01064072735607624, -0.02847883105278015, -0.0245953556150198, -0.004870527423918247, -0.03342379257082939, 0.0067896125838160515, 0.01689312607049942, -0.018627744168043137, 0.005291237495839596, -0.01443358976393938, -0.1611384153366089, 0.012181173078715801, 0.00025990980793721974, -0.025436775758862495, 0.034692395478487015, 0.0037249017041176558, 0.055197153240442276, 0.006893171928822994, 0.0024821890983730555, 0.03422637656331062, -0.013164986856281757, 0.03704836964607239, -0.0197539534419775, -0.041941553354263306, -0.010381828993558884, -0.017190858721733093, 0.003266975050792098, 0.005786380730569363, 0.018239397555589676, 0.015223230235278606, 0.02091899700462818, -0.03308722376823425, 0.026019295677542686, -0.025151986628770828, 2.0226443666615523e-05, -0.0017847813433036208, -0.0029611513018608093, -0.008019380271434784, -0.022575946524739265, -0.005679585039615631, -0.005498356185853481, -0.005038811359554529, 0.006964368745684624, 0.023598596453666687, -0.00021116406423971057, -0.03676358237862587, -0.010381828993558884, 0.007385078817605972, -0.007669867016375065, 0.04132019728422165, -0.006854337174445391, 0.027883365750312805, 0.016543611884117126, 0.031197266653180122, -0.015016111545264721, 0.011164996773004532, -0.004592211451381445, -0.00586405023932457, -0.0151455607265234, 0.00437538418918848, 0.005841396749019623, -0.015533908270299435, -0.005711947567760944, -0.013164986856281757, 0.012595410458743572, -0.017747489735484123, -0.006718415301293135, -0.010686034336686134, 0.004291241988539696, -0.0031747424509376287, 0.0006792039494030178, -0.01599992625415325, 0.023080797865986824, 0.005854341667145491, 0.006333303637802601, -0.021320289000868797, -0.03704836964607239, 0.013385050930082798, -0.008297695778310299, -0.0015315462369471788, 0.009508046321570873, 0.00608411431312561, 0.01045949850231409, 0.009495101869106293, -0.006893171928822994, 0.006524241529405117, -0.0018915769178420305, 0.010355939157307148, 0.03197396174073219, 0.023598596453666687, -0.0005962755531072617, 0.03590921685099602, -0.028918959200382233, 0.03531375154852867, 0.014912552200257778, -0.0005048520397394896, -0.018045224249362946, 0.02251122146844864, -0.02881539985537529, 0.002234617481008172, 0.020750712603330612, -0.012860781513154507, -0.0021472391672432423, -0.004067942034453154, 0.024724803864955902, 0.006951423827558756, 0.028142264112830162, 0.01004526112228632, 0.00015604701184201986, 0.013333271257579327, -0.004255643580108881, -0.01143683958798647, -0.03536553308367729, 0.033993370831012726, 0.023184359073638916, 0.014058186672627926, -0.003789626294746995, 0.017941664904356003, 0.011093799956142902, 0.011106744408607483, -0.0029854229651391506, 0.03754027932882309, -0.0018915769178420305, 0.016038760542869568, 0.004634282551705837, -0.00038672960363328457, 0.036970701068639755, -0.0037993350997567177, 0.00786404125392437, 0.02006463147699833, 0.004614865407347679, -0.028116373345255852, -0.005624569486826658, 0.027883365750312805, -0.009520991705358028, -0.03565032035112381, -0.10583769530057907, 0.014097021892666817, 0.0063494849018752575, 0.028918959200382233, -0.015054945833981037, 0.02164391241967678, 0.0021278217900544405, 0.02561800368130207, -0.0011318717151880264, 0.013132625259459019, 0.005339780822396278, -0.013320326805114746, -0.008912580087780952, -0.006469225510954857, 0.0033462627325206995, 0.007482165936380625, 0.012686025351285934, -0.026019295677542686, 0.0019757188856601715, 0.03145616501569748, 0.00369901186786592, -0.008543649688363075, -0.00010932392615359277, 0.001690930686891079, -0.01572808250784874, -0.013449775986373425, -0.021825140342116356, 0.013423886150121689, 0.002186073921620846, 0.0009425522293895483, 0.009682802483439445, -0.018135838210582733, -0.006022625602781773, -0.000867309863679111, -0.02033647522330284, -0.00975400023162365, 0.013540389947593212, -0.008232971653342247, 0.022485332563519478, -0.0327247679233551, 0.0031698881648480892, -0.013359161093831062, 0.00851775985211134, -0.02886717952787876, 0.005786380730569363, 0.007132652681320906, -0.026744212955236435, 0.021022556349635124, 0.01438181009143591, -0.003938492853194475, -0.004818747751414776, -0.005087355151772499, -0.0014999930281192064, -0.01282194722443819, 0.017851049080491066, -0.02183808572590351, 0.002517787739634514, -0.011087327264249325, -0.01919732242822647, -0.008977304212749004, 0.00721679488196969, -0.006970841437578201, -0.005498356185853481, 0.04178621247410774, 0.01609054021537304, -0.005281528923660517, -0.01655655726790428, -0.012375347316265106, 0.015598633326590061, -0.004174737725406885, 0.014860772527754307, -0.01347566582262516, 0.005074410233646631, 0.016686007380485535, -0.01260188315063715, 0.019145542755723, -0.014006407000124454, -0.01496433187276125, 0.0063494849018752575, -0.012537158094346523, -0.008453034795820713, -0.00608411431312561, -0.024711858481168747, -0.02450473979115486, 0.003608397440984845, 0.02736556902527809, 0.006297705229371786, -0.006139129865914583, -0.006893171928822994, -0.04331371560692787, -0.00869898870587349, 0.01839473657310009, -0.0011682793265208602, 0.009359179995954037, -0.027080779895186424, 0.008394783362746239, -0.004155320581048727, -0.006262106820940971, 0.03567621111869812, -0.009851086884737015, -0.033501461148262024, -0.009300927631556988, -0.06746894121170044, -0.0015509636141359806, -0.002744323806837201, -0.0016682770801708102, 0.011061437427997589, -0.011792825534939766, 0.008129412308335304, 0.019132597371935844, -0.004019398707896471, 0.01343683060258627, -0.001765363966114819, 0.004488652106374502, -0.013566279783844948, -0.006025861948728561, 0.013236184604465961, -0.019262045621871948, 0.027831586077809334, -0.011915802024304867, 0.015287954360246658, 0.0047346060164272785, -0.02161802165210247, 0.014446535147726536, -0.02723611891269684, 0.002380247926339507, -0.009048501960933208, 0.019779842346906662, 0.024970756843686104, 0.027054890990257263, -0.015210284851491451, -0.005436867941170931, -0.0038090436719357967, -0.003118108492344618, -0.010323576629161835, 0.0438573993742466, -0.012809001840651035, -0.014912552200257778, 0.032388199120759964, 0.01958566904067993, 0.013216766528785229, -0.020711878314614296, 0.010660144500434399, -0.012446544133126736, 0.04595447704195976, 0.020181136205792427, -0.01497727632522583, -0.006414209492504597, 0.0010372119722887874, -0.002498370362445712, 0.01803227886557579, 0.02261478081345558, 0.009728110395371914, 0.013035537675023079, 4.1717037674970925e-05, -0.012220008298754692, -0.01875719428062439, 0.008964359760284424, 0.004504833370447159, -0.00965691264718771, 0.02801281400024891, -0.027961034327745438, 0.04937193915247917, 0.018796028569340706, -0.01614231988787651, -0.012860781513154507, -0.0034304047003388405, 0.016232933849096298, 0.0005084927543066442, -0.0012160137994214892, -0.030109891667962074, -0.021436793729662895, -0.03668591380119324, 0.018122892826795578, 0.014472424983978271, 0.004110013134777546, 0.01781221479177475, 0.02522965520620346, -0.015598633326590061, -0.002338176826015115, -0.014511259272694588, 0.025708617642521858, 0.013514500111341476, 0.002305814530700445, -0.010828428901731968, 0.023236138746142387, -0.003870532149448991, 0.021307343617081642, -0.008666626177728176, 0.0001122567628044635, -0.007559835445135832, -0.0012969195377081633, 0.011326808482408524, -0.0040808869525790215, 0.0261746346950531, -0.03176684305071831, 0.006724887993186712, 0.031197266653180122, 0.004207100253552198, 0.018524184823036194, -0.00994817353785038, 0.010705451481044292, 0.02656298317015171, 0.0010299304267391562, -0.002517787739634514, -0.007928765378892422, -0.0061714923940598965, -0.04157909378409386, -0.011275028809905052, -0.04132019728422165, -0.01234945748001337, 0.005883467849344015, -0.0075857252813875675, -0.013993462547659874, 0.011508037336170673, 0.006504823919385672, -0.017268527299165726, 0.0056310417130589485, -0.02770213596522808, -0.037773288786411285, -0.006686053238809109, 0.004572794307023287, 0.01425236091017723, 0.028970738872885704, 0.01781221479177475, -0.01401935238391161, 0.002365684835240245, -0.0011124543379992247, 0.04522956162691116, 0.014886662364006042, -0.025721563026309013, 0.003983800299465656, 0.015805751085281372, 0.019249102100729942, -0.016491832211613655, -0.019909292459487915, -0.0012127775698900223, 0.01570219174027443, 0.022550057619810104, 0.03453705459833145, -0.024802474305033684, 0.07155954092741013, 0.008368893526494503, -0.010064678266644478, 0.01182518806308508, 0.01739797741174698, 0.024750694632530212, -0.025074316188693047, 0.017773380503058434, -0.0034789482597261667, -0.03899011015892029, 0.013708674348890781, -0.00804527010768652, -0.003459530882537365, -0.00025121241924352944, -0.03197396174073219, -0.010258851572871208, 0.0028300839476287365, 0.011676320806145668, -0.03823930397629738, 0.0013948155101388693, 0.013876957818865776, 0.030938368290662766, 0.003186069428920746, -0.00911322608590126, -0.03590921685099602, -0.012407708913087845, 0.013106735423207283, -0.020271750167012215, -0.008116466924548149, -0.017605096101760864, -0.013307381421327591, 0.00922325812280178, -0.02295134961605072, -0.002999986056238413, -0.004174737725406885, -0.007469221018254757, 0.00739155150949955, -0.027883365750312805, 0.00544010428711772, 0.0489577017724514, 0.003375388914719224, 0.015404459089040756, -0.033579133450984955, -0.007935238070786, -0.02222643420100212, 0.02139795944094658, 0.01870541460812092, -0.021139061078429222, -0.01425236091017723]} +{"id": "test:10000053", "text": "\"Meaningful, reliable and valid mRNA expression analyses by real-time quantitative PCR (RT-qPCR) can only be achieved, if suitable reference genes are chosen for normalization and if appropriate RT-qPCR quality standards are met. Human periodontal ligament (hPDL) fibroblasts play a major mediating role in orthodontic tooth movement and periodontitis. Despite corresponding in-vitro gene expression studies being a focus of interest for many years, no information is available for hPDL fibroblasts on suitable reference genes, which are generally used in RT-qPCR experiments to normalize variability between samples. The aim of this study was to identify and validate suitable reference genes for normalization in untreated hPDL fibroblasts as well as experiments on orthodontic tooth movement or periodontitis (Aggregatibacter actinomycetemcomitans). We investigated the suitability of 13 candidate reference genes using four different algorithms (geNorm, NormFinder, comparative Delta C-q and BestKeeper) and ranked them according to their expression stability. Overall PPIB (peptidylprolyl isomerase A), TBP (TATA-box-binding protein) and RPL22 (ribosomal protein 22) were found to be most stably expressed with two genes in conjunction sufficient for reliable normalization. This study provides an accurate tool for quantitative gene expression analysis in hPDL fibroblasts according to the MIQE guidelines and shows that reference gene reliability is treatment-specific.\"", "vector_field": [-0.03432755917310715, -0.01961190439760685, -0.0037091788835823536, -0.033385973423719406, -0.01961190439760685, 0.005128284450620413, -0.016262546181678772, 0.015253703109920025, -0.029135379940271378, -0.013962384313344955, 0.0015351225156337023, -0.005407397635281086, 0.013666457496583462, 0.01747315749526024, -0.024467801675200462, -0.004805454518646002, 0.024158421903848648, -0.006611283402889967, -0.0063590724021196365, 0.01649121753871441, -0.021293308585882187, 0.0034737822134047747, -0.006544027011841536, -0.014540787786245346, -0.02875874564051628, 0.025718767195940018, 0.024427447468042374, -0.025812925770878792, -0.01986747793853283, -0.01895279437303543, 0.014769459143280983, 0.006143852602690458, -0.013451237231492996, 0.017917048186063766, -0.013922031037509441, -0.03174491971731186, -0.014150702394545078, -0.004623862914741039, 0.028274500742554665, 0.0076739308424293995, 0.01844164729118347, -0.011413374915719032, 0.0076537542045116425, -0.014123799279332161, -0.0004033269651699811, -0.002866795053705573, 0.026203010231256485, -0.0014426452107727528, -0.01877792738378048, 0.01655847392976284, 0.003320774296298623, 0.0048189060762524605, 0.00011496604565763846, -0.0046541281044483185, -0.002493523061275482, -0.004173246677964926, 0.01604732684791088, 0.030050065368413925, -0.011668948456645012, -0.01263743732124567, -0.024239130318164825, -0.009045957587659359, -0.024615764617919922, 0.0006355709629133344, -0.007976584136486053, -0.002004234353080392, -0.0008650827221572399, 0.021723749116063118, 0.02465611882507801, -0.01107709389179945, 0.03198704123497009, 0.01314858440309763, 0.001898305956274271, 0.006853405386209488, 0.03279411792755127, -0.016316350549459457, -0.014742556028068066, -0.01224062591791153, -0.01131249126046896, -0.0016864489298313856, 0.003105554496869445, -0.016545021906495094, 0.00900560338050127, 0.016033874824643135, 0.002708743093535304, -0.00880383513867855, -0.003988292068243027, 0.01685440167784691, -0.0017015815246850252, 0.0009272947208955884, 0.015966618433594704, 0.04172573983669281, 0.011991778388619423, 0.005420848727226257, -0.02988865040242672, 0.006987918168306351, -0.0027995388954877853, 0.014043091796338558, 0.013168761506676674, -0.025140363723039627, -0.0017015815246850252, 0.026418231427669525, -0.013444512151181698, -0.006348984315991402, -0.009543652646243572, -0.016356704756617546, 0.005753766745328903, -0.008050565607845783, 0.01526715513318777, -0.008366669528186321, -0.022692237049341202, 0.024830983951687813, -0.010861874558031559, -0.050684262067079544, 0.030991651117801666, -0.029108477756381035, 0.015509276650846004, -0.010787892155349255, -0.0033005974255502224, 0.006584380753338337, 0.022355956956744194, 0.032229166477918625, 0.019060403108596802, -0.020862869918346405, 0.02349931187927723, 0.012052308768033981, 0.003924398683011532, 0.009503299370408058, -0.02567841298878193, 0.00822543166577816, 0.002892016200348735, 0.0071358815766870975, 0.010182586498558521, 0.009019054472446442, -0.016464315354824066, -0.007055174093693495, -0.0022715777158737183, -0.03392402082681656, -0.01935633085668087, -0.03518843650817871, -0.0019285711459815502, 0.023539666086435318, -0.024413995444774628, -0.0018226427491754293, 0.002357329474762082, 0.007182960864156485, 0.002819715766236186, 0.0036923647858202457, 0.01957155019044876, -0.004869347903877497, -0.01710997335612774, -0.006671813782304525, 0.0023270640522241592, 0.015630338340997696, 0.0009525157511234283, -0.006907210685312748, -0.0035275870468467474, 0.02164304070174694, -0.024212228134274483, -0.015347861684858799, -0.010639929212629795, -0.004095901735126972, -0.016477765515446663, 0.025947436690330505, 0.008729853667318821, 0.023472409695386887, -0.005666333716362715, 0.0004682712024077773, 0.0009651263244450092, -0.006093410775065422, -0.011373021639883518, 0.02854352630674839, -0.045115452259778976, 0.015307508409023285, 0.014675300568342209, -0.009536927565932274, -0.017769085243344307, 0.003272013505920768, -0.015320959500968456, -0.009617635048925877, -0.012980444356799126, 0.02222144417464733, 0.013626103289425373, -0.0059118191711604595, -0.0015662284567952156, 0.009193920530378819, 0.017957402393221855, -0.0014897246146574616, 0.0004972754395566881, -0.02298816479742527, 0.002685203216969967, -0.008501182310283184, 0.016437413170933723, -0.015119191259145737, -0.6400637030601501, -0.02875874564051628, 0.04304395988583565, -0.0063657984137535095, 0.009476397186517715, 0.03142208978533745, 0.0023539667017757893, -0.014500434510409832, -0.019598452374339104, 0.029350601136684418, -0.020782161504030228, 0.02127985842525959, -0.0002875202044378966, -0.01986747793853283, -0.022813299670815468, -0.025839827954769135, -0.005363680887967348, -0.0252345222979784, 0.020728357136249542, 0.02091667428612709, -0.03421994671225548, 0.0034048445522785187, -0.0029710421804338694, 0.019369782879948616, 0.01604732684791088, 0.019383233040571213, 0.022638432681560516, -0.03279411792755127, 0.011689125560224056, 0.02843591570854187, -0.03370880335569382, 0.02661999873816967, 0.0012232218869030476, 0.004781914874911308, 0.05724846571683884, -0.001957155065611005, -0.019154561683535576, 0.018939342349767685, 0.028570428490638733, 0.015818655490875244, -0.015025032684206963, 0.0008785339305177331, 0.014957776293158531, 0.005094656255096197, -0.007566321175545454, 0.03830912336707115, -0.007169509772211313, 0.014850166626274586, -0.0016217147931456566, 0.010760989971458912, -0.015549630858004093, -0.0032030760776251554, -0.022355956956744194, -0.01786324381828308, 0.008117821998894215, -0.02629716880619526, 0.020634198561310768, -0.0065877437591552734, 0.023432055488228798, -0.012610535137355328, 0.024104617536067963, 0.02136056497693062, 0.004778552334755659, -0.014661849476397038, -0.004173246677964926, -0.010794618166983128, 0.010108605027198792, -0.002244675299152732, 0.009980818256735802, -0.0026532565243542194, 0.020782161504030228, 0.04387793689966202, 0.003450242569670081, 0.0005300628254190087, 0.018831733614206314, 0.03588790073990822, 0.01873757317662239, 0.008891267701983452, 0.006493585184216499, 0.021508529782295227, 0.02458886243402958, 0.0029424582608044147, -0.013505042530596256, 0.01156806480139494, 0.014608044177293777, 0.017849791795015335, -0.02698318287730217, 0.032551996409893036, 4.316481135901995e-05, 0.004751649685204029, -0.004808817524462938, -0.003551126690581441, -0.00033165706554427743, -0.042828742414712906, -0.032229166477918625, 0.027534684166312218, -0.00891817081719637, -0.013666457496583462, 0.002532195532694459, -0.017338644713163376, -0.02277294546365738, -0.01040453277528286, 0.021777553483843803, 0.006584380753338337, 0.005420848727226257, -0.00038693324313499033, -0.021521979942917824, 0.028409013524651527, 0.020069247111678123, -0.02382214181125164, -0.0126172611489892, -0.026929378509521484, 0.0018495451658964157, -0.02004234492778778, -0.010249842889606953, -0.02843591570854187, 0.023512763902544975, -0.01437937282025814, -0.021696846932172775, -0.018508903682231903, -0.0016091043362393975, -0.021737199276685715, -0.005393946077674627, 0.0074049062095582485, 0.0176076702773571, 0.0034468795638531446, -0.015778301283717155, -0.0019941460341215134, 0.01771528087556362, 0.009483122266829014, -0.0033308626152575016, -0.004603686276823282, 0.02204657904803753, -0.02563805878162384, 0.011904344893991947, 0.01002117246389389, 0.012919913977384567, 0.008985426276922226, -0.0006860131397843361, -0.04046132415533066, -0.01390857994556427, -0.004021920263767242, 0.016827497631311417, -0.022826749831438065, -0.014339019544422626, -0.029538918286561966, -0.028355209156870842, 0.017015814781188965, 0.043232277035713196, 0.0012753454502671957, 0.011675674468278885, 0.00401183171197772, -0.02988865040242672, 0.0018024658784270287, -0.00029718829318881035, -0.009960641153156757, -0.02629716880619526, -0.023378251120448112, 0.002629716880619526, -0.005676422268152237, -0.007290570996701717, 0.0002992900263052434, -0.00046028452925384045, -0.01815917156636715, 0.008628969080746174, -0.006322081666439772, 0.004321210086345673, 0.010794618166983128, 0.005367043893784285, -0.0334128737449646, 0.009328433312475681, 0.002463257871568203, -0.0016814046539366245, 0.0018411381170153618, 0.02458886243402958, 0.000548558309674263, 0.010135507211089134, -0.005377132445573807, -0.0066079203970730305, -0.007122430484741926, 0.008178352378308773, -0.012973718345165253, -0.005269522313028574, -0.008998878300189972, 0.01587245985865593, 0.025812925770878792, -0.002385913161560893, -0.0025120186619460583, 0.009119939059019089, 0.028274500742554665, -0.01721758395433426, 0.021226052194833755, -0.02040552720427513, 0.01016913540661335, -0.00802366342395544, -0.023432055488228798, 0.003991654608398676, 0.015132642351090908, 0.005114832893013954, 0.017338644713163376, 0.013040974736213684, -0.028166892006993294, -0.0001940971560543403, -0.016531571745872498, 0.0026465309783816338, -0.017997756600379944, 0.008440651930868626, -0.026498937979340553, 0.0034166143741458654, 0.006880308035761118, -0.001790696056559682, 0.012778676114976406, -0.014675300568342209, 0.002905467292293906, -0.009503299370408058, 0.003830239875242114, -0.017769085243344307, 0.008676048368215561, 0.004445633850991726, -0.0005456158542074263, -0.003435109741985798, 0.016652632504701614, 0.010882051661610603, 0.012879559770226479, -0.013679908588528633, -0.007102253381162882, -0.007310747634619474, 0.0029844932723790407, 0.0039008588064461946, 0.005915181711316109, -0.01158151589334011, 0.011541161686182022, -0.0035914804320782423, 0.017405901104211807, 0.01274504791945219, 0.03198704123497009, 0.027790257707238197, -0.004553243983536959, 0.0341392420232296, -0.002912192838266492, -0.005948809906840324, 0.014043091796338558, 0.01016913540661335, -0.007478888146579266, 0.0022211356554180384, -0.002365736523643136, 0.021145345643162727, 0.011083819903433323, -0.006799600552767515, 0.003907584585249424, 0.001203885767608881, 0.022355956956744194, -0.0062010204419493675, 0.009792501106858253, 0.023808689787983894, -0.01662573032081127, 0.00747216260060668, 0.006883671041578054, 0.045814916491508484, 0.011110722087323666, 0.01092913094907999, 0.021871712058782578, 0.004109353292733431, -0.027870964258909225, 0.012099388055503368, -0.011393197812139988, 0.011998504400253296, -0.015509276650846004, -0.027736451476812363, 0.01094930712133646, -0.007539418525993824, 0.013827872462570667, 0.03384331241250038, -0.02702353708446026, 0.0030214842408895493, 0.003325818572193384, 0.01521334983408451, 0.02262498252093792, 0.0043380241841077805, 0.013484865427017212, -0.0015376446535810828, -0.03572648763656616, 0.033063143491744995, 0.02916228398680687, -0.01754041388630867, -0.02200622484087944, -0.029350601136684418, -0.0010222940472885966, -0.02208693139255047, -0.0068197776563465595, 0.00391431013122201, 0.008702950552105904, 0.009146841242909431, -0.0006153941503725946, 0.024064263328909874, -0.0035242242738604546, 0.022382859140634537, -0.01113089919090271, 0.02582637593150139, -0.0028298040851950645, -0.004142981022596359, 0.0019689248874783516, 0.003917673137038946, -0.04796711355447769, 0.03478490188717842, 0.024279484525322914, -0.013646280393004417, -0.00434138672426343, 0.008487731218338013, -0.008285962045192719, 0.013229291886091232, -0.0005687351222150028, 0.012415491975843906, 0.001834412571042776, -0.010175861418247223, 0.017970852553844452, 0.0056999619118869305, 0.022934360429644585, 0.041268397122621536, -0.01961190439760685, -0.02025756426155567, -0.03930451720952988, 0.001714192098006606, 0.009422591887414455, 0.02448125183582306, 0.012980444356799126, 0.006053057033568621, 0.016719888895750046, -0.014756008051335812, 0.014540787786245346, -0.005192177835851908, -0.005975712090730667, 0.01289301086217165, -0.01982712373137474, 0.0017402538796886802, -0.01276522409170866, 0.004506164696067572, -0.007391455117613077, -0.0044758995063602924, -0.0012198591139167547, -0.0022127286065369844, -0.02813998982310295, 0.007075351197272539, 0.022759493440389633, -0.019665708765387535, 0.007351101376116276, 0.021629590541124344, 0.041241493076086044, 0.02811308577656746, -0.020163405686616898, 0.021979322656989098, 0.005380494985729456, 0.007990035228431225, -0.011083819903433323, -0.016719888895750046, 0.026862122118473053, -0.004455722402781248, 0.004785277880728245, -0.0065877437591552734, -0.018239878118038177, 0.017298292368650436, -0.017311742529273033, 0.025180716067552567, 0.01899314671754837, 0.006204383447766304, 0.0355919748544693, 0.015092289075255394, 0.0004527182027231902, -0.00035540692624635994, -0.01572449691593647, -0.019450489431619644, -0.008871091529726982, 0.010000995360314846, -0.016948560252785683, 0.030157674103975296, -0.02730601280927658, -0.015038483776152134, -0.022651884704828262, 0.02570531517267227, -0.012267529033124447, 0.002283347537741065, -0.012886285781860352, 0.018979696556925774, -0.016571924090385437, 0.0117093026638031, -0.026028145104646683, 0.01975986734032631, -0.019948184490203857, -0.023660726845264435, -0.014581141993403435, 0.012671065516769886, -0.0036990903317928314, -0.013585750013589859, -0.0112586859613657, -0.01873757317662239, -0.023270640522241592, -0.014715653844177723, 0.0028415739070624113, -0.0009180469787679613, -0.0068197776563465595, 0.013249468989670277, 0.0017957402160391212, 0.011211606673896313, 0.015025032684206963, -0.03992327302694321, -0.020943576470017433, 0.00983958039432764, -0.030668821185827255, -0.021804455667734146, -0.009039231576025486, -0.009308256208896637, 0.002871839329600334, 0.005242619663476944, -0.010000995360314846, 0.019262172281742096, 0.0014729105168953538, -0.007492339238524437, -0.015737948939204216, -0.003961389418691397, 0.023916300386190414, 0.0461108423769474, 0.040003981441259384, 0.014567689970135689, -0.009368786588311195, -0.009207372553646564, -0.0395735427737236, -0.016504667699337006, -0.03602241352200508, -0.005447751376777887, 0.020728357136249542, -0.008440651930868626, 0.018656866624951363, -0.02313612960278988, -0.029189186170697212, 0.0013930437853559852, -0.009288080036640167, -0.02193896844983101, 0.023512763902544975, 0.038981687277555466, 0.024790631607174873, -0.03123377449810505, 0.022315602749586105, 0.025880180299282074, -0.004230414051562548, 0.013478140346705914, -0.002128658350557089, 0.030695723369717598, -0.0019369781948626041, -0.020311368629336357, 0.01677369326353073, -0.007411631755530834, -0.027736451476812363, -0.01276522409170866, -0.005360318347811699, -0.008521359413862228, 0.05219080299139023, 0.008978701196610928, -0.014540787786245346, -0.02164304070174694, -0.00010072664736071602, -0.0029575908556580544, -0.009664714336395264, -0.013605927117168903, -0.014298665337264538, -0.0038638680707663298, 0.01920836791396141, 0.012482748366892338, -0.016545021906495094, 0.004876073449850082, -0.028086183592677116, -0.021925516426563263, 0.029431307688355446, 0.007848797366023064, 0.015926266089081764, -0.009866482578217983, 0.018643414601683617, -0.007848797366023064, -0.005901730619370937, -0.003752895398065448, -0.008380120620131493, -0.010908953845500946, -0.007263668347150087, 0.037932489067316055, 0.03021148033440113, 0.03873956575989723, -0.00491979019716382, 0.001859633601270616, 0.02390284836292267, 0.0025557351764291525, -0.005874827969819307, -0.006786149460822344, 0.006856768392026424, -0.0230554211884737, 0.012227174825966358, 0.01638360694050789, -0.00747216260060668, 0.015563081949949265, -0.02683521993458271, -0.0009499936713837087, 0.025086557492613792, -0.027359817177057266, -0.010882051661610603, 0.012590358033776283, -0.025342131033539772, -0.043851032853126526, 0.007122430484741926, -0.01164204627275467, -0.008615517988801003, -0.041268397122621536, -0.001721758395433426, 0.00918719545006752, -0.02110499143600464, 0.0288663562387228, -0.015038483776152134, 0.022181089967489243, -0.028785647824406624, 0.02636442519724369, -0.00041236449033021927, 0.012314608320593834, -0.004032008349895477, -0.019800221547484398, -0.01158151589334011, -0.0013804332120344043, 0.022181089967489243, 0.006214471999555826, 0.02945820987224579, -0.017526961863040924, 0.0066549996845424175, -0.019840575754642487, -0.001984057482331991, -0.00848100520670414, -0.02222144417464733, -0.01946394145488739, -0.018656866624951363, -0.01144027803093195, -0.016652632504701614, -0.027171500027179718, -0.020526587963104248, -0.004196786321699619, 0.009240999817848206, -0.007734461687505245, 0.016356704756617546, 0.003961389418691397, -0.00926117692142725, 0.03298243507742882, 0.005393946077674627, 0.04347439855337143, -0.001610785722732544, 0.012072485871613026, 0.01094930712133646, -0.013673183508217335, -0.02390284836292267, -0.02501930110156536, 0.008359944447875023, 0.0016679534455761313, 0.02520762011408806, 0.007808443624526262, -0.011070368811488152, -0.03890097886323929, 0.013868225738406181, 0.011574789881706238, -0.010727361775934696, -0.02549009583890438, 0.021777553483843803, -0.003213164396584034, -0.01651811972260475, -0.001036586007103324, 0.008797109127044678, -0.04078415408730507, 0.0038672308437526226, -0.011480631306767464, 0.009725244715809822, -0.007761364337056875, 0.0067928750067949295, 0.016840949654579163, 0.012926639057695866, -0.0012610534904524684, 0.009395689703524113, -0.025261424481868744, -0.02189861424267292, -0.004899613093584776, -0.007909327745437622, -0.02640477940440178, 0.00491979019716382, -0.004674305208027363, 0.028973964974284172, -7.981417729752138e-05, -0.0016469359397888184, 0.010337276384234428, -0.011278863064944744, 0.007330924738198519, -0.0070417230017483234, 0.00312741263769567, 0.03325146064162254, -0.014016189612448215, 0.01732519455254078, 0.030803333967924118, 0.004546518437564373, 0.0045431554317474365, -0.00731747318059206, 0.0005275407456792891, -0.01541511807590723, -0.019840575754642487, -0.0024649393744766712, 0.0009180469787679613, 0.009563829749822617, -0.004913064651191235, 0.012630712240934372, -0.02266533486545086, 0.008319590240716934, -0.029861748218536377, -0.021078089252114296, 0.012052308768033981, -0.055150073021650314, -0.01775563322007656, 0.012146467342972755, 0.016894754022359848, 0.020714905112981796, -0.00784207135438919, 0.008689499460160732, -0.020001990720629692, 0.026135753840208054, -0.03349358215928078, 0.03505392372608185, -0.000711234170012176, -0.01067355740815401, -0.03572648763656616, -0.029216088354587555, -0.0021269768476486206, -0.026135753840208054, 0.018724123015999794, -0.028005477041006088, -0.013881676830351353, 0.01815917156636715, -0.0010155683849006891, 0.02179100550711155, 0.010370904579758644, -0.03707161173224449, 0.0018310496816411614, 0.019114209339022636, 0.012516376562416553, -0.0014880431117489934, -0.010122056119143963, 0.004559969529509544, 0.0031375011894851923, 0.008951799012720585, 0.019894380122423172, -0.0034048445522785187, -0.009523476473987103, 0.002513699932023883, -0.023364799097180367, 0.01327637117356062, -0.007297296542674303, -0.01692165620625019, -0.010525593534111977, 0.024925142526626587, -0.030130771920084953, -0.025180716067552567, -0.007021545898169279, -0.0009735333151184022, -0.015576533041894436, 0.02390284836292267, 0.01372698787599802, -0.015670692548155785, -0.0016814046539366245, 0.02294781059026718, -0.0010441523045301437, 0.01276522409170866, -0.00796985812485218, -0.03039979748427868, -0.013431061059236526, -0.020647650584578514, -0.010397806763648987, 0.006517124827951193, -0.009072859771549702, 0.01229443121701479, 0.01971951499581337, -0.03925071284174919, -0.02266533486545086, -4.403178536449559e-05, -0.035242240875959396, -0.040219198912382126, 0.0018545894417911768, -0.007909327745437622, 0.011749655939638615, -0.007290570996701717, -0.008259059861302376, 0.019087305292487144, 0.00456669507548213, 0.020755259320139885, 0.01158824097365141, -0.029242990538477898, 0.011366295628249645, -0.009483122266829014, 0.018320586532354355, 0.028920160606503487, -0.0297810398042202, -0.003907584585249424, 0.029431307688355446, -0.003894133260473609, -0.021078089252114296, 0.025624608621001244, -0.01436592172831297, 0.025476643815636635, -0.014782910235226154, 0.029027771204710007, 0.015011581592261791, -0.00012557991431094706, 0.013962384313344955, -0.028893258422613144, 0.007781540974974632, 0.010323825292289257, 0.009900110773742199, -0.005387220531702042, 0.008420474827289581, 0.02106463722884655, -0.0007049289415590465, 0.013592476025223732, -0.006695353426039219, -0.011534436605870724, 0.031502798199653625, -0.0004159374802839011, 0.018091915175318718, -0.003440154017880559, -0.02450815588235855, 0.016141485422849655, 0.0015275562182068825, -0.011157801374793053, -0.0042472281493246555, -0.016571924090385437, -0.013067876920104027, -0.021306760609149933, 0.023620372638106346, -0.008561712689697742, -0.004802091978490353, -0.012428943999111652, 0.006426328793168068, 0.003897496033459902, -0.003517498727887869, -0.018132267519831657, -0.0265123900026083, -0.023082323372364044, 0.0035544894635677338, 0.018145719543099403, 3.706971983774565e-05, -0.020943576470017433, -0.001995827304199338, -0.00982612930238247, -0.012314608320593834, -0.004526341333985329, -0.007996760308742523, -0.0016049008117988706, -0.002515381434932351, 0.000501478963997215, -0.01263743732124567, 0.012792127206921577, 0.007983309216797352, -0.02604159526526928, -0.010135507211089134, 0.006453231442719698, 0.19197607040405273, -0.015859009698033333, 0.01067355740815401, 0.002224498428404331, 0.0007225836743600667, -0.008440651930868626, -0.00039513010415248573, -0.010518867522478104, -0.01203885767608881, 0.024642666801810265, 0.029834844172000885, -0.004317847080528736, 0.0007789107039570808, -0.0036015689838677645, 0.002522106980904937, 0.0010693733347579837, -0.02193896844983101, -0.022813299670815468, -0.033385973423719406, -0.006362435407936573, 0.0010458336910232902, -0.009691616520285606, 0.008770206943154335, 0.006856768392026424, 0.015065385960042477, 0.01340415794402361, -0.014984678477048874, -0.001987420255318284, 0.01133266743272543, 0.01060630101710558, -0.020688002929091454, 0.005098019260913134, 0.010135507211089134, -0.0006095091812312603, 0.0012005229946225882, 0.020028892904520035, -0.0032585624139755964, -0.0007864770595915616, 0.014903970994055271, -0.010317099280655384, 0.024050813168287277, -0.010115331038832664, -0.010074976831674576, 0.0021017559338361025, 0.012832480482757092, -0.01238186378031969, 0.0015351225156337023, 0.008238882757723331, -0.006402789149433374, 0.013417609967291355, -0.03249818831682205, 0.009631086140871048, -0.015280606225132942, 0.01411034818738699, -0.010982935316860676, -0.014836715534329414, 0.008548261597752571, 0.009503299370408058, 0.008339767344295979, 0.029969356954097748, -0.00744525995105505, 0.0033308626152575016, -0.016289448365569115, 0.007411631755530834, -0.034623485058546066, 0.013538670726120472, -0.009321707300841808, 0.009853031486272812, -0.010935856029391289, -0.02865113690495491, 0.016316350549459457, -0.00801021233201027, 0.0014317161403596401, 0.01598007045686245, -0.025032753124833107, -0.013081328943371773, 0.030345991253852844, 0.009879933670163155, 0.007236765697598457, 0.03061501681804657, 0.005091293249279261, -0.0023337898310273886, -0.038470540195703506, -0.007082076743245125, -0.024521606042981148, -0.008561712689697742, 0.02538248524069786, 0.00010981674131471664, -0.009429317899048328, -0.012428943999111652, 0.018603062257170677, -0.0019285711459815502, -0.016370156779885292, 0.014231409877538681, -0.004422094207257032, -0.004159795120358467, -0.012879559770226479, -0.004442271310836077, -0.007694107946008444, -0.02822069637477398, -0.043017059564590454, 0.07613400369882584, 0.01997508853673935, 0.01819952391088009, -0.008057291619479656, 0.009960641153156757, -0.0064969477243721485, 0.009879933670163155, -0.004485987592488527, -0.008985426276922226, 0.009207372553646564, -0.030076967552304268, 0.014581141993403435, -0.021696846932172775, 0.015038483776152134, 0.009584006853401661, 0.018683768808841705, -0.04024610295891762, 0.01526715513318777, -0.01801120676100254, -0.00983958039432764, -0.008709676563739777, 0.003199713071808219, 0.018091915175318718, -0.017782535403966904, -0.004082450643181801, -0.012738321907818317, 0.02134711481630802, -0.004718021489679813, -0.032525092363357544, -0.00033396901562809944, 0.004549880977720022, 0.015401666983962059, -0.0021017559338361025, 0.004401917569339275, 0.011225057765841484, -0.0030013073701411486, -0.0064969477243721485, -0.023405153304338455, 0.0046339514665305614, -0.003947938326746225, -0.0013459644978865981, 0.001525034080259502, -0.006648274138569832, 0.0239701047539711, 0.004741561133414507, 0.017257938161492348, 0.015401666983962059, -0.011036740615963936, -0.00893162190914154, -0.026122303679585457, -0.015253703109920025, -0.015347861684858799, -0.01786324381828308, 0.012953542172908783, -0.004529704339802265, -0.038362931460142136, -0.011083819903433323, 0.01732519455254078, -0.004122804384678602, -0.018912440165877342, -0.01158824097365141, 0.013014072552323341, -0.005299787502735853, 0.01274504791945219, 0.0029239628929644823, -0.17045408487319946, 0.018925892189145088, 0.02499239891767502, -0.029000869020819664, 0.015509276650846004, 0.011897619813680649, 0.005124921444803476, -0.018280232325196266, 0.003038298338651657, -0.008285962045192719, 0.017513511702418327, 0.004119441378861666, -0.03435445949435234, -0.02505965530872345, -0.01411034818738699, 0.017204133793711662, 0.0021051187068223953, 0.027521232143044472, 0.012859383597970009, 0.013162036426365376, 0.009705067612230778, -0.0036957275588065386, -0.0038067002315074205, -0.008252333849668503, -0.005713413003832102, -0.018643414601683617, -0.004334661178290844, -0.006779423914849758, -0.014742556028068066, -0.014298665337264538, -0.016840949654579163, 0.027870964258909225, 0.02497894875705242, 0.0036923647858202457, 0.007862248457968235, -0.008373395539820194, 0.013935482129454613, -0.004563332535326481, -0.0009247725829482079, 0.022207994014024734, 0.02803237922489643, 0.02019030787050724, -0.005908456165343523, -0.0020546766463667154, 0.005938721355050802, 0.04366271570324898, 0.012025406584143639, -0.02076871134340763, 0.018804829567670822, -0.010438160039484501, 0.012307882308959961, 0.004734835587441921, 0.04702552780508995, 0.026243364438414574, 0.013296548277139664, 0.010586123913526535, -0.009436042979359627, 0.02403736114501953, -0.005777306389063597, -0.016531571745872498, -0.0060227918438613415, -0.017042718827724457, 0.02702353708446026, -0.014392823912203312, -0.009906836785376072, -0.02132021076977253, -0.03061501681804657, 0.008164901286363602, -0.02465611882507801, 0.0031845804769545794, -0.017567316070199013, -0.022826749831438065, 0.011870717629790306, -0.017352096736431122, 0.01662573032081127, 0.02142782136797905, -0.012529827654361725, 0.007835346274077892, 0.0009205690585076809, 0.0002019787352764979, -0.014285214245319366, 0.027010085061192513, -0.025691863149404526, -0.001713351346552372, 0.0018495451658964157, -0.00045944383600726724, 0.019692612811923027, 0.03717922046780586, -0.023109225556254387, -0.002301843138411641, 0.04455050081014633, -0.0006099295569583774, 0.02001544088125229, -0.009846306405961514, 0.018253330141305923, 0.013572298921644688, -0.0054040346294641495, -0.007505790796130896, -0.020862869918346405, 0.005071116611361504, 0.005888279061764479, -0.016867851838469505, -0.016289448365569115, 0.025409387424588203, 0.02520762011408806, 0.0036318341735750437, -0.03443516790866852, -0.012153193354606628, 0.05001170188188553, -0.009294805116951466, -0.018239878118038177, 0.0003898757277056575, -0.003847053973004222, 0.018979696556925774, -0.010323825292289257, 0.03214845806360245, -0.0076739308424293995, -1.5237729712680448e-05, 0.012630712240934372, -0.032847922295331955, 0.07694108039140701, 0.0038840449415147305, -0.000342586194165051, 0.038120806217193604, -0.003766346490010619, -0.012610535137355328, -0.13634175062179565, 0.0012921595480293036, 0.0176076702773571, 0.017244486138224602, -0.012267529033124447, 0.01203885767608881, -0.003320774296298623, 0.0005103063303977251, -0.006513761822134256, 0.025503545999526978, -0.011393197812139988, 0.003628471400588751, -0.005034125875681639, -0.0023875946644693613, 0.010754264891147614, 0.007351101376116276, 0.000321358471410349, 0.0021068002097308636, -0.017594218254089355, 0.01340415794402361, -0.002313612960278988, -0.008124547079205513, -0.009550378657877445, -0.01372698787599802, 0.023580020293593407, -0.021374017000198364, -0.05124921724200249, 0.012913187965750694, -0.0019252083729952574, 0.00757304672151804, -0.009536927565932274, -0.02425258234143257, 0.05033453181385994, -0.0010693733347579837, -0.01920836791396141, -0.029189186170697212, -0.00879038404673338, -0.018804829567670822, 0.03187943249940872, 0.0015073793474584818, 0.01359920110553503, 0.022423213347792625, 0.02066110074520111, -0.01269124262034893, 0.00887781661003828, -0.014756008051335812, -0.008904719725251198, -0.0002459054521750659, 0.0026835219468921423, 0.002236268250271678, -0.024212228134274483, -0.012206997722387314, -0.005239257123321295, 0.008911444805562496, 0.023620372638106346, -0.014877068810164928, 0.002454850822687149, 0.025194168090820312, 0.006207745987921953, -0.016249094158411026, -0.01229443121701479, 0.014258312061429024, -0.025113461539149284, 0.002892016200348735, 0.02219454199075699, -0.018616512417793274, -0.014984678477048874, -0.028462819755077362, 0.020634198561310768, -0.020365172997117043, -0.003877319162711501, 0.014782910235226154, -0.0005166116170585155, -0.01939668506383896, -0.011393197812139988, 0.03427375480532646, -0.023593470454216003, -0.020432429388165474, -0.011245234869420528, -0.017257938161492348, -0.013747164979577065, -0.022961262613534927, 0.023257190361618996, -0.03424685075879097, 0.036264535039663315, 0.01390857994556427, -0.00312741263769567, 0.00491979019716382, -0.013760616071522236, -0.0027692734729498625, -0.027010085061192513, 0.029108477756381035, 0.006601194851100445, 0.015697594732046127, -0.01526715513318777, 0.01662573032081127, -0.012415491975843906, -0.0010340638691559434, 0.014096897095441818, 0.013740438967943192, -0.02883945405483246, -0.01736554689705372, -0.05079187452793121, -0.0038201515562832355, -0.0038672308437526226, -0.012543278746306896, 0.003890770487487316, -0.029000869020819664, 0.0020244112238287926, 0.025247972458600998, -0.0012610534904524684, -0.009563829749822617, -0.02926989272236824, 0.0025590979494154453, -0.02945820987224579, -0.014177604578435421, -0.01801120676100254, -0.03309004381299019, 0.05805554240942001, -0.009395689703524113, 0.019235270097851753, 0.038658857345581055, 0.00424050260335207, 0.005703324917703867, 0.015388215892016888, 0.0330362394452095, -0.029108477756381035, -0.01022294070571661, -0.022759493440389633, 0.024857887998223305, -0.013289823196828365, 0.0013081328943371773, -0.010518867522478104, -0.040730345994234085, -0.02048623561859131, -0.008992152288556099, 0.002416178584098816, -0.03505392372608185, 0.00887781661003828, 0.0012055671541020274, 0.02448125183582306, 0.016679534688591957, -0.015630338340997696, -0.034811802208423615, 0.016289448365569115, -0.0037428068462759256, 0.017042718827724457, 0.015778301283717155, -0.029000869020819664, 0.007647028658539057, 0.016975462436676025, -0.012280980125069618, 0.04315156862139702, 0.01022294070571661, 0.01619528979063034, -0.011608418077230453, -0.0026583008002489805, 0.009476397186517715, 0.0055620865896344185, -0.007761364337056875, 0.012960267253220081, 0.0011895938077941537, 0.03661426901817322, 0.0037629837170243263, -0.004630588460713625, -0.001298044458962977, -0.008084193803369999, 0.022355956956744194, -0.009859757497906685, 0.0046339514665305614, -0.003497321857139468, -0.029296794906258583, -0.03613002598285675, 0.011904344893991947, 0.007243491243571043, -0.0005422530230134726, 0.024575410410761833, 0.005992526188492775, -0.008299414068460464, -0.009577280841767788, -0.00446581095457077, 0.00873657874763012, 0.028489721938967705, 0.001126541174016893, -0.014285214245319366, 0.020620746538043022, 0.0297810398042202, 0.004889525007456541, -0.0010618070373311639, 0.02582637593150139, 0.01572449691593647, 0.0076739308424293995, 0.014083446003496647, 0.026525840163230896, 0.005814297590404749, 0.02136056497693062, 0.0063590724021196365, 0.005124921444803476, -0.01895279437303543, 0.010922404937446117, -0.011675674468278885, 0.03672187775373459, -0.0014476894866675138, -0.014554238878190517, -0.00489288754761219, -0.014984678477048874, -0.026445133611559868, 0.0014670256059616804, -0.01196487620472908, -0.017567316070199013, 0.011648771353065968, -0.0011122492142021656, 0.01274504791945219, 0.000629686051979661, 0.009779050014913082, -0.002081579063087702, 0.006053057033568621, -0.007122430484741926, -0.012032131664454937, -0.03021148033440113, -0.013323450461030006, 0.04640676826238632, 0.026391327381134033, 0.013619378209114075, 0.005992526188492775, 0.012516376562416553, 0.0051081073470413685, 0.00891817081719637, 0.025584254413843155, -0.014204506762325764, 0.019437039270997047, -0.03198704123497009, 0.010633203200995922, -0.001169416937045753, -0.033385973423719406, -0.017298292368650436, 0.01844164729118347, -0.013377255760133266, -0.010700459592044353, 0.029027771204710007, -0.026498937979340553, 0.09781739860773087, 0.008131273090839386, -0.022167639806866646, 0.009530201554298401, -0.04008468985557556, 0.02320338413119316, -0.005632705986499786, 0.008628969080746174, -0.022651884704828262, -0.053697340190410614, 0.02277294546365738, -0.023768337443470955, 0.01928907446563244, -0.028516624122858047, -0.007768089883029461, 0.019329428672790527, 0.0002717570459935814, 0.010760989971458912, -0.017338644713163376, -0.010902227833867073, 0.011574789881706238, -0.01965225860476494, -0.010855148546397686, 0.0027844063006341457, -0.014392823912203312, -0.005165275186300278, 0.031610406935214996, -0.017096523195505142, 0.006981192156672478, -0.015334410592913628, 0.027548134326934814, 0.00014764834486413747, -0.017553865909576416, -0.007156058214604855, 0.026754511520266533, -0.01779598742723465, -0.025301778689026833, -0.02080906555056572, 0.013713536784052849, 0.02610885165631771, 0.006940838415175676, 0.0047012073919177055, -0.0016091043362393975, -0.013962384313344955, -0.0012526465579867363, -0.005531821399927139, -0.002816352993249893, -0.007848797366023064, -0.013545396737754345]} +{"id": "test:10000054", "text": "\"Know Buckeye Trail Class of 2001 graduates that are NOT on this List? Help us Update the 2001 Class List by adding missing names.\\nMore 2001 alumni from Buckeye Trail HS have posted profiles on Classmates.com\u00ae. Click here to register for free at Classmates.com\u00ae and view other 2001 alumni.\\nAlumni from the Buckeye Trail High School class of 2001 that have been added to this alumni directory are shown on this page. All of the people on this page graduated in '01 from Buckeye Trail . You can register for free to add your name to the BTHS alumni directory.\"", "vector_field": [-0.02488471008837223, -0.02446204051375389, -0.017435146495699883, -0.02763206698000431, -0.019680581986904144, 0.04150093346834183, -0.0002916177036240697, 0.004930051974952221, -0.01509725209325552, -0.017910651862621307, -0.013505634851753712, -0.02637726441025734, -0.008684552274644375, -0.01771252416074276, 0.010289378464221954, -0.01544067170470953, 0.008123193867504597, 0.007759961299598217, 0.02744714729487896, 0.011729098856449127, -0.002187648555263877, -0.003292204812169075, 0.0036587391514331102, -0.0036950623616576195, -0.015242544934153557, -0.005772090516984463, 0.025413047522306442, -0.007244831882417202, 0.00921949464827776, -0.023180820047855377, -0.0012036195257678628, 0.030062420293688774, -0.023352529853582382, -0.008136401884257793, -0.003602603217586875, -0.012435751035809517, -0.006521669682115316, 0.016589807346463203, 0.03764406591653824, 0.013182028196752071, 0.02201847732067108, 0.0023378946352750063, -0.03154176473617554, -0.017884233966469765, -0.009113826788961887, -0.004256421234458685, 0.004362089093774557, -0.015638798475265503, -0.016933226957917213, 0.035979803651571274, -0.01282539963722229, -0.01736910454928875, -0.002085283165797591, -0.03819882124662399, 0.010672423057258129, 0.003787521505728364, 0.022507190704345703, 0.017804984003305435, -0.02145051397383213, -0.015823716297745705, 0.015004793182015419, 0.015004793182015419, -0.03737989813089371, -0.021622223779559135, -0.005484806839376688, -0.0029768531676381826, -0.009648769162595272, -0.012706523761153221, 0.001506588188931346, -0.0029273214749991894, 0.026284804567694664, -0.006653754040598869, 0.012528209947049618, 0.023788409307599068, 0.026324430480599403, -0.00990633387118578, 0.0068551828153431416, -0.01851823925971985, 0.009463850408792496, -0.009846895933151245, 0.019482456147670746, -0.024937544018030167, -0.009952562861144543, 0.011577202007174492, 0.030300172045826912, 0.006122114136815071, 0.03236068785190582, 0.03629680722951889, -0.001684902235865593, -0.03558355197310448, -0.008460008539259434, 0.0021975550334900618, -0.016629431396722794, 0.010870549827814102, 0.005923987366259098, 0.011643243953585625, -0.0042366087436676025, 0.020631590858101845, 0.011755515821278095, -0.011676264926791191, -0.0006567073869518936, 0.010619589127600193, -0.011623430997133255, -0.0008214001427404583, -0.024673374369740486, -0.0040516904555261135, 0.0073306867852807045, -0.007456167135387659, 0.00038118744851090014, -0.008043942973017693, -0.03278335928916931, 0.031700268387794495, -0.0035299567971378565, -0.014859500341117382, -0.013432987965643406, 0.009305349551141262, -0.019759833812713623, -0.0035893949680030346, 0.014423621818423271, -0.04641447588801384, 0.043376531451940536, -0.008790220133960247, 0.009173264726996422, -0.013380154967308044, -0.01789744198322296, 0.011114906519651413, -0.013472613878548145, 0.010857341811060905, 0.009794062003493309, 0.0012193045113235712, 0.017567232251167297, -0.0006104778149165213, 0.020182503387331963, 0.0007986981654539704, 0.011828162707388401, 0.020473089069128036, 0.018531447276473045, 0.011504555121064186, -0.03146251663565636, -0.02522812969982624, 0.005696141626685858, 0.011610222980380058, 0.017197394743561745, -0.0024419112596660852, -0.02205810323357582, 0.03484387695789337, 0.03936116397380829, 0.008651531301438808, 0.01771252416074276, -0.015718048438429832, -0.0036521349102258682, 0.0059173833578825, 0.004659278783947229, -0.006769327912479639, -0.006174948066473007, -0.007931671105325222, 0.0076741063967347145, 0.009708207100629807, 0.0017336084274575114, -0.009364787489175797, -0.00465597677975893, 0.0026235273107886314, 0.03531938046216965, -0.013003713451325893, 0.008235465735197067, 0.022493980824947357, 0.004613049328327179, -0.02538663148880005, 0.026667850092053413, -0.005778694525361061, 0.012600855901837349, 0.019522082060575485, -0.04512004554271698, -0.002194252796471119, 0.005157897714525461, 0.00818923581391573, -0.013921700417995453, 0.023999743163585663, -0.014727415516972542, -0.01618034392595291, 8.67320122779347e-05, 0.0015618986217305064, 0.047391898930072784, 0.04263686016201973, -0.03389286994934082, 0.01771252416074276, 0.0058876643888652325, -0.0004697253170888871, 0.022190187126398087, -0.028609491884708405, 0.005646610166877508, 0.003609207458794117, -0.009027971886098385, 0.0035464675165712833, -0.6356960535049438, -0.008559072390198708, 0.005663120653480291, -0.019152244552969933, 0.009496871381998062, 0.029798250645399094, 0.007106143049895763, 0.013829241506755352, -0.03193802013993263, 0.005441879387944937, -0.022427938878536224, 0.03365511819720268, -0.00334999174810946, -0.001226734253577888, -0.010447879321873188, -0.00602965522557497, 0.03389286994934082, 0.001115288003347814, 0.023880867287516594, -0.025848926976323128, -0.008915700018405914, -0.01376319956034422, 0.013155611231923103, 0.013386758975684643, 0.0035695822443813086, 0.013842450454831123, 0.025241337716579437, -0.02591496892273426, -0.018755991011857986, 0.0012539767194539309, -0.015031210146844387, 0.02607347071170807, 0.019746625795960426, 0.008922304026782513, 0.051011014729738235, 0.0028860452584922314, -0.035477884113788605, 0.02362990751862526, 0.005124876741319895, 0.027737734839320183, -0.0156520064920187, -0.028767991811037064, 0.025360213592648506, 0.013195236213505268, -0.002473281230777502, 0.039070580154657364, 0.0002608667709864676, -0.0012605809606611729, 0.014357579872012138, -0.040206506848335266, 0.02355065755546093, 0.002052262192592025, -0.02423749677836895, 0.012363104149699211, 0.0012201301287859678, -0.008275090716779232, 0.019323954358696938, 0.014965168200433254, -0.013816033490002155, -0.003804031992331147, -0.024131828919053078, 0.025954594835639, 0.0008709318353794515, 0.008255277760326862, -0.0063895853236317635, -0.009371391497552395, -0.04578046873211861, 0.02839815616607666, 0.03946683183312416, 0.0016015239525586367, 0.010315795429050922, -0.0014644863549619913, 0.01488591730594635, 0.022071311250329018, 0.013234861195087433, 0.0011780281784012914, 0.008182631805539131, -0.017276646569371223, -0.024409206584095955, 0.014965168200433254, 0.007581647485494614, -0.027922652661800385, -0.015202919952571392, -0.031198346987366676, 0.009034575894474983, -0.013789616525173187, 0.00997897982597351, -0.001182981301099062, -0.0042465147562325, 0.046889979392290115, -0.006293823942542076, -0.011597014032304287, -0.0025211619213223457, -0.03764406591653824, -0.0028398155700415373, 0.002874487778171897, -0.010282774455845356, 0.026971643790602684, 0.014555705711245537, -0.015704840421676636, -0.002702777972444892, -0.008935512974858284, 0.015348212793469429, -0.002816700842231512, -0.006115510128438473, 0.025439465418457985, 0.004365391097962856, 0.017276646569371223, 0.05600380524992943, -0.0005411334568634629, -0.010870549827814102, -0.026747100055217743, 0.0075288135558366776, 0.007092935033142567, -0.0024848387110978365, -0.026879185810685158, 0.0008486425504088402, 0.004854103550314903, 0.014608539640903473, -0.023352529853582382, -0.002783679636195302, 0.009701603092253208, -0.018121985718607903, -0.027658483013510704, 0.030987011268734932, 0.019139036536216736, 0.023378947749733925, -0.0014991584466770291, -0.013604697771370411, 0.00041193838114850223, 0.006211271043866873, -0.028028320521116257, 0.026747100055217743, -0.01633884571492672, -0.01805594377219677, 0.01904657669365406, -0.006343355402350426, 0.005953706335276365, 0.004269629716873169, -0.013380154967308044, -0.0071853939443826675, 0.0005535163800232112, 0.003909699618816376, -0.0336022824048996, -0.0006022225134074688, -0.00470550823956728, -0.0017550720367580652, 0.002453468507155776, -0.024713000282645226, -0.008109984919428825, -0.0029025557450950146, 0.0012292108731344342, -0.020948592573404312, -0.0040781074203550816, -0.003097380278632045, -0.01920507848262787, -0.015044418163597584, 0.0038931891322135925, 0.019931543618440628, 0.0017649783985689282, 0.02916424535214901, 0.03946683183312416, -0.001916875597089529, 0.01916545256972313, 0.02030137926340103, -0.04068201035261154, 0.016748307272791862, 0.02377520129084587, 0.02359028160572052, -0.02874157577753067, 0.030907761305570602, -0.0107846949249506, -0.013195236213505268, 0.008592093363404274, 0.03003600426018238, -0.006049467716366053, -0.0004276233958080411, 0.0007392601692117751, -0.009496871381998062, -0.0029801554046571255, -0.01935037225484848, 0.002030798466876149, -0.014846292324364185, 0.0021117001306265593, -0.0037545005325227976, 0.016365263611078262, -0.009437433443963528, 0.017844608053565025, -0.013109381310641766, 0.02045988105237484, -0.015308587811887264, 0.000203389412490651, -0.02374878339469433, 0.026086678728461266, 0.009569518268108368, -0.0025294171646237373, 0.012581043876707554, 0.007746752817183733, 0.0018656927859410644, 0.02240152284502983, 0.00990633387118578, 0.012818795628845692, 0.049029745161533356, -0.015585964545607567, 0.019905125722289085, -0.024567706510424614, -0.022533606737852097, -0.028001902624964714, 0.017818192020058632, -0.010639402084052563, 0.010837528854608536, -0.014344370923936367, -0.022731732577085495, -0.025214921683073044, 0.005593776237219572, 0.01809556968510151, 0.008618510328233242, 0.027077311649918556, -0.03188518434762955, -0.015678424388170242, 0.00898834690451622, -0.016008634120225906, 0.0030676613096147776, 0.024554498493671417, 0.02205810323357582, 0.024554498493671417, 0.019667373970150948, 0.021397680044174194, -0.007535417564213276, 0.003373106475919485, 0.02573005110025406, 0.005431972909718752, 0.018306903541088104, 0.029666166752576828, 0.02324686199426651, -0.01599542610347271, 0.042346272617578506, -0.006518367677927017, 0.031779516488313675, 0.012964088469743729, 0.006640545558184385, 0.015070835128426552, -0.008915700018405914, 0.01236970815807581, 0.03851582482457161, -0.005933893844485283, -0.01690680906176567, -0.0034110809210687876, 0.014436829835176468, -0.006178250070661306, 0.011497951112687588, 0.02672068402171135, 0.007654293905943632, -0.008341132663190365, 0.013505634851753712, -0.015361420810222626, 0.003972439561039209, 0.021648641675710678, 0.004441339522600174, 0.028873659670352936, -0.0031056355219334364, 0.0027539606671780348, 0.026694266125559807, -0.04218777269124985, 0.01240273006260395, 0.032730527222156525, -0.006254198495298624, 0.01091677974909544, -0.021595807746052742, -0.021886393427848816, 0.004315859172493219, -0.023867659270763397, -0.005098459776490927, 0.006769327912479639, 0.017527606338262558, -0.01112811453640461, 0.010949800722301006, 0.0021793933119624853, 0.014251912012696266, -0.012376313097774982, -0.023444989696145058, -0.013908492401242256, -0.010903570801019669, 0.02217697910964489, -0.016021843999624252, -0.00910061877220869, 0.00917986873537302, -0.007264644373208284, -0.022916652262210846, 0.003780917264521122, -0.02950766496360302, 0.014238703064620495, -0.003160120453685522, 0.022454356774687767, 0.03291544318199158, 0.0026532462798058987, 0.015942592173814774, -0.0022256230004131794, -0.0032674390822649, -0.0028381645679473877, -0.018386155366897583, 0.012455563060939312, -0.028292488306760788, -0.029534082859754562, 0.02668105810880661, -0.015585964545607567, -0.012231020256876945, 0.01024975348263979, 0.01055354718118906, 0.014740624465048313, 0.013551863841712475, -0.014014160260558128, 0.005303190555423498, 0.006888203788548708, -0.02565079927444458, 0.012508396990597248, -0.0037578025367110968, 0.0009369740728288889, 0.02145051397383213, 0.033417366445064545, -0.019918333739042282, -0.02419787086546421, -0.049716584384441376, 0.006686775013804436, 0.09304028749465942, 0.0020836321637034416, -0.017237020656466484, 0.005072042811661959, -0.02993033640086651, 0.008849658071994781, -0.015810508280992508, -0.026047052815556526, -0.019601332023739815, 0.00027201141347177327, -0.016880393028259277, -0.002176091307774186, -0.007997713051736355, 0.006901412270963192, -0.00426632771268487, 0.0017088425811380148, 0.0064721377566456795, -0.0021282106172293425, -0.008664739318192005, -0.02244114689528942, -0.021041052415966988, 0.02195243537425995, 0.0027506586629897356, 0.03133043274283409, 0.004137545358389616, 0.002230576006695628, 0.017157768830657005, -0.0022388314828276634, 0.04477662593126297, -0.0071325600147247314, -0.014595331624150276, 0.0007772344397380948, 0.006280615460127592, 0.01170268189162016, -0.009708207100629807, 0.028873659670352936, 0.003262485843151808, 0.009424225427210331, 0.031013429164886475, 0.002836513565853238, 0.019099410623311996, 0.002537672407925129, -0.0014735670993104577, -0.014476455748081207, 0.016365263611078262, -0.018755991011857986, 0.004299348685890436, -0.0011466580908745527, 0.032387107610702515, -0.007892046123743057, 0.00978745799511671, 0.012448959052562714, -0.005260263103991747, -0.00999879278242588, -0.007542022038251162, -0.023009110242128372, 0.014172661118209362, 0.012105539441108704, -0.013452800922095776, -0.017871025949716568, -0.03751198202371597, -0.04622955620288849, 0.021516555920243263, -0.009384600445628166, -0.010692236013710499, -0.018188027665019035, -0.015969010069966316, -0.005795205011963844, -0.0040252734906971455, -0.0060461657121777534, 0.008446800522506237, 0.004728623200207949, -0.007522209081798792, -0.007495792582631111, 0.012739544734358788, 0.018993742763996124, 0.016669057309627533, 0.0006158437463454902, 0.027315063402056694, 0.03148893266916275, -0.016246387735009193, -0.03339094668626785, 0.006822161842137575, -0.01748798042535782, 0.017435146495699883, 0.021292012184858322, -0.01205931045114994, -0.0073042698204517365, -0.004124336875975132, 0.030696425586938858, -0.012964088469743729, -0.035398632287979126, -0.0017682805191725492, -0.0036653433926403522, 0.006330146919935942, -0.0071853939443826675, 0.03373436629772186, 0.024871502071619034, 0.0069608502089977264, -0.009529893286526203, -0.006617430597543716, -0.003962533548474312, 0.02477904222905636, -0.03275694325566292, -0.0005745673552155495, 0.010111064650118351, -0.01057336013764143, 0.011075280606746674, -0.004685695748776197, -0.006693379487842321, 0.017923859879374504, 0.0007792982505634427, 0.013895283453166485, 0.007462771143764257, -0.02286381833255291, 0.004718716721981764, 0.014833083376288414, 0.009661977179348469, -0.004688997752964497, -0.027816984802484512, 0.005431972909718752, -0.019403206184506416, 0.02736789733171463, 0.003929512109607458, -0.044486042112112045, 0.004652674775570631, -0.011339450255036354, -0.019601332023739815, 0.011537576094269753, 0.003955929074436426, -0.020208921283483505, -0.01179514080286026, 0.012118748389184475, -0.013254674151539803, -0.013366946019232273, -0.006006540264934301, -0.017276646569371223, 0.04247835651040077, 0.02019571140408516, 0.01522933691740036, -0.016748307272791862, 0.011755515821278095, -0.0021381170954555273, -0.0007124305120669305, -0.016497347503900528, -0.015969010069966316, -0.004748435690999031, 0.00042679786565713584, 0.026350846514105797, 0.011709285899996758, 0.00686178682371974, -0.004121034871786833, -0.031436096876859665, -0.014687790535390377, 0.007066518068313599, -0.04865990951657295, -0.03671947494149208, 0.015335004776716232, -0.0030775675550103188, -0.012871629558503628, -0.002133163856342435, -0.03370795026421547, 0.0036884581204503775, 0.0029025557450950146, -0.0017583741573616862, 0.021939227357506752, -0.03851582482457161, 0.00759485550224781, -0.01591617614030838, 0.02068442478775978, -0.011445117183029652, 0.010705444030463696, 0.024448830634355545, 0.020552340894937515, -0.013241466134786606, 0.017778566107153893, -0.02057875692844391, -0.00692782923579216, -0.0048970310017466545, -0.06482704728841782, 0.0010187013540416956, -0.001809556968510151, -0.0024881407152861357, 0.025558341294527054, -0.023907285183668137, 0.008829845115542412, 0.03413062170147896, 0.0021744403056800365, -0.015176502987742424, -0.011517764069139957, 0.03159460052847862, -0.0036851561162620783, 0.028028320521116257, -0.004629559814929962, -0.0007433877908624709, 0.005682933144271374, -0.010527130216360092, -0.04025933891534805, 0.0012011429062113166, 0.002321384148672223, 7.723844464635476e-05, 6.552627019118518e-05, 0.008565676398575306, -0.011900808662176132, -0.008763803169131279, 0.010969613678753376, -0.03875357657670975, 0.01305654738098383, -0.009609143249690533, 0.007238227874040604, -0.002256992971524596, -0.03661380708217621, -0.02641689032316208, -0.006822161842137575, 0.028873659670352936, -0.014648165553808212, 0.0015709793660789728, 0.013565072789788246, 0.004900333005934954, -0.005953706335276365, 0.013089568354189396, -0.03037942200899124, 0.027816984802484512, 0.028450990095734596, 0.02580930106341839, 0.013320716097950935, -0.014304745942354202, -0.01874278299510479, -0.004054992459714413, 0.007872233167290688, 0.008783616125583649, 0.016206761822104454, 0.012151769362390041, -0.000869280775077641, 0.002220669761300087, 0.010480901226401329, 0.0021992060355842113, -0.006442418787628412, -0.017738942056894302, 0.0020671216771006584, 0.03304752707481384, -0.005448483396321535, -0.0041837748140096664, -0.021516555920243263, -0.04179151728749275, 0.02415824495255947, -0.0002621050807647407, 0.0078061907552182674, -0.008882679045200348, -0.0229034423828125, -0.018716366961598396, 0.03571563586592674, -0.029956752434372902, 0.025096045807003975, -0.006016446743160486, -0.0023428478743880987, -0.00323441787622869, -0.027420731261372566, 0.02744714729487896, -0.020869342610239983, -0.0069872671738266945, 0.019442830234766006, -0.02510925382375717, 0.020486297085881233, 0.026179136708378792, 0.0003813938528764993, -0.015427463687956333, -0.003787521505728364, 0.01553313061594963, 0.025069627910852432, -0.007099539041519165, -0.005712652578949928, -0.00624099001288414, 0.016286011785268784, -0.0004218447138555348, 0.022626066580414772, -8.337830513482913e-05, -0.014040576294064522, -0.032651275396347046, -0.010850737802684307, 0.03640247508883476, 0.010157293640077114, 0.018214445561170578, 0.00015736623026896268, -0.005431972909718752, -0.014648165553808212, -0.013347133062779903, -0.005422066431492567, -0.011293220333755016, -0.04408978670835495, -0.011385679244995117, -0.011002634651958942, 0.0058744559064507484, 0.026047052815556526, -0.0030032701324671507, 0.013776407577097416, 0.009113826788961887, 0.009206285700201988, -0.051777102053165436, 0.010408254340291023, -0.005957008805125952, 0.011266803368926048, -0.030987011268734932, 0.006478742230683565, 0.010817715898156166, -0.017052102833986282, 0.002595459343865514, -0.019403206184506416, -0.027473565191030502, -0.007997713051736355, 0.001893760752864182, 0.002887696260586381, 0.016893601045012474, -0.01011766865849495, 0.026958435773849487, -0.01479345839470625, 0.0065612951293587685, -0.008479821495711803, -0.04575405269861221, 0.028239654377102852, -0.013895283453166485, 0.00042349574505351484, -0.010177106596529484, 0.007059913594275713, 0.002684616483747959, 0.0015189711702987552, 0.02294306829571724, 0.008076963946223259, -0.022388312965631485, -0.020710840821266174, -0.009503476321697235, 0.00014271312102209777, 0.013459404930472374, -0.00312049500644207, -0.013882075436413288, -0.014872708357870579, -0.0008560723508708179, 0.02351103164255619, -0.019495664164423943, -0.008922304026782513, -0.0005299888434819877, 0.012607460841536522, 0.012211207300424576, 0.03851582482457161, 0.005478202365338802, -0.01847861334681511, 0.017884233966469765, -0.02443562261760235, -0.016550181433558464, 0.0019234797218814492, 0.00025962849031202495, 0.018069151788949966, 0.011200761422514915, -0.02900574542582035, 0.008090171962976456, -0.02019571140408516, -0.039598915725946426, -0.013578280806541443, 0.01385565847158432, 9.127241355599836e-05, 0.0096355602145195, 0.013043339364230633, -0.006359866354614496, 0.006247594486922026, -0.008466613478958607, 0.00805715098977089, -0.026179136708378792, -0.006756119430065155, 0.014819875359535217, 0.02950766496360302, 0.004989489912986755, 0.006359866354614496, -0.02377520129084587, 0.006637243553996086, 0.006254198495298624, -0.003949325066059828, -0.007667502388358116, 0.010758277960121632, -0.03730064630508423, 0.002258643973618746, 0.01194703858345747, -0.03500237688422203, -0.03053792379796505, 0.02298269420862198, -0.0025756466202437878, -0.038991328328847885, -0.014595331624150276, -0.002022542990744114, -0.007310874294489622, 0.005940497852861881, -0.0033268770202994347, -0.0035860927309840918, 0.009450642392039299, 0.008460008539259434, 0.0020539131946861744, -0.00720520643517375, -0.009893124923110008, 0.006587711628526449, -0.00032752816332504153, 0.01398774329572916, -0.013604697771370411, 0.019865501672029495, 0.010190315544605255, -0.01664264127612114, 0.010500713251531124, 0.003338434500619769, 0.0024105410557240248, -0.00596361281350255, -0.0030181296169757843, -0.02366953343153, 0.009767645038664341, -0.006742910947650671, -0.012515000998973846, -0.044486042112112045, 0.0068419743329286575, -0.007885441184043884, -0.0026796632446348667, -0.035636384040117264, 0.016497347503900528, 0.023867659270763397, -0.02645651437342167, -0.020512714982032776, -0.004289442207664251, -0.045357801020145416, 0.011742307804524899, -0.01931074634194374, -0.010850737802684307, -0.0213712640106678, 0.00908740982413292, 0.023920493200421333, -0.01656338945031166, -0.02225622907280922, 0.009761041030287743, -0.009034575894474983, -0.022995902225375175, 0.009120430797338486, 0.20097969472408295, 0.001971360296010971, 0.011346054263412952, 0.030934177339076996, -0.0014801713405176997, 0.008486425504088402, -0.002867883536964655, 0.015902968123555183, -0.03294186294078827, 0.013697157613933086, -0.014608539640903473, 0.0213712640106678, -0.024290330708026886, 0.0033516427502036095, -0.002456770744174719, -0.027684900909662247, -0.01607467792928219, -0.021014636382460594, -0.01614071987569332, 0.003062708070501685, -0.006297125946730375, 0.005461691878736019, -0.0020506109576672316, -0.00315351621247828, 0.05463012680411339, 0.01621996983885765, -0.000579520536120981, 0.030564341694116592, 0.028926493600010872, 0.009873312897980213, -0.023999743163585663, -0.01398774329572916, -0.008499634452164173, -0.019403206184506416, -0.02969258464872837, 0.014978376217186451, 0.023497823625802994, -0.011253595352172852, -0.0004519764625001699, 0.013049943372607231, 0.004646070301532745, 0.013459404930472374, -0.009694998152554035, -0.012851816602051258, -0.015282170847058296, 0.025743259117007256, 0.011762119829654694, -0.012356500141322613, -0.0029372279532253742, 0.006756119430065155, -0.028688741847872734, -0.009160056710243225, 0.006531575694680214, -0.0013324018800631166, -0.010712048970162868, 0.0008325447561219335, 0.04115751385688782, 0.011346054263412952, -0.014357579872012138, 0.014185870066285133, -0.021582597866654396, 0.03732706606388092, -0.00999879278242588, 0.03730064630508423, -0.018372947350144386, -0.0006381329731084406, -0.020565548911690712, -0.011068676598370075, -0.005171106196939945, -0.008024130016565323, -0.008255277760326862, 0.007641085423529148, 0.006174948066473007, -0.006756119430065155, -0.0006389585323631763, -0.008466613478958607, -0.02114672027528286, 0.013617906719446182, 0.0036917603574693203, 0.011048863641917706, -0.027684900909662247, -0.026892393827438354, -0.004520590417087078, -0.024171452969312668, -0.015427463687956333, -0.02786981873214245, 0.029877502471208572, -0.016391679644584656, 0.02702447772026062, 0.0027126844506710768, 0.00851284246891737, 0.03354945033788681, -0.020935384556651115, -0.0025690426118671894, 0.006369772367179394, 0.030247338116168976, -0.0053362115286290646, 0.015401046723127365, 4.715827526524663e-05, 0.03405136987566948, -0.0012614064617082477, 0.017237020656466484, -0.01882203295826912, -0.011933829635381699, -0.0015751069877296686, -0.026786725968122482, -0.023194028064608574, 0.005976821295917034, 0.0036719476338475943, -0.014436829835176468, 0.006749515421688557, -0.02374878339469433, 0.0031931414268910885, -0.005194220691919327, 0.01295748446136713, 0.001910271355882287, -0.004365391097962856, 0.010910175740718842, 0.02813398651778698, -0.015216127969324589, 0.023352529853582382, -0.015308587811887264, 0.00420028530061245, 0.03500237688422203, -0.019865501672029495, -0.01135265827178955, -0.01785781793296337, 0.011035655625164509, 0.011213969439268112, -0.040180087089538574, 0.006901412270963192, -0.0063268449157476425, 0.0030742655508220196, 0.0007318303687497973, -0.005966914817690849, -0.02813398651778698, -0.01870315708220005, -0.01522933691740036, -0.021648641675710678, 0.006518367677927017, -0.004613049328327179, -0.014397204853594303, -0.01935037225484848, -0.0006158437463454902, -0.01682755909860134, -0.041131097823381424, 0.02187318541109562, -0.035979803651571274, -0.008691156283020973, -0.021041052415966988, 0.013102777302265167, -0.005197523161768913, -0.024514872580766678, 0.008843054063618183, 0.0370364785194397, -0.023537447676062584, -0.03962533548474312, -0.04179151728749275, -0.016589807346463203, 0.01053373422473669, 0.008658135309815407, 0.017355896532535553, 0.0272886473685503, -0.04403695464134216, -0.027499981224536896, -0.006359866354614496, -0.16589806973934174, 0.006254198495298624, -0.011781932786107063, -0.011544181033968925, 0.01091677974909544, -0.0005213207914493978, 0.025782885029911995, -0.007654293905943632, 0.006954246200621128, 0.011412096209824085, 0.02205810323357582, 0.001182981301099062, -0.005329607520252466, -0.004048388451337814, -0.04195002093911171, 0.0009782504057511687, -0.0030098743736743927, -0.00635656388476491, 0.045701220631599426, 0.0009881567675620317, 0.016893601045012474, 0.028530240058898926, 0.022507190704345703, -0.00107813929207623, 0.03019450418651104, 0.003605905454605818, 0.001317542395554483, 0.028847243636846542, 0.005299888551235199, -0.009628956206142902, 0.005990029778331518, 0.0035794884897768497, 0.013591489754617214, 0.0012407682370394468, -0.008347736671566963, -0.004005460999906063, -0.00908740982413292, -0.029058577492833138, -0.016167135909199715, 0.02831890620291233, -0.011398888193070889, -0.013228257186710835, 0.03236068785190582, -0.008770407177507877, 0.0096355602145195, 0.0027440544217824936, 0.02446204051375389, 0.00855246838182211, -0.006482044234871864, -0.041236765682697296, -0.011398888193070889, -0.013934909366071224, 0.024567706510424614, -0.0030049211345613003, -0.008810033090412617, 0.018267279490828514, 0.019072994589805603, -0.004101221915334463, 0.003629020182415843, 0.013697157613933086, -0.010969613678753376, -0.02355065755546093, -0.010163898579776287, 0.006102301646023989, -0.01874278299510479, -0.018372947350144386, -0.013135798275470734, -0.030141670256853104, -0.020222129300236702, 0.011464930139482021, -0.018452197313308716, -0.04472379386425018, 0.011808349750936031, 0.0188880767673254, -0.000535767525434494, 0.02068442478775978, -0.026879185810685158, -0.0023263373877853155, -0.0015817112289369106, 0.01912582851946354, -0.02504321187734604, 0.04305953159928322, -0.043191615492105484, 0.016880393028259277, -0.016312429681420326, -0.0005559929995797575, -0.010639402084052563, 0.002823305083438754, 0.014502872712910175, -0.018716366961598396, -0.02419787086546421, -0.036772310733795166, -0.028609491884708405, 0.0002546753385104239, 0.025148877874016762, 0.013591489754617214, -0.00887607503682375, -0.013749990612268448, 0.01135265827178955, -0.013842450454831123, 0.02439599670469761, 0.006822161842137575, -0.001906969235278666, 0.00756843900308013, 0.005422066431492567, -0.013267883099615574, 0.011861183680593967, 0.020473089069128036, 0.023801617324352264, 0.012125352397561073, 0.011438513174653053, -0.005157897714525461, 0.007905254140496254, 0.007297665812075138, 0.0014322907663881779, -0.004926749970763922, -0.016655849292874336, -0.010084647685289383, 0.01573125831782818, -0.00015117478324100375, 0.04522571340203285, -0.0073438952676951885, -0.009728019125759602, -0.00978745799511671, -0.0048574055545032024, -0.010546943172812462, -0.09409695863723755, 0.004850801546126604, 0.01732947863638401, 0.013261278159916401, -0.01001860573887825, 0.011616826988756657, 0.0013571677263826132, 0.014014160260558128, -0.04142168164253235, 0.025928176939487457, 0.0015585965011268854, -0.03201726824045181, 0.008611906319856644, -0.02839815616607666, 0.013241466134786606, 0.03304752707481384, 0.00931195355951786, -0.013162215240299702, -0.014700998552143574, 0.02332611382007599, 0.001479345839470625, -0.0026978247333317995, 0.020182503387331963, -0.011286616325378418, 0.012917858548462391, -0.020948592573404312, -0.032043687999248505, 0.005177710205316544, 0.020618382841348648, -0.010546943172812462, 0.0038700744044035673, -0.001883854391053319, 0.022969486191868782, -0.01181495375931263, -0.020618382841348648, 0.0038766786456108093, -0.023695949465036392, 0.007542022038251162, 0.011663056910037994, -0.011464930139482021, -0.009985583834350109, 0.005233846139162779, 0.03193802013993263, -0.00997897982597351, -0.006092395167797804, -0.027499981224536896, -0.004292744677513838, -0.008882679045200348, 0.007713731843978167, -0.04305953159928322, -0.021741099655628204, -0.005273471586406231, -0.024131828919053078, -0.018637115135788918, -0.008843054063618183, 0.002955389441922307, -0.025822509080171585, 0.019680581986904144, -0.013333925046026707, -0.00896853394806385, 0.0058876643888652325, 0.015744466334581375, -0.00761466845870018, 0.011200761422514915, 0.03209652006626129, 0.011015842668712139, 0.006874995306134224, -0.013934909366071224, 0.01882203295826912, -0.0038700744044035673, -0.0059702168218791485, 0.017606856301426888, -0.0038964911364018917, 0.015638798475265503, -0.008109984919428825, 0.021133512258529663, -0.028081152588129044, -0.020631590858101845, 0.010375233367085457, -0.02332611382007599, -0.006541482172906399, -0.015942592173814774, -0.0025723446160554886, -0.017382312566041946, -0.006098999176174402, -0.011861183680593967, -0.0007772344397380948, -0.003137005725875497, -0.0016130813164636493, -0.02718297950923443, -0.005976821295917034, 0.0048970310017466545, -0.002176091307774186, 0.0065348781645298, -0.02201847732067108, 0.009133639745414257, 0.03981025144457817, -0.011042259633541107, -0.012983901426196098, 0.02106746844947338, -0.01036202535033226, -0.004781457129865885, -0.07666181027889252, 0.022005269303917885, -0.007046705111861229, -0.028583073988556862, 0.007733544334769249, -0.0018772502662613988, 0.0025030001997947693, 0.01702568493783474, -0.001954849809408188, 0.024594124406576157, 0.002689569490030408, 0.012574438937008381, 0.013406571000814438, 0.015044418163597584, -0.017646482214331627, 0.0023395458701997995, 0.0362703874707222, -0.006184854079037905, 0.024950752034783363, 0.00841377954930067, -0.0038964911364018917, -0.01488591730594635, 0.01706531085073948, 0.016708683222532272, -0.01828048750758171, 0.011999871581792831, -0.0036785518750548363, 0.026628224179148674, -0.010421463288366795, -0.004375297110527754, 0.0240525770932436, -0.004794665612280369, 0.0004276233958080411, 0.007509001065045595, -0.0005935544613748789, -0.013360342010855675, 0.017646482214331627, 0.0108771538361907, 0.0033153195399791002, 0.015321795828640461, -0.009463850408792496, -0.005266867112368345, 0.007390124723315239, -0.01372357364743948, 0.0026400377973914146, 0.0030412443447858095, 0.005682933144271374, 0.002948785200715065, 0.009661977179348469, -0.006138624623417854, -0.0026796632446348667, 0.01828048750758171, 0.011497951112687588, -0.031119095161557198, 0.004243212752044201, -0.022586440667510033, -0.010348816402256489, -0.01931074634194374, 0.003282298566773534, -0.014119827188551426, 0.03806673735380173, 0.007291061338037252, 0.009536497294902802, 0.006501856725662947, 0.01595580205321312, 0.009404412470757961, 0.01102905161678791, 0.021239178255200386, -0.006564597133547068, -0.0013406571233645082, 0.000834195816423744, 0.015374629758298397, 0.007931671105325222, 0.018293695524334908, -0.008816637098789215, 0.02786981873214245, -0.010388441383838654, -0.01587655022740364, -0.027843400835990906, 0.023960119113326073, 0.007561834529042244, 0.012792378664016724, -0.04382561892271042, 0.025241337716579437, 0.013221653178334236, -0.019839083775877953, -0.014582122676074505, -0.014238703064620495, 0.028081152588129044, 0.0213712640106678, -0.028371740132570267, 0.01181495375931263, 0.01467458251863718, 0.021476931869983673, 0.009140243753790855, -0.027156561613082886, -0.004137545358389616, -0.011960246600210667, 0.004401714075356722, 0.011445117183029652, -0.033575866371393204, 0.01744835637509823, -0.007667502388358116, 0.009457246400415897, -0.022956276312470436, 0.0019366882042959332, -0.021173136308789253, -0.018267279490828514, -0.023828035220503807, -0.005190918687731028, 0.03415703773498535, -0.0263376384973526, -0.029534082859754562, 0.016655849292874336, -0.035477884113788605, -0.00965537317097187, 0.00669998349621892, -0.010758277960121632, 0.008651531301438808, 0.018425781279802322, 0.01372357364743948, 0.009648769162595272, 0.01637847162783146, 0.008664739318192005, 0.030141670256853104, -0.026007426902651787, 0.0261527206748724, -0.00481778010725975, -0.018465405330061913, 0.0069608502089977264, 0.0033351322636008263, -0.02133163809776306, -0.0017864421242848039, -0.011392283253371716, 0.029798250645399094, 0.005890966393053532, -0.015295378863811493, 0.040866926312446594, -0.00908740982413292, 0.04720698297023773, 0.031779516488313675, -0.0015189711702987552, 0.002382473321631551, -0.0007054135203361511, 0.017765358090400696, 0.007033496629446745, 0.01147153414785862, 0.00349693582393229, 0.0014603586168959737, 0.005498015321791172, -0.02294306829571724, -9.065327321877703e-05, -0.007376916240900755, 0.0013505634851753712, 0.0033532939851284027, 0.014383995905518532, -0.008975137956440449, 0.03336453065276146, -0.004860707558691502, -0.012429146096110344, -0.007832608185708523, 0.04305953159928322, 0.0009757738444022834, 0.014819875359535217, 0.009648769162595272, 0.013816033490002155, -0.010903570801019669, -0.0014009206788614392, -0.014846292324364185, 0.03502879664301872, 0.0013654229696840048, -0.015387837775051594, 0.001081441412679851, -0.014529288746416569, -0.045331381261348724, -0.008829845115542412, 0.013360342010855675, 0.014740624465048313, -0.028952911496162415, 0.0050423238426446915, -0.016352053731679916, -0.014278328977525234, -0.01238291710615158, -0.005389045458287001, -0.01599542610347271, -0.019878709688782692, -0.018306903541088104, -0.01250179298222065]} +{"id": "test:10000055", "text": "\"Irish crime thriller at its best, Stuart Neville is a fantastic writer. Careful, this will grab you and not let go. The Ghosts of Belfast is the first in the Jack Lennon series but this is one of those series that is loosley connected so reading out of order is not a problem.\\nHighly recommend any of his books!\\nFegan has been a \\\"hard man,\\\" an IRA killer in northern Ireland. Now that peace has come, he is being haunted day and night by twelve ghosts: a mother and infant, a schoolboy, a butcher, an RUC constable, and seven other of his innocent victims. In order to appease them, he's going to have to kill the men who gave him orders.\\n\\\"Perfect for summer\u2014especially if you want to be reminded of what a blessing it is to live in relatively peaceful times.\\\"\\n\\\"Stuart Neville is Ireland's answer to Henning Mankell.\\\"\"", "vector_field": [0.013146813958883286, -0.009400277398526669, -0.01932995580136776, -0.018244003877043724, -0.00641730148345232, 0.0019784700125455856, -0.015162613242864609, -0.019913656637072563, -0.040723223239183426, -0.023687342181801796, 0.007370903622359037, 0.01798609085381031, -0.010106146335601807, 0.0004797704750671983, 0.013560833409428596, 0.006556438747793436, 0.03491337597370148, 0.01122603565454483, -0.011327843181788921, -0.013655854389071465, -0.022642111405730247, 0.030895352363586426, -0.008660471998155117, -0.020348036661744118, 0.029103530570864677, -0.012963559478521347, 0.005127732641994953, -0.01444317027926445, -0.018949873745441437, -0.0021159108728170395, -0.004829095676541328, 0.010112933814525604, 0.006023643538355827, -0.012603837996721268, -0.04911220818758011, -0.011612906120717525, -0.026130734011530876, -0.019737189635634422, 0.01673724502325058, -0.0035429203417152166, 0.01736166700720787, -0.00013288072659634054, 0.006668427959084511, 0.0030220025219023228, 0.0016238385578617454, -0.0018834490329027176, 0.01688656397163868, 0.0025401110760867596, -0.011653629131615162, 0.0027861471753567457, 0.015393378213047981, 0.006637885235249996, 0.01090024970471859, -0.007391265127807856, -0.0035972180776298046, -0.02057880163192749, 0.001622141688130796, 0.013757661916315556, -0.00077162025263533, -0.024936186149716377, -0.006149206776171923, -0.02793613076210022, -0.03803548961877823, -0.005704645067453384, 0.009223810397088528, -0.004398107994347811, -0.02507193200290203, -0.02638864889740944, -0.008565451018512249, -0.01810825988650322, 0.04094041511416435, 0.011008844710886478, 0.0027589984238147736, 0.024352487176656723, 0.016805116087198257, -0.019112765789031982, -0.011443225666880608, -0.0049003614112734795, -0.010947760194540024, -0.0032391929998993874, 0.0068007782101631165, -0.010106146335601807, -0.032442834228277206, 0.04056033119559288, 0.014063086360692978, 0.024651125073432922, -0.00694670295342803, 0.02260138839483261, -0.006461418233811855, 0.006149206776171923, 0.01295677199959755, 0.018977021798491478, 0.016397884115576744, -0.0040248120203614235, -0.005215966142714024, 0.014660360291600227, -0.016682947054505348, 0.007866369560360909, 0.0009256049524992704, -0.015393378213047981, 0.008090347051620483, -0.00800890102982521, -0.013302919454872608, -0.00743877561762929, -0.016981584951281548, -0.01641145907342434, 0.006634491961449385, -0.01033691130578518, 0.02492261305451393, 0.035510651767253876, -0.012210180051624775, -0.0004048991249874234, 0.025506312027573586, -0.020497355610132217, 0.006719331722706556, -0.020809566602110863, 0.032768622040748596, -0.011986201629042625, -0.023538023233413696, -0.020619524642825127, 0.008592600002884865, -0.019995102658867836, -0.01022831629961729, -0.0381169356405735, 0.027094518765807152, 0.03412606194615364, -0.016940860077738762, -0.01166041661053896, -0.01292283646762371, -0.00861974898725748, 0.017443113029003143, -0.007812071591615677, 0.016180694103240967, -0.004788372199982405, -0.011395715177059174, 0.029402166604995728, -0.019099190831184387, -0.006396939512342215, -0.026958772912621498, -0.022614963352680206, 0.021990541368722916, 0.020171569660305977, -0.014619637280702591, -0.006054185796529055, 0.022913599386811256, 0.02853340469300747, -0.0032833099830895662, 0.01505401823669672, -0.005982920061796904, -0.010493016801774502, 0.018474768847227097, -0.02921212464570999, 0.0011750346748158336, -0.0034054794814437628, 0.010377634316682816, -0.006851682439446449, -0.004177523776888847, -0.0011487342417240143, -0.038877103477716446, -0.0235244482755661, -0.0055722943507134914, 6.850833597127348e-05, 0.0055824751034379005, 0.010784867219626904, 0.027175964787602425, 0.021216798573732376, 0.0026622808072715998, -0.007513434626162052, 0.004924116190522909, -0.0062917377799749374, 0.018963446840643883, 0.013336855918169022, -0.027148814871907234, 0.009441000409424305, 0.010940972715616226, 0.032687172293663025, -0.0035089843440800905, -0.00037202361272647977, -0.020565228536725044, -0.011802947148680687, 0.01411738432943821, 0.03830697759985924, 0.02980939857661724, 0.03314870223402977, -0.018868425861001015, 0.00017646730702836066, -0.008850513957440853, 0.0023042557295411825, 0.0019292626529932022, -6.866741750854999e-05, 0.000808949873317033, -0.013628705404698849, 0.0072962441481649876, -0.008497579023241997, -0.6324588060379028, -0.009841445833444595, 0.005280444398522377, 0.0022143253590911627, -0.0004920722567476332, 0.017972515895962715, 0.026334350928664207, 0.01487755123525858, -0.006220472510904074, -0.0033002779819071293, 0.0075677321292459965, 0.012006564065814018, 0.022506367415189743, -0.01586848311126232, -0.036922387778759, -0.013995214365422726, -0.03263287618756294, -0.004784978926181793, 0.01335721742361784, 0.004591543227434158, -0.026470094919204712, 0.05090402811765671, 0.0031967731192708015, -0.011450013145804405, 0.026429371908307076, 0.03222564235329628, 0.0006405423628166318, -0.016669372096657753, 0.0028082055505365133, 0.03472333401441574, -0.0405060313642025, 0.03154692426323891, -0.0004899512859992683, 0.02284572832286358, 0.04267793893814087, -0.005484060849994421, -0.019519997760653496, 0.03070531040430069, -0.0018596939044073224, 0.019940804690122604, 0.01616711914539337, -0.025112655013799667, 0.029320720583200455, -0.012149094603955746, -0.00043777463724836707, 0.009312043897807598, 0.017130902037024498, 0.004364171996712685, -0.014253128319978714, -0.008395771495997906, 0.0006278163637034595, -0.006929735187441111, -0.002467148704454303, -0.01026225183159113, -0.005870931316167116, -0.02087743952870369, 0.016343586146831512, -0.008226091042160988, -0.003439415479078889, 0.0056062303483486176, -0.0064580244943499565, 0.013676215894520283, -0.033365894109010696, -0.028316214680671692, -0.024800442159175873, 0.019900081679224968, -0.0017901250394061208, -0.00838219653815031, 0.032985810190439224, -0.01506759226322174, 0.01596350409090519, 0.03570069372653961, -0.021108204498887062, -0.003814408555626869, 0.021433990448713303, 0.031492624431848526, 0.040696073323488235, 0.015515548177063465, 0.032361388206481934, 0.0038245893083512783, -0.005989707540720701, -0.022560665383934975, 0.00642408849671483, 0.005884505808353424, 0.03656945377588272, 0.015284783206880093, 0.0026368286926299334, 0.013404727913439274, 0.009155938401818275, -0.004849457181990147, -0.018067536875605583, -0.0069908201694488525, -0.0007542280363850296, -0.0150404442101717, 0.0029948537703603506, 0.020986033603549004, -0.014755381271243095, 0.0016501389909535646, 0.0030033376533538103, -0.053293123841285706, 0.011830096133053303, 0.009060917422175407, 0.019316382706165314, -0.005670708604156971, -0.012149094603955746, 0.01611282117664814, 0.01745668798685074, -0.006620917469263077, 0.015732739120721817, -0.011646841652691364, 0.010133295319974422, 0.007493073120713234, 0.008327899500727654, -0.01935710571706295, -0.02391810715198517, -0.03100394643843174, 0.00021719052165281028, 0.022017689421772957, -0.0019631986506283283, -0.018814129754900932, 0.023578746244311333, -0.009916105307638645, 0.006502141244709492, -0.006264589261263609, 0.00533134862780571, 0.027773238718509674, -0.009481723420321941, -0.007269095163792372, -0.03214419633150101, -0.019967954605817795, -0.004116439260542393, -0.002972795395180583, 0.031085394322872162, -0.00694670295342803, 0.034234657883644104, 0.02120322547852993, 0.00912878941744566, -0.0162349920719862, 0.03966442123055458, -0.041021861135959625, -0.0019581082742661238, 0.023157939314842224, 0.008850513957440853, -0.0009680249495431781, -0.024338912218809128, -0.019397828727960587, -0.0030813906341791153, -0.024949761107563972, -0.010784867219626904, -0.005422975867986679, -0.007425201125442982, 0.0006638733902946115, -0.0032561609987169504, -0.027053793892264366, -0.017280220985412598, -0.011741862632334232, 0.0052057853899896145, -0.018053961917757988, -0.027108091861009598, -0.02474614605307579, 0.014972571283578873, 0.01706303097307682, -0.01893629878759384, -0.004398107994347811, -0.003164533758535981, -0.013818747363984585, 0.029076380655169487, 0.0034886228386312723, -0.016058525070548058, -0.02915782853960991, -0.006631098221987486, 0.008660471998155117, -0.0021125171333551407, 0.010893462225794792, 0.014361723326146603, -0.020809566602110863, -0.02364661917090416, 0.010045061819255352, 0.008470430038869381, -2.8527465474326164e-05, 0.012841389514505863, -0.0009680249495431781, -0.00013585013221018016, -0.001119040185585618, 0.02269640937447548, 0.024080999195575714, 0.006600555963814259, 0.0005747913382947445, -0.010995270684361458, 0.01586848311126232, 0.009033768437802792, -0.00026024685939773917, -0.019262084737420082, -0.012162669561803341, -0.0011046173749491572, 0.027257410809397697, -0.0008730040863156319, -0.004255576990544796, 0.013757661916315556, 0.018868425861001015, 0.00865368451923132, 0.011707927100360394, 0.014850402250885963, 0.0019513211445882916, 0.015325506217777729, -0.022017689421772957, 0.036650899797677994, -0.02030731365084648, 0.036080777645111084, -0.008789429441094398, -0.00617974903434515, -0.026307202875614166, -0.016520055010914803, 0.01242058351635933, -0.0016679553082212806, 0.007459137123078108, 0.0027793599292635918, 0.023402279242873192, -0.001985257025808096, -0.02534341998398304, 0.011891181580722332, -0.0015313628828153014, 0.006115270778536797, 0.014334574341773987, -0.022343475371599197, 0.0059489840641617775, 0.015244060195982456, -0.004934297408908606, 0.024909038096666336, -0.020918162539601326, -0.005725006572902203, 0.012739581987261772, 0.0014202224556356668, 0.019221361726522446, 0.0069433096796274185, -0.005321167875081301, 0.008341473527252674, -0.0113142691552639, 0.033338744193315506, -0.03138403221964836, 0.004062141291797161, 0.026551540940999985, 0.013913768343627453, -0.021094629541039467, 0.008938747458159924, 0.021895520389080048, -0.00869440846145153, -0.018148982897400856, -0.009237384423613548, 0.02412172220647335, 0.002658887067809701, 0.01808111183345318, -0.041591987013816833, -0.014185256324708462, 0.02424389310181141, -0.017076604068279266, 0.003478442085906863, -0.005758942570537329, 0.013397940434515476, 0.005148094147443771, -0.0015652988804504275, -0.011205673217773438, 0.020212292671203613, -0.010981695726513863, -0.011239609681069851, 0.028506256639957428, -0.025804949924349785, -0.006244227755814791, -0.03559209778904915, -0.017076604068279266, 0.0005820027436129749, -0.008945534937083721, 0.013601556420326233, 0.011945478618144989, 0.032741472125053406, 0.0009849929483607411, 0.013364003971219063, 0.016777968034148216, 0.013832321390509605, 0.016777968034148216, -0.03135688230395317, -0.02239777334034443, 0.0125088170170784, 0.0012810847256332636, -0.025234824046492577, -0.02070097252726555, -0.007961390540003777, -0.010479442775249481, -0.0009298469522036612, 0.008083559572696686, 0.0048867869190871716, 0.01923493668437004, -0.030270928516983986, -0.00023521903494838625, -0.0015907508786767721, -0.0037227813154459, 0.026198606938123703, -0.03355593606829643, -0.004577969200909138, 0.001919081900268793, -0.003363059600815177, -0.009441000409424305, -0.012970346957445145, -0.021216798573732376, 0.008097134530544281, -0.001966592390090227, -0.022709984332323074, -0.0030746033880859613, 0.030949650332331657, -0.009759998880326748, 0.011551820673048496, -0.012115159071981907, -0.022207731381058693, 0.000516675878316164, -0.0022533517330884933, 0.006295131519436836, 0.014782530255615711, -0.013187536969780922, 0.0428951270878315, -0.010011125355958939, 0.0160856731235981, -0.0236737672239542, -0.01783677190542221, -0.00396712077781558, 0.13205184042453766, 0.00946814939379692, 0.011483948677778244, 0.009345979429781437, 0.023687342181801796, -0.0315740741789341, 0.012339136563241482, -0.02000867761671543, 0.013913768343627453, -0.015610569156706333, 0.020741695538163185, 0.0037024198099970818, -0.001026564626954496, 0.011022418737411499, 0.00904055591672659, 0.007167287170886993, -0.014035937376320362, -0.03635226562619209, -0.01745668798685074, -0.012529178522527218, 0.008674046956002712, 0.0464516244828701, 0.021162502467632294, -0.003225618740543723, -0.004269151017069817, -0.030895352363586426, 0.018963446840643883, 0.007628817111253738, -0.021094629541039467, 0.00472728768363595, -0.021108204498887062, -0.004004450514912605, 0.01813540793955326, -0.01968289166688919, -0.00784600805491209, 0.009916105307638645, 0.00838219653815031, -0.024528954178094864, 0.005589262582361698, -0.01521691121160984, 0.023252960294485092, -0.007852794602513313, 0.008809790946543217, -0.009033768437802792, 0.04172772914171219, -0.027583196759223938, -0.0020039218943566084, 0.033365894109010696, 0.017755325883626938, -0.001036745379678905, 0.04202636703848839, 0.008490792475640774, -0.00957674439996481, -0.014402446337044239, -0.01337079145014286, 0.006539470981806517, -0.01584133319556713, 0.020022250711917877, 0.0021820859983563423, 0.03553779795765877, -0.023565171286463737, -0.010024700313806534, -0.0189091507345438, 0.002487510209903121, -0.00414019450545311, -0.045284222811460495, -0.02439321018755436, -0.0005378859350457788, -0.022465644404292107, -0.040967561304569244, 0.011626480147242546, -0.023700915277004242, -0.022465644404292107, -0.012318775057792664, 0.036433711647987366, 0.014958997257053852, 0.022642111405730247, 0.02254709042608738, 0.020470207557082176, 0.012895687483251095, 0.01082559023052454, -0.0015440888237208128, -0.00853151548653841, -0.02599499002099037, 0.0012607231037691236, 0.0093052564188838, 0.00135150202549994, 0.0018902362789958715, 0.004944478161633015, -0.01658792607486248, 0.03923003748059273, 0.0247189961373806, -0.0031153266318142414, -0.02442036010324955, -0.0015508760698139668, -0.0020887618884444237, -0.01487755123525858, -0.0032391929998993874, 0.024637550115585327, 0.02599499002099037, 0.01658792607486248, -0.0022007508669048548, -0.010472655296325684, -0.018420470878481865, 0.01360834389925003, -0.014768955297768116, -0.00869440846145153, 0.004099471028894186, -0.010493016801774502, 0.011015632189810276, -0.000530250312294811, -0.025601333007216454, 0.023538023233413696, 0.02355159819126129, -0.011606118641793728, 0.040397439152002335, 0.015420527197420597, 0.03225279226899147, -0.0150404442101717, -0.003227315377444029, -0.008782641962170601, -0.028071874752640724, 0.017266646027565002, 0.01067627128213644, -0.00760845560580492, 0.01199977658689022, 0.0285877026617527, -0.0007733170641586185, -0.014755381271243095, 0.0008475520880892873, -0.01870553381741047, 0.018773406744003296, 0.00694670295342803, -0.0011572182411327958, -0.03594503179192543, -0.020714545622467995, -0.03537490591406822, 0.019044894725084305, 0.00845685601234436, -0.008626536466181278, -0.028696298599243164, -0.030895352363586426, -0.007180861663073301, 0.00904055591672659, 0.0177146028727293, -0.018026813864707947, -0.004289512988179922, -0.007031543180346489, -0.007683114614337683, 0.019574295729398727, -0.0384155735373497, -0.017130902037024498, -0.018339024856686592, 0.01337079145014286, -0.005192210897803307, -0.03985445946455002, -0.02108105458319187, -0.022316325455904007, 0.013764449395239353, 0.003025396028533578, 0.04509418085217476, -0.005114158149808645, 0.0016755909891799092, 0.008925173431634903, -0.020918162539601326, 0.01721234992146492, 0.0011521278647705913, -0.010547314770519733, -0.039447229355573654, 0.02227560244500637, 0.030216632410883904, 0.0007470166310667992, 0.009936466813087463, 0.0014159803977236152, -0.0009383309516124427, 0.007703476585447788, 0.018977021798491478, -0.00784600805491209, 0.007126564159989357, -0.048352040350437164, -0.014090235345065594, -0.002787844045087695, -0.027746088802814484, 0.006712544709444046, -0.020171569660305977, -0.027827536687254906, 0.019139915704727173, -0.012692071497440338, 0.005239721387624741, 0.0043675657361745834, 0.02477329410612583, -0.003308761864900589, -0.014158107340335846, -0.0008687620866112411, -0.010968121699988842, -0.009624254889786243, 0.009631042368710041, -0.015352655202150345, 0.004130013752728701, -0.0032391929998993874, 0.019112765789031982, -0.011382141150534153, -0.03347449004650116, -0.004269151017069817, 0.007873156107962132, 0.005633379332721233, -0.010309762321412563, 0.012868538498878479, -0.012868538498878479, -0.026795880869030952, -0.02177334949374199, -0.00743877561762929, -0.00990253034979105, -0.009060917422175407, -0.006644672714173794, -0.00904055591672659, 0.0014753683935850859, -0.011219248175621033, 0.006359610240906477, -0.011938692070543766, -0.002076884498819709, 0.01429385133087635, 0.009943253360688686, -0.0024620580952614546, -0.004689957946538925, 0.011382141150534153, -0.009556382894515991, -0.05014386400580406, 0.007676327601075172, 0.02055165357887745, 0.003790653310716152, 0.0044150762259960175, 0.0014287064550444484, 0.0027420304249972105, -0.015732739120721817, 0.013784810900688171, 0.004927509929984808, -0.03399031609296799, -0.03100394643843174, 0.003352878615260124, 0.004686564207077026, 0.0034445058554410934, -0.01828472688794136, 0.0013464115327224135, -0.001975076273083687, 0.008450068533420563, 0.0005684283096343279, 0.02284572832286358, -0.0238230861723423, -0.013981640338897705, -0.04566430673003197, -0.006376578006893396, -0.02638864889740944, 0.016248565167188644, 0.0006613282021135092, -0.0049003614112734795, -0.010893462225794792, -0.00829396303743124, 0.00559265585616231, -0.0046967449598014355, 0.004143587779253721, 0.0005752155557274818, -0.025954267010092735, -0.00694670295342803, 0.015596994198858738, -0.011069929227232933, 0.019560720771551132, -0.003973907791078091, 0.013465812429785728, 0.022343475371599197, -0.015596994198858738, -0.010133295319974422, 0.002577440580353141, 0.011775799095630646, -0.011782585643231869, 0.0027250624261796474, -0.004706925712525845, 0.007832433097064495, 0.01387983188033104, -0.012949985451996326, -0.021556159481406212, 0.016153546050190926, -0.02057880163192749, 0.0065428647212684155, -0.0026385255623608828, -0.007180861663073301, 0.003715994069352746, -0.02200411446392536, 0.023755213245749474, -0.014823253266513348, -0.021488286554813385, -0.024528954178094864, -0.013099303469061852, -0.0030508481431752443, -0.006875437684357166, 0.01444317027926445, 0.0014634908875450492, 0.04455120489001274, -0.004174130503088236, -0.0032459802459925413, -0.006071154028177261, -0.003946759272366762, -0.0052057853899896145, 0.0025536855682730675, 0.008090347051620483, -0.002350069349631667, 0.008517940528690815, -0.014538190327584743, 0.00014539463154505938, -0.0211489275097847, -0.008796215988695621, -0.001029958133585751, -0.0002432788460282609, -0.025397716090083122, 0.011809734627604485, 0.020266590639948845, 0.017117328941822052, 0.009033768437802792, -0.03257858008146286, 0.01810825988650322, -0.006912766955792904, -0.0043777464888989925, 0.027569621801376343, 0.009678552858531475, 0.019995102658867836, 0.0006524199852719903, 0.009834658354520798, 0.006403726991266012, -0.00889123696833849, -0.022031264379620552, -0.009088066406548023, 0.008192155510187149, 0.0025570790749043226, -0.02042948454618454, -0.0012853267835453153, -0.009732849895954132, -0.01046586874872446, 0.026008564978837967, -0.010411570779979229, -0.002675855066627264, 0.006295131519436836, 0.015977077186107635, -0.0043302359990775585, 0.010893462225794792, 0.01207443606108427, 0.002518052700906992, 0.013323280960321426, -0.007092628162354231, -0.025397716090083122, -0.014918274246156216, -0.007140138652175665, 0.039311483502388, 0.01923493668437004, -0.019547147676348686, -0.0008581571164540946, 0.018393322825431824, -0.030651012435555458, 0.011843671090900898, -0.0390128493309021, 0.0309767983853817, 0.02828906662762165, 0.021488286554813385, -0.00022843183251097798, 0.014497467316687107, 0.0024756325874477625, 0.011456799693405628, -0.004890180192887783, -0.02030731365084648, 0.0038653125520795584, 0.02891348861157894, 0.0019479275215417147, 0.008402558043599129, -0.02469184808433056, -0.0048155211843550205, 0.019112765789031982, 0.033881720155477524, -0.004272544756531715, 0.005796272307634354, -0.005670708604156971, 0.011965840123593807, -0.01628929004073143, -0.006793991196900606, -0.015746312215924263, -0.008877662941813469, 0.013214685954153538, -0.022357050329446793, -0.027569621801376343, -0.020592376589775085, 0.01252239104360342, -0.01445674430578947, 0.019275659695267677, 0.02641579695045948, -0.0018902362789958715, -0.0021023363806307316, -0.014578914269804955, -0.020972460508346558, 0.0019089010311290622, 0.007676327601075172, 0.030542416498064995, 0.022262029349803925, -0.0031933793798089027, 0.02975510247051716, 0.013757661916315556, -0.008796215988695621, 0.0027844503056257963, -0.008334686048328876, -0.015583420172333717, -0.0007873156573623419, -0.00029439496574923396, -0.003186592133715749, 0.028506256639957428, 0.011687564663589, 0.020795993506908417, -0.031845562160015106, -0.008843726478517056, -0.03382742404937744, 0.00869440846145153, -0.024080999195575714, 0.019519997760653496, 0.022682834416627884, -0.021814072504639626, 0.006003282032907009, -0.02596784196794033, 0.01118531171232462, -0.012827815487980843, -0.024108147248625755, -0.003593824338167906, -0.038225531578063965, -0.012128733098506927, -0.0002956675598397851, 0.005124338902533054, -0.009047342464327812, 0.006698970217257738, -0.01413095835596323, -0.0011673991102725267, 0.024501806125044823, 0.2295703887939453, -0.020375186577439308, 0.0045100972056388855, 0.015773462131619453, -0.02474614605307579, 0.0007071418222039938, 0.014918274246156216, -0.011483948677778244, -0.010207954794168472, 0.0053008063696324825, -0.0118368836119771, 0.00904055591672659, -0.0028692905325442553, 0.004964839667081833, -0.01166041661053896, -0.029917994514107704, -0.03168266639113426, -0.020986033603549004, -0.013167175464332104, -0.027148814871907234, 0.004523671232163906, -0.0013998608337715268, 0.004971626680344343, -0.014361723326146603, 0.014931848272681236, 0.013228259980678558, 0.012732794508337975, 0.007859582081437111, -0.011307481676340103, 0.022669261321425438, -0.01143643818795681, -0.0021922667510807514, -0.02224845439195633, -0.022492794319987297, -0.02885919064283371, 0.006570013239979744, -0.0046526282094419, -0.010621974244713783, -0.022384198382496834, -0.014918274246156216, -0.01505401823669672, 0.01279387902468443, -0.021243948489427567, 7.271057256730273e-06, -0.005114158149808645, 0.041646283119916916, -0.025411291047930717, 0.003756717313081026, -0.005032711662352085, 0.016207842156291008, -0.03958297148346901, -0.040994711220264435, 0.02302219532430172, 0.015854908153414726, -0.007133351173251867, -0.005881112068891525, -0.003933184780180454, 0.02391810715198517, -0.006709150969982147, 0.00456439470872283, -0.0035904308315366507, 0.016343586146831512, -0.005287231877446175, 0.007384477648884058, -0.02132539451122284, 0.033664532005786896, -0.031166840344667435, 0.012407008558511734, 0.0187598317861557, -0.012189818546175957, 0.010445506311953068, -0.00292189116589725, -0.02314436435699463, 0.01487755123525858, -0.014008788391947746, -0.01953357271850109, 0.038877103477716446, 0.009610680863261223, 0.01352011039853096, 0.03154692426323891, 0.008246452547609806, 0.01396806538105011, 0.013764449395239353, -0.003055938519537449, -0.014809679239988327, -0.046750258654356, -0.0004895270685665309, -0.0021125171333551407, 0.00550102861598134, -0.014768955297768116, 0.004686564207077026, -0.006478385999798775, -0.005378859117627144, -0.0014193740207701921, 0.01882770285010338, -0.0005612169043160975, 0.013343642465770245, 0.011321055702865124, -0.010133295319974422, 0.00034063280327245593, -0.021936243399977684, 0.005137913394719362, 0.011735075153410435, -0.009848232381045818, -0.030868202447891235, 0.002146453130990267, -0.01521691121160984, 0.012705645523965359, -0.0022312933579087257, 0.0030525450129061937, -0.00342923472635448, -0.011551820673048496, -0.007791710086166859, 0.008375409990549088, 0.0029575240332633257, 0.03187270835042, -0.02376878820359707, -0.024311764165759087, 0.0321713462471962, -0.03784544765949249, -0.006094909273087978, 0.003719387808814645, 0.026524392887949944, -0.021963391453027725, 0.015108316205441952, 0.014416021294891834, -0.022886451333761215, 0.008972683921456337, -0.010438719764351845, -0.04848778620362282, -0.002689429558813572, -0.0069433096796274185, 0.015339081175625324, 0.005317774135619402, -0.022642111405730247, 0.01932995580136776, -0.008741918951272964, -0.014497467316687107, -0.028424810618162155, -0.005083615891635418, 0.009047342464327812, 0.004829095676541328, 0.0035666755866259336, 0.007045117672532797, -0.0003376633976586163, -0.02093173749744892, 0.02885919064283371, -0.005446731112897396, -0.0315740741789341, -0.02349730022251606, 0.0015712376916781068, -0.01766030490398407, -0.01521691121160984, -0.0075677321292459965, 0.009793935343623161, -0.028940636664628983, -0.014212405309081078, -0.026252904906868935, -0.006301918998360634, 0.01658792607486248, -0.04452405869960785, 0.009956828318536282, 0.02746102772653103, -0.020144421607255936, -0.024366062134504318, -0.019424976781010628, -0.17353522777557373, 0.022262029349803925, 0.016492905095219612, -0.028370512649416924, 0.041266199201345444, -0.005813240073621273, 0.01953357271850109, 0.013581194914877415, -0.023605894297361374, -0.002097246004268527, -0.014416021294891834, -0.001053713378496468, -0.025411291047930717, -0.0030016410164535046, -0.015854908153414726, 0.0005171000957489014, -0.01628929004073143, 0.027393154799938202, -0.002524839947000146, -0.003970514051616192, 0.034234657883644104, -0.030162334442138672, 0.012637773528695107, -0.03165552020072937, 0.031818412244319916, -0.014497467316687107, 0.008714769966900349, -0.0007644088473170996, 0.0094545753672719, -0.015909206122159958, 0.005857356823980808, -0.02499048411846161, 0.0006681153899990022, -0.02093173749744892, 0.014538190327584743, 0.0238230861723423, 0.021393267437815666, -0.009026980958878994, -0.007038330193608999, 0.02170547842979431, 0.04292227700352669, 0.017198774963617325, 0.014307425357401371, 0.02105390653014183, 0.024678273126482964, 0.02796328067779541, 0.00752022210508585, -0.015596994198858738, 0.016492905095219612, -0.0012047287309542298, 0.0019886507652699947, -0.020592376589775085, 0.007479498628526926, -0.0004852850688621402, 0.028967786580324173, 0.007160500157624483, -0.0011368566192686558, 0.005433156620711088, 0.014429595321416855, -0.005507816094905138, -0.02474614605307579, -0.016126396134495735, -0.025533460080623627, 0.0017527954187244177, 0.012155882082879543, -0.03445184603333473, -0.006108483299612999, 0.015827760100364685, -0.02057880163192749, -0.0025740470737218857, -0.02159688249230385, -0.019818635657429695, -0.02469184808433056, -0.01855621486902237, 0.028723446652293205, -0.002706397557631135, -0.027569621801376343, -0.004917329177260399, -0.0030525450129061937, 0.016832266002893448, 0.00828038901090622, 0.05218002200126648, -0.020660249516367912, -0.022112710401415825, -0.010520165786147118, -0.010723781771957874, 0.025180526077747345, 0.001344714779406786, -0.0012463002931326628, -0.012155882082879543, 0.013391152955591679, -0.006916160695254803, 0.006984032690525055, 0.013655854389071465, 0.019262084737420082, 0.01599065214395523, 0.0208502896130085, 0.00593540957197547, 0.01903131976723671, -0.011341417208313942, 0.010907036252319813, -0.0026368286926299334, -0.020090123638510704, 0.011171737685799599, 0.02820761874318123, 0.007750986609607935, 0.0209996085613966, 0.011558608151972294, 0.006427482236176729, -0.035754989832639694, -0.03651515766978264, 0.006909373681992292, 0.006549651734530926, 0.023958830162882805, -0.01380517240613699, 0.0019784700125455856, 0.015705589205026627, -0.017999663949012756, 0.016492905095219612, -0.002093852497637272, 0.043139468878507614, 0.015827760100364685, -0.021461138501763344, 0.0001308657811023295, 0.015800610184669495, -0.016805116087198257, -0.09339192509651184, -0.003446202725172043, 0.00480873417109251, 0.03534775599837303, -0.018895575776696205, 0.035157717764377594, 0.010493016801774502, -0.004513490479439497, -0.005599443335086107, 0.014850402250885963, -0.002217718865722418, -0.01709017902612686, 0.010520165786147118, 0.0288863405585289, 0.0018495130352675915, -0.00040956534212455153, -0.02070097252726555, -0.013710151426494122, -0.006912766955792904, 0.02207198739051819, -0.0068482886999845505, 0.023931680247187614, -0.008972683921456337, -0.01613997109234333, -0.01586848311126232, -0.0033715434838086367, -0.012515603564679623, 0.017796048894524574, 0.0074048396199941635, 0.012278052046895027, -0.0017001945525407791, 0.01014008279889822, 0.02123037353157997, -0.012440945021808147, 0.010438719764351845, 0.00921702291816473, -0.016547203063964844, 0.007927454076707363, -0.013275770470499992, -0.026103585958480835, -0.002757301554083824, 0.006203504279255867, 0.028044726699590683, -0.008538302034139633, -0.01371693890541792, 0.008341473527252674, -0.03868706151843071, 0.037954043596982956, 0.018678385764360428, -0.0211489275097847, -0.007187648676335812, -0.004574575461447239, 0.005718219093978405, -0.021393267437815666, 0.03350163996219635, -0.002972795395180583, 0.012454519048333168, 0.008823364973068237, -0.012461306527256966, -0.0023721277248114347, 0.006607342977076769, 0.002015799516811967, -0.0004333205579314381, 0.017565283924341202, -0.005497635342180729, 0.009590319357812405, -0.004208066500723362, -0.004374352749437094, -0.0018020026618614793, -0.010370847769081593, -0.02686375379562378, 0.016655799001455307, -0.0001618323876755312, -0.0033681499771773815, -0.012088010087609291, -0.0035089843440800905, -0.01920778676867485, 0.00480873417109251, 0.012990708462893963, 0.004130013752728701, -0.02638864889740944, -0.016574351117014885, 0.0013124755350872874, -0.014022363349795341, 0.03523916378617287, -0.008436494506895542, -0.00565713457763195, 0.011361779645085335, -0.002870987169444561, -0.0004992836620658636, -0.0022075381129980087, 0.022737132385373116, 0.009658191353082657, -0.006532683502882719, -0.03285006806254387, 0.025804949924349785, -0.0059964945539832115, 0.008898024447262287, 0.018719108775258064, 0.006722725462168455, -0.017443113029003143, 0.0036311540752649307, -0.07232444733381271, 0.028777744621038437, -0.00617296202108264, 0.003186592133715749, 0.0023008622229099274, -0.016004227101802826, 0.013180749490857124, 0.0018359386594966054, 0.0024824198335409164, 0.03290436416864395, -0.01673724502325058, 0.011843671090900898, 0.010031486861407757, -0.011945478618144989, -0.012841389514505863, -0.013201111927628517, 0.008646897971630096, 0.01628929004073143, 0.00889123696833849, 0.004167343024164438, -0.02180049940943718, 0.017266646027565002, -0.00018261819786857814, 0.026103585958480835, -0.002925284905359149, 0.006162781268358231, -0.00249769096262753, 0.02513980306684971, -0.007696689106523991, 0.0020344643853604794, 0.015678441151976585, -0.034587591886520386, -0.006912766955792904, 0.007065479177981615, -0.013696577399969101, -0.042813681066036224, 0.013649066910147667, -0.0010681361891329288, 0.015244060195982456, 0.010893462225794792, -0.0077713485807180405, 0.0046492344699800014, 0.01843404583632946, 0.01066269725561142, 0.021610457450151443, -0.0055112093687057495, -0.004516884218901396, 0.017768898978829384, 0.02915782853960991, 0.01780962385237217, 0.021936243399977684, 0.013248622417449951, -0.01042514480650425, -0.016058525070548058, -0.020348036661744118, -0.0004708622582256794, 0.013248622417449951, 0.0023110429756343365, -0.012664922513067722, -0.038198381662368774, 0.05060539022088051, -0.02733885683119297, 0.007560945115983486, 0.012936410494148731, 0.02123037353157997, 0.009529233910143375, -0.009060917422175407, 0.005219359882175922, 0.014782530255615711, -0.018216855823993683, -0.009882168844342232, -0.02921212464570999, 0.010886674746870995, 0.023157939314842224, 0.009169512428343296, -0.01428027730435133, 0.014212405309081078, -0.004530458711087704, -0.013201111927628517, 0.01042514480650425, 0.01658792607486248, -0.0039162165485322475, -0.03347449004650116, 0.005772517062723637, 0.035456351935863495, 0.012278052046895027, -0.008674046956002712, 0.014185256324708462, 0.020565228536725044, 0.009434212930500507, 0.020361611619591713, 0.01396806538105011, -0.0039026422891765833, -0.03249713033437729, 0.0017527954187244177, 0.02042948454618454, -0.017293795943260193, -0.011253183707594872, 0.016940860077738762, 0.018800554797053337, -0.007099415175616741, 0.014266702346503735, 0.004062141291797161, -0.003990876022726297, -0.0160856731235981, 0.04875927418470383, -0.017429539933800697, -0.03434325009584427, -0.001296355971135199, -0.011477162130177021, 0.01090024970471859, -0.009841445833444595, -0.011015632189810276, 0.021108204498887062, -0.03374597802758217, 0.024488231167197227, 0.022560665383934975, -0.0077238380908966064, -0.00659716222435236, 0.03510341793298721, 0.0035361333284527063, -0.008436494506895542, 0.020823141559958458, -0.03447899594902992, 0.023361556231975555, 0.00745234964415431, 0.02799042873084545, 0.0005442489054985344, 0.002796327928081155, 0.009135575965046883, -0.003709206823259592, -0.02215343341231346, -0.0050734346732497215, -0.012149094603955746, 0.010628761723637581, -0.013798385858535767, -0.013112877495586872, 0.03686809167265892, -0.04064177721738815, 0.04357384890317917, 0.002601195825263858, -0.003923004027456045, 0.004337023478001356, -0.029103530570864677, 0.026877326890826225, -3.3114130928879604e-05, 0.0014354935847222805, -0.012067648582160473, -0.010438719764351845, 0.007459137123078108, -0.012121946550905704, 0.012698858045041561, -0.030162334442138672, -0.0019801666494458914, 0.0038551317993551493, -0.013038218952715397, -0.0027454239316284657, -0.016031375154852867, -0.0017901250394061208, 0.010723781771957874, 0.00923059694468975, 0.00184102903585881, 0.01738881692290306, -0.0013540472136810422, -0.0013396244030445814, 0.038524169474840164, 0.0048630316741764545, -0.007085840683430433, -0.02007654868066311, 0.003268038621172309, 0.004384533502161503, -0.006478385999798775, -0.01127354521304369, 0.024515381082892418, -0.015746312215924263, -0.012101584114134312, -0.023456577211618423, 0.009671765379607677, 0.0375196635723114, -0.032442834228277206, 0.01143643818795681, -0.010459081269800663, -0.036705199629068375, 0.04552856460213661, -0.008266814053058624, -0.028777744621038437, 0.01790464296936989, -0.002480722963809967]} +{"id": "test:10000056", "text": "\"The Center for the Study of Culture, Politics and Society (CECUPS) is devoted to the sociological analysis of cultural and political subjects, in particular subjects placed in the confluence of those two realms or dimensions of social reality. In this way, and mainly from a comparative perspective, we make research about cultural policy, the relationship between culture and territory, citizenship and the Welfare State, the cultural construction of Europe or the reflexive configuration of modernity.\\nArturo Rodr\u00edguez Morat\u00f3. Mariano Mart\u00edn Zamorano. 2018. Cultural Policy in Ibero-America \u2013 A Regional Perspective. Taylor & Francis.\"", "vector_field": [-0.0033879834227263927, 0.007753527723252773, 0.05227939784526825, -0.03896850720047951, -0.01818530261516571, 0.013505064882338047, -0.007130834739655256, 0.007177704479545355, -0.01334436982870102, -0.02948750928044319, 0.004824194125831127, -0.00025882755289785564, 0.010779947973787785, 0.0008210503729060292, -0.007204486522823572, -0.0014772210270166397, 0.0289518591016531, -0.0054736691527068615, 0.0036290257703512907, -0.012581069022417068, -0.021640243008732796, -0.004332066047936678, -0.0073986598290503025, -0.02820195071399212, -0.0013675802620127797, 0.010887077078223228, 0.0345761813223362, -0.011208467185497284, 0.0003634465974755585, -0.013270718045532703, -0.0008679197053425014, -0.017716608941555023, 0.0012470592046156526, -0.020234161987900734, -0.0083427419885993, -0.016779223456978798, -0.0024606401566416025, -0.026367349550127983, 0.008048134855926037, -0.008483350276947021, 0.002944398671388626, 0.008315959945321083, 0.003167028073221445, -0.011523161083459854, 0.013022980652749538, 0.03436192125082016, 0.03187115117907524, -0.0029795507434755564, -0.017743391916155815, 0.015761489048600197, 0.02383640594780445, 0.021251896396279335, -0.03219253942370415, -0.008168656378984451, 0.017863912507891655, -0.027639517560601234, -0.043066225945949554, 0.02986246347427368, 0.013103327713906765, -0.024921096861362457, 0.0014588080812245607, 0.005855319555848837, -0.018131738528609276, 0.025885265320539474, -0.003115137107670307, -0.014810710214078426, -0.0050719319842755795, 0.011677160859107971, 0.0024790531024336815, -0.005818493664264679, 0.023769449442625046, 0.026862826198339462, -0.00302474619820714, -0.03146941214799881, 0.03414766117930412, 0.005440190900117159, -0.026648566126823425, -0.010190732777118683, -0.012768547050654888, 0.020783202722668648, -0.004044154193252325, -0.008222220465540886, -0.023421278223395348, -0.0043956744484603405, -0.00589549308642745, 0.022510673850774765, 0.003685938660055399, 0.004261761903762817, -0.01718096062541008, 0.0005168181378394365, -0.012246288359165192, 0.008329350501298904, 0.011328988708555698, -0.007063878700137138, -0.0284429918974638, 0.01373941171914339, 0.003608939005061984, 0.010813425295054913, 0.011255336925387383, -0.01744878478348255, 0.009963082149624825, 0.009983168914914131, -0.03168367221951485, -0.016323920339345932, -0.014797319658100605, 0.0035955477505922318, -0.00388680724427104, -0.0012319940142333508, 0.03859355300664902, 0.002520900685340166, -0.012206114828586578, 0.008249003440141678, 0.0006524044438265264, -0.00576492864638567, 0.018479909747838974, -0.020247554406523705, -0.001143277040682733, -0.010204124264419079, 0.007853961549699306, -0.00782717950642109, 0.02129206992685795, 0.021010855212807655, 0.01411436591297388, -0.02481396682560444, 0.02696995623409748, 0.00444923946633935, -0.015574011020362377, -0.008523523807525635, -0.002746877958998084, -0.004767281003296375, -0.032165758311748505, 0.0001868496328825131, -0.0005887960433028638, 0.028496557846665382, -0.00018465262837707996, -0.015158883295953274, -0.011034381575882435, 0.016524789854884148, -0.008878391236066818, -0.0647064670920372, 0.029139336198568344, 0.011944985017180443, -0.02822873182594776, -0.02421136014163494, -0.002415444701910019, 0.03173723816871643, -0.002597900340333581, 0.024599706754088402, 0.005647755227982998, 0.016096269711852074, 0.02292580157518387, -0.0222160667181015, -0.0005996764521114528, 0.020488595589995384, -0.0016211769543588161, 0.011697247624397278, -0.0025192268658429384, 0.010492036119103432, 0.006501446478068829, -0.013183674775063992, -0.013270718045532703, -0.009106042794883251, -0.0010286145843565464, -0.003521895967423916, 0.007686571218073368, 0.028148384764790535, 0.04866376519203186, 0.0067826625891029835, 0.007271443028002977, 0.004322022665292025, -0.01956460066139698, 0.02659500204026699, -0.021238505840301514, 0.0008047297596931458, -0.01756930537521839, 0.028925076127052307, -0.007820483297109604, 0.01602931320667267, -0.017971042543649673, 0.0032457015477120876, 0.002214576117694378, 0.02117154933512211, 0.024425620213150978, 0.019711904227733612, -0.00012156734737800434, -0.014837493188679218, 0.013632281683385372, 0.02694317325949669, 0.007994569838047028, -0.003481722204014659, 0.014275060966610909, 0.02046181447803974, 0.022256240248680115, -0.01756930537521839, -0.6483502388000488, -0.021774154156446457, 0.012935937382280827, -0.02595222182571888, -0.0048643676564097404, 0.007579441647976637, 0.029942810535430908, 0.009119434282183647, -0.024023883044719696, -0.015332968905568123, 0.0016287094913423061, 0.011389249004423618, 0.0006021872977726161, -0.014743754640221596, -0.034067314118146896, -0.018895039334893227, -0.004221588373184204, -0.007532571908086538, 0.020006511360406876, -0.013063154183328152, -0.014034018851816654, 0.025644224137067795, 0.0031988322734832764, 0.025764744728803635, 0.01184455119073391, 0.010217515751719475, 0.030130287632346153, -0.00432537030428648, 0.0031653542537242174, 0.0107732517644763, -0.014971405267715454, -0.0016797635471448302, -0.012949327938258648, 0.0026246828492730856, 0.029541073366999626, 0.018961993977427483, 0.0038667202461510897, -0.006842923350632191, 0.0083427419885993, 0.023782841861248016, -0.02155989594757557, 0.0002669878304004669, 0.02129206992685795, -0.012594460509717464, 0.01693991757929325, 0.02243032492697239, 0.0190691240131855, -0.025121964514255524, -0.0059122322127223015, 0.0016236878000199795, -0.004265109542757273, -0.01322384923696518, 0.007552659139037132, 0.023635536432266235, 0.023876579478383064, -0.012299853377044201, 0.016256963834166527, -0.033558446913957596, 0.003305962309241295, 0.014328625984489918, -0.02508179098367691, 0.019457470625638962, -0.03797755390405655, -0.02230980433523655, 0.022283021360635757, 0.024050666019320488, -0.037174079567193985, 0.0016471224371343851, 0.01995294727385044, 0.0044325003400444984, 0.00751918088644743, 0.011683856137096882, -0.023756058886647224, -0.023648928850889206, 0.014020627364516258, 0.008623957633972168, 0.016350703313946724, -0.020783202722668648, -0.013558629900217056, -0.01297611091285944, -0.0048643676564097404, -0.0044325003400444984, -0.009380563162267208, 0.009634996764361858, -0.008449872024357319, -0.022631194442510605, -0.023247191682457924, 0.02181432954967022, 0.0019266644958406687, -0.0021610113326460123, 0.021104592829942703, -0.0008645718917250633, -0.010639339685440063, -0.018386172130703926, -0.006832879967987537, 0.014944623224437237, -0.007479007355868816, -0.005647755227982998, -0.0012771894689649343, -0.006190100219100714, 0.004623325541615486, -0.013458195142447948, -0.003351157531142235, -0.011409335769712925, 0.01602931320667267, 0.018279042094945908, -0.010418384335935116, 0.0017358394106850028, 0.05640390142798424, -0.024358663707971573, 0.012648025527596474, 0.02935359627008438, 0.010492036119103432, -0.019778860732913017, -0.010204124264419079, -0.02421136014163494, 0.014060800895094872, 0.017636261880397797, 0.01605609618127346, -0.004874411039054394, -0.004981541074812412, 0.0036524604074656963, 0.005413408391177654, -0.022135717794299126, -0.01009029895067215, 0.0280948206782341, -0.0009239955106750131, -0.004737150855362415, -0.02682265266776085, -0.016752440482378006, 0.009628301486372948, -0.0103447325527668, 0.045690909028053284, -0.01631052978336811, 0.03746868669986725, 0.004941367078572512, 0.020876942202448845, -0.027010129764676094, -0.01127542369067669, -0.016015922650694847, -0.006929966155439615, -0.024519359692931175, 0.0024204663932323456, -0.023394495248794556, 0.007003617938607931, -0.010800034739077091, -0.006504794582724571, -0.015480272471904755, -0.018332606181502342, -0.018881646916270256, -0.005600885953754187, -0.019885990768671036, -0.006029405631124973, 0.011724029667675495, -0.0038767638616263866, -0.008074916899204254, -0.01592218317091465, 0.0051723662763834, -0.02145276591181755, -0.017502348870038986, 0.011355770751833916, 0.015225838869810104, -0.06347447633743286, 0.00429524015635252, 0.004248370882123709, 0.004984888713806868, -0.0019718599505722523, -0.0014881014358252287, -0.010478644631803036, -0.035299308598041534, 0.006729097571223974, 0.015882009640336037, 0.012648025527596474, 0.018279042094945908, -0.0028673990163952112, 0.007418746594339609, -0.013819758780300617, 0.015975749120116234, 0.004720411729067564, -0.008670827373862267, -0.0016780897276476026, -0.006046144757419825, -0.021747373044490814, -0.004757237620651722, 0.0038667202461510897, -0.006478012073785067, 0.01628374680876732, -0.0056979721412062645, 0.004944715183228254, 0.007699962705373764, 0.004877758678048849, 0.013310891576111317, -0.020997462794184685, 0.007291529793292284, 0.018587039783596992, 0.026032568886876106, -0.013525151647627354, 0.0063005778938531876, 0.009936299175024033, 0.01943068765103817, 0.042209185659885406, -0.02445240318775177, 0.004492760635912418, -0.003036463400349021, -0.00808161310851574, -0.014087583869695663, -0.009795691817998886, -0.022108936682343483, 0.019457470625638962, 0.009755517356097698, 0.018131738528609276, -0.047083597630262375, -0.028791164979338646, -0.01071968674659729, -0.015855226665735245, 0.02782699465751648, 0.008074916899204254, 0.005731450393795967, -0.026112915948033333, 0.015051753260195255, -0.0015508729266002774, -0.007418746594339609, 0.00021007507166359574, 0.005928971339017153, -0.02295258454978466, 0.018104955554008484, -0.013304196298122406, 0.0190691240131855, -0.009701953269541264, 0.002104098442941904, -0.014716971665620804, 0.011911507695913315, 0.0002897947852034122, 0.02884472906589508, -0.008670827373862267, 0.02897864207625389, 0.0013868302339687943, -0.018493302166461945, 0.018841473385691643, -0.007612919434905052, 0.018386172130703926, 0.04502134770154953, -0.004154631868004799, 0.0172880906611681, 0.01656496338546276, -0.017114004120230675, 0.023863188922405243, 0.013692541979253292, -0.00031553106964565814, -0.003963806666433811, -0.008182046934962273, 0.0022296414244920015, -0.028175167739391327, -0.011864637956023216, 0.01359880343079567, 0.0063574910163879395, -0.015440098941326141, 0.005841928068548441, 0.023595362901687622, 0.028523338958621025, -0.0177969578653574, -0.0030264200177043676, 0.015185665339231491, 0.004342109430581331, -0.006950053386390209, 0.006909879390150309, -0.00984256062656641, -0.04129858314990997, -0.03647773712873459, -0.0142214959487319, -0.025041617453098297, -0.02455953322350979, 0.01549366395920515, -0.005041801370680332, 0.0025995743926614523, 0.009360476396977901, -0.00567118963226676, -0.01097412034869194, 0.027666300535202026, -0.0011867985595017672, -0.013143501244485378, -0.026420915499329567, 0.015948966145515442, 0.030290983617305756, 0.0073651815764606, -0.021894676610827446, -0.027773430570960045, 0.005872058216482401, -0.023166844621300697, 0.02196163311600685, 0.0036658516619354486, 0.023233799263834953, 0.00410106685012579, 0.016752440482378006, -0.009668475016951561, -0.0012085593771189451, -0.0015709598083049059, 0.014783928170800209, 0.018225476145744324, 0.013592108152806759, -0.0019685120787471533, -0.008302568458020687, -0.014087583869695663, -0.005604233592748642, 0.001713241683319211, 0.022363370284438133, -0.006354142911732197, -0.028684034943580627, -0.006484707817435265, -0.01298280619084835, 0.000518492073751986, 0.007512485142797232, 0.0004172207845840603, 0.013310891576111317, 0.00410106685012579, 0.009059173054993153, -0.013210457749664783, 0.0013232218334451318, 0.0391559824347496, 0.0016010900726541877, 0.006461272947490215, -0.04488743469119072, -0.01329080481082201, -0.014328625984489918, 0.10134489834308624, 0.0020840116776525974, 0.0008394633186981082, 0.03312992677092552, 0.004278501030057669, -0.0289518591016531, -0.010646034963428974, -0.009762213565409184, 0.007679875940084457, -0.016256963834166527, -0.034308355301618576, 0.010813425295054913, 0.024974660947918892, -0.008389611728489399, 0.017140787094831467, 0.018720952793955803, -0.005841928068548441, -0.016136443242430687, 0.011288815177977085, 0.00030737079214304686, 0.005309626460075378, -0.03294244781136513, 0.017475567758083344, 0.027050303295254707, -0.0043755872175097466, -0.02218928374350071, 0.017502348870038986, 0.021586677059531212, -0.003997284919023514, -0.017140787094831467, -0.02044842205941677, 0.008838217705488205, -0.005410060752183199, 0.020917115733027458, -0.037816859781742096, 0.02182772010564804, 0.03733477368950844, 0.014248278923332691, 0.023032931610941887, 0.009554648771882057, 0.0042115445248782635, 0.02431849017739296, 0.008309263736009598, -0.013264022767543793, -0.003367896657437086, -0.030532024800777435, 0.016859570518136024, 0.02572457119822502, 0.014007235877215862, -0.029621420428156853, -0.0017709913663566113, -0.003947068005800247, -0.009501084685325623, -0.027144042775034904, 0.013766193762421608, -0.00661192461848259, 0.006206839345395565, 0.012641330249607563, -0.014797319658100605, -0.011014293879270554, -0.015172273851931095, -0.010552296414971352, 0.00027159106684848666, -0.0006846270989626646, -0.009246651083230972, -0.009427432902157307, -0.0027452041395008564, -0.008469958789646626, -0.02281867153942585, 0.0066186198964715, 0.006963444408029318, -0.03562069684267044, -0.007445529103279114, 0.018720952793955803, 0.0010671144118532538, 0.0023585320450365543, 0.0190691240131855, -0.0015349708264693618, -0.01941729709506035, 0.018640605732798576, -0.01159681286662817, -0.008510132320225239, -0.008443176746368408, -0.038566768169403076, 0.007345094811171293, 0.0018530127126723528, -0.008791348896920681, -0.0016554920002818108, -0.011737421154975891, 0.012574373744428158, -0.0076597887091338634, -0.002542661502957344, 0.006822836585342884, -0.014583059586584568, 0.004074284341186285, 0.012259679846465588, 0.018091564998030663, 0.006354142911732197, 0.008677522651851177, -0.00901899952441454, -0.01041168812662363, -0.0286036878824234, -0.0025543789379298687, -0.018212085589766502, 0.008275785483419895, 0.043200138956308365, -0.020501988008618355, -0.007237964775413275, -0.007325008045881987, 0.012206114828586578, 0.009025695733726025, -0.031924713402986526, 0.007345094811171293, 0.0030080070719122887, 0.0005327202379703522, 0.004599890671670437, -0.016899744048714638, 0.008697610348463058, 0.011034381575882435, -0.026420915499329567, -0.037950772792100906, -0.016993483528494835, 0.002231315243989229, 0.00895873922854662, -0.02431849017739296, 0.03594208508729935, -0.017475567758083344, -0.016323920339345932, -0.006976835895329714, -0.025389790534973145, -0.021519722416996956, 0.03524574264883995, -0.014382191002368927, -0.004630020819604397, -0.031148022040724754, 0.018359389156103134, -0.016203399747610092, 0.010512122884392738, -0.01880129985511303, -0.012246288359165192, -0.01818530261516571, 0.014368799515068531, -0.0059189279563724995, -0.009313606657087803, 0.027505606412887573, -0.03645095229148865, 0.017890695482492447, 0.015373142436146736, -0.0035921998787671328, 0.009735430590808392, -0.015306186862289906, 0.005677885375916958, 0.00907925982028246, 0.030398113653063774, -0.008530219085514545, -0.019511034712195396, -0.03784364089369774, -0.04625334218144417, 0.03146941214799881, 0.022363370284438133, 0.04898515343666077, 0.009748822078108788, -2.4650866180309094e-05, 0.012273070402443409, 0.0035118525847792625, -0.006909879390150309, 0.011509770527482033, 0.009996560402214527, -0.013076544739305973, 0.03296923264861107, 0.01730148121714592, 0.005165670532733202, -0.019243210554122925, -0.013143501244485378, 0.007653093431144953, 0.018399562686681747, -0.011081250384449959, -0.0023953579366207123, -0.009313606657087803, -0.02722438983619213, -0.021988414227962494, -0.0036524604074656963, -0.017716608941555023, 0.015426707454025745, -0.011898116208612919, -0.0004946389235556126, 0.009501084685325623, 0.026929782703518867, -0.002490770537406206, -0.005570755340158939, 0.028523338958621025, -0.023930145427584648, 0.03634382411837578, -0.005647755227982998, -0.022108936682343483, 0.0050786277279257774, -0.019028950482606888, -0.03112124092876911, 0.020033294335007668, 0.026728913187980652, 0.007338399067521095, 0.020970679819583893, 0.012507417239248753, -0.005155627150088549, 0.020100250840187073, 0.038057900965213776, -0.004693629220128059, 0.004074284341186285, 0.01932355761528015, -0.02731812745332718, 0.015962356701493263, 0.005038453731685877, -0.004301935434341431, -0.014783928170800209, 0.001566775026731193, -0.0008231427636928856, -0.02079659514129162, 0.024880923330783844, -0.0042182402685284615, 0.004024067427963018, 0.010893773287534714, 0.0027301388327032328, 0.025175530463457108, -0.02622004598379135, 0.018747735768556595, 0.010565687902271748, 0.0008566208416596055, -0.023849796503782272, 0.028550121933221817, 0.014690189622342587, -0.013766193762421608, 0.027103867381811142, 0.0008356970502063632, 0.0038700681179761887, -0.00030423220596276224, 0.002750225830823183, -0.007291529793292284, 0.002922638086602092, -0.021345635876059532, 0.006354142911732197, 0.0032021801453083754, -0.0027485517784953117, -0.022269630804657936, 0.013049762696027756, -0.0019099254859611392, -0.010605861432850361, -0.0014328625984489918, -0.015560620464384556, -0.0009290172019973397, -0.005831884685903788, -0.0009055825648829341, 0.03369235619902611, -0.00579505879431963, 0.027800213545560837, -0.001461319043301046, -0.03564747795462608, 0.01766304485499859, -0.014395582489669323, -0.007813788019120693, 0.015011578798294067, -0.0021241854410618544, 0.006555011495947838, 0.005627668462693691, 0.005205844063311815, 0.002308314898982644, -0.027130650356411934, 0.017020264640450478, -0.002045511733740568, -0.00884491391479969, 0.03280853480100632, -0.0026129656471312046, -0.01869416981935501, -0.003262440674006939, 0.0019182950491085649, 0.017087221145629883, -0.01929677650332451, -0.00011926572187803686, -0.03296923264861107, -0.0019049037946388125, -0.016015922650694847, -0.0007499093771912158, 0.004861020017415285, -0.02168041653931141, 0.001878121285699308, 3.7532085116254166e-05, 0.003064919961616397, -0.021211722865700722, -0.008891782723367214, 0.01744878478348255, 0.023233799263834953, 0.002715073758736253, 0.002437205519527197, -0.012333331629633904, 0.025403181090950966, -0.0003504738269839436, 0.007304920814931393, 0.0011416031047701836, 0.012366809882223606, -0.011837855912744999, 0.009507779963314533, -0.026059351861476898, 0.010250994004309177, -0.019136080518364906, -0.0006657956982962787, -0.033183492720127106, -0.03650451824069023, 0.04178066551685333, -0.002082337625324726, -0.013665759935975075, -0.023220408707857132, 0.016832787543535233, 0.024144403636455536, -0.0004691118374466896, -0.0065449681133031845, -0.0027552475221455097, 0.005179062020033598, 0.01023090723901987, -0.03246036544442177, -0.023756058886647224, 0.037816859781742096, -0.0002512949868105352, -0.01278193760663271, 0.02193485014140606, -0.032888881862163544, 0.0206894651055336, -0.025871874764561653, 0.015574011020362377, 0.024131013080477715, -0.02206876315176487, 0.0035085047129541636, 0.0008570392965339124, 0.01908251643180847, -0.04103075712919235, 0.010378210805356503, -0.004198153503239155, -0.012199418619275093, -0.0037830250803381205, 0.02771986462175846, 0.015989139676094055, -0.0020907071884721518, 0.002663182793185115, -0.006123144179582596, -0.004854324273765087, 0.01917625404894352, 0.02257763035595417, -0.0028523339424282312, -0.0031971584539860487, -0.004954758565872908, -0.021211722865700722, 0.0010177341755479574, -0.01442236453294754, 0.033906616270542145, 0.002400379627943039, -0.005597537849098444, -0.017368437722325325, 0.000584611261729151, -0.03248714655637741, -0.012366809882223606, 0.013310891576111317, 0.028898295015096664, 0.020542161539196968, -0.006481359712779522, 0.0019601427484303713, -0.004663499072194099, -0.011201771907508373, 0.025389790534973145, -0.026394132524728775, -0.033906616270542145, 0.0003605172678362578, -0.007425442337989807, 0.024666663259267807, 0.003722764551639557, -0.016725657507777214, 0.015132100321352482, 0.02785377763211727, 0.01365906372666359, -0.0026179873384535313, 0.04625334218144417, -0.01215924508869648, 0.006310621742159128, -0.0033628749661147594, -0.008798044174909592, -0.02243032492697239, -0.008228916674852371, 0.024291707202792168, -0.023622145876288414, 0.008516828529536724, 0.017529131844639778, -0.0052962349727749825, -0.01815851964056492, -0.0021141418255865574, 0.001336613087914884, -0.03160332515835762, 0.037066951394081116, -0.013531846925616264, -0.010672817938029766, 0.030505243688821793, 0.0007135019404813647, 0.017770174890756607, 0.001246222178451717, -0.010840208269655704, 0.024050666019320488, 0.022657977417111397, -0.0071910955011844635, -0.04826202616095543, 0.016270356252789497, -0.026902999728918076, 0.012560982257127762, 0.0015659380005672574, -0.004435847979038954, 0.02169380709528923, -0.031040892004966736, -0.0037830250803381205, -0.017006874084472656, -0.012788633815944195, -0.04202171042561531, 0.02457292377948761, -0.01931016705930233, 0.0056611462496221066, 0.04298587888479233, -0.02671552263200283, 0.0025175530463457108, -0.003719416679814458, 0.009420736692845821, -0.026929782703518867, -0.011831159703433514, -0.007358485832810402, -0.020769812166690826, -0.0017659696750342846, -0.01882808282971382, -0.009956386871635914, 0.005125497002154589, -0.019711904227733612, 0.013451499864459038, -0.005758232902735472, 0.023099888116121292, 0.1924053281545639, -0.002910920651629567, 0.00746561586856842, 0.018345998600125313, -0.0039001984987407923, 0.02682265266776085, 0.003006333252415061, 0.009601518511772156, 0.01385993231087923, 0.02407744899392128, -0.0388345941901207, -0.0073919640854001045, -0.005359843373298645, 0.007847266271710396, -0.011697247624397278, -0.038325726985931396, -0.029407162219285965, -0.03195149824023247, 0.0009934625122696161, -0.006022709887474775, 0.0017776869935914874, 0.0004954758333042264, 0.005969144869595766, -0.011034381575882435, 0.022162500768899918, -0.006662141531705856, -0.005088671110570431, -0.029433943331241608, 0.02469344437122345, 0.006464620586484671, -0.01980564370751381, -0.020984072238206863, 0.007405355107039213, -0.005932318978011608, -0.02044842205941677, 0.005754885263741016, 0.013143501244485378, -0.012319940142333508, 0.020542161539196968, -0.015989139676094055, -0.025363007560372353, 0.02719760686159134, 0.0034515918232500553, -0.01549366395920515, 0.012996197678148746, 0.027184216305613518, -0.0061197965405881405, -0.007887439802289009, 0.006016014143824577, -0.008389611728489399, -0.03160332515835762, 0.013176979497075081, 0.0270770862698555, 0.04927976056933403, 0.02745204046368599, 0.0067893583327531815, 0.011576726101338863, 0.01843973621726036, -0.0010938968043774366, 0.0018429693300276995, -0.0011575052049010992, 0.031040892004966736, -0.019524427130818367, 0.011067858897149563, 0.02518892101943493, 0.013940280303359032, 0.021626850590109825, 0.0005791711155325174, 0.015279403887689114, -0.01929677650332451, 0.016739049926400185, -0.03511182963848114, -0.014877666719257832, -0.00027054487145505846, -0.007338399067521095, -0.018238868564367294, 0.03945058956742287, 0.02070285566151142, 0.012889067642390728, 0.0451284758746624, 0.015011578798294067, -0.006193448323756456, -0.03312992677092552, 0.009681865572929382, 0.01342471782118082, -0.03058559074997902, 0.026621783152222633, -0.013833150267601013, -0.00705718295648694, -0.020488595589995384, -0.007894136011600494, -0.006732445675879717, -0.011014293879270554, -0.023756058886647224, 0.015734706073999405, 0.0011792660225182772, 0.020850159227848053, -0.007311616558581591, -0.027398476377129555, 0.007880744524300098, -0.010163950733840466, 0.07654432207345963, 0.030799850821495056, 0.005758232902735472, 0.012340026907622814, 0.017100611701607704, -0.016631919890642166, 0.010123777203261852, 0.018988776952028275, -0.01665870100259781, -0.0034582875669002533, -0.022403543815016747, 0.0002205369673902169, -0.007097356952726841, -0.001256265677511692, 0.019243210554122925, 0.019899381324648857, -0.03677234426140785, 0.01628374680876732, 0.004472673870623112, -0.026621783152222633, -0.01592218317091465, 0.0009591475245542824, 0.0211983323097229, 0.007083965465426445, 0.007251356262713671, -0.025376398116350174, -0.001329080550931394, -0.019912773743271828, -0.003813155461102724, -0.006541620474308729, -0.021372416988015175, 0.012534200213849545, -0.003046507015824318, 0.006036100909113884, -0.0021794242784380913, 0.013779585249722004, -0.011074555106461048, -0.0018714256584644318, 0.019457470625638962, -0.0098358653485775, 0.0012520808959379792, -0.0037997642066329718, 0.006541620474308729, -0.015989139676094055, -0.01091386005282402, 0.008101699873805046, 0.00014081725385040045, -0.01604270376265049, -0.020649291574954987, -0.0022162501700222492, 0.011047772131860256, -0.01009699422866106, -0.033531662076711655, 0.02932681329548359, -0.010317949578166008, -0.01944408006966114, -0.02044842205941677, -0.01744878478348255, 0.027773430570960045, -0.021158158779144287, 0.009166303090751171, 0.026635175570845604, 0.017341654747724533, -0.0014403951354324818, 0.006186752580106258, -0.17055082321166992, 0.01979225128889084, 0.018975386396050453, -0.009869343601167202, -2.5108572572207777e-06, 0.01116829365491867, -0.009032391011714935, 0.01629713736474514, -0.01782373897731304, -0.004683585837483406, 0.016002530232071877, 0.02884472906589508, -0.03425478935241699, -0.018627213314175606, -0.007887439802289009, 0.0014077540254220366, -0.038807813078165054, 0.00388680724427104, -0.007010313682258129, 0.026635175570845604, 0.03738833963871002, -0.0235819723457098, 0.01091386005282402, 0.012152549810707569, 0.010927251540124416, 0.02897864207625389, -0.0016061117639765143, 0.014194713905453682, -0.0005850297748111188, -0.00933369342237711, 0.0003209712740499526, -0.0029544420540332794, 0.04111110419034958, -0.014971405267715454, -0.030906980857253075, 0.01122855395078659, -0.015882009640336037, -0.0056812334805727005, -0.00041408222750760615, 0.021024245768785477, 0.02833586186170578, 0.011864637956023216, 0.007425442337989807, -0.007545963395386934, -0.008275785483419895, 0.056243203580379486, 0.007351790554821491, -0.0011089619947597384, 0.008115091361105442, -0.016832787543535233, 0.008623957633972168, 0.017127394676208496, 0.016498006880283356, 0.0006561707123182714, 0.008998912759125233, 0.0147303631529212, 0.0010453535942360759, 0.007458920124918222, -0.003382961731404066, 0.015199056826531887, -0.004452587105333805, -0.02431849017739296, 0.025363007560372353, -0.007237964775413275, -0.0011005924316123128, -0.014275060966610909, 0.0027535734698176384, 0.011255336925387383, -0.0345761813223362, 0.002743530087172985, 0.020609118044376373, 0.020019901916384697, -0.012654720805585384, -0.03371914103627205, 0.01148298755288124, -0.0032021801453083754, -0.032165758311748505, -0.0137929767370224, -0.004030763171613216, -0.01744878478348255, 0.006347447633743286, 0.002338445046916604, -0.018131738528609276, -0.006257056724280119, -0.009621605277061462, -0.015078535303473473, -0.007673180196434259, 0.026353958994150162, -0.0008532730280421674, -0.012420374900102615, 0.004164675250649452, -0.022403543815016747, -0.007412050850689411, -0.00352859147824347, 0.019470861181616783, 0.010016647167503834, -0.02006007730960846, 0.014154540374875069, -0.021412592381238937, -0.011717334389686584, 0.01329080481082201, -0.006370882038027048, -0.03398696705698967, 0.003960459027439356, 0.03211219236254692, 0.004476021509617567, -0.0031954844016581774, -0.0032908970024436712, 0.027371693402528763, 0.0010805056663230062, -0.004489412996917963, 0.02469344437122345, 0.027800213545560837, 0.017863912507891655, -0.0030732895247638226, 0.029005425050854683, 0.003910241648554802, -0.009909517131745815, 0.006129839923232794, -0.018466519191861153, 0.04844950512051582, -0.018627213314175606, 0.004643412306904793, -0.0071977912448346615, 0.008784652687609196, -0.011878029443323612, -0.10032716393470764, -0.007686571218073368, -0.0006402686121873558, 0.0161766167730093, 0.0008603871101513505, 0.010204124264419079, -0.01993955485522747, 0.0137929767370224, -0.0016027640085667372, 0.029300032183527946, -0.010538904927670956, -0.0423966646194458, 0.002202858915552497, -0.01247393898665905, 0.025041617453098297, 0.005527234170585871, 0.02056894265115261, -0.009501084685325623, -0.02771986462175846, 0.013565325178205967, -0.002403727499768138, 0.011074555106461048, -0.019524427130818367, 0.007900831289589405, -0.007900831289589405, 0.014395582489669323, -0.023515015840530396, 0.015466880984604359, 0.024050666019320488, -0.0029778769239783287, -0.0323532335460186, -0.014288452453911304, 0.02455953322350979, -0.006236969493329525, 0.004646759945899248, 0.004750542342662811, 0.005962449125945568, -0.016096269711852074, 0.013551933690905571, -0.005400017369538546, -0.0032172452192753553, -0.00721118226647377, 0.0011005924316123128, -0.031174805015325546, -0.02482735738158226, -0.0308266319334507, -0.03639739006757736, -0.008001265116035938, 0.02785377763211727, -0.01881469041109085, -0.028898295015096664, -0.011944985017180443, -0.020140424370765686, 0.006484707817435265, 0.007271443028002977, -0.014462538063526154, -0.005095366388559341, -0.002236336935311556, -0.030317766591906548, 0.0018027955666184425, -0.00020599491836037487, -0.0005360680515877903, -0.02055555209517479, -0.0019283384317532182, 0.025041617453098297, -0.012346722185611725, -0.0451284758746624, -0.006652098149061203, 0.002440553391352296, -0.03760259971022606, -0.016765831038355827, -0.006026057526469231, -0.029514292255043983, 0.009460910223424435, -0.016698874533176422, 0.001822882448323071, -0.003880111500620842, -0.015132100321352482, 0.02884472906589508, -0.025537094101309776, -0.008784652687609196, -0.022296413779258728, 0.0021610113326460123, -0.019899381324648857, 0.02181432954967022, 0.0004540467052720487, 0.016484616324305534, 0.005972492508590221, 0.010418384335935116, 0.007171008735895157, 0.03238001838326454, 0.012139158323407173, 0.039557721465826035, -0.02659500204026699, -0.020153814926743507, 0.0032021801453083754, -0.005654450971633196, -0.021131375804543495, 0.007847266271710396, 0.00445593474432826, -0.02782699465751648, 0.0016136443009600043, -0.06052840128540993, 0.007445529103279114, 0.007874048314988613, -0.00420150114223361, -0.019778860732913017, 0.0034382008016109467, -0.009487693198025227, 0.0021057722624391317, -0.014141148887574673, 0.019885990768671036, -0.03953093662858009, 0.015132100321352482, 0.016002530232071877, -0.002919290214776993, -0.03524574264883995, -0.007063878700137138, 0.025537094101309776, 0.005282843951135874, -0.007566050160676241, 0.005785015411674976, 0.004077632445842028, 0.0029494203627109528, 0.02130546234548092, 0.007974483072757721, -0.01980564370751381, -0.008443176746368408, -0.007559354417026043, -0.005008323583751917, -0.02569778822362423, -0.013297501020133495, 0.011416031047701836, -0.008443176746368408, 0.017006874084472656, 0.03851320594549179, 0.0014805688988417387, -0.023662319406867027, 0.010224211029708385, 0.038138248026371, 0.029755333438515663, 0.04927976056933403, -0.03521895781159401, -0.02683604322373867, -0.006595185492187738, -0.011195075698196888, 0.006481359712779522, 0.00576492864638567, -0.018761126324534416, 0.0031904627103358507, -0.009809082373976707, 0.026742305606603622, 0.0132841095328331, 0.0225642379373312, -0.005500451661646366, -0.019390514120459557, 0.0016554920002818108, -0.0063440995290875435, 0.004874411039054394, 0.024251533672213554, -0.013665759935975075, -0.0185602568089962, 0.03425478935241699, 0.03409409523010254, 0.030237417668104172, -0.007003617938607931, -0.001493960153311491, 0.0039001984987407923, -0.009561344981193542, -0.01843973621726036, -0.019778860732913017, -0.013525151647627354, -0.01578827202320099, -0.00239703175611794, -0.0023987058084458113, 0.023983709514141083, -0.004760585725307465, -0.022269630804657936, 0.002341792918741703, 0.01953781768679619, 0.017971042543649673, 0.014663406647741795, -0.003354505402967334, 0.014449147507548332, -0.000551551696844399, 0.02455953322350979, 0.030773067846894264, 0.009601518511772156, -0.014288452453911304, -0.006534924730658531, -0.006173361092805862, 0.00260124821215868, -0.0027602692134678364, 0.023113278672099113, -0.007525876630097628, -0.019738687202334404, -0.025229094550013542, -0.004492760635912418, 0.003324375255033374, -0.01908251643180847, 0.020876942202448845, 0.02932681329548359, 1.412357232766226e-05, -0.0016261986456811428, 0.0017048721201717854, -0.016832787543535233, -0.012386896647512913, -0.0073852683417499065, -0.049092281609773636, -0.026782479137182236, -0.021385809406638145, 0.012420374900102615, -0.013009589165449142, 0.000668306543957442, -0.0030682676006108522, -0.016524789854884148, -0.008034743368625641, 0.006026057526469231, 0.010184037499129772, 0.012560982257127762, -0.005915579851716757, 0.0037830250803381205, -0.00626375200226903, -0.007880744524300098, 0.0037495470605790615, 0.0147303631529212, 0.005239322315901518, 0.016082879155874252, 0.019671730697155, 0.003481722204014659, 0.0021911414805799723, 0.015306186862289906, 0.01590879261493683, -0.003203853964805603, -0.017114004120230675, -0.012875676155090332, -0.01854686625301838, -0.0009290172019973397, 0.0032875491306185722, 0.006056188140064478, -0.0147303631529212, 0.04968149960041046, -0.0010386579670011997, 0.002658161101862788, 0.003083332907408476, -0.0042182402685284615, 0.030264200642704964, 0.002435531700029969, -0.018345998600125313, 0.013371152803301811, -0.030290983617305756, -0.0023032932076603174, -0.0021024246234446764, 0.004553021397441626, -0.0376293808221817, -0.007720049470663071, 0.003679242916405201, -0.02181432954967022, 0.022028587758541107, 0.0037428513169288635, 0.004563064780086279, 0.032540712505578995, -0.01498479675501585, 0.02619326487183571, 0.010050125420093536, -0.01707383058965206, 0.0031251804903149605, 0.02407744899392128, 0.009996560402214527, -0.01605609618127346, -0.012875676155090332, -0.004603238310664892, -0.0006879749125801027, -0.027505606412887573, -0.01845312863588333, 0.008677522651851177, -0.008516828529536724, -0.008490045554935932, -0.0023284016642719507, 0.010585774667561054, 0.012634634040296078, 0.00777361448854208, 0.006230274215340614, -0.02822873182594776, -0.012326635420322418, -0.0023568582255393267, -0.016631919890642166, 0.006856314372271299, -0.013551933690905571, -0.005108757875859737]} +{"id": "test:10000057", "text": "\"We made it through another Queensland summer \u2013 even though we are breaking records this March with a dry and hot Autumn on the horizon! I hope you are all keeping cool and making the most of it.\\nAs we roll into the final financial quarter, now is the time for businesses to really hit the ground running to make sure the foundations are strong, (vision, systems, processes, resource both people & financial) to ensure growth and financial goals are achieved for the year and beyond.\\nThere have been more legislative changes that affect all small businesses around the need to implement single touch payroll and for those who have a builders license, whereby the changes to the reporting rules have been increased. There is an article in this newsletter to explain more about what you need to do to make sure you are not in breach of the new rules.\\nOur first event for the year will be next Tuesday 19th, 5:30 pm at the Ovolo Incholm Hotel, we encourage anyone in business to come along, we have a variety of experts in their field to answer all of your questions whether it be buying, selling a business, strategic marketing and even Estate Planning which is usually overlooked due to the morbid discussions it requires. There will be a marketing expert who can provide some tips on effective ways to promote your business giving you the best bang for your buck and also automation experts who will help you identify ways to save you time in your business.\\nThis is what we do at Advivo and business growth will be the subject of our May event. It is our motivation for helping people achieve their business goals and drive success. I will, of course, be in attendance at our \u201cSpeed Dating With The Business Experts\u201d to talk about these workshops and much more, so register here, come along, say hi, have a glass of wine and see how we can help you!\\nWe understand businesses and our clients are confronted & challenged by an enormous range of issues and changes while also maintaining the day to day operations, not to mention the legislative changes and compliance that applies to different industries.\\nRegister here today to speak with a range of experts about the issues that are keeping you awake at night.\\nSingle Touch Payroll (STP) \u2013 The new GST!\\nEveryone should be aware of the changes regarding Single Touch Payroll (STP) and the reporting requirements that come with it. This includes ensuring your business accounting software is ready.\\nClick here to find out how this may affect you and if you have any queries contact your Advivo accountant today!\\nWhen looking at your business records, do you ever find yourself scratching your head over a transaction because it was a long time ago? OR do you already have enough going on in your business that you don\u2019t have time for the daily processing of creditors and bank reconciliations?\\nClick here to find out how Advivo implemented AI to help improve these processes.\\nAre you eligible for the Small Business Instant Asset Write-off?\\nIn January this year, the government announced they will introduce a new legislation to increase the $20,000 instant asset write off threshold to $25,000 from 29 January until 30 June 2020. This is a proposal and not yet law. It has sparked interest for small business owners and an increase in calls from our clients who want to know if they can take advantage of the asset write-off.\\nRead more here and find out if this is something you can take advantage of!\\nA fringe benefit is a benefit provided in respect of employment. This effectively means a benefit provided to an employee (or their associate) because they are an employee. Read more here about the different types of FBT and some of the strategies you can put in place to reduce the amount of FBT you have to pay.\\nDo you have a Queensland Building Construction Commission (QBCC) license? Are you aware of the new reporting laws that have been implemented from 1st January 2019?\\nFor those with a financial year ending 30 June 2018 your R&D Tax Incentive lodgements are almost due! Be sure to lodge with the Department of Industry, Innovation & Science and then your tax return with the ATO.\\nI often speak with inventors who tell me \u201cI know what the market wants\u201d as they are explaining their innovation, despite having conducted no market research.\\nFebruary\u2019s Charity of the Month!\\nThis month we have decided to donate our monthly contribution to GIVIT. GIVIT is working in partnership with the Queensland Government to manage donated funds and all offers of goods and services after major flooding in North and Far North Queensland. Donations made will help support Queensland residents severely impacted by monsoonal trough. Emergency workers and local councils are currently assessing need and will work with GIVIT to provide critical and immediate support as well as providing longer recovery support to re-establish homes and help communities recover.\\nPlease CLICK HERE to vote on this month\u2019s recipient of our contribution.\\n21 March \u2013 Lodge and pay February 2019 monthly business activity statement.\\n31 March \u2013 Lodge tax return for companies and super funds with total income of more than $2 million in the latest year lodged (excluding large/medium taxpayers), unless the return was due earlier.\\n31 March \u2013 Lodge tax return for the head company of a consolidated group (excluding large/medium), with a member who had a total income in excess of $2 million in their latest year lodged, unless the return was due earlier.\\n31 March \u2013 Lodge tax return for individuals and trusts whose latest return resulted in a tax liability of $20,000 or more, excluding large/medium trusts.\"", "vector_field": [-0.016594737768173218, 0.0032414868474006653, -0.005369911901652813, -0.057502370327711105, -0.003475264646112919, 0.004305699374526739, -0.0010755525436252356, -0.02824873849749565, -0.004476671107113361, -0.035450492054224014, 0.02013978734612465, 0.011521409265697002, 0.0004954694886691868, 0.021451734006404877, -0.00141051784157753, 0.0013285210588946939, 0.014710558578372002, -0.007536718621850014, 0.023838361725211143, -0.008422981016337872, -0.005586243700236082, 0.0026029592845588923, -0.0205864068120718, 0.030370185151696205, -0.004005626309663057, 0.004085878375917673, -0.003386289579793811, -0.0035345815122127533, -0.010697953402996063, 0.01226810272783041, 0.001728037022985518, 0.007466934155672789, -0.010321117006242275, 0.016050418838858604, -0.033356957137584686, -0.027732333168387413, -0.004556923173367977, 0.006619053427129984, 0.029002409428358078, -0.03254745900630951, 0.006095670163631439, 0.00891147181391716, -0.003817208344116807, 0.013782424852252007, -0.019106976687908173, 0.018367262557148933, 0.01519904937595129, -0.03620416298508644, -0.016455169767141342, 0.007138947490602732, 0.015031566843390465, 0.02667160890996456, -0.0431826077401638, 0.0032502098474651575, -0.004856996238231659, -0.006535312160849571, 0.00596656883135438, 0.02227519080042839, 0.00046275800559669733, -0.0066993054933846, -0.005568797700107098, 0.014850126579403877, -0.03304990753531456, 0.007159882690757513, -0.007306430023163557, -0.016734305769205093, -0.0263506006449461, -0.020167700946331024, -0.009581402875483036, 0.009623273275792599, 0.03425019979476929, 0.02655995450913906, -0.0020028131548315287, -0.009539531543850899, 0.001809161389246583, -0.033747751265764236, -0.0105723412707448, -0.0008941130363382399, 0.008094994351267815, 0.011549323797225952, 0.00019605063425842673, -0.01147953886538744, -0.03427811339497566, 0.016538910567760468, 0.003028644248843193, 0.0033601203467696905, -0.017836900427937508, 0.0005831361631862819, -0.0018109059892594814, 0.014905954711139202, 0.002194720320403576, 0.010767737403512001, 0.027397368103265762, 0.027732333168387413, -0.009099889546632767, 0.003255443647503853, -0.02194022573530674, 0.03620416298508644, 0.0040300507098436356, -0.008192691951990128, 0.0033409297466278076, -0.01167493499815464, -0.01350328791886568, -0.017557762563228607, -0.021577347069978714, 0.005422250367701054, 0.025527145713567734, 0.007362257689237595, 0.010286225005984306, -0.013545158319175243, -0.028723271563649178, 0.021242380142211914, -0.002011536154896021, -0.012093641795217991, 0.009023127146065235, -0.015603798441588879, -0.0038032515440136194, -0.0013102027587592602, -0.01934424415230751, -0.014026670716702938, 0.019204676151275635, 0.00284720491617918, -0.006954018492251635, -0.005666495766490698, 0.01709718629717827, 0.0003639694186858833, -0.023280085995793343, -0.016399340704083443, 0.0034299048129469156, -0.024061672389507294, -0.007418084889650345, 0.005753726232796907, 0.010265289805829525, 0.012323930859565735, -0.0006010184297338128, 0.016734305769205093, -0.023726707324385643, -0.007697222754359245, -0.018911581486463547, -0.024675775319337845, 9.426350152352825e-05, 0.02435476705431938, -0.010760759003460407, -0.008332260884344578, 0.018241649493575096, 0.02396397292613983, 0.023280085995793343, -0.010111764073371887, 0.03148673474788666, -0.006556247361004353, 0.02288929373025894, -0.0037753377109766006, 0.009183631278574467, -0.010118742473423481, 0.021172596141695976, 0.01322415005415678, -0.005659517366439104, -0.0016661032568663359, -0.02267993986606598, -0.01572941057384014, 0.003302548313513398, 0.013217171654105186, 0.023615051060914993, 0.03464099019765854, 0.013252063654363155, 0.01617603190243244, 0.014933868311345577, -0.008834709413349628, -0.0001790406822692603, -0.0032536990474909544, 0.013621920719742775, -0.00042939232662320137, -0.018129995092749596, 0.020614320412278175, 0.02429893985390663, 0.02256828546524048, -0.029197806492447853, -0.0009115591528825462, -0.0076483734883368015, -0.023824404925107956, 0.008974278345704079, -0.01849287375807762, 0.030928460881114006, 0.027885857969522476, -0.014515161514282227, -0.007376214489340782, 0.008325282484292984, 0.0005460631800815463, 0.002430242719128728, -0.0053524659015238285, -0.0026395961176604033, 0.0276346355676651, 0.006346893962472677, 0.0015588097739964724, -0.6449198126792908, 0.011130617000162601, 0.0015561928739771247, -0.02013978734612465, -0.020558493211865425, -0.004933759104460478, 0.02177274227142334, -0.001520428340882063, -0.033189475536346436, 0.021116768941283226, 0.002796611050143838, 0.011326013132929802, 0.011095724999904633, 0.0017952044727280736, -0.008450894616544247, -0.026950746774673462, 0.010977091267704964, -0.0024284981191158295, -0.0032100838143378496, 0.023489439859986305, -0.0038974604103714228, 0.041033245623111725, -0.018032297492027283, 0.02064223401248455, 0.02166108787059784, -0.0053838687017560005, -0.0009490682859905064, -0.03475264832377434, -0.01561775617301464, 0.003086216514930129, -0.02566671371459961, -0.0027791650500148535, 0.015185092575848103, 0.008946363814175129, 0.05102637782692909, 0.0010799140436574817, -0.03483638912439346, 0.041089072823524475, 0.004574369639158249, 0.03196126967668533, -0.02273576706647873, -0.005949122831225395, 0.019148847088217735, 0.013014796189963818, 0.016790134832262993, 0.01849287375807762, 0.022289147600531578, 0.0008901876281015575, 0.017571719363331795, -0.004159152042120695, 0.006053799297660589, 0.00024359127564821392, -0.014180196449160576, -0.0008531147032044828, 0.014347678981721401, -0.006123583763837814, 0.00683887442573905, 0.007962403818964958, 0.001162783126346767, 0.009469747543334961, -0.0005390847218222916, 0.010802630335092545, -0.0017838645726442337, -0.02836039289832115, -0.020656190812587738, -0.004797679837793112, -0.01553401444107294, 0.009574423544108868, 0.020628277212381363, -0.02120050974190235, -0.004323145374655724, 0.029421117156744003, -0.0016521464567631483, -0.015645669773221016, 0.027662549167871475, 0.016734305769205093, 0.0060119288973510265, -0.001288395025767386, 0.009002191945910454, 0.016148118302226067, -0.005579265300184488, -0.009148739278316498, -0.01263098232448101, -0.008869601413607597, 0.024173326790332794, -0.004658110905438662, -0.024996783584356308, -0.030872631818056107, 0.022973034530878067, 0.011039896868169308, 0.0005111709469929338, 0.021242380142211914, 0.008255498483777046, 0.0004679918347392231, 0.020391011610627174, 0.0038032515440136194, -0.014194153249263763, -0.007222688756883144, 0.006001461297273636, -0.024633904919028282, -0.02261015586555004, -0.004134727641940117, 0.006765600759536028, 0.015659626573324203, 0.04393627867102623, 0.03896762803196907, -0.001769035356119275, 0.0005962207214906812, 0.02852787636220455, -0.038493093103170395, -0.0008701246115379035, -0.026378514245152473, 0.0033165051136165857, -0.007522761821746826, -0.01928841695189476, -0.029114065691828728, 0.032017096877098083, -0.014026670716702938, 0.011409754864871502, 0.010774715803563595, 0.0037229994777590036, 0.007753050420433283, 0.018241649493575096, -0.04214979708194733, 0.00792751181870699, 0.014284873381257057, -0.0005068094469606876, -0.0017620568396523595, 0.005139623302966356, -0.009372049011290073, -0.005017500836402178, -0.004658110905438662, 0.038660574704408646, -0.0039672451093792915, 0.011570258997380733, -0.021842526271939278, -0.0036811288446187973, 0.0007628310704603791, 0.016510996967554092, -0.04178691655397415, -0.016622651368379593, -0.024494335055351257, 0.008248520083725452, -0.010453707538545132, -0.016203945502638817, -0.03338487073779106, 0.005635092966258526, 0.00046319415559992194, -0.011437668465077877, 0.0015622989740222692, -0.031151769682765007, 0.007815856486558914, -0.006723729893565178, 0.013028752990067005, -0.0015832342905923724, 0.0012744382256641984, -0.008311325684189796, -0.0196373388171196, -0.003921885043382645, -0.014361635781824589, -0.021814612671732903, 0.022191449999809265, -0.01595272123813629, -0.027146143838763237, -0.014026670716702938, -0.015589841641485691, -0.007878662087023258, 0.008925428614020348, -0.0008827730780467391, -0.022010009735822678, 0.014891997911036015, -0.00706567382439971, 0.008841687813401222, 0.00578164029866457, -0.0009857050608843565, 0.008534636348485947, -0.00561066810041666, -0.007913554087281227, 0.004867464303970337, -0.0057921078987419605, -0.005300127435475588, 0.00017173512605950236, -0.01258213259279728, -0.008192691951990128, 0.012121555395424366, -0.00103280961047858, 0.010984069667756557, 0.009337157011032104, -0.016427254304289818, 0.012868248857557774, 0.00916967447847128, 0.016845962032675743, 0.01051651407033205, -0.00482210423797369, -0.021228423342108727, 0.010663061402738094, -0.0033077821135520935, 0.024005845189094543, -0.00583746749907732, -0.002264504786580801, 0.03609250858426094, 0.01215644832700491, 0.012302995659410954, 0.023307999595999718, 0.0026762329507619143, -0.015561928041279316, 0.016720348969101906, -0.01652495376765728, 0.029756082221865654, 0.014431420713663101, 0.00933017861098051, -0.025359662249684334, -0.011060832068324089, -0.010314138606190681, -0.003907928243279457, 0.01617603190243244, -0.005593222100287676, 0.004836061038076878, -0.008101972751319408, 0.010558384470641613, 0.015310704708099365, -0.016734305769205093, -0.003893971210345626, -0.015101350843906403, -0.013210193254053593, 0.01572941057384014, -0.014305808581411839, 0.013307890854775906, -0.018939495086669922, -0.019665252417325974, -0.013126451522111893, 0.0032414868474006653, 0.009630251675844193, 0.01737632416188717, 0.018199779093265533, 0.02554110251367092, 0.03204501047730446, -0.019023235887289047, 0.031207596883177757, -0.003743934677913785, 0.01843704655766487, 0.009204566478729248, 0.021786699071526527, -0.005673474166542292, 0.027201971039175987, 0.0037962731439620256, -0.00776700722053647, -0.004256850108504295, -0.015520057640969753, 0.007906575687229633, 0.020321225747466087, 0.017557762563228607, -0.019441941753029823, 0.032240405678749084, 0.03726488724350929, -0.025862110778689384, -0.007452977355569601, 0.008555571548640728, 0.01771128922700882, 0.019595468416810036, 0.001060723327100277, -0.0051012421026825905, -0.01122133620083332, -0.03235206380486488, 0.03729280084371567, 0.016050418838858604, 0.0011854630429297686, -0.023489439859986305, 0.013168321922421455, -0.0016966339899227023, -0.0072924732230603695, 0.008953342214226723, 0.01217738352715969, -0.01663660816848278, 0.012903141789138317, 0.0014619838912039995, -0.0036811288446187973, 0.011060832068324089, -0.01826956495642662, 0.016273729503154755, 0.00916967447847128, -0.034082718193531036, 0.009441833943128586, 0.003914906643331051, 0.0053105950355529785, -0.018423089757561684, 0.0016661032568663359, 0.013510266318917274, 0.0036253011785447598, 0.0030879611149430275, -0.01120040100067854, -0.005502502433955669, -0.00011601661390159279, -0.008981256745755672, 0.0026413407176733017, 0.002800100250169635, 0.014654730446636677, -0.023936059325933456, 0.0015169390244409442, 0.0018614997388795018, -0.0010659572435542941, 0.00029592961072921753, 0.01277055125683546, -0.026880962774157524, 0.02448037825524807, -0.005422250367701054, -0.013105516321957111, -0.013761489652097225, -0.027257798239588737, 0.00317170238122344, 0.036008767783641815, -0.02470368891954422, -0.01759963296353817, 0.02396397292613983, 0.01917676255106926, 0.016762221232056618, 0.016399340704083443, 0.005216386169195175, 0.04067036509513855, -0.025415489450097084, -0.010621190071105957, -0.01015363447368145, -0.029197806492447853, 0.007934490218758583, 0.12494204938411713, 0.016148118302226067, 0.005324552301317453, 0.02357318066060543, -0.003480498446151614, -0.011633064597845078, -0.015324661508202553, -0.027118230238556862, 0.028807014226913452, -0.005052392836660147, -0.005649049766361713, 0.0030391120817512274, -0.017948556691408157, -0.006552758160978556, 0.020600363612174988, 0.004591815639287233, 0.00044138653902336955, -0.0006524844211526215, 0.008395067416131496, 0.0010476388270035386, -0.010739823803305626, 0.012707744725048542, -0.018032297492027283, 0.044969089329242706, 0.02628081664443016, 0.02108885534107685, 0.022875336930155754, 0.010523492470383644, 0.015226962976157665, -0.00792751181870699, -0.01381731778383255, 0.021563390269875526, 0.0075855678878724575, 0.020725976675748825, -0.0032641668803989887, -0.003981201909482479, 0.0004030051059089601, -0.004574369639158249, 0.009832626208662987, 0.013028752990067005, 0.007704201154410839, 0.004504585172981024, 0.018255608156323433, -0.015645669773221016, 0.015240919776260853, -0.011737741529941559, -0.0065283337607979774, 0.018925538286566734, -0.010502557270228863, 0.0004006062517873943, 0.01945590041577816, 0.00027303159004077315, -0.023447569459676743, 0.0063050235621631145, 0.005432717967778444, 0.00381371914409101, -0.0068528312258422375, -0.007445998955518007, 0.023321956396102905, -0.034613076597452164, -0.02939320169389248, -0.03935841843485832, -0.004773254971951246, -0.0038590789772570133, -0.006301534362137318, -0.009204566478729248, -0.01986064948141575, 0.021116768941283226, -0.015324661508202553, 0.007355279289186001, -0.006615564227104187, -0.041368212550878525, -0.02266598306596279, -0.013063645921647549, 0.020851587876677513, 0.006249195896089077, 0.019414028152823448, 0.01669243536889553, 0.003328717313706875, -0.001659124856814742, -0.0037055532447993755, -0.012330909259617329, -0.014145304448902607, -0.03226831927895546, -0.009916367940604687, 0.000644197512883693, 0.0022470587864518166, 0.011332991532981396, -0.03478056192398071, 0.02425706759095192, -0.012651917524635792, -0.002117957454174757, 0.0029780506156384945, -0.01156328059732914, -0.005240811035037041, 0.0046197292394936085, 0.01088637113571167, 0.015143221244215965, 0.009532553143799305, 0.004225447308272123, -0.004096345975995064, -0.011011983267962933, -0.016385383903980255, -0.021046984940767288, 0.0031577455811202526, 0.001398305525071919, 0.010446729138493538, -0.004508074373006821, -0.008234563283622265, -0.04000043496489525, 0.022945120930671692, -0.035785455256700516, -0.00786470528692007, 0.015771280974149704, -0.016594737768173218, 0.0035834305454045534, 0.020572450011968613, -0.0036008767783641815, 0.0035276031121611595, -0.0025785346515476704, -0.001980133354663849, -0.04036331549286842, 0.02025144174695015, 0.004124260041862726, -0.00031446610228158534, 0.0020132807549089193, 0.013231128454208374, -0.013663792051374912, 8.477718438371085e-06, -0.008276433683931828, 0.027997514232993126, 0.008988235145807266, 1.12650068331277e-05, -0.020628277212381363, -0.02989565022289753, -0.001809161389246583, -0.02295907773077488, -0.006416678428649902, -0.03090054541826248, -0.019260503351688385, -0.027718376368284225, 0.00956046674400568, 0.013356740586459637, -0.01404760591685772, 0.0399446077644825, -0.031040115281939507, 0.005669984966516495, -0.02143777720630169, 0.01725071109831333, 0.024605991318821907, -0.016385383903980255, -0.027257798239588737, -0.01873014122247696, -0.007341322023421526, -0.005443185567855835, -0.024326853454113007, -0.0215913038700819, -0.030872631818056107, 0.0007414596038870513, 0.020544536411762238, 0.03570171445608139, -0.024005845189094543, -0.004647643305361271, 0.011116660200059414, -0.009211544878780842, 0.0011566769098863006, 0.0033077821135520935, -0.033691924065351486, -0.0137894032523036, -0.007142436690628529, 0.0018772012554109097, 0.029253633692860603, -0.01413134764879942, -0.008108951151371002, 0.005359444301575422, 0.02537361904978752, -0.01568754017353058, 0.003663682611659169, -0.024522248655557632, -0.03444559499621391, -0.01404760591685772, -0.00459530483931303, 0.005820021498948336, 0.02644830010831356, -0.03924676403403282, -0.00956046674400568, 0.05071932449936867, 0.02121446654200554, 0.01962338201701641, 0.008234563283622265, 0.00743204215541482, -0.013580050319433212, 0.03553423285484314, 0.019079063087701797, 0.005432717967778444, -0.000601890729740262, -0.015575884841382504, -0.04281972721219063, -0.0058653815649449825, 0.03157047554850578, 0.010028023272752762, -0.006364339962601662, 0.0040300507098436356, 0.028695357963442802, -0.01838121935725212, 0.012163426727056503, -7.638124225195497e-05, -0.0007580333622172475, -0.016873875632882118, -0.011640042997896671, -0.005872359964996576, -0.008946363814175129, 0.004856996238231659, -0.006130562163889408, 0.02538757584989071, -0.0035031784791499376, 0.0061584762297570705, 0.007355279289186001, 0.02295907773077488, 0.013949908316135406, 0.007913554087281227, -0.019427984952926636, 0.023056775331497192, 0.007857726886868477, 0.01322415005415678, 0.02166108787059784, 0.019888563081622124, -0.024187283590435982, -0.006151497829705477, 0.011591194197535515, -0.00333395111374557, 0.014501204714179039, 0.01860453002154827, -0.028667444363236427, -0.018283521756529808, 0.025192180648446083, 0.019930433481931686, -0.028862841427326202, -0.011228314600884914, 0.03614833578467369, 0.00885564461350441, -0.002252292586490512, -0.006531822960823774, -0.029867736622691154, -0.0033304619137197733, 0.026867005974054337, -0.008018231019377708, 0.005547862499952316, -0.02852787636220455, -0.020628277212381363, -0.00496167317032814, 0.008185713551938534, -0.00899521354585886, 0.02796960063278675, 0.012728679925203323, -0.006744665093719959, -0.0052059185691177845, -0.0023517352528870106, -0.0042603397741913795, -0.010328095406293869, -0.014305808581411839, 0.017669418826699257, -0.013028752990067005, 0.01617603190243244, -0.00045839647646062076, 0.015938764438033104, -0.01333580445498228, -0.010949177667498589, 0.025122394785284996, 0.02283346652984619, -0.022973034530878067, -0.014389549382030964, -0.009937303140759468, 0.009044062346220016, -0.008053123019635677, -0.010125720873475075, -0.003342674346640706, -0.009867518208920956, -0.008478808216750622, -0.007913554087281227, -0.0023430122528225183, 0.005858403164893389, -0.03639955818653107, -0.008583485148847103, -0.0011479539098218083, -0.030984288081526756, 0.023321956396102905, -0.00491980230435729, -0.004874442704021931, -0.016762221232056618, -0.02487117052078247, -0.0012726936256513, -0.002044683787971735, -0.007913554087281227, -0.007086609024554491, -0.011584215797483921, -0.008136864751577377, 0.018827838823199272, -0.007934490218758583, 0.021968139335513115, 0.011186444200575352, 0.007201753556728363, -0.033636096864938736, -0.002569811651483178, -0.0015117052244022489, -0.003040856681764126, 0.020949285477399826, 0.0011270185932517052, -0.012589110992848873, -0.018688270822167397, 0.0197489932179451, 0.007962403818964958, -0.03176587447524071, 0.0024738581851124763, 0.006556247361004353, 0.018911581486463547, -0.009141760878264904, 0.02143777720630169, -0.027383411303162575, 0.014124369248747826, -0.004448757506906986, -0.003736956277862191, 0.014612860046327114, -0.014598903246223927, 0.030202701687812805, 0.02796960063278675, 0.01142371166497469, -0.019316330552101135, -0.039107196033000946, 0.0015954466070979834, -0.005474588833749294, 0.006702794693410397, -0.011765655130147934, -0.013014796189963818, -0.01539444550871849, -0.010132699273526669, -0.011291121132671833, 0.019986260682344437, -0.003524113679304719, -0.0020307269878685474, 0.00037487325607798994, 0.021074898540973663, 0.02120050974190235, -0.012456521391868591, 0.002939669182524085, 0.008611398749053478, -0.028890755027532578, 0.017111143097281456, -0.03531092405319214, -0.014249981380999088, -0.02012583054602146, 0.05119385942816734, 0.009064997546374798, -0.033077821135520935, -0.014501204714179039, 0.006580671761184931, -0.04265224561095238, -0.024103542789816856, -0.006039842497557402, 0.03034227155148983, 0.03824186697602272, 0.0005425739800557494, 0.004888399504125118, 0.03570171445608139, 0.007718157954514027, 0.022986991330981255, -0.029225720092654228, -0.012002922594547272, -0.003663682611659169, -0.009532553143799305, -0.0033723325468599796, -0.010279246605932713, -0.0403912290930748, 0.021730871871113777, 0.026992619037628174, -0.01203781459480524, 0.017781073227524757, 0.015561928041279316, -0.011277164332568645, -0.007536718621850014, 0.004494117572903633, 0.016552867367863655, 0.010530470870435238, 0.01497573871165514, 0.016957616433501244, -0.0417310893535614, -0.016999486833810806, 0.022749723866581917, -0.005073328036814928, -0.02668556571006775, 0.013998757116496563, 0.008464851416647434, 0.0012430351926013827, 0.006521354895085096, 0.002700657583773136, -0.013859188184142113, -0.007355279289186001, -0.006489952094852924, 0.02329404279589653, 0.011186444200575352, 0.009762842208147049, -0.007850748486816883, 0.019721079617738724, -0.007013335358351469, 0.005708366632461548, 0.007459955755621195, -0.026266859844326973, -0.0008164778701029718, -0.0032170622143894434, -0.020111873745918274, 0.028723271563649178, -0.02273576706647873, 0.0003002911398652941, 0.019441941753029823, -0.017836900427937508, -0.018939495086669922, 0.0005639454466290772, -0.002032471587881446, 0.008374132215976715, -0.012079684995114803, -0.01956755481660366, -0.005509480834007263, -0.013231128454208374, -0.01469660084694624, 0.0031333209481090307, -0.020684106275439262, -0.003611344378441572, -0.011535366997122765, -0.014612860046327114, 0.021354036405682564, -0.018939495086669922, -0.009253415279090405, -0.023824404925107956, -0.001703612389974296, 0.0006642605294473469, 0.007271537557244301, 0.22866962850093842, 0.00983960460871458, 0.020628277212381363, 0.026755351573228836, 0.007571610622107983, 0.010956156067550182, 0.016050418838858604, 0.006678370293229818, -0.007383192889392376, 0.016287686303257942, -0.020237484946846962, 0.005593222100287676, -0.020893458276987076, -0.004127749241888523, -0.010683996602892876, -0.02674139477312565, -0.049770258367061615, -0.016217902302742004, -0.02318238839507103, 0.022763680666685104, 0.03921885043382645, 0.015547971241176128, -0.006353872362524271, -0.0063782972283661366, 0.022414758801460266, 0.014961781911551952, 0.0008457000949420035, -0.0013407333754003048, 0.030091047286987305, 0.014180196449160576, -0.01286127045750618, 0.005118688102811575, 0.003883503610268235, -0.01128414273262024, -0.008150821551680565, 0.006552758160978556, 0.0001980133238248527, -0.020725976675748825, 0.01888366788625717, 0.0105723412707448, 0.009462769143283367, 0.008374132215976715, 0.001769035356119275, -0.05122177302837372, 0.016845962032675743, 0.011402776464819908, -0.0062352390959858894, -0.0013389887753874063, -0.014487247914075851, 0.006563225761055946, -0.043685052543878555, -0.01399177871644497, 0.021619217470288277, 0.01060723327100277, 0.016203945502638817, -0.006360850762575865, 0.018032297492027283, 0.00913478247821331, -0.011542345397174358, -0.010760759003460407, -0.007166861090809107, 0.02481534332036972, -0.01709718629717827, -2.154866888304241e-05, -0.014096454717218876, 0.009937303140759468, -0.011995944194495678, 0.03260328620672226, 0.010914284735918045, -0.014515161514282227, 0.017781073227524757, 0.0033269727136939764, -0.02251245826482773, 0.007194774691015482, -0.02752297930419445, -0.01641329750418663, 0.014236023649573326, -0.0037962731439620256, 0.03645538538694382, 0.029197806492447853, -0.009672122076153755, 0.0200839601457119, 0.011102703399956226, -0.023447569459676743, -0.010530470870435238, -0.038548920303583145, 0.0196373388171196, -0.007376214489340782, -0.013042709790170193, -0.00039886165177449584, 0.006793514359742403, 0.015575884841382504, -0.012847313657402992, -0.01741819456219673, 0.005956101231276989, -0.00638527562841773, 0.0004675556847359985, 0.013091559521853924, -0.021312166005373, 0.008346217684447765, -0.019079063087701797, 0.028611617162823677, 0.0207538902759552, 0.0026919343508780003, 0.000987449660897255, 0.003138554748147726, 0.0024040737189352512, 0.015129264444112778, 0.018018340691924095, -0.031207596883177757, 0.017529848963022232, -0.012924076989293098, -0.007009846158325672, -0.00657020416110754, -0.007955425418913364, 0.011626086197793484, -0.0004195788933429867, 0.0020097915548831224, 0.00936507061123848, -0.00933017861098051, 0.01591085083782673, -0.014173218049108982, -0.002997241448611021, -0.012951990589499474, 0.0021040006540715694, -0.011786590330302715, -0.02093532867729664, -0.0069784433580935, -0.006741175893694162, -0.023698793724179268, 0.018520787358283997, 0.003182169981300831, -0.014515161514282227, -0.03438976779580116, -0.017446108162403107, -0.012121555395424366, 0.03045392595231533, -0.021074898540973663, -0.01720884069800377, 0.01607833243906498, -0.008569528348743916, 0.001304968842305243, -0.02249850146472454, -0.02069806307554245, 0.00739017128944397, -0.021647131070494652, 0.03159838914871216, -0.018199779093265533, -0.02492699958384037, 0.00776700722053647, -0.0037299778778105974, -0.0005949122714810073, -0.005411782767623663, -0.017850857228040695, 0.027160100638866425, -0.013761489652097225, -0.030928460881114006, -0.03497595712542534, 0.012107598595321178, 0.006475995294749737, -0.030202701687812805, 0.00030726956902071834, 0.028974495828151703, -0.011605150997638702, -0.02012583054602146, -0.013684727251529694, -0.18043462932109833, 0.011367883533239365, 0.017348410561680794, -0.01888366788625717, 0.028164995834231377, -0.008115929551422596, 0.01341954618692398, 0.011849396862089634, -0.009797734208405018, 0.01776711642742157, 0.003736956277862191, 0.011123638600111008, -0.012009900994598866, 0.0013791148085147142, -0.005725812632590532, 0.014019692316651344, -0.04011209309101105, 0.046839311718940735, 0.02194022573530674, 0.009413919411599636, 0.013621920719742775, -0.012421628460288048, 0.010342053137719631, -0.015324661508202553, 0.011095724999904633, 0.0197489932179451, -0.010321117006242275, 0.00772513635456562, -0.0020830652210861444, -0.01322415005415678, -0.011291121132671833, 0.02781607396900654, 0.013091559521853924, 0.008562549948692322, -0.009162696078419685, 0.004815125837922096, -0.01142371166497469, -0.01923258975148201, -0.010028023272752762, 0.04778837785124779, 0.04011209309101105, 0.0199164766818285, -0.0017986936727538705, 0.018827838823199272, -0.003072259481996298, 0.03148673474788666, 0.0023465014528483152, 0.004504585172981024, 0.00899521354585886, -2.3892989702289924e-05, 0.005443185567855835, -0.01567358337342739, 0.01967920921742916, 0.003239742247387767, 0.01189824566245079, 0.0008941130363382399, -0.019721079617738724, -0.013810339383780956, 0.002850694116204977, -0.021046984940767288, 0.005338509101420641, -0.010307160206139088, 0.010725867003202438, -0.005087284836918116, -0.0017463554395362735, -0.030258528888225555, 0.008360174484550953, 0.0032467206474393606, -0.02379649132490158, 0.025638800114393234, -0.00868118368089199, -0.024857213720679283, 0.007676287554204464, -0.012316952459514141, -0.010809608735144138, -0.03416645899415016, -0.009441833943128586, 0.006130562163889408, 0.008806794881820679, -0.004410375840961933, -0.011828460730612278, 0.023601094260811806, -0.014375592581927776, 0.004138216841965914, -0.005031457636505365, -0.012603068724274635, 0.024159369990229607, 0.021465690806508064, 0.0205864068120718, -0.01720884069800377, -0.012456521391868591, -0.009986151941120625, 0.0005966568714939058, 0.003281612880527973, -0.004850017838180065, 0.02831852249801159, -0.00390443904325366, 0.008011252619326115, -0.0015998081071302295, 0.012379758059978485, 0.01491991151124239, 0.005265235435217619, -0.03137508034706116, 0.02531779184937477, 0.011584215797483921, 0.0009063252946361899, -0.023168431594967842, 0.019427984952926636, 0.026769308373332024, -0.005596711300313473, -0.027606721967458725, 0.019427984952926636, 0.020614320412278175, 0.010495578870177269, 0.0030966841150075197, 0.03391523286700249, -0.014263938181102276, -0.018367262557148933, 0.020879501476883888, -0.007976360619068146, 0.04376879706978798, -0.010781695134937763, 0.014598903246223927, 0.005331530701369047, 0.0015823619905859232, -0.00922550167888403, -0.12617024779319763, -0.026308730244636536, 0.019804822281003, 0.01833934895694256, 0.015017610043287277, 0.0015457251574844122, 0.014333722181618214, 0.04122864082455635, -0.008032187819480896, 0.02492699958384037, -0.005558330100029707, -0.015492144040763378, 0.002069108420982957, 0.00014818288036622107, 0.020293312147259712, -0.006493441294878721, 0.008806794881820679, -0.011584215797483921, -0.00942089781165123, 0.01591085083782673, -0.01553401444107294, -0.028946582227945328, -0.00248083658516407, -0.003834654577076435, -0.03405480086803436, -0.015031566843390465, -0.027090316638350487, 0.01404760591685772, -0.012533283792436123, 0.01574336737394333, -0.010453707538545132, -0.021019071340560913, 0.010872414335608482, -0.011172487400472164, 0.010844500735402107, 0.019386114552617073, -0.02826269529759884, -0.016566824167966843, 0.003569473745301366, -0.018129995092749596, 0.004745341371744871, 0.0034630524460226297, 0.03681826591491699, -0.03807438537478447, 0.017460064962506294, -0.019079063087701797, -0.024619948118925095, 0.013726597651839256, 0.0136777488514781, -0.018423089757561684, -0.025820240378379822, 0.003422926412895322, -0.006744665093719959, -0.0017760137561708689, 0.006242217496037483, -0.02233101800084114, 0.003932352643460035, 0.007815856486558914, -0.010453707538545132, 0.0021231912542134523, -0.008192691951990128, -0.011172487400472164, -0.010321117006242275, 0.016832005232572556, 0.01698553003370762, -0.011193422600626945, -0.004232425708323717, -0.02149360440671444, 0.04290346801280975, 0.00017369781562592834, -0.018464960157871246, 0.019162805750966072, -0.019218632951378822, 0.01697157323360443, -0.01945590041577816, 0.0008291262784041464, -0.023419655859470367, -0.028611617162823677, -0.003178680781275034, 0.012498391792178154, -0.026936789974570274, -0.029253633692860603, -0.006015418097376823, -0.012693787924945354, 0.022149577736854553, 0.014061562716960907, -0.0022993970196694136, -0.0020743422210216522, 0.00445573590695858, -0.02798355743288994, 0.01277055125683546, 0.013887101784348488, 0.01663660816848278, -0.002362203085795045, 0.007166861090809107, 0.001223844476044178, 0.002836737083271146, 0.0033112713135778904, 0.016497040167450905, 0.025401532649993896, -0.002777420450001955, -0.030035220086574554, -0.055911287665367126, 0.013182278722524643, -0.011346948333084583, -0.00661207502707839, -0.004141706041991711, 0.010921263135969639, 0.0030146874487400055, 0.001784736872650683, -0.00964420847594738, -0.00583746749907732, -0.0408378504216671, 0.03776733577251434, -0.02064223401248455, -0.01900927908718586, -0.01986064948141575, -0.004155662842094898, 0.014766385778784752, -0.010349031537771225, 0.00015679690113756806, 0.0036462366115301847, -0.017529848963022232, -0.016943659633398056, 0.017641503363847733, -0.0009944281773641706, 0.011039896868169308, 0.01843704655766487, 0.0020970222540199757, -0.018367262557148933, 0.004361527040600777, -0.011786590330302715, 0.011172487400472164, -0.011570258997380733, 0.009016148746013641, 0.024605991318821907, 0.0008334877784363925, -0.024829300120472908, -0.0036078551784157753, 0.022805552929639816, 0.010474642738699913, -0.012170405127108097, -0.0043266345746815205, -0.019260503351688385, -0.008206648752093315, -0.030649323016405106, -0.013252063654363155, -0.0070307813584804535, 0.0063782972283661366, 0.02136799320578575, 0.014612860046327114, -0.014752428978681564, 0.039051368832588196, 0.011437668465077877, 0.0017899706726893783, -0.0077949208207428455, -0.017669418826699257, 0.001915582688525319, 0.005806064698845148, 0.0024616457521915436, -0.009155717678368092, -0.02397793158888817, 0.04658808559179306, 0.00661207502707839, 0.03215666487812996, -0.01464077364653349, 0.021563390269875526, -0.0004566518764477223, -0.027788160368800163, -0.0022069327533245087, 0.00610613776370883, -0.0392746776342392, -0.008004274219274521, -0.012777529656887054, 0.00021120694873388857, 0.03304990753531456, 0.022819509729743004, -0.0024564119521528482, -0.004895377904176712, -0.003524113679304719, -0.007606503088027239, 0.026545997709035873, 0.010230397805571556, -0.012330909259617329, -0.013775446452200413, 0.03162630274891853, 0.01804625429213047, 0.005226853769272566, -0.0022505479864776134, 0.003211828414350748, -0.024899085983633995, 0.018813882023096085, -0.005157069303095341, 0.0072924732230603695, -0.005732791032642126, -0.03388731926679611, 0.03271494060754776, 0.026699524372816086, 0.019386114552617073, 0.0025104950182139874, -0.008841687813401222, 0.010321117006242275, 0.013168321922421455, -0.004877931904047728, -0.011821482330560684, -0.019302373751997948, -0.013489330187439919, 0.0013965609250590205, -0.01122133620083332, -0.01585502363741398, 0.0024162859190255404, 0.00794844701886177, 0.01709718629717827, 0.010035001672804356, 0.03277076780796051, 0.0201956145465374, -0.011005004867911339, -0.005055882036685944, -0.015422359108924866, -0.025010740384459496, -0.02132612280547619, 0.040586624294519424, 0.00734830042347312, -0.008946363814175129, 0.007501826621592045, 0.0044382899068295956, 0.03430602699518204, -0.00045883262646384537, 0.021758785471320152, -0.003082727314904332, -0.004815125837922096, -0.010146656073629856, -0.0036357687786221504, 0.010851479135453701, -0.0280254278331995, -0.015701496973633766, 0.0036776396445930004, -0.030035220086574554, -0.002979795215651393, 0.025806283578276634, -0.008025209419429302, 0.06531822681427002, 0.028220824897289276, -0.025638800114393234, 0.023252172395586967, 0.0007650118204765022, 0.015715453773736954, 0.0008657631115056574, -0.007711179554462433, -0.008311325684189796, -0.02509448118507862, 0.010286225005984306, 0.003346163546666503, -0.016873875632882118, -0.04566693305969238, -0.01923258975148201, -0.0010947432601824403, 0.0038555897772312164, 0.004665089305490255, -0.024605991318821907, -0.004096345975995064, 0.0031804253812879324, 0.01189824566245079, 0.009462769143283367, 0.019888563081622124, -0.008639312349259853, -0.024494335055351257, -0.005446674767881632, 0.005446674767881632, -0.006793514359742403, -0.029979391023516655, -0.000506373296957463, -0.0005700515466742218, -0.019148847088217735, -0.0050907745026052, 0.00762743828818202, 0.009183631278574467, 0.0012979904422536492, -0.014543075114488602, 0.021005114540457726, 0.027550892904400826, -0.008506721816956997, -0.0007545441621914506, -0.017571719363331795, -0.023447569459676743, -0.03090054541826248, 0.0033810557797551155, 0.0029449029825627804, -0.0007135458290576935, -0.04725801572203636]} +{"id": "test:10000058", "text": "\"Here are your morning links!\\nJanczak looked solid in his second start since returning from injury.\\nTCU broke through in the sixth inning with a pair of run. Jake Guenther and Alex Isola singled to open the inning. A wild pitch had both runners in scoring position and Johnny Rizer delivered a two-run single up the middle to make it a 6-2 ballgame. The Frogs didn\u2019t go quietly, scoring a run on a Zach Humphreys groundout in the eighth. With two outs in the ninth, the Frogs put the tying run on base with walks to Josh Watson and Isola around a Guenther base hit. UTA closer Andrew Gross wiggled off the hook with a ground out to end the game. Jared Janczak (0-2) tossed 4 1/3 innings. He allowed six runs (two earned) on six hits, walked one and struck out two. Haylen Green and Matt Rudis allowed one combined runner over the last 3 2/3 innings of the game. The pair fanned seven.\\nThe defensive end position is loaded, name-wise, but not very experienced.\\nThis is a spot with a ton of turnover and, in turn, opportunities for new faces. Mathis, who some have compared to Jerry Hughes when it comes to potential, appears to be on his way to a starting role after continuing to earn rave reviews through spring. Patterson hinted that 2019 could finally be Bowen\u2019s time too, so don\u2019t take your eyes off the 6-foot-4 Byron Nelson High School product when fall camp arrives down the stretch.\\nThe cornerback would fill a massive need with Jeff Gladney and Julius Lewis set to graduate.\\nJenkins is getting recruited by Jeremy Modkins mostly because he\u2019s the cornerbacks coach for the Frogs. However, Jenkins says he\u2019s built a strong relationship with each of the coaches on staff.\\nJenkins was hosted on his visit by redshirt freshman Atanza Vongor. The four-star says he got along with the defensive back as well as the rest of the team.\\nEquestrian has put together a solid season.\\nTCU News: Joe Broadnax\u2019s career is over, will Justin Rogers play this year?\"", "vector_field": [-0.006660254672169685, -0.011339228600263596, 0.029424408450722694, -0.006708243861794472, -0.016357557848095894, 0.012847470119595528, -0.0009606466046534479, -0.01681002974510193, 0.013567311689257622, -0.006598553620278835, 0.028300082311034203, 0.011743711307644844, -0.00470639718696475, -0.029177604243159294, 0.012916026636958122, -0.002353198593482375, 0.02409072034060955, 0.011017014272511005, 0.0017121961573138833, -0.014328287914395332, -0.016097042709589005, 0.019895067438483238, -0.03559448570013046, -0.008836920373141766, 0.0017636134289205074, 0.012065926566720009, 0.020169293507933617, -0.0020704034250229597, -0.021444443613290787, -0.002966778352856636, 0.005553068593144417, 0.021732378751039505, -0.0450826920568943, -0.0009066584170795977, -0.01860620826482773, -0.023021239787340164, 0.006420306861400604, 0.010777066461741924, 0.01390323881059885, -0.010406862013041973, 0.01833198219537735, -0.007979964837431908, 0.014849316328763962, -0.012148194946348667, -0.01564457081258297, 0.009611607529222965, 0.007356102112680674, 0.00859011709690094, -0.017591573297977448, 0.0030696128960698843, 0.030082549899816513, 0.01244298741221428, -0.013430199585855007, 0.00010674446093617007, 0.009645885787904263, -0.024584325030446053, 0.030219662934541702, 0.026860399171710014, 0.0082062017172575, -6.25041575403884e-05, -0.01805775612592697, -0.002869085408747196, -0.013656435534358025, 0.003965987823903561, 0.0022109439596533775, -0.007328679319471121, -0.003777457866817713, -0.020100736990571022, -0.038254473358392715, -0.018482806161046028, 0.024255255237221718, 0.01460251398384571, 0.0025554399471729994, -0.01831827126443386, 0.031590789556503296, -0.004058538936078548, -0.04511011391878128, -0.010495984926819801, 0.01223046239465475, 0.0042367856949567795, -7.353745604632422e-05, -0.02040238492190838, -0.04681031033396721, 0.01781095378100872, 0.0027628231327980757, 0.008267901837825775, -0.00232234806753695, 0.006019252352416515, -0.008219912648200989, 0.0017704691272228956, 0.009974956512451172, 0.04239527881145477, 0.02026527188718319, -7.251981878653169e-05, -0.012484121136367321, 0.006050102412700653, -0.04269692674279213, 0.027120912447571754, -0.00781542994081974, -0.03191985934972763, 0.007973109371960163, -0.011195260100066662, 0.01430086512118578, -0.01653580367565155, -0.036197781562805176, -0.017015699297189713, 0.003880292410030961, 0.011394074186682701, 0.023542268201708794, -0.007520637474954128, -0.008720374666154385, -0.007623471785336733, -0.005278842989355326, -0.05213028937578201, -0.018948988988995552, -0.01397179439663887, 0.02717575803399086, -0.0018715898040682077, 0.008315891958773136, -0.027120912447571754, 0.01936032809317112, 0.013738702982664108, 0.013416487723588943, -0.037239838391542435, -0.0017756108427420259, 0.0010232042986899614, -0.010180626064538956, 0.00974186509847641, 0.009865266270935535, -0.008494138717651367, 0.010975879617035389, 0.01630271226167679, 0.027902456000447273, 0.00891918782144785, 0.0002240937465103343, 0.006660254672169685, -0.025914320722222328, -0.019278060644865036, -0.02424154430627823, -0.04486330971121788, 0.019922491163015366, 0.0059164175763726234, -0.014054062776267529, -0.01493158470839262, 0.023377733305096626, 0.016851164400577545, 0.009481350891292095, 0.012360719032585621, 0.012203039601445198, 0.0033712610602378845, 0.015946218743920326, -0.0019144375110045075, -0.016206733882427216, -0.005556496325880289, -0.0034740956034511328, -7.803646440152079e-05, -0.0219243373721838, -0.001134608406573534, -0.028492040932178497, -0.012833758257329464, -0.013190251775085926, -0.01460251398384571, 0.02744998410344124, 0.026380503550171852, 0.019799089059233665, 0.03246831148862839, 0.011894536204636097, -0.0014791043940931559, -0.005974690429866314, -0.007993676699697971, 0.0011166124604642391, 0.00015200239431578666, -0.015589726157486439, 0.046590931713581085, 0.0044664498418569565, 0.01230587437748909, -0.011243250221014023, 0.01409519650042057, -0.03680107742547989, -0.01160659920424223, 0.014067773707211018, -0.024433501064777374, -0.0006988468230701983, 0.024049585685133934, -0.03260542452335358, -0.016508381813764572, 0.01860620826482773, -0.011311806738376617, -0.023514846339821815, 0.016576938331127167, 0.006272910628467798, 0.030932648107409477, 0.005268559325486422, 0.015987353399395943, -0.6226018071174622, 0.0041202399879693985, 0.033812016248703, -0.0038871481083333492, 0.028135547414422035, -0.0018921566661447287, -0.0005137445405125618, 0.005412527825683355, -0.009577329270541668, 0.008165067993104458, -0.01676889695227146, 0.017207657918334007, 0.018565073609352112, -0.0003755605430342257, 0.0063894568011164665, -0.009563618339598179, 0.020045891404151917, -0.008692951872944832, 0.007225845009088516, -0.0032667126506567, -0.035813864320516586, 0.0146162249147892, 0.007513781543821096, -0.03660912066698074, 0.032632846385240555, 0.01805775612592697, 0.014369421638548374, -0.006259199697524309, 0.0012888603378087282, 0.010153203271329403, -0.03488149866461754, 0.018647341057658195, -0.0029067914001643658, -0.03205697238445282, 0.04190167412161827, -0.010557685978710651, -0.018098890781402588, 0.0443422794342041, -0.024172987788915634, 0.028409773483872414, -0.01160659920424223, -0.00143368577118963, 0.04085961729288101, 0.01589137502014637, 0.0016153602628037333, 0.027367716655135155, -0.009858410805463791, 0.00045161531306803226, -0.012456698343157768, -0.02947925217449665, 0.00721213361248374, -0.014259731397032738, 0.02473515085875988, -0.009104290045797825, -0.007726306561380625, -0.008706662803888321, 0.0010009234538301826, -0.004624129272997379, 0.01627529039978981, 0.004024260677397251, -0.019127236679196358, 0.020704032853245735, -0.02651761658489704, 0.02036125212907791, -0.0010437712771818042, 0.007616616319864988, -0.019031256437301636, 0.009330525994300842, 0.03729468211531639, -0.007369813043624163, 0.002248649951070547, 4.161588003626093e-05, -0.004778381437063217, 0.01986764557659626, 0.027408849447965622, 0.02639421448111534, 0.013992361724376678, -0.02088228054344654, 0.007650894578546286, 0.028437195345759392, 0.011010157875716686, -0.017509305849671364, -0.010626242496073246, -3.403718437766656e-05, 0.03167305886745453, 0.02743627317249775, -0.016823740676045418, -0.004702968988567591, -0.016055909916758537, -0.03447015956044197, 0.003504946129396558, 0.02703864499926567, -0.008500994183123112, -0.004528150428086519, 0.010228615254163742, 0.01642611436545849, -0.019017545506358147, 0.005793015938252211, 0.028300082311034203, -0.04291630908846855, 0.0019829939119517803, -0.01249783206731081, -0.003965987823903561, -0.014081484638154507, 0.016069620847702026, 0.005148585885763168, 0.013012005016207695, -0.01460251398384571, 0.053665950894355774, -0.026366792619228363, -0.015425190329551697, -0.01460251398384571, 0.004027688875794411, 0.001184311811812222, 0.017865799367427826, -0.03156336769461632, 0.02551669254899025, 0.008021099492907524, 0.018482806161046028, -0.03551221638917923, 0.01922321505844593, -0.007870274595916271, 0.015932507812976837, -0.026627305895090103, -0.011983659118413925, 0.02551669254899025, 0.011592887341976166, -0.054817698895931244, -0.019922491163015366, -0.013382209464907646, 0.012641800567507744, -0.009282536804676056, 0.03600582107901573, -0.003993410617113113, 0.01160659920424223, -0.0244609247893095, 0.01012578047811985, 0.01676889695227146, 0.01257324405014515, -0.03394912928342819, -0.028683999553322792, -0.012868036516010761, -0.004397893324494362, -0.015397767536342144, -0.008850631304085255, -0.021211350336670876, -0.018496517091989517, 0.008542127907276154, -0.012264740653336048, -0.011867113411426544, 0.006759661249816418, 0.008555838838219643, -0.002279500477015972, -0.0018390255281701684, -0.00409967266023159, 0.011490053497254848, -0.010941602289676666, -0.01166144385933876, -0.025818340480327606, -0.028492040932178497, 0.020347539335489273, 0.035813864320516586, -0.015946218743920326, -0.002248649951070547, -0.0011200401932001114, -0.014684781432151794, -0.0026754136197268963, -0.0010849051177501678, -0.025544116273522377, -0.009076867252588272, 0.0013899810146540403, -0.027765342965722084, 0.005086884833872318, 0.004617273807525635, -0.01988135650753975, 0.0006127228261902928, -0.023240620270371437, 0.012120772153139114, 0.013368498533964157, 0.01665920577943325, 0.012175616808235645, 0.006865923758596182, 0.007445225492119789, -0.02306237444281578, 0.0038357307203114033, -0.0004991763271391392, -0.01782466471195221, 0.009817277081310749, -0.01830456033349037, 0.021979182958602905, -0.006721955258399248, -0.0043841819278895855, -0.023610824719071388, 0.0005077458336018026, -0.01639869064092636, -0.011627165600657463, 0.0019658547826111317, 0.030466465279459953, 0.028135547414422035, 0.037760864943265915, 0.014698492363095284, 0.005275415256619453, 0.016371268779039383, -0.02562638372182846, 0.004264208022505045, -0.03334583342075348, 0.016384979709982872, -0.012120772153139114, 0.024776283651590347, -0.009015167132019997, 0.013190251775085926, -0.00960475206375122, -0.01818115822970867, -0.009645885787904263, 0.014890450984239578, 0.028300082311034203, -0.010701654478907585, -0.0004098387435078621, -0.009357948787510395, 0.01846909523010254, 0.012785769067704678, -0.005971262697130442, 0.04412290081381798, 0.004747530911117792, -0.020045891404151917, 0.005861572455614805, -0.002488597296178341, 0.01031773816794157, -0.011304950341582298, -0.011250105686485767, 0.0006692818715237081, 0.010235470719635487, 0.006458013318479061, -0.0028433767147362232, 0.003952276427298784, 0.0018938706489279866, 0.002120106713846326, -0.011250105686485767, 0.04009178280830383, -0.016330135986208916, 0.016042198985815048, 0.015850240364670753, 0.00721213361248374, 0.006691104732453823, 0.053885333240032196, -0.011682011187076569, 0.02871142141520977, 0.0003136455488856882, 0.005350827239453793, 0.003585499944165349, 0.011668299324810505, 0.029534097760915756, -0.009988667443394661, -0.005806727334856987, 0.028437195345759392, -0.02450205758213997, -0.007205277681350708, -0.002647991059347987, 0.0015733694890514016, 0.0361703597009182, -0.02078630030155182, 0.0004006264789495617, 0.01493158470839262, -0.00019388608052395284, 0.02909533679485321, 0.0006002970039844513, 0.001603362848982215, -0.00701674772426486, -0.008089655078947544, -0.006005540955811739, -0.0017670412780717015, -0.004805803764611483, -0.0015236660838127136, -0.011791701428592205, 0.007068165112286806, -0.01909981295466423, -0.000296720681944862, -0.013862104155123234, -0.007232700474560261, -0.01243613101541996, 0.005803299602121115, -0.035046033561229706, 0.011620310135185719, -0.004507583566009998, -0.010975879617035389, -0.02654503844678402, 0.021595267578959465, -0.004209362901747227, -0.0171528123319149, 0.013896382413804531, -0.0007374098058789968, -0.0085147051140666, 0.01403349544852972, -0.00561476917937398, 0.006955047138035297, -0.00839815940707922, 0.011003302410244942, -0.030411619693040848, -0.01667291671037674, -0.01937403902411461, 0.0007875416777096689, -0.007979964837431908, 0.005933556705713272, 0.005182864144444466, 0.01960713043808937, 0.021883204579353333, -0.014246020466089249, -0.02330917678773403, -0.012141338549554348, -0.014492823742330074, 0.028656575828790665, -0.010468563064932823, -0.017591573297977448, 0.005138302221894264, 0.007417802698910236, -0.012004226446151733, -0.007362957578152418, -0.017769819125533104, 0.02705235593020916, -0.004493872169405222, -0.013251952826976776, -0.021362174302339554, -0.023007528856396675, 0.0029804895166307688, 0.10848364979028702, 0.03205697238445282, -0.005505078937858343, 0.032769959419965744, 0.00416137371212244, 0.009008311666548252, 0.0010729077039286494, -0.02191062644124031, 0.02561267279088497, -0.016878586262464523, -0.007472647819668055, 0.020937126129865646, 0.001655637170188129, -0.003969415556639433, -0.008548983372747898, -0.0006529997335746884, -0.028354927897453308, -0.02702493406832218, 0.005460517480969429, 0.0069996085949242115, 0.008864343166351318, -0.004195651970803738, -0.01693343184888363, 0.021334752440452576, 0.01020119246095419, -0.016755184158682823, 0.011572320945560932, 0.012257885187864304, 0.017358481884002686, 0.00890547689050436, 0.000667996471747756, 0.004726964049041271, 0.008165067993104458, -0.0010206334991380572, -0.011490053497254848, 0.01833198219537735, 0.02089599147439003, -0.009138568304479122, 0.008747796528041363, -0.009728153236210346, -0.007726306561380625, 0.02038867399096489, 0.01639869064092636, -0.011777989566326141, 0.016892297193408012, -0.022349387407302856, 0.003612922504544258, 0.018427960574626923, -0.007664605975151062, -0.02165011130273342, 0.019072391092777252, 0.002641135361045599, -0.005038895644247532, 0.011065003462135792, -0.0065334253013134, -0.002168096136301756, -0.015288077294826508, -0.008809497579932213, -0.00789084192365408, -0.007356102112680674, -0.025215044617652893, -0.013457621447741985, 0.006817934103310108, -0.017769819125533104, 0.0057176039554178715, -0.03419593349099159, -0.0187021866440773, -0.01064680889248848, -0.030164817348122597, -0.023172063753008842, -0.011620310135185719, -0.02435123361647129, -0.0161655992269516, 0.01300514955073595, 0.02587318606674671, 0.014246020466089249, 0.024406079202890396, 0.04102415218949318, -0.007218989077955484, 0.00878893118351698, 0.016508381813764572, -0.01831827126443386, 0.00740409130230546, -0.02821781486272812, 0.03345552459359169, -0.015027563087642193, 0.01447911188006401, 0.02256876789033413, -0.02256876789033413, 0.0027748204302042723, -0.014767048880457878, -0.009721297770738602, 0.035046033561229706, -0.024392368271946907, 0.0073081124573946, -0.021608978509902954, 0.03202955052256584, 0.00839815940707922, 0.00637574540451169, -0.009693874977529049, 0.01896270178258419, -0.054159559309482574, 0.005731315352022648, -0.034662116318941116, 0.008692951872944832, 0.010783921927213669, 0.009049445390701294, -0.0025640095118433237, -0.01844167150557041, 0.0037945967633277178, -0.01255953311920166, -0.03230377659201622, -0.002121820580214262, 0.0330716073513031, 0.016864875331521034, 0.014643647707998753, -0.006708243861794472, -0.012669223360717297, -0.0065574198961257935, -0.015603437088429928, 0.004843509756028652, -0.02551669254899025, 0.029918013140559196, -0.002699408447369933, -0.008960321545600891, 0.038007669150829315, 0.01805775612592697, -0.03984498232603073, -0.003965987823903561, 0.01064680889248848, -0.011867113411426544, 0.02603772096335888, -0.011243250221014023, -0.008021099492907524, -0.008377592079341412, -0.03822705149650574, -0.01372499205172062, 0.0033952558878809214, -0.025201333686709404, -0.0025520119816064835, -0.02846461907029152, -0.011304950341582298, -0.002272644778713584, -0.008734085597097874, -0.002147529274225235, -0.024529481306672096, -0.00509374076500535, 0.005590774584561586, 0.026764418929815292, 0.010173769667744637, -0.014328287914395332, -0.002946211490780115, -0.013491899706423283, 0.003832302987575531, -0.00859011709690094, -0.007260122802108526, -0.0003847728075925261, -0.005302837584167719, 0.013862104155123234, 0.02407700940966606, 0.01083191204816103, 0.002272644778713584, -0.0033935420215129852, 0.02677813172340393, 0.00468240212649107, 0.013382209464907646, 0.008274758234620094, -0.004305342212319374, -0.03880292549729347, 0.006108375266194344, 0.024762572720646858, 0.01830456033349037, 0.006283194292336702, -0.012299018912017345, 0.004312197677791119, 0.01936032809317112, -0.019922491163015366, 0.0018133168341591954, -0.02536586858332157, -0.025187622755765915, -0.0032941352110356092, 0.0026531328912824392, -0.018811875954270363, -0.01397179439663887, -0.015973642468452454, 0.004233357962220907, 0.03965302184224129, -0.010482273995876312, 0.01705683209002018, -0.006005540955811739, 0.024968242272734642, -0.029561521485447884, 0.013423343189060688, -0.030219662934541702, 0.003633489366620779, 0.011236394755542278, -0.020155582576990128, -0.016234155744314194, -0.026613594964146614, -0.008871198631823063, 0.0030798963271081448, 0.01805775612592697, 0.004768097773194313, 0.021608978509902954, 0.0054022446274757385, 0.01224417332559824, -0.017879510298371315, -0.015493746846914291, 0.016097042709589005, 0.003534082556143403, -0.011908247135579586, -0.03137141093611717, -0.014273443259298801, -0.017015699297189713, 0.029013069346547127, -0.004253924824297428, -0.009830988012254238, -0.008850631304085255, -0.007904552854597569, -0.01543890219181776, 0.009070011787116528, 0.024433501064777374, 0.0428614616394043, 0.007273834198713303, 0.0077880071476101875, 0.02629823610186577, 0.012299018912017345, -0.0007472647703252733, 0.0021321040112525225, 0.0032204370945692062, 0.013306797482073307, -0.0005660188035108149, 0.0002935070951934904, 0.001902440213598311, -0.0071984222158789635, 0.00513144675642252, 0.013128550723195076, -0.036718808114528656, -0.03869323432445526, 0.03644458204507828, 0.038254473358392715, -0.024049585685133934, -0.030164817348122597, 0.00012361576955299824, -0.024268966168165207, -0.01988135650753975, -0.01973053254187107, 0.01859249547123909, -0.014026639983057976, -0.02155413292348385, -0.029314717277884483, 0.05435151606798172, -0.002713119611144066, 0.0038151638582348824, -0.006204354576766491, -0.007527492940425873, -0.01494529563933611, -0.016576938331127167, 0.007945687510073185, -0.01920950412750244, -0.007506926078349352, 0.040557969361543655, 0.005326832644641399, -0.009700731374323368, -0.005950695835053921, 0.012868036516010761, -0.005419383756816387, -0.005998685024678707, 0.011311806738376617, 0.03957075625658035, -0.00458642328158021, 0.004593278747051954, -0.0014054062776267529, 0.012648656032979488, 0.0005373107851482928, -0.0051074521616101265, 0.0021766657009720802, -0.00771945109590888, -0.02296639420092106, 0.009070011787116528, 0.020470941439270973, 0.0005947267636656761, -0.0275596734136343, -0.003911142703145742, -0.006636259611696005, -0.004116811789572239, -0.0162890013307333, -0.005134874489158392, -0.021458154544234276, -0.04382125288248062, 0.010537118650972843, -0.007075020577758551, 0.013875816017389297, 0.0032701403833925724, -0.00038198771653696895, -0.014753337949514389, -0.01064680889248848, 0.018537651747465134, -0.0054673729464411736, 0.02884853444993496, -0.029396984726190567, -0.011613454669713974, -0.019415173679590225, 0.004535005893558264, 0.03255058079957962, -0.016864875331521034, 0.015726838260889053, -0.011010157875716686, -0.013814114965498447, 0.005878711584955454, -0.004137379117310047, -0.016494670882821083, -0.024159276857972145, 0.0020824007224291563, 0.0395159088075161, 0.0034175366163253784, 0.013505611568689346, -0.011181549169123173, -0.011743711307644844, 0.03137141093611717, 0.01781095378100872, 0.004716680385172367, 0.025914320722222328, 0.01793435402214527, 0.013437055051326752, -0.006214637774974108, 0.013574168086051941, 0.005631908308714628, -0.004754386376589537, -0.013039427809417248, -0.0009709300356917083, 0.018373114988207817, -0.004487016703933477, -0.014643647707998753, -0.008315891958773136, -0.010358872823417187, -0.016069620847702026, 0.02075887843966484, 0.005354254972189665, 0.011339228600263596, 0.009357948787510395, -0.0030027704779058695, -0.003014767775312066, 0.020566919818520546, -0.012998294085264206, -0.008775219321250916, 0.0018613063730299473, -0.008274758234620094, -0.03219408541917801, -0.01364958006888628, 0.002222941257059574, 0.03411366418004036, 0.006139225792139769, -0.04003693908452988, -0.008041665889322758, 0.024543192237615585, -0.014917872846126556, -0.02395360730588436, -0.009261970408260822, 0.018825588747859, 0.023542268201708794, 0.009830988012254238, 0.0027559674344956875, 0.021965472027659416, -0.01770126260817051, 0.004504155833274126, -0.00838444847613573, -0.023542268201708794, -0.0019195792265236378, 0.02258247882127762, 0.012792624533176422, 0.002120106713846326, -0.05106080695986748, 0.009577329270541668, 0.017413325607776642, 0.013793548569083214, 0.00484008202329278, 0.015301789157092571, -0.0028022429905831814, 0.011538042686879635, -0.030301930382847786, -0.015671994537115097, 0.009961245581507683, 0.01050284132361412, 0.0374043732881546, -0.039790134876966476, -0.010667376220226288, 0.015301789157092571, -0.014712204225361347, -0.0027833899948745966, 0.00721213361248374, 0.0021698102355003357, -0.006176931783556938, 0.030932648107409477, -0.02616112306714058, -0.006036391016095877, -0.0017327630193904042, -0.012676078826189041, 0.04245012626051903, 0.005971262697130442, -0.009625318460166454, 0.004857221152633429, 0.020868569612503052, 0.0094127943739295, -0.01911352574825287, -0.0009263684041798115, -0.01205221563577652, -0.0006045817863196135, 0.012532110325992107, 0.007767440285533667, 0.0063894568011164665, -0.010146347805857658, 0.015836529433727264, -0.021348463371396065, -0.024310100823640823, 0.00445273844525218, 0.012504687532782555, -0.014725915156304836, 0.01094845775514841, 0.0036437727976590395, -0.02642163820564747, 0.004109956324100494, -0.0071984222158789635, 0.005299409851431847, -0.0005591631634160876, -0.005998685024678707, -0.027737921103835106, -0.01767384074628353, -0.020470941439270973, -0.01083191204816103, -0.009844698943197727, -0.014506534673273563, -0.024954531341791153, 0.002665130188688636, -0.018907856196165085, 0.02052578702569008, 0.2375890612602234, 0.016700340434908867, 0.007698883768171072, 0.028903380036354065, -0.0028776549734175205, 0.010859333910048008, 0.017481882125139236, 0.0013882671482861042, -0.010118925012648106, -0.008562694303691387, 0.007733162026852369, 0.011956236325204372, -0.008967177011072636, -0.012881748378276825, 0.026435349136590958, 0.009282536804676056, -0.0292324498295784, -0.018825588747859, -0.03973529115319252, -0.014506534673273563, -0.0002866514550987631, 0.0025828625075519085, 0.01805775612592697, -0.01705683209002018, 0.025064220651984215, -0.002078972989693284, 0.011634021066129208, 0.004768097773194313, 0.02961636520922184, 0.020717745646834373, -0.030987493693828583, 0.020100736990571022, 0.009104290045797825, 0.010105214081704617, -0.009728153236210346, -0.014520245604217052, -0.004535005893558264, 0.011538042686879635, 0.020841145887970924, -0.008884909562766552, 0.0026274241972714663, -0.034552425146102905, -0.00884377583861351, -0.011901391670107841, 0.0061803595162928104, -0.003681478789076209, -0.005450233817100525, -0.014177463948726654, -0.021471865475177765, 0.026051433756947517, -0.027107201516628265, 0.013505611568689346, 0.01391694974154234, 0.029287295415997505, 0.003962560091167688, -0.010927890427410603, 0.02895822376012802, -0.01601477526128292, 0.005690181627869606, 0.018482806161046028, -0.006444301921874285, 0.025064220651984215, -0.001974424347281456, 0.027148336172103882, -0.025722362101078033, 0.017879510298371315, -0.000988069106824696, 0.04234043508768082, 0.006961902603507042, 0.0007575482595711946, -0.009536195546388626, -0.005954123567789793, -0.004061966668814421, -0.0008209629449993372, -0.032742537558078766, -0.030466465279459953, 0.017358481884002686, 0.014698492363095284, 0.02038867399096489, 0.0009237975464202464, 0.01973053254187107, -0.005049178842455149, 0.020128158852458, -0.013711280189454556, -0.005419383756816387, -0.02821781486272812, 0.01160659920424223, -0.01820858009159565, -0.007911409251391888, 0.0085147051140666, 0.015384056605398655, 0.0034638121724128723, -0.012100204825401306, -0.007959398441016674, 0.025311022996902466, -0.009865266270935535, 0.016837451606988907, 0.033016763627529144, -0.025832053273916245, 0.020553208887577057, -0.01630271226167679, -0.006677393801510334, 0.04999132826924324, 0.015246943570673466, -0.018098890781402588, 0.014821894466876984, 0.009968101046979427, -0.00682479003444314, 0.019922491163015366, -0.016467247158288956, -0.016960853710770607, -0.021183928474783897, 0.02152671106159687, 0.0004404748906381428, 0.001087475917302072, -0.00370204565115273, -0.02295268326997757, 0.013896382413804531, 0.008884909562766552, 0.005607913713902235, -0.0018321698298677802, -0.03394912928342819, 0.009399082511663437, -0.015233232639729977, 0.0024817418307065964, -0.022088872268795967, -0.014643647707998753, -0.017221368849277496, 0.01563085988163948, -0.017742397263646126, -0.005535929463803768, -0.03770602121949196, 0.021485576406121254, -0.01667291671037674, 0.01447911188006401, -0.026695862412452698, -0.008939755149185658, 0.005460517480969429, -0.01442426722496748, 0.023034950718283653, 0.013279375620186329, -0.014739626087248325, 0.008795786648988724, 0.006951619405299425, 0.007520637474954128, -0.021211350336670876, 0.033812016248703, -0.018770743161439896, -0.04214847460389137, -0.016508381813764572, 0.0073766689747571945, 0.03910457342863083, 0.007733162026852369, 0.0062420605681836605, 0.016316423192620277, -0.01107871439307928, -0.011243250221014023, -0.0356767512857914, -0.012072782032191753, 0.024680305272340775, -0.050457511097192764, 0.009844698943197727, 0.022116295993328094, -0.0038391584530472755, -0.01678260788321495, -0.013341075740754604, -0.1753946989774704, 0.028656575828790665, 0.009433360770344734, -0.012189328670501709, 0.013896382413804531, 0.011188404634594917, 0.017399614676833153, 0.0062934779562056065, -0.0185787845402956, -0.026983799412846565, 0.007849708199501038, 0.0219243373721838, -0.029013069346547127, -0.011236394755542278, -0.00553935719653964, 0.01833198219537735, -0.0017584717134013772, 0.0075754825957119465, 0.03614293411374092, 0.006752805784344673, 0.017207657918334007, -0.023871339857578278, 0.02424154430627823, -0.02152671106159687, 0.00756177119910717, 0.019538573920726776, 0.015466324053704739, 0.0021612406708300114, 0.00878893118351698, -0.009152280166745186, -0.03246831148862839, -0.0008466715808026493, -0.011284383945167065, -0.01988135650753975, 0.02293897233903408, 0.0021029675845056772, -0.012388141825795174, 0.0048777880147099495, -0.0006812792271375656, 0.02126619592308998, 0.014863028191030025, -0.019895067438483238, -0.009577329270541668, 0.009769286960363388, -0.016947142779827118, 0.02808070182800293, -0.00621806550770998, 0.0170294102281332, 0.016604360193014145, -0.023871339857578278, 0.010461706668138504, -0.005446806084364653, 0.02407700940966606, -0.003996838349848986, 0.009501917287707329, 0.015356633812189102, -0.016220444813370705, 0.014246020466089249, 0.019401462748646736, -0.007568626664578915, -0.004487016703933477, 0.002173237968236208, -0.017331058159470558, -0.0007931118598207831, -0.023665670305490494, -0.020278984680771828, -0.03447015956044197, -0.0251464881002903, -0.046728044748306274, 0.006663682404905558, -0.013519322499632835, -0.0035957833752036095, -0.012045360170304775, -0.00385972554795444, 0.019003834575414658, 0.0017841804074123502, 0.0010703367879614234, -0.008350170217454433, 0.000641430844552815, -0.006207782309502363, -0.018140023574233055, 0.03309903293848038, -0.019799089059233665, -0.015274366363883018, -0.018482806161046028, -0.015096119605004787, -0.00909743458032608, 0.001680488814599812, 0.0325780026614666, -0.014369421638548374, 0.000369990331819281, -0.00852841604501009, -0.005004617385566235, -0.004963483661413193, 0.003729468211531639, 0.015562303364276886, 0.0033027047757059336, 0.007068165112286806, -0.007781151682138443, -0.002224655356258154, 0.005415956024080515, -0.021321041509509087, -0.030082549899816513, 0.0002645848726388067, 0.0012785769067704678, 0.016330135986208916, 0.0027628231327980757, 0.00948820635676384, 0.013382209464907646, -0.016714051365852356, -0.019908778369426727, 0.0203749630600214, 0.006790511775761843, 0.009926967322826385, -0.0009589326800778508, 0.015054985880851746, -0.006252343766391277, -0.00923454761505127, 0.02385762892663479, -0.03957075625658035, 0.07393122464418411, 0.01512354239821434, -0.020320117473602295, 0.0021029675845056772, 0.003756890771910548, -0.016467247158288956, -0.09345608949661255, -0.013231385499238968, 0.011887679807841778, 0.03767859935760498, -0.0036917622201144695, 0.01654951646924019, -0.01301886048167944, 0.029177604243159294, -0.02254134602844715, 0.02628452517092228, -0.016851164400577545, -0.002788531593978405, 0.021046815440058708, 0.02254134602844715, 0.007568626664578915, 0.007822285406291485, 0.01020119246095419, -0.014684781432151794, 0.005957551300525665, 0.015370345674455166, 0.00439446559175849, 0.011983659118413925, -0.002695980481803417, -0.004202507436275482, 0.017769819125533104, -0.011311806738376617, -0.036088090389966965, 0.05627109482884407, -0.003602638840675354, -0.01430086512118578, 0.014780760742723942, 0.00320672569796443, 0.01962084323167801, -0.036197781562805176, -0.010420572943985462, 0.00740409130230546, -0.026709575206041336, -0.014054062776267529, 0.0397627130150795, -0.023679381236433983, -0.015192098915576935, 0.002795387292280793, 0.004075678065419197, -0.03795282542705536, 0.0026565606240183115, -0.007349246181547642, -0.02258247882127762, 0.013697569258511066, -0.0055702077224850655, -0.025187622755765915, -0.01719394512474537, -0.006615692749619484, -0.013738702982664108, -0.01805775612592697, 0.04269692674279213, -0.004524722695350647, 0.013670146465301514, 0.02898564748466015, -0.00936480425298214, -0.0004820371977984905, 0.0010034943697974086, 0.0007442654459737241, -0.028162969276309013, -0.013121695257723331, -0.010667376220226288, -0.003307846374809742, -0.014506534673273563, -0.014218597672879696, -0.00833645835518837, -0.010742788203060627, -0.02026527188718319, -0.002219513524323702, -0.034141089767217636, 0.016508381813764572, -0.023775359615683556, 0.012175616808235645, -0.03792540356516838, -0.01276520173996687, 0.017235079780220985, -0.02308979630470276, 0.000618721533101052, -0.015027563087642193, 0.02668215148150921, -0.0027713924646377563, 0.02664101868867874, 0.00832274742424488, 0.007266978733241558, -0.005114307627081871, 0.024694016203284264, -0.015603437088429928, -0.014012929052114487, 0.011627165600657463, 0.019278060644865036, -0.024694016203284264, -0.012011081911623478, 0.010612531565129757, 0.017070544883608818, -0.011421496979892254, -0.016042198985815048, -0.0033438384998589754, -0.008233623579144478, -0.023802783340215683, -0.0718471109867096, 0.02590060792863369, 0.006163220386952162, -0.018386827781796455, -0.010667376220226288, -0.00454871729016304, 0.006526569370180368, -0.00458642328158021, -0.019579708576202393, 0.012319585308432579, -0.03013739362359047, 0.003952276427298784, -0.016384979709982872, -0.007218989077955484, -0.014767048880457878, -0.013889526948332787, 0.018743321299552917, -0.02487226203083992, -0.0032032979652285576, -0.010763355530798435, 0.0014893878251314163, 0.016453536227345467, 0.0025365869514644146, 0.035073455423116684, -0.006070669274777174, 0.005498223472386599, -0.0005235995049588382, 0.027216892689466476, -0.0032461455557495356, -0.03255058079957962, 0.02550298161804676, -0.012785769067704678, -0.004740675445646048, 0.020169293507933617, 0.0020909702870994806, -0.02972605638206005, 0.0029325000941753387, 0.03271511569619179, 0.0030439042020589113, 0.004031116608530283, -0.03408624231815338, -0.019689397886395454, 0.01545261312276125, -0.006348323076963425, 0.00666711013764143, -0.025201333686709404, -0.028382349759340286, 0.011311806738376617, -0.0009237975464202464, 0.009926967322826385, 0.02732658199965954, 0.00871351920068264, -0.012847470119595528, -0.02281557023525238, -0.019716821610927582, -0.0026788413524627686, 0.0011457488872110844, -0.002834807150065899, -0.0031570224091410637, -0.005628480575978756, 0.02895822376012802, 0.039680447429418564, 0.004291630815714598, -0.00400369381532073, 0.04977194964885712, -0.014575091190636158, 0.006718527525663376, 0.019716821610927582, 0.025845764204859734, -0.01885301060974598, -0.03334583342075348, -0.019648265093564987, 0.017632706090807915, 0.022390520200133324, -0.008247335441410542, 0.025434425100684166, -0.011867113411426544, 0.005779304541647434, -0.014012929052114487, 0.02574978396296501, 0.0026274241972714663, -0.007548059802502394, -0.004918921738862991, 0.002493739128112793, 0.02241794392466545, 0.0034689540043473244, -0.003948848694562912, 0.010859333910048008, -0.019031256437301636, 0.008775219321250916, -0.0037157570477575064, 0.008226768113672733, 0.0009417935507372022, 0.008926043286919594, 0.0017396187176927924, 0.020566919818520546, -0.010530263185501099, 0.03038419783115387, 0.03523799031972885, 0.0018647341057658195, 0.0011166124604642391, 0.012977726757526398, 0.0021355319768190384, -0.027230603620409966, -0.02258247882127762, 0.003263284685090184, -0.013402776792645454, -0.019949913024902344, 0.021225063130259514, 0.017385903745889664, 0.008836920373141766, -8.248192170867696e-05, 0.0060638138093054295, 0.011044436134397984, -0.011908247135579586, 0.01745446026325226, 0.015863951295614243, -0.0409967303276062, -0.012374430894851685, 0.036088090389966965, 0.029451830312609673, -0.0017001987434923649, -0.024721438065171242, 0.00360606680624187, 0.0029839174821972847, -0.00781542994081974, 0.02218485251069069, -0.010022945702075958, 0.008350170217454433, 0.004630985204130411, 0.01090046763420105, 0.016755184158682823, -0.02629823610186577, 0.0024765999987721443, -0.006187215447425842, -0.006091236136853695, 0.002908505266532302, 0.016453536227345467, -0.020964547991752625, 0.07288916409015656, 0.012778913602232933, -0.004024260677397251, 0.03063100017607212, -0.014629935845732689, 0.01676889695227146, -0.004072250332683325, 0.014383133500814438, 0.026846686378121376, -0.02961636520922184, -0.007417802698910236, 0.004113384056836367, -0.006691104732453823, -0.01925063692033291, -0.024323811754584312, 0.002200660528615117, 0.02140330895781517, 0.00040405429899692535, -0.0032958490774035454, 0.0171528123319149, 0.015027563087642193, 0.0013385637430474162, 0.008521560579538345, 0.024255255237221718, -0.015425190329551697, 0.005758737679570913, -0.002834807150065899, -0.020827434957027435, -0.004271063953638077, -0.02614741213619709, 0.004970339126884937, -0.01793435402214527, -0.02447463572025299, -0.0012674365425482392, 0.015959929674863815, -0.012820047326385975, -0.025064220651984215, 0.008240479975938797, 0.00852841604501009, -0.007568626664578915, 0.0011954522924497724, 0.01319710724055767, -0.012772058136761189, -0.030658423900604248, 0.01770126260817051, 0.005559924058616161, -0.002522875554859638, -0.00017696121358312666, -0.01886672154068947]} +{"id": "test:10000059", "text": "\"Grace. What is it? What does it mean? How do you get it and how do you KNOW you got it?\\nHonestly, I don\u2019t know the answers to these questions. I have no way of describing something I don\u2019t understand BUT believe deeply in my soul that it exists. How do you get these graces? Sanctifying grace is IN your soul. It\u2019s the free and underserved gift from God. Actual grace would come from our knowledge and belief of all things right and wrong \u2013 following the Ten Commandments, making sure we follow a path that is pure and justified. Justified means to be sinless, and to be sinless means you are in favor with God. When you are baptized you are sinless and therefore have favor with God. When you go to reconciliation, you are once again sinless and in favor with God. You are in a state of grace. Once sanctifying grace is in your soul, you can increase it by every good action you do \u2013 receiving Communion, saying prayers, performing corporal works of Mercy. The most important thing to think about is NOT what you are going to do or how you are going to do it but to recognize God\u2019s love. The reason you can do any good work is because God LOVES YOU FIRST in Grace. You must continually seek God\u2019s grace, continually respond to the actual graces God is working within you, inclining you to turn to Him and do good.\\nHave I answered the question as to what Grace is? I\u2019ve read through all the study guides provided to me and compiled all the information, but I don\u2019t believe that I\u2019ve properly explained what Grace is.\\nTo me Grace is a feeling. Sometimes it\u2019s butterflies in the stomach, and it takes my breath away. I feel this way every time I think of my husband. We met and married 38 years ago. I see God\u2019s grace in our meeting, getting married, and living our life together as husband and wife. Sometimes it\u2019s a jolt, like an electric shock and then an overwhelming wave of love, like the enormous amount of love I felt when I held my first baby in my arms for the first time. This happened again with my 2nd and 3rd child, and then God blessed us with twins to make 4 and 5. Then sometimes it feels like a raging fight going on inside me, making hard decisions that breaks my heart in two. I have no power to stop things from happening and my only recourse is just to constantly pray. Mostly it\u2019s a feeling of ultimate peace and an over-whelming calmness that comes over me at different times during the day when God allows me glimpses of HIS love for me: when my dog and cat are doing something cute together; when I see the birds eating the seed I put out for them; when a certain little girl comes over to the house and calls me \u201cGrandma\u201d. I just breathe a sigh of thanksgiving and relax in the comfort of His hand.\\nWhere does Grace come from?\"", "vector_field": [0.026145223528146744, -0.0010026380186900496, 0.016399836167693138, -0.025596007704734802, -0.011514371261000633, 0.0010920452186837792, -0.0068971277214586735, -0.01146966777741909, -0.04018215462565422, -0.02371845580637455, 0.021483276039361954, 0.012012497521936893, -0.060439273715019226, 0.012433988973498344, -0.0035635160747915506, 0.012057201005518436, 0.043451905250549316, 0.01287463866174221, 0.0007591629982925951, -0.01735777221620083, 0.011923090554773808, 0.012529782019555569, 0.005374012049287558, 0.0030398452654480934, -0.019414138048887253, -0.0058849104680120945, 0.0162848848849535, -0.01868610642850399, -0.0007607595762237906, -0.030500631779432297, -0.021879222244024277, 0.013551578857004642, -0.03371929004788399, -0.013027908280491829, -0.010205194354057312, -0.00569970952346921, -0.011175901629030704, -0.01609329879283905, -0.02098514884710312, 0.007241983897984028, 0.018328478559851646, 0.014905459247529507, 0.0007739311549812555, -0.0007296266849152744, -0.011239763349294662, 0.024293215945363045, -0.03471554443240166, -0.047820083796978, -0.005377205088734627, 0.021444957703351974, 0.049199510365724564, 0.021036239340901375, -0.02282438427209854, -0.025212833657860756, -0.025442738085985184, -0.013998614624142647, -0.018711652606725693, 0.016450926661491394, 0.0470537394285202, -0.013296129181981087, 0.015122591517865658, 0.0022814804688096046, -0.008832154795527458, 0.01023073960095644, 0.00395946204662323, 0.003458143211901188, -0.010633071884512901, 0.01091406587511301, -0.024740252643823624, -0.017472723498940468, 0.007880606688559055, 0.026209086179733276, 0.0037072061095386744, 0.027588510885834694, 0.02094683237373829, -0.015620716847479343, -0.008500071242451668, -0.0018695686012506485, -0.003911565523594618, 0.005677357781678438, -0.015748441219329834, -0.0018935169791802764, 0.001732264761812985, 0.015556855127215385, -0.0226838868111372, 0.005789116956293583, 0.011316398158669472, 0.03458781912922859, -0.004614050500094891, -0.0053708190098404884, 0.03152242675423622, 0.019401364028453827, 0.035175349563360214, 0.007388867437839508, 0.004160628654062748, 0.019222550094127655, -0.021674862131476402, 0.02422935515642166, 0.02611967734992504, -0.004776899702847004, 0.004722616635262966, 0.009968903847038746, -0.006315981037914753, -0.006079690530896187, -0.030296271666884422, -0.047079283744096756, -0.00592003483325243, -0.01583784818649292, 0.0018663755618035793, -0.021968629211187363, -0.01853283680975437, 0.020487023517489433, -0.002717340597882867, -0.01657865196466446, 0.03887936472892761, 0.002428363775834441, -0.005287798121571541, -0.007835903204977512, -0.01076718233525753, -0.005961545277386904, -0.015403585508465767, -0.014803280122578144, 0.01509704627096653, -0.02141941338777542, 0.0011000280501320958, 0.004313897807151079, -0.021112874150276184, -0.0060349865816533566, -0.012491464614868164, 0.009228101000189781, 0.0400288850069046, 0.0018695686012506485, -0.01045425795018673, 0.025621552020311356, 0.003451756900176406, 0.016361519694328308, -0.03318284824490547, 0.009649592451751232, 0.010243511758744717, -0.005204776767641306, 0.027920594438910484, -0.00980286207050085, 0.012146607972681522, -0.014292381703853607, 0.008953494019806385, -0.0018823411082848907, 0.02748633176088333, 0.00959850288927555, 0.014407333917915821, 0.022773293778300285, 0.035686250776052475, 0.021866450086236, -0.002849854761734605, 0.0019334309035912156, 0.029172295704483986, 0.004677913151681423, -0.018775515258312225, 0.028405947610735893, 0.015199226327240467, -0.008710816502571106, 0.016744693741202354, 0.02168763428926468, 0.008155214600265026, 0.005852979142218828, -0.013640985824167728, 0.018073029816150665, -0.011686800047755241, -0.01357712410390377, -0.029146751388907433, -0.004888658411800861, -0.005540053825825453, 0.045418862253427505, -0.001054526073858142, 0.023897269740700722, 0.010173263028264046, 0.012536168098449707, 0.03604387864470482, 0.01167402695864439, -0.02385895326733589, 0.025583235546946526, -0.02071692794561386, 0.024024995043873787, 0.001170276547782123, 0.02301597036421299, -0.01931195706129074, -0.017076777294278145, 0.016182705760002136, -0.020231574773788452, 0.007720951456576586, -0.011725117452442646, -0.018660562112927437, 0.005214356351643801, 0.0016133212484419346, -0.02420380897819996, -0.635148823261261, 0.0036050265189260244, 0.009834793396294117, -0.024855203926563263, 0.02024434693157673, -0.004396918695420027, 0.012427601963281631, 0.014254064299166203, -0.034613363444805145, 0.02664334885776043, -0.025953635573387146, 0.018545610830187798, -0.00521754939109087, -0.012031655758619308, -0.02342468872666359, -0.019158687442541122, 0.005201583728194237, -0.015582399442791939, -0.024357078596949577, -0.012721369042992592, -0.024548666551709175, 0.004978065844625235, -0.041536036878824234, -0.00864056870341301, 0.009758158586919308, 0.020065532997250557, 0.004901431035250425, -0.012370126321911812, -0.00021573480626102537, 0.006750244647264481, -0.022402891889214516, 0.026132451370358467, 0.018673334270715714, -0.014675554819405079, 0.04350299388170242, -0.010850204154849052, -0.024982929229736328, 0.019439682364463806, -0.006877969019114971, 0.03374483436346054, -0.008979038335382938, 0.025915319100022316, 0.016042208299040794, -0.009681523777544498, -0.021151192486286163, 0.009783703833818436, -0.008908789604902267, 0.015109818428754807, 0.004013745114207268, 0.04243010655045509, 0.001952589605934918, 0.004681106191128492, -0.0055336677469313145, 0.0072803013026714325, 0.017677083611488342, 0.0012660699430853128, 0.03290185332298279, -0.011495213024318218, 0.010831044986844063, -0.0012708596186712384, 0.030832715332508087, 0.011080107651650906, -0.01745995134115219, -0.028916846960783005, -0.02845703810453415, 0.025468282401561737, 0.011878387071192265, -0.014650010503828526, 0.0055783712305128574, 0.026694439351558685, 0.034204643219709396, 0.021623773500323296, 0.00726114260032773, -0.01616993360221386, 0.01675746589899063, -0.001816882286220789, 0.010882134549319744, -0.012625575065612793, 0.020768018439412117, 0.01691073551774025, -0.013526033610105515, -0.012753300368785858, -0.005632654298096895, -0.011131198145449162, 0.0239355880767107, -0.006169097498059273, -0.043451905250549316, -0.001955782761797309, 0.010051924735307693, 0.002907330868765712, 0.01002638041973114, 0.027435241267085075, -0.005211163312196732, -0.016029436141252518, 0.0049046240746974945, 0.04217465966939926, -0.01983562856912613, -0.018443429842591286, -0.02079356275498867, -0.013768710196018219, 0.016157159581780434, -0.02812495455145836, 0.00024766597198322415, 0.03216104954481125, 0.01806025579571724, -0.03315730392932892, -0.028686942532658577, -0.002675830153748393, 0.03678468242287636, -0.022990426048636436, 0.025302240625023842, 0.01222962886095047, 0.0037806478794664145, 0.021585455164313316, -0.017511041834950447, -0.013130087405443192, 0.004655561409890652, 0.013883662410080433, 0.00014778133481740952, -0.022300712764263153, 0.025659870356321335, 0.0008541581919416785, -0.017843125388026237, 0.015199226327240467, -0.0025624744594097137, 0.009343053214251995, -0.013832572847604752, 0.007574067916721106, -0.0027604475617408752, -0.014381788671016693, 0.019490772858262062, 0.0032186596654355526, 0.009055673144757748, 0.007778427563607693, 0.023782318457961082, -0.010249897837638855, 0.0021840904373675585, -0.007906151935458183, 0.012919342145323753, -0.036503687500953674, -0.006743858102709055, -0.011744275689125061, 0.017536586150527, -0.009330281056463718, -0.005335694644600153, -0.010332918725907803, -0.02738415077328682, -0.025838684290647507, -0.04380953311920166, -0.03170124441385269, 0.007350550033152103, -0.01880105957388878, -0.029989732429385185, 0.019337503239512444, -0.018826603889465332, -0.013245039619505405, -0.015671806409955025, -0.04378398880362511, 0.009062059223651886, -0.02408885769546032, 0.0027604475617408752, 0.025353331118822098, -0.024293215945363045, 0.014305153861641884, -0.02442094124853611, -0.020065532997250557, -0.0134110813960433, 0.011105652898550034, -0.047309186309576035, -0.04684937745332718, -0.016144387423992157, 0.01864778995513916, 0.0068971277214586735, 0.021585455164313316, 0.013870890252292156, 0.02631126530468464, -0.031471338123083115, -0.004467167425900698, -0.0049046240746974945, 0.0014584551099687815, 0.006699154619127512, -0.020039988681674004, -0.011999725364148617, -0.0038955998606979847, 0.01030737441033125, -0.007931697182357311, 0.01749826781451702, -0.008455367758870125, -0.016106070950627327, 0.026413444429636, 0.02235180325806141, -0.0005224734195508063, -0.008238235488533974, 0.0058593652211129665, -0.03142024949193001, 0.013947525061666965, -0.015160908922553062, 0.005655006039887667, -0.003908372484147549, 0.0243443064391613, -0.013347219675779343, 0.009164239279925823, 0.00549535034224391, 0.011278080753982067, 0.005894489586353302, -0.03282522037625313, 0.010665003210306168, -0.025736505165696144, 0.021968629211187363, -0.016897963359951973, 0.006367070600390434, -0.01654033362865448, 0.008589478209614754, -0.01601666398346424, -0.0014193394454196095, 0.027281971648335457, 0.001094440114684403, 0.0026774266734719276, 0.007471888326108456, 0.002950438065454364, 0.029070116579532623, 0.011993338353931904, 0.004489519167691469, -0.013743165880441666, -0.033617112785577774, 0.027614055201411247, 0.011169515550136566, 0.0063127875328063965, -0.012759686447679996, -0.015786759555339813, 0.0017785648815333843, -0.002053172793239355, 0.015211998485028744, 0.016003889963030815, -0.0012157784076407552, 0.009317508898675442, -0.03407692164182663, -0.028073864057660103, 0.04480578377842903, -0.015454675070941448, 0.012542554177343845, -0.01167402695864439, -0.0023022356908768415, -0.030577266588807106, -0.004958907142281532, 0.012989590875804424, 0.031471338123083115, -0.00713341822847724, 0.012459533289074898, 0.004569347016513348, 0.0030701798386871815, 0.014330699108541012, -0.012676665559411049, -0.0012405250454321504, -0.0036050265189260244, -0.042762190103530884, -0.003905179211869836, 0.0011120021808892488, 0.025327784940600395, 0.022428438067436218, 0.004617244005203247, 0.025034019723534584, 0.006111621391028166, -0.012593644671142101, 0.003502846695482731, 0.015569627285003662, 0.009477164596319199, -0.036248236894607544, -0.04235347360372543, -0.025583235546946526, -0.009170625358819962, -0.016080524772405624, 0.008365960791707039, 0.007414412219077349, 0.02364182099699974, -0.0040680281817913055, 0.003035055473446846, -0.010773569345474243, 0.009036514908075333, 0.00854477472603321, -0.016591424122452736, -0.009777316823601723, 0.01768985576927662, 0.019120370969176292, -0.006724699400365353, 0.010262670926749706, -0.017855897545814514, -0.005581564269959927, -0.018124118447303772, 0.0038349307142198086, -0.009234488010406494, 0.018826603889465332, 0.004955714102834463, -0.008634181693196297, 0.0013203528942540288, -0.03443454951047897, -0.0009723034454509616, 0.0035635160747915506, 0.024663617834448814, 0.012817162089049816, 3.327824379084632e-05, -0.023910041898489, -0.020078305155038834, -0.005099404137581587, 0.034766633063554764, -0.0007048800471238792, 0.001100826309993863, -0.013168404810130596, -0.009387757629156113, -0.018124118447303772, 0.0093813706189394, 0.0003863668825943023, -0.015850622206926346, -0.03060281090438366, 0.01708954945206642, 0.006453284993767738, -0.027435241267085075, 0.008947107009589672, 0.009815634228289127, -0.028073864057660103, 0.001641260925680399, -0.008487299084663391, -0.014931004494428635, 0.009853951632976532, 0.09293241053819656, 0.013717620633542538, -0.0001376032887492329, 0.027843959629535675, -0.007095100823789835, 0.02545551024377346, -0.010760796256363392, -0.019324729219079018, 0.009745386429131031, -0.017740944400429726, 0.012759686447679996, -0.004406498279422522, 0.007388867437839508, -0.01182091049849987, 0.02260725200176239, -0.010754410177469254, 0.024957384914159775, 0.0035443573724478483, 0.01280438993126154, 0.008717203512787819, 0.030091913416981697, -0.030704990029335022, -0.00357948150485754, 0.03170124441385269, -0.023079833015799522, 0.00259440578520298, 0.024561438709497452, 0.004039289895445108, 0.004623630084097385, -0.01530140545219183, -0.006890741642564535, -0.0018025132594630122, 0.017562130466103554, -0.022364575415849686, -0.016936279833316803, 0.013640985824167728, 0.0022926563397049904, 0.027000978589057922, 0.006673609837889671, -0.007637930102646351, 0.020768018439412117, 0.02537887543439865, 0.022696658968925476, -0.017472723498940468, -0.008251008577644825, -0.0354563444852829, -0.019490772858262062, 0.029095660895109177, 0.010645844042301178, -0.014943776652216911, 0.011744275689125061, -0.01159739214926958, -0.014560602605342865, -0.035200897604227066, 0.007663475349545479, -0.018328478559851646, -0.014151884242892265, 0.01305983867496252, -0.008046649396419525, -0.00375510286539793, -0.020154939964413643, -0.028584763407707214, 0.01853283680975437, -0.008155214600265026, -0.018673334270715714, -0.017932532355189323, -0.009502708911895752, 0.01820075325667858, -0.03348938748240471, 0.0009451619698666036, -0.01315563265234232, -0.017638765275478363, -0.04932723566889763, 0.011393032968044281, 0.028073864057660103, 0.0011543110013008118, 0.009777316823601723, 0.004224490839987993, 0.01146966777741909, 0.0036465369630604982, 0.008736361749470234, -0.021598227322101593, 0.003911565523594618, -0.003927531186491251, -0.00925364624708891, 0.0076443166472017765, 0.002814730629324913, 0.00667999591678381, 0.006497988477349281, 0.005169652868062258, 0.005575178191065788, 0.007127032149583101, 0.026924343779683113, -0.011405806057155132, 0.01950354501605034, 0.029248930513858795, 0.010977928526699543, -0.008487299084663391, 0.012900182977318764, -0.029504379257559776, 0.03793420270085335, -0.04069305211305618, 0.018749969080090523, -0.003086145268753171, 0.018009167164564133, -0.0029616139363497496, -0.012089132331311703, -0.0029296826105564833, -0.029913097620010376, -0.031164798885583878, 0.018967101350426674, -0.033795926719903946, 0.01167402695864439, 0.00206754170358181, -0.004856727551668882, 0.01964404247701168, 0.012791617773473263, 0.0536443255841732, -0.007574067916721106, -0.015531309880316257, -0.02405053935945034, -0.03862391412258148, 0.03525198623538017, 0.020742472261190414, -0.005147300660610199, -0.014982094056904316, -0.0005236708093434572, 0.010262670926749706, -0.001725878450088203, -0.0020388036500662565, 0.02779287099838257, 0.0026630577631294727, -0.030143002048134804, -0.02153436467051506, 0.0023708876688033342, -0.008742747828364372, -0.021700408309698105, 0.023411916568875313, -0.014458423480391502, 0.0016141195083037019, -0.0058721378445625305, -0.010588368400931358, -0.028661398217082024, -0.007369708735495806, 0.0317523330450058, -0.028738033026456833, -0.006922672502696514, 0.0019988897256553173, 0.012140221893787384, 0.011182287707924843, -0.013526033610105515, -0.013232267461717129, -0.031062619760632515, 0.013666531071066856, -0.00438414653763175, -0.014445650391280651, -0.002632722957059741, -0.015480220317840576, 0.025519372895359993, 0.01591448299586773, 0.007356936112046242, 0.008985424414277077, 0.004642788786441088, 0.023360827937722206, -0.006012634839862585, 0.01661696843802929, 0.01786866970360279, -0.01406247727572918, -0.002862627385184169, 0.036350417882204056, -0.010952383279800415, 0.012325422838330269, 0.007388867437839508, -0.0018232684815302491, 0.00987949687987566, 0.021406641229987144, -0.0009244066895917058, -0.028814667835831642, -0.016668058931827545, -0.0577315129339695, 0.010550050996243954, 0.02178981527686119, 0.0001793133415048942, 0.004029710777103901, -0.003512426046654582, -0.01509704627096653, 0.010147718712687492, -0.02461252734065056, 0.00975177250802517, 0.008819382637739182, 0.040795233100652695, -0.0072036669589579105, 0.04176593944430351, 0.008959880098700523, 0.015480220317840576, 0.005904069170355797, -0.013296129181981087, -0.007906151935458183, -0.0033016805537045, 0.00514410762116313, -0.005655006039887667, 0.003004720900207758, 0.01378148328512907, -0.01260003075003624, -0.03241650015115738, -0.012267946265637875, -0.011629323475062847, -0.012619188986718655, 0.010818272829055786, 0.0026854095049202442, -0.015863394364714622, -0.009726227261126041, -0.030143002048134804, 0.011220605112612247, 0.0024778570514172316, 0.017217274755239487, 0.0037774546071887016, 0.005102597177028656, -0.0001432910212315619, -0.010249897837638855, 0.004230876918882132, -0.004368180874735117, 0.004681106191128492, 0.013921979814767838, 0.022249622270464897, 0.003946689888834953, 0.006216994486749172, 0.001351485843770206, 0.026438988745212555, -0.006986535154283047, -0.006315981037914753, 0.028559217229485512, -0.02057643048465252, -0.0005939193069934845, -0.0034357914701104164, 0.012759686447679996, -0.01489268708974123, -0.047973353415727615, -0.02171318046748638, 0.014790507033467293, 0.03142024949193001, -0.00793808326125145, -0.010958769358694553, 0.012644734233617783, 0.013628213666379452, 0.0025832296814769506, 0.03249313682317734, 0.011022632010281086, -0.015965573489665985, 0.009125921875238419, -0.03377038240432739, 0.010269057005643845, 0.0179453045129776, 0.01786866970360279, 0.004620437044650316, -0.0040999590419232845, -0.01924809440970421, -0.010096628218889236, -0.007574067916721106, 0.013513261452317238, -0.0008988617919385433, 0.018187981098890305, -0.003298487514257431, -0.02705206722021103, 0.028201589360833168, 0.013232267461717129, -0.006846037693321705, 0.022837156429886818, -0.002428363775834441, 0.02486797794699669, -0.008129670284688473, -0.0054634190164506435, -0.019733449444174767, 0.004978065844625235, 0.005000417586416006, 0.004218104295432568, 0.004856727551668882, 0.009636820293962955, -0.021381095051765442, -0.010997086763381958, 0.02746078558266163, 0.011131198145449162, -0.002995141549035907, -0.0317523330450058, 0.015454675070941448, -0.006054145283997059, -0.01579953171312809, 0.0196057241410017, 0.0024507155176252127, 0.01056282315403223, -0.02212189882993698, -0.026336809620261192, -0.02245398238301277, 0.01809857413172722, -0.005278218537569046, -0.007420798763632774, 0.01335999183356762, 0.025353331118822098, -0.013072611764073372, 0.009368598461151123, 0.009311122819781303, -0.0018296546768397093, -0.03936471790075302, -0.025774821639060974, 0.009917814284563065, -0.03323393687605858, 0.014943776652216911, 0.04069305211305618, -0.01720450259745121, -0.017191728577017784, -0.00014059683599043638, 0.01983562856912613, 0.001786547712981701, -0.01576121337711811, -0.01964404247701168, 0.018187981098890305, -0.004955714102834463, -0.0014927810989320278, 0.0038828274700790644, -0.007778427563607693, 0.010696934536099434, -0.01426683645695448, 0.01924809440970421, -0.03348938748240471, 0.014752189628779888, 0.02067861147224903, -0.005712482146918774, 0.023488551378250122, -0.005517702084034681, -0.021470503881573677, -0.010869362391531467, -0.005019576288759708, 0.0002680220641195774, 0.003905179211869836, 0.015352495014667511, -0.00178974075242877, -0.02194308489561081, 0.03223768621683121, -0.008576706051826477, -0.009611275047063828, -0.008500071242451668, 0.014036932028830051, 0.008921562694013119, -3.874146204907447e-05, -0.009477164596319199, 0.0031244626734405756, -0.036094967275857925, -0.011801752261817455, -0.018034711480140686, -0.01285547949373722, -0.00738248135894537, 0.05183063820004463, -0.003522005397826433, -0.0030334589537233114, -0.005121755879372358, 0.0018999032909050584, -0.03581397607922554, 0.0021777041256427765, 0.0025465090293437243, 0.016553105786442757, 0.020768018439412117, 0.019401364028453827, 0.012338194996118546, 0.0004901430802419782, -0.02009107731282711, 0.0037710685282945633, -0.023182012140750885, 0.002415591152384877, -0.0015334932832047343, -0.0035539367236196995, 0.034204643219709396, -0.011808138340711594, -0.020372072234749794, -0.0222751684486866, 0.032212141901254654, -0.021023467183113098, -0.004709844011813402, 0.014100794680416584, 0.0005591941880993545, 0.0068971277214586735, -0.006108428351581097, 0.008110511116683483, 0.027256427332758904, 0.0003614206798374653, 0.021189508959650993, -0.025289468467235565, -0.001671595498919487, 0.008672499097883701, 0.0206147488206625, -0.019324729219079018, 0.016629740595817566, 0.00549535034224391, -0.0044863261282444, 0.012663892470300198, 0.00017262776964344084, -0.010447870939970016, -0.010882134549319744, 0.005310149863362312, 0.026541169732809067, 0.0072355978190898895, -0.01572289690375328, 0.028738033026456833, 0.011405806057155132, -0.01411356683820486, 0.003137235064059496, 0.01502041146159172, 0.010326532647013664, 0.03655477613210678, -0.015263088047504425, -0.03170124441385269, 0.004850341007113457, -0.026260174810886383, -0.0012109887320548296, -0.009068445302546024, 0.012433988973498344, 0.0008334029116667807, 0.006801334209740162, -0.03471554443240166, 0.01735777221620083, 0.01983562856912613, 0.007637930102646351, 0.013730392791330814, -0.006485215853899717, -0.02863585203886032, -0.015148135833442211, -0.03027072735130787, -0.003217063145712018, -0.0008996600518003106, 0.00819353200495243, 0.036503687500953674, -0.010716092772781849, -0.012255174107849598, -0.02738415077328682, 0.021623773500323296, -0.003917951602488756, 0.0023485359270125628, 0.21846012771129608, 0.006986535154283047, -0.01809857413172722, 0.02560877986252308, -0.017779262736439705, 0.00670554069802165, 0.024599755182862282, -0.006686381995677948, -0.023105377331376076, 0.0048599205911159515, -0.03044954128563404, 0.018456202000379562, -0.025148971006274223, 0.0010784744517877698, -0.015671806409955025, -0.011776207014918327, -0.019414138048887253, -0.007708178833127022, 0.018890466541051865, -0.02830376848578453, 0.01772817224264145, -0.007740110158920288, -0.017140639945864677, -0.008934334851801395, 0.018302934244275093, -0.006820492912083864, 0.00455976789817214, -0.008385119028389454, 0.010051924735307693, -0.015850622206926346, -0.008021104149520397, -0.012121063657104969, 0.011903931386768818, 0.010166876949369907, -0.03093489445745945, 0.001009822473861277, 0.005632654298096895, 0.007395253516733646, 0.010147718712687492, -0.001725878450088203, 0.016808556392788887, 0.011865613982081413, -0.013500489294528961, -0.0004498300259001553, -0.01021158043295145, 0.04378398880362511, -0.01850729249417782, -0.00751020573079586, -0.006290435791015625, -0.009988063015043736, -0.02520006150007248, 0.00153828295879066, 0.004479940049350262, -0.002182493917644024, -0.0075868405401706696, 6.166702951304615e-05, -0.0025656677316874266, -0.003636957611888647, -0.00874913390725851, -0.006596975028514862, -0.018366795033216476, 0.02205803617835045, -0.020039988681674004, 0.04324754700064659, -0.01252339594066143, 0.023590730503201485, -0.009687909856438637, -0.002022838220000267, -0.0014584551099687815, -0.03438345715403557, -0.021585455164313316, 0.02963210456073284, 0.005581564269959927, 0.010671389289200306, -0.00017901399405673146, 0.022581707686185837, 0.020359300076961517, 0.013034294359385967, 0.016272112727165222, 0.009004583582282066, -0.01496932189911604, 0.0119550209492445, -0.003847703104838729, -0.03655477613210678, -0.01002638041973114, -0.019631268456578255, 0.007363322656601667, 0.0076443166472017765, 0.0008884841809049249, 0.010869362391531467, -0.0022655148059129715, 0.011514371261000633, -0.0034964606165885925, -0.02830376848578453, 0.0058753308840096, -0.02486797794699669, -0.005833820439875126, 0.01583784818649292, -0.022006945684552193, 0.0049493275582790375, 0.007152576930820942, 0.024216581135988235, 0.01761322095990181, 0.021662089973688126, 0.0028674169443547726, 0.011016245931386948, 0.0003029467479791492, 0.021853676065802574, -0.004403305239975452, -0.014586147852241993, -0.010460644029080868, -0.03397474065423012, 0.00027620443142950535, -0.015735669061541557, -0.004205332137644291, 0.012644734233617783, -0.016233794391155243, -0.02500847354531288, 0.03144579380750656, -0.006114814430475235, -0.008576706051826477, -0.007376094814389944, -0.018954329192638397, 0.015556855127215385, -8.940721454564482e-05, 0.004016938153654337, -0.014586147852241993, 0.016553105786442757, 0.00035443573142401874, -0.04948050528764725, 0.02937665581703186, -0.01579953171312809, -0.005894489586353302, 0.022875472903251648, -0.001554248621687293, 0.005862558726221323, -0.0008198322029784322, -0.0033016805537045, -0.03144579380750656, -0.013334446586668491, -0.02707761339843273, 0.010409553535282612, 0.0075868405401706696, 0.022505072876811028, 0.0119550209492445, -0.026796618476510048, 0.001586977974511683, -0.014739417470991611, -0.0014919828390702605, -0.025723731145262718, 0.0018999032909050584, 0.003119673114269972, 0.0179453045129776, -0.006322367116808891, 0.035073172301054, -0.017549358308315277, -0.026260174810886383, -0.020180484279990196, -0.0027077612467110157, 0.005587950814515352, -0.033872559666633606, 0.012631962075829506, 0.03489435836672783, -0.010345691815018654, 0.006590588483959436, -0.0167191494256258, -0.1575610488653183, 0.009368598461151123, 0.002064348664134741, -0.010134945623576641, 0.02597918175160885, 0.020691383630037308, 0.020180484279990196, 0.011233377270400524, -0.03578842803835869, 0.010128559544682503, 0.011022632010281086, -0.018009167164564133, -0.007663475349545479, -0.013385537080466747, -0.01258087158203125, 0.00029236957198008895, -0.0002544513263273984, 0.037832021713256836, 0.017102321609854698, 0.012089132331311703, 0.030807171016931534, -0.01476496271789074, 0.03841955587267876, -0.011258922517299652, 0.013270584866404533, 0.009145080111920834, -0.0222751684486866, 0.006699154619127512, -0.022377347573637962, -0.021891994401812553, 0.0063990019261837006, -0.004751354455947876, 0.0031100937630981207, -0.02153436467051506, -0.0012812372297048569, -0.024037767201662064, 0.027409696951508522, 0.01035846397280693, -0.013832572847604752, 0.013934752903878689, 0.0005791511503048241, -0.025762049481272697, 0.005738026928156614, -0.007350550033152103, 0.0016276901587843895, 0.017855897545814514, -0.008404278196394444, -0.003522005397826433, 0.011226991191506386, -0.0015789952594786882, 0.011642096564173698, 0.0036401506513357162, -0.0196057241410017, 0.010058310814201832, -0.011150356382131577, -0.00417978735640645, -0.02246675454080105, 0.017485495656728745, -0.00625211838632822, -0.04232792928814888, -0.0024235739838331938, -0.002391642890870571, 0.01481605228036642, -0.006006248760968447, 0.02153436467051506, 0.008199918083846569, 0.022211305797100067, -0.011942248791456223, -0.036171603947877884, 0.012427601963281631, -0.024216581135988235, -0.014458423480391502, 0.00975177250802517, -0.03305512294173241, 0.002375677227973938, -0.0011271694675087929, -0.0032793288119137287, 0.003004720900207758, -0.014215746894478798, -0.0030797591898590326, 0.0028674169443547726, 0.009336667135357857, -0.012299877591431141, -0.02853367291390896, 0.0023389565758407116, 0.013500489294528961, 0.0019446068909019232, -0.014956548810005188, -0.005910455249249935, -0.004352215211838484, 0.005932806991040707, -0.0024571018293499947, -0.012644734233617783, -0.013768710196018219, 0.003512426046654582, 0.0002374879113631323, 0.018175208941102028, 0.009438847191631794, -0.00847452599555254, -0.02112564630806446, 0.0367080457508564, -0.005894489586353302, -0.02245398238301277, 0.007708178833127022, 0.023807862773537636, 0.018481748178601265, 0.0115463025867939, -0.004572540055960417, 0.03474108874797821, -0.0006186659447848797, -0.020410388708114624, 0.008040262386202812, 0.006386229302734137, 0.01224878802895546, -0.018251843750476837, 0.02560877986252308, 0.003220256185159087, -0.004403305239975452, 0.021930310875177383, -0.023897269740700722, 0.0395946204662323, -0.015033183619379997, 0.015326950699090958, 0.006462864112108946, -0.021342778578400612, -0.020142167806625366, -0.12394393980503082, 0.0005560010904446244, 0.010671389289200306, 0.039390262216329575, 0.009343053214251995, -0.0006046961061656475, -0.006089269649237394, -0.012791617773473263, -0.018916010856628418, 0.022594479843974113, -0.0034804949536919594, -0.017434407025575638, -0.006303208414465189, 0.010894907638430595, -0.005961545277386904, 0.00395946204662323, 0.008021104149520397, -0.020282665267586708, 0.0015398795949295163, 0.006376650184392929, -0.009374984540045261, 0.0075996131636202335, -0.035609614104032516, -0.013072611764073372, 0.0013067822437733412, 0.00451506394892931, -0.007069556042551994, -0.007848676294088364, 0.011910317465662956, 0.018583927303552628, -0.01872442476451397, -0.0020340140908956528, -0.015748441219329834, -0.00740802614018321, 0.01927364058792591, -0.006960989907383919, -0.03019409254193306, -0.00542190857231617, 0.011667640879750252, -0.019082052633166313, -0.008078579790890217, -0.0036625026259571314, 0.002109052147716284, -0.00834680162370205, -0.00965597853064537, 0.010671389289200306, -0.033872559666633606, 0.032620858401060104, 0.021227827295660973, -0.014547830447554588, -0.0115463025867939, -0.007184508256614208, -0.03384701535105705, -0.026004726067185402, 0.01307899784296751, 0.0091770114377141, 0.012363740243017673, -0.022147443145513535, -0.023846181109547615, 0.005897682625800371, -0.0001569615415064618, 0.012108290567994118, -0.014547830447554588, -0.00011016245844075456, 0.002388449851423502, -0.013130087405443192, -0.013219494372606277, -0.019669586792588234, 0.011680413968861103, -0.01913314312696457, 0.009668751619756222, -0.018954329192638397, -0.02239011973142624, 0.032620858401060104, -0.03992670774459839, 0.02797168493270874, -0.01293211430311203, -0.0025385261978954077, 0.00227988394908607, -0.005022769328206778, -0.02375677414238453, -0.00791892409324646, 0.014254064299166203, -0.023322509601712227, 0.019005417823791504, -0.0023229909129440784, 0.025928091257810593, -0.002680619712918997, 0.003055810695514083, -0.01890323869884014, -0.0072930739261209965, 0.02246675454080105, 0.012900182977318764, -0.0007595621282234788, -0.003518812358379364, -0.023514095693826675, -0.005613495595753193, -0.025468282401561737, -0.010779955424368382, 0.026924343779683113, 0.0057539925910532475, -0.021853676065802574, -0.01857115514576435, -0.001602145261131227, -0.012893796898424625, -0.01099070068448782, 0.004907817114144564, -0.002013258868828416, 0.0019126756815239787, -0.008736361749470234, -0.018698880448937416, 0.02141941338777542, -0.03558406978845596, 0.024957384914159775, -0.009240874089300632, -0.00847452599555254, -0.02649007923901081, 0.006833265535533428, -0.00910037662833929, 0.023437460884451866, -0.02053811401128769, 0.011156742461025715, -0.012389284558594227, -0.004719423595815897, -0.0024379431270062923, 0.01654033362865448, -0.017843125388026237, 0.016450926661491394, -0.009994449093937874, 0.018749969080090523, -0.01613161526620388, -0.015186453238129616, 0.012919342145323753, -0.017970848828554153, 0.008532002568244934, 0.07045288383960724, -0.019529089331626892, 0.004141469951719046, 0.0006541893817484379, 0.02276052162051201, 0.01576121337711811, 0.031803421676158905, -0.015339722856879234, -0.024625301361083984, 0.011661254800856113, -0.0028785928152501583, 0.0117506617680192, 0.002490629442036152, -0.005054700654000044, 0.026132451370358467, 0.0005540053825825453, 0.010320146568119526, 0.012753300368785858, 0.019286412745714188, -0.017881441861391068, -0.006130780093371868, -0.02420380897819996, -0.019043736159801483, 0.017804807052016258, 0.010352077893912792, 0.004454394802451134, -0.021151192486286163, 0.028891300782561302, 0.01657865196466446, 0.016808556392788887, -0.0026678473223000765, 0.014956548810005188, -0.006331946235150099, -0.015416357666254044, 0.03078162483870983, 0.011603779159486294, -0.033208392560482025, -0.02338637225329876, 0.012893796898424625, -0.0004809628881048411, 0.022620024159550667, 0.015544082038104534, 0.016783010214567184, -0.027281971648335457, -0.0018887273035943508, 0.004518257454037666, 0.019082052633166313, 0.021636545658111572, -0.0033687360119074583, 0.0017466337885707617, 0.00413508340716362, 0.04457588121294975, 0.007625157944858074, 0.008646954782307148, 0.01468832790851593, 0.008040262386202812, -0.00039933889638632536, 0.025877000764012337, -0.0052686394192278385, -0.002527350327000022, -0.00594877265393734, 0.02075524628162384, 0.022888246923685074, 0.004748161416500807, -0.005303763318806887, -0.00980286207050085, 0.03637596219778061, -0.011150356382131577, 0.0007232404896058142, 0.01348771620541811, 0.022249622270464897, -0.006609747186303139, 0.011482440866529942, -0.018034711480140686, -0.03951798751950264, -0.025851456448435783, 0.0016037418972700834, 0.030321817845106125, -0.0024986122734844685, 0.021598227322101593, 0.0050802454352378845, -0.014049704186618328, 0.019963353872299194, 0.004585312679409981, -0.011367488652467728, -0.0247530248016119, 0.029964188113808632, 0.0038508963771164417, -0.010901293717324734, 0.0355074368417263, -0.02779287099838257, -0.004805637523531914, 0.03655477613210678, 0.017881441861391068, -0.010779955424368382, 0.004256421700119972, 0.004665140528231859, -9.978483285522088e-05, -0.026771074160933495, -0.006628905888646841, -0.01084381714463234, 0.0025353331584483385, -0.00030733729363419116, 0.020474251359701157, 0.015671806409955025, -0.006571430247277021, 0.032467588782310486, 0.02116396464407444, 0.017766490578651428, 0.0158761665225029, 0.013602668419480324, 0.025889774784445763, 0.029478834941983223, 0.0026438990607857704, -0.017638765275478363, -0.016744693741202354, 0.013091770000755787, -0.007797586265951395, 0.019069280475378036, -0.03962016850709915, -0.047411367297172546, 0.014292381703853607, -0.014905459247529507, 0.004240456502884626, -0.02087019756436348, 0.004783285781741142, 0.019414138048887253, -0.0007807165384292603, -0.0033112599048763514, -0.018852148205041885, -0.013973070308566093, -0.010862976312637329, 0.01252339594066143, 0.006079690530896187, 0.024433713406324387, -0.043681807816028595, 0.018928783014416695, 0.03305512294173241, -0.02397390455007553, -0.026668893173336983, 0.02282438427209854, 0.011935862712562084, -0.003917951602488756, -0.017255591228604317, 0.012178539298474789, 0.005259059835225344, 0.003220256185159087, 0.010997086763381958, -0.013526033610105515, -0.03900709003210068, 0.028201589360833168, 0.010741638019680977, -0.02564709633588791, 0.006232959683984518, -0.026796618476510048]} +{"id": "test:10000060", "text": "\"Everyone seems to want a brighter smile, and teeth whitening is a great option for boosting your confidence. If food and drinks have stained your pearly whites, you may be tempted to rush to the drug store and pick up the first over-the-counter whitening product you find. However, while side effects from whitening products are rare, you should follow these tips before bleaching your teeth to ensure your safety.\\nIf your teeth appear stained, don\u2019t assume you need to bleach them. It\u2019s possible a good teeth cleaning will restore the sparkle to your smile. During an oral exam, your dentist will also check for cavities and gum issues, which should be treated before bleaching your teeth.\\nAt-home whiteners most often use carbamide peroxide in concentrations ranging from 10 to 20 percent. Carbamide peroxide is hydrogen peroxide with urea added. This product remains active for up to 10 hours at a low enough acidity level that enamel etching is not a concern.\\nStart with a product containing 10 percent carbamide peroxide. If the whitener doesn\u2019t bother your mouth, but you don\u2019t see the results you want, try another over-the-counter product with a higher concentration.\\nWhen you choose an at-home teeth bleaching kit, it\u2019s your responsibility to follow the instructions on the package. Don\u2019t leave the gel or strips on your teeth longer than advised or you could experience tooth sensitivity and sore gums. Also, avoid drinking soda, sports drinks or other sugary/acidic beverages for several hours after whitening to prevent etching your teeth.\\nAt-home whitening kits are inexpensive and widely available, but that doesn\u2019t make them the best choice. Professional in-office teeth bleaching can lighten your smile up to eight shades in a single 45-minute visit. Combined with professional-grade take-home treatments, patients see an average improvement of 11 shades. In comparison, over-the-counter whitening requires daily 30-minute applications for more than two weeks to achieve a six-shade improvement.\\nIn-office whitening uses LED light-activated technology to maximize results in the shortest amount of time without causing tooth sensitivity. It\u2019s effective both for people who simply want to lighten their smile and for patients with severely discolored teeth.\\nPregnant women and nursing mothers should postpone teeth bleaching.\\nTranslucent teeth don\u2019t respond well to whitening. A tooth-colored composite placed on the lingual side of a see-through tooth is a more effective way to improve its appearance.\\nZen Dental Center in Seattle offers Philips Zoom in-office teeth bleaching for safe, professional results. To learn more about this treatment, or to schedule an appointment, please contact us at 206-324-1100 today.\"", "vector_field": [0.007713858038187027, 0.0029867878183722496, 0.018966183066368103, -0.03749488294124603, 0.003773292526602745, 0.012114424258470535, 0.004281545057892799, -0.026969552040100098, -0.0097404345870018, -0.0033132918179035187, 0.012255962938070297, 0.0117412768304348, -0.03579641878604889, -0.0005158924032002687, -0.017190515995025635, -0.004281545057892799, 0.0404028594493866, 0.0024061575531959534, 0.007334277033805847, -0.006333855912089348, -0.03018634021282196, -0.012429669499397278, -0.013156663626432419, 0.030057668685913086, -0.02191276103258133, -0.007347144186496735, 0.04027418792247772, -0.007224906235933304, 0.00269406009465456, 0.0014025196433067322, 0.007430780678987503, -0.011606171727180481, -0.018605902791023254, 0.0015915059484541416, -0.025013744831085205, 0.02186129242181778, 0.0010269596241414547, -0.009457357227802277, 0.012107990682125092, -0.00568084791302681, -0.009830504655838013, -0.014166735112667084, 0.006620150059461594, 0.005841687321662903, -0.009322252124547958, 0.02115359902381897, -0.010184351354837418, -0.014758624136447906, -0.00683889165520668, 0.011831346899271011, 0.008389383554458618, 0.006826024502515793, 0.0071155354380607605, -0.011149387806653976, 0.004509937018156052, -0.0389617383480072, -0.005889939144253731, 0.03901320695877075, -0.012262396514415741, -0.01191498339176178, 0.014346875250339508, 0.015028834342956543, -0.002788955345749855, 0.004966720938682556, 0.014848694205284119, -0.0037636421620845795, 0.004175391048192978, 0.02256898581981659, -0.009624630212783813, -0.007578752934932709, 0.010801974684000015, 0.030803963541984558, 0.028590813279151917, 0.0133303701877594, 0.03636257350444794, -0.0007933403830975294, -0.0232766792178154, -0.015118904411792755, 0.009779036045074463, -0.011387430131435394, 0.004333013668656349, 0.004429517313838005, -0.034097954630851746, 0.01559498906135559, 0.005150077864527702, 0.020831920206546783, 0.012249529361724854, 0.013111628592014313, -0.02271052449941635, 0.0010261554270982742, -0.008453719317913055, 0.014359742403030396, 0.012699879705905914, -0.006539730355143547, -0.007623787969350815, 0.018052615225315094, 0.004410216584801674, 0.04529237747192383, -0.00830574706196785, -0.002554129809141159, -0.008659593760967255, 0.016688697040081024, -0.018695972859859467, -0.011130087077617645, -0.016071073710918427, -0.004242943599820137, -0.005185462534427643, -0.004390915855765343, 0.00977260246872902, -0.007424347102642059, 0.0028307735919952393, 0.03366047143936157, 0.0035738516598939896, -0.03790663182735443, -0.011020716279745102, -0.004452034831047058, -0.0082542784512043, -0.018708840012550354, -0.012165892869234085, -0.015003100037574768, -0.01279638335108757, 0.016740165650844574, 0.015401981770992279, -0.015466317534446716, 0.012461837381124496, 0.013960860669612885, 0.002345038577914238, -0.010962814092636108, 0.00906490907073021, 0.002552521415054798, 0.0051918961107730865, 0.015479184687137604, 0.021282270550727844, -0.005510358139872551, -0.006150498986244202, 0.029054030776023865, 0.01305372640490532, -0.018142685294151306, -0.019249260425567627, -0.017486460506916046, 0.026918083429336548, 0.0048895180225372314, 0.0038826633244752884, -0.020072758197784424, -0.002552521415054798, -0.0019332896918058395, 0.005153294652700424, 0.023649826645851135, -0.009476657956838608, -0.0035320334136486053, 0.010711904615163803, -0.028436407446861267, 0.0036703553050756454, -0.0022356677800416946, 0.000433462206274271, 0.004268677905201912, 0.02907976508140564, 0.011927850544452667, -0.031318649649620056, 0.02094772458076477, 0.014913029968738556, -0.015530653297901154, 0.013703517615795135, 0.008839733898639679, 0.0009521692991256714, 0.02217010408639908, -0.014012329280376434, 0.010518897324800491, -0.0042558107525110245, -0.010711904615163803, -0.020433038473129272, 0.04670776426792145, -0.03898747265338898, 0.037031665444374084, -0.016122542321681976, 0.010004211217164993, -0.002880633808672428, 0.013870790600776672, 0.0041142720729112625, -0.004304062575101852, -0.018104083836078644, 0.018142685294151306, 0.028590813279151917, 0.00274552870541811, 0.02085765451192856, -0.012378200888633728, 0.020021289587020874, -0.0026779761537909508, 0.011651206761598587, -0.04382552206516266, 0.0005770113784819841, 0.0194808691740036, -0.018309958279132843, -0.02115359902381897, -0.6435635089874268, -0.023559756577014923, 0.004094971343874931, 0.016071073710918427, 0.010711904615163803, 0.010358057916164398, 0.031318649649620056, -0.021114997565746307, -0.016186878085136414, 0.015685059130191803, -0.004664342850446701, 0.01872170716524124, -0.02291639894247055, -0.003049515187740326, -0.0005693715065717697, -0.0018110517412424088, 0.014835827052593231, -0.006240569055080414, 0.004693293944001198, -0.01297008991241455, -0.017692334949970245, 0.018760308623313904, -0.03360900282859802, 0.008337914943695068, 0.029671654105186462, 0.01229456439614296, 6.242579547688365e-05, -0.024293184280395508, 0.0038601458072662354, 0.015813730657100677, 0.0038858801126480103, 0.032682567834854126, -0.006430359557271004, 0.01580086350440979, 0.042487338185310364, -0.008794698864221573, 0.009347986429929733, 0.022195838391780853, 0.0032666483893990517, 0.005343085154891014, -0.020677514374256134, -0.011464633047580719, 0.030675292015075684, -0.01747359335422516, 0.009984910488128662, -0.014835827052593231, 0.003956649452447891, 0.023688428103923798, 0.02372702956199646, 0.022903531789779663, 0.010692603886127472, 0.013420440256595612, -0.0021970663219690323, -0.010094281286001205, 0.018940448760986328, -0.014346875250339508, 0.025875844061374664, -0.008543789386749268, 0.005603644996881485, -0.0075851865112781525, -0.0011282884515821934, 0.019274994730949402, 0.0007848963141441345, -0.013130929321050644, -0.028230533003807068, 0.03237375617027283, -0.024601995944976807, -0.0036735720932483673, 0.019905485212802887, 0.01863163709640503, 0.02923417091369629, 0.010898478329181671, -0.019094854593276978, 0.00880756601691246, 0.01610967516899109, 0.01721625030040741, 0.028101861476898193, -0.0007209626492112875, 0.010062113404273987, -0.001125875860452652, 0.02599164843559265, 0.0037797261029481888, -0.011773444712162018, -0.012326732277870178, 0.02473066747188568, 0.0028902841731905937, 0.004995672032237053, -0.02529682219028473, -0.0023595141246914864, 0.004001684486865997, 0.03764928877353668, 0.017396390438079834, 0.0006775360088795424, -0.040917545557022095, -0.03358326852321625, 0.004201125353574753, -0.0032232217490673065, -0.01670156419277191, 0.003982383757829666, -0.004001684486865997, -0.00749511644244194, -0.01827135682106018, -0.005584344267845154, 0.039450690150260925, 0.006658751517534256, -0.013510510325431824, -0.012944355607032776, -0.013857923448085785, 0.023032203316688538, -0.0011298968456685543, -0.007456514984369278, 3.829988418146968e-05, -0.013433307409286499, -0.013253167271614075, -0.0038054604083299637, -0.011194422841072083, 0.020535975694656372, -0.0001744102337397635, 0.0015633590519428253, -0.041226357221603394, -0.0035320334136486053, -0.00333259254693985, -0.026918083429336548, 0.007900431752204895, 0.030958369374275208, -0.014205336570739746, 0.013317503035068512, 0.002887067385017872, 0.020973458886146545, -0.013201698660850525, 0.013459041714668274, -0.0038601458072662354, 0.011953584849834442, 0.007643088698387146, 0.0012368550524115562, 0.02397150546312332, 0.012629110366106033, -0.009019874036312103, 0.004751196131110191, -0.013870790600776672, -0.006568681448698044, 0.015389114618301392, -0.006021827459335327, -0.016186878085136414, -0.007906865328550339, -0.003979166969656944, -0.019828282296657562, -0.010866310447454453, -0.01802688091993332, 0.002251751720905304, 0.009553860872983932, -0.00754658505320549, -0.027021020650863647, 0.012300997972488403, 0.014243938028812408, 0.005182245746254921, -0.025837242603302002, -0.004712594673037529, -0.004728678613901138, 0.0014515756629407406, 0.002740703523159027, 0.02807612717151642, -0.0023755980655550957, 0.0025396542623639107, -0.010216519236564636, -0.000457588117569685, -0.006877493113279343, 0.011824913322925568, 0.009373720735311508, -0.04127782583236694, 0.014269672334194183, 0.0015882891602814198, -0.010287288576364517, 0.02564423531293869, 0.0018094433471560478, 0.0018094433471560478, -0.04765993356704712, -0.014552749693393707, -0.011040017008781433, 0.015157505869865417, 0.02422884851694107, -0.003750775009393692, -0.023302413523197174, 0.011213723570108414, 0.011767011135816574, 0.014681421220302582, -0.0024978360161185265, 0.007951900362968445, -0.03963083028793335, 0.03973376750946045, 0.016946040093898773, 0.017434991896152496, 0.016289815306663513, -0.014707155525684357, -0.007900431752204895, -0.02034296840429306, -0.02281346172094345, -0.010892044752836227, 0.001630107406526804, 0.00820280984044075, 0.022633321583271027, 0.0036960896104574203, 0.0001688813790678978, -0.021037794649600983, 0.02205429971218109, 0.010338757187128067, 0.009039174765348434, -0.011902116239070892, 0.0231737419962883, 0.024151645600795746, -0.000780473230406642, -0.02856507897377014, -0.002208325080573559, -0.024460457265377045, -0.007527284324169159, 0.015414848923683167, -0.01918492466211319, 0.026068851351737976, 0.014629952609539032, -0.009547427296638489, 0.00891050323843956, -0.0327083021402359, 0.01459135115146637, -0.00020768388640135527, -0.03070102632045746, 0.01747359335422516, 0.01832282543182373, -0.012223795056343079, 0.0018094433471560478, -0.010988548398017883, -0.002975529059767723, 0.009386587888002396, -0.005809519439935684, -0.012185193598270416, 0.04104621708393097, 0.004374831914901733, 0.025219619274139404, -0.036516979336738586, 0.008447285741567612, 0.011721976101398468, 0.009495958685874939, 0.04120062291622162, 0.0322965532541275, -0.00234825536608696, 0.022877797484397888, 0.037829428911209106, 0.020214296877384186, 0.007990501821041107, -0.0076302215456962585, 0.03453543782234192, -0.0064367931336164474, 0.030263543128967285, -0.03013487160205841, 0.0046772100031375885, -0.005761267617344856, -0.025271087884902954, 0.030906900763511658, 0.011117219924926758, 0.017048977315425873, 0.022144369781017303, 0.005700148642063141, 0.020368702709674835, 0.028487876057624817, -0.03185907006263733, -0.00183035247027874, -0.011162254959344864, -0.001780492253601551, -0.017447859048843384, -0.007932599633932114, -0.0072313398122787476, -0.0016091982834041119, -0.0052626654505729675, 0.030212074518203735, -0.02034296840429306, 0.018438629806041718, 0.009148545563220978, -0.0327083021402359, 0.013163097202777863, 0.020214296877384186, 0.023289546370506287, -0.028359204530715942, -0.024460457265377045, 0.011889249086380005, 0.011007849127054214, -0.027201160788536072, -0.0015633590519428253, -0.007250640541315079, -0.015981003642082214, 0.007469382137060165, 0.01746072620153427, 0.0032296553254127502, 0.024164512753486633, 0.0008556656539440155, 0.02241457998752594, 0.010364491492509842, -0.04724818468093872, 0.0029642703011631966, 0.020728982985019684, 0.028539344668388367, 0.004664342850446701, 0.004365181550383568, 0.001512694638222456, -0.03716033697128296, -0.021192200481891632, 0.0040917545557022095, -0.00464504212141037, -0.0003307260340079665, -0.002528395503759384, 0.015003100037574768, -0.02871948480606079, 0.01246827095746994, -0.0006220464129000902, 0.011528968811035156, 0.007739592343568802, 0.004236510023474693, -0.016315549612045288, -0.013754986226558685, -0.008222110569477081, 0.011265192180871964, -0.00968896597623825, 0.011181555688381195, -0.025103814899921417, 0.013471908867359161, 0.006806723773479462, 0.061813801527023315, 0.004850916564464569, 0.0019365064799785614, 0.03800956904888153, -0.0165085569024086, 0.008518055081367493, 0.001383218914270401, -0.009972043335437775, 0.0011701066978275776, -0.0019365064799785614, 0.015672191977500916, -0.0020716115832328796, 0.020664647221565247, -0.005899589508771896, 0.009399455040693283, -0.005671197548508644, 0.01737065613269806, -0.026261858642101288, 0.020561710000038147, 0.008794698864221573, -0.003863362595438957, -0.01453988254070282, 0.015736527740955353, -0.003374410793185234, 0.028101861476898193, -0.0014113658107817173, 0.030057668685913086, 0.009907707571983337, 0.022285908460617065, -0.025168150663375854, -0.02462773025035858, 0.00951525941491127, 0.0034065786749124527, -0.0033390261232852936, -0.011644773185253143, -0.013960860669612885, -0.0320906788110733, -0.025412626564502716, 0.044803425669670105, -0.003705739974975586, 0.021333739161491394, 0.009071342647075653, -0.013870790600776672, -0.011471066623926163, -0.02998046576976776, -0.021243669092655182, 0.03553907573223114, 0.006517212837934494, -0.007591620087623596, -0.021012060344219208, 0.005867421627044678, -0.009450923651456833, -0.016122542321681976, -0.031627461314201355, -0.001523149199783802, -0.01847723126411438, -0.0014258413575589657, -0.011767011135816574, -0.011606171727180481, -0.024267449975013733, 0.0042075589299201965, -0.015273310244083405, -0.009637497365474701, -0.003165319561958313, -0.0009787078015506268, -0.023907169699668884, -0.015131771564483643, 0.005548959597945213, 0.0012465054169297218, -0.007610920816659927, 0.02397150546312332, -0.01295078918337822, -0.015582121908664703, 0.0013414006680250168, -0.012738481163978577, 0.00379902683198452, -0.00473189540207386, -0.008717495948076248, 0.010911345481872559, 0.04503503441810608, 0.002226017415523529, -0.0007555431220680475, -0.010975681245326996, -0.029620185494422913, -0.02004702389240265, 0.00027965952176600695, 0.026969552040100098, -0.025322556495666504, -0.01131022721529007, -0.001756366342306137, -0.0130215585231781, 0.022298775613307953, -0.016276948153972626, -0.004484202712774277, -0.005622945725917816, 0.008196376264095306, 0.006401408463716507, 0.018502965569496155, 0.0050053223967552185, -0.008614558726549149, -0.009682532399892807, 0.0068581923842430115, -0.006134415045380592, -0.023186609148979187, -0.001478114165365696, 0.008196376264095306, 0.0025428710505366325, -0.006449660286307335, -0.0021198634058237076, 0.011979319155216217, -0.0019188141450285912, -0.02044590562582016, 0.002602381631731987, -0.0020072758197784424, 0.017782405018806458, 0.02139807492494583, -0.033840611577034, 0.011857081204652786, -0.003070424310863018, -0.0200984925031662, 0.018567301332950592, -0.04382552206516266, 0.03600229322910309, 0.014462679624557495, -0.019763946533203125, 0.0165085569024086, 0.016933172941207886, -0.03376340866088867, -0.011342395097017288, -0.007366444915533066, 0.023534022271633148, 0.031061306595802307, -0.004487419500946999, 0.005394553765654564, -0.04084034264087677, -0.007038332521915436, -0.01519610732793808, 0.006025044247508049, -0.02130800485610962, -0.030315011739730835, -0.00689036026597023, 0.009483091533184052, 0.0005657526198774576, -0.01837429404258728, 0.03510159254074097, -0.020162828266620636, 0.00792616605758667, -0.021089263260364532, 0.0011886032298207283, 0.003844061866402626, 0.0036928728222846985, -0.0032923826947808266, -0.04068593680858612, -0.01944226771593094, 0.0037636421620845795, -0.017589397728443146, 0.0048895180225372314, -0.026068851351737976, 0.012629110366106033, 0.024396121501922607, 0.025605633854866028, -0.009605329483747482, -0.022388845682144165, 0.008839733898639679, -0.003776509314775467, -0.004062803462147713, -0.01918492466211319, -0.0006501933094114065, -0.033300191164016724, 0.01105288416147232, -0.023534022271633148, 0.011625472456216812, 0.012603376060724258, 0.016920305788517, 0.01690743863582611, 0.03028927743434906, -0.0019863666966557503, -0.028359204530715942, -0.023649826645851135, -0.019866883754730225, 0.002084478735923767, -0.005832036957144737, -0.024653464555740356, 0.025155283510684967, -0.003284340724349022, 0.0010961205698549747, 0.03517879545688629, 0.03749488294124603, 0.0034065786749124527, 0.010409526526927948, 0.014810092747211456, -0.0010985331609845161, 0.032991379499435425, -0.006057212129235268, -0.024241715669631958, 0.012391068041324615, -0.018695972859859467, -0.02211863547563553, -0.018747441470623016, -0.00749511644244194, 0.042435869574546814, -0.02433178573846817, 0.011194422841072083, -0.01241680234670639, -0.03893600404262543, -0.0049570705741643906, -0.0003146420931443572, -0.024293184280395508, -0.011072184890508652, -0.014861561357975006, -0.018618769943714142, 0.007868263870477676, 0.017499327659606934, 0.010699037462472916, -0.028487876057624817, -0.004667559638619423, -0.011779878288507462, 0.023817099630832672, -0.0061762332916259766, -0.030469417572021484, -0.0014009112492203712, -0.015530653297901154, 0.0027809133753180504, -0.017023243010044098, 0.02277486026287079, 0.023585490882396698, -0.004828399047255516, 0.019120588898658752, 0.01158687099814415, -0.015234708786010742, -0.008955538272857666, 0.024923674762248993, -0.0068581923842430115, 0.012043654918670654, -0.0013220999389886856, 0.01115582138299942, -0.030366480350494385, -0.018708840012550354, -0.053836166858673096, -0.00689036026597023, -0.004686860367655754, 0.000720158452168107, -0.01070547103881836, -0.0034998655319213867, -0.01675303280353546, 0.03028927743434906, 0.011844214051961899, -0.0040435027331113815, 0.007096234709024429, -0.01954520493745804, -0.015440583229064941, 0.018361426889896393, 0.008563090115785599, 0.004120705649256706, -0.005188679322600365, 0.00802910327911377, -0.03574495017528534, -0.007134836167097092, -0.026068851351737976, 0.008318614214658737, 0.0008178683929145336, 0.012635543942451477, 0.008434418588876724, 0.01686883717775345, 0.004690077155828476, 0.0019397232681512833, -0.005690498277544975, -0.00797763466835022, 0.01039665937423706, 0.02944004535675049, -0.03070102632045746, 0.01559498906135559, 0.0013293377123773098, 0.015311911702156067, -0.01868310570716858, -0.03458690643310547, -0.024267449975013733, -0.005626162514090538, -0.01690743863582611, -0.010563932359218597, 0.025219619274139404, 0.023186609148979187, -0.017190515995025635, -0.0024318918585777283, 0.0034644808620214462, 0.01241680234670639, -0.014076665043830872, -0.013471908867359161, 0.03463837504386902, -0.008183509111404419, 0.019802547991275787, -0.015118904411792755, 9.593819413566962e-06, 0.024524793028831482, -0.005571477115154266, 0.002084478735923767, -0.016920305788517, 0.009656798094511032, -0.013459041714668274, 0.01737065613269806, 0.014475546777248383, 0.029954731464385986, -0.018207021057605743, -0.03221935033798218, -0.00880756601691246, -0.035513341426849365, 0.043182164430618286, 0.012178760021924973, -0.03705739974975586, -0.012481138110160828, 0.019622407853603363, 0.002528395503759384, 0.0016453871503472328, -0.018863245844841003, 0.0014604218304157257, 0.020535975694656372, 0.01942940056324005, -0.0034869983792304993, -0.01751219481229782, -0.00972113385796547, -0.014218203723430634, 0.00913567841053009, 0.012635543942451477, -0.026918083429336548, -0.009386587888002396, -0.015015967190265656, -0.009425189346075058, -0.0019847583025693893, -0.0030093053355813026, -0.0331200510263443, -0.02913123369216919, 0.016367018222808838, 0.01695890724658966, -0.012629110366106033, -0.008363649249076843, 0.00683889165520668, -0.01680450141429901, 0.017949678003787994, -0.022440314292907715, 0.0029160184785723686, 0.008009802550077438, -0.0008765747770667076, -0.0080484040081501, 0.021282270550727844, -0.026866614818572998, 0.0046772100031375885, -0.03376340866088867, -0.018438629806041718, 0.019776813685894012, -0.013169530779123306, 0.016482822597026825, 0.007057633250951767, 0.019223526120185852, -0.039862439036369324, 0.006864625960588455, -0.0034290961921215057, -0.03515306115150452, 0.006620150059461594, -0.015878066420555115, -0.007617354393005371, 0.011458199471235275, -0.009373720735311508, 0.005619728937745094, 0.022285908460617065, -0.006780989468097687, 0.031627461314201355, -0.013587713241577148, -0.009418755769729614, 0.0003180599305778742, -0.008003368973731995, 0.025168150663375854, 0.021642550826072693, -0.020471639931201935, 0.021938495337963104, 0.028230533003807068, -0.020175695419311523, -0.00015319953672587872, 0.025271087884902954, -0.005288399755954742, 0.018284223973751068, 0.017679467797279358, 0.011014282703399658, 0.04174104332923889, 0.0020748283714056015, -0.009090643376111984, -0.013060159981250763, 0.020574577152729034, 0.021925628185272217, 0.0025702137500047684, 0.013819321990013123, -0.02125653624534607, 0.015286177396774292, -0.010814841836690903, -0.008871901780366898, -0.01529904454946518, -0.010917779058218002, -0.034200891852378845, 0.0038569290190935135, 0.011548269540071487, -0.002246926538646221, -0.012236662209033966, 0.00716700404882431, -0.011136520653963089, -0.008215676993131638, 0.0015287785790860653, 0.011110786348581314, -0.0017451075837016106, -0.014604218304157257, 0.012506872415542603, -0.01236533373594284, 0.01509317010641098, -0.007881131023168564, 5.518802208825946e-05, -0.021166466176509857, 0.019879750907421112, -0.0008114348165690899, -0.008119173347949982, -0.007128402590751648, 0.022942133247852325, 0.01564645767211914, -0.011921416968107224, 0.0046997275203466415, -0.015710793435573578, -0.014153867959976196, 0.01600673794746399, -0.015440583229064941, -0.0028323819860816, -0.017833873629570007, 0.0013687433674931526, -0.004712594673037529, 0.0026152487844228745, -0.03790663182735443, 0.028796687722206116, -0.010029945522546768, -0.013266034424304962, -0.0033004246652126312, 0.22378551959991455, -0.011175122112035751, -0.024113044142723083, 0.05877715349197388, 0.016225479543209076, -0.0046997275203466415, 0.033248722553253174, 0.01716478168964386, -0.001828744076192379, -0.005153294652700424, 0.01093064621090889, -0.025772906839847565, -0.03867866098880768, -0.017177648842334747, 0.017782405018806458, -0.010364491492509842, -0.017589397728443146, -0.021475277841091156, 0.014861561357975006, -0.009109944105148315, -0.028282001614570618, -0.031370118260383606, -0.0074436478316783905, 0.012159459292888641, 0.004345880821347237, 0.019249260425567627, 0.00842798501253128, -0.004622524604201317, 0.043439507484436035, 0.012429669499397278, 0.005153294652700424, -0.003186228685081005, 0.0008653160184621811, 0.021874159574508667, -0.008402250707149506, 0.012281697243452072, 0.00837651640176773, -0.01604533940553665, -0.009142111986875534, 0.013845056295394897, 0.0018432196229696274, 0.023148007690906525, -0.04814888536930084, -0.0075594522058963776, 0.0014234287664294243, 0.025052346289157867, -0.0059188902378082275, 0.005603644996881485, -0.02180982381105423, 0.007212039083242416, -0.016174010932445526, -0.029465779662132263, 0.00936085358262062, -0.008447285741567612, 0.008382949978113174, -0.016624361276626587, -0.0020297933369874954, 0.003121892921626568, 0.005330218002200127, 0.002674759365618229, -0.021925628185272217, 0.022633321583271027, -0.006282387301325798, 0.0327083021402359, -0.04053153097629547, 0.01994408667087555, -0.00880756601691246, 0.008337914943695068, -0.012410368770360947, 0.013420440256595612, 0.015286177396774292, 0.005056791007518768, -0.01459135115146637, 0.01529904454946518, -0.0132274329662323, -0.021616816520690918, -0.004477769136428833, 0.031781867146492004, 0.026532068848609924, 0.026840880513191223, 0.0025026611983776093, 0.00623413547873497, -0.011831346899271011, 0.015685059130191803, -0.0051468610763549805, -0.016984641551971436, 0.010287288576364517, 0.0073278434574604034, 0.02735556662082672, 0.01812981814146042, 0.0010526939295232296, 0.015530653297901154, -0.0014105616137385368, -0.0029417527839541435, -0.00661371648311615, -0.025168150663375854, -0.014964498579502106, 0.009959176182746887, 0.004632174968719482, 0.008119173347949982, -0.016341283917427063, 0.021848425269126892, -0.004339447245001793, -0.02498801052570343, -0.0022115418687462807, 0.006639450788497925, -0.007385745644569397, 0.016315549612045288, -0.009373720735311508, -0.000462815398350358, -0.0006409450434148312, -0.023392483592033386, 0.01806548237800598, -0.003725040704011917, 0.0017981845885515213, 0.02473066747188568, 0.009309384971857071, -0.03296564519405365, 0.012609809637069702, -0.02483360469341278, 0.02060031145811081, -0.02003415673971176, 0.010467428714036942, 0.010859876871109009, -0.011368129402399063, 0.00016033678548410535, -0.043182164430618286, 0.021024927496910095, -0.014835827052593231, -0.03716033697128296, -0.007540151476860046, -0.013845056295394897, -0.01443694531917572, 0.017434991896152496, -0.0006329030729830265, 0.009476657956838608, -0.012481138110160828, 0.006137631833553314, 0.019506603479385376, -0.01737065613269806, 0.006404625251889229, -0.01196645200252533, 0.0008701412007212639, -0.030366480350494385, 0.0004234097432345152, -0.035255998373031616, -0.0073278434574604034, -0.005156511440873146, -0.00045597972348332405, -0.012667711824178696, -0.027123957872390747, 0.010589666664600372, -0.01782100647687912, -0.022697657346725464, 0.011522535234689713, -0.006173016503453255, -0.01372925192117691, -0.008852601051330566, 0.0008074138313531876, 0.017126180231571198, -0.018940448760986328, 0.004188258200883865, 0.033892080187797546, -0.006916094571352005, 0.009367287158966064, -0.019673876464366913, -0.1595526933670044, 0.01737065613269806, 0.015865199267864227, -0.012989390641450882, 0.024576261639595032, 0.013130929321050644, 0.01459135115146637, -0.004641825333237648, 0.02331528067588806, 0.018760308623313904, 0.003329375758767128, -0.008189942687749863, -0.039141878485679626, -0.035719215869903564, 0.0066008493304252625, 0.015260443091392517, -0.019004784524440765, 0.011606171727180481, 0.04557545483112335, 0.005040707066655159, 0.011574003845453262, 0.024820737540721893, 0.01726771891117096, -0.0025637801736593246, -0.0038118939846754074, -0.010840576142072678, -0.0033454596996307373, 0.013561978936195374, -0.011715542525053024, 0.012944355607032776, -0.012198060750961304, 0.0029192352667450905, -0.014411211013793945, 0.007282808423042297, 0.006311338394880295, -0.04560118913650513, 0.013356104493141174, -0.009939875453710556, 0.0010390225797891617, 0.02479500323534012, 0.014372609555721283, 0.005970358848571777, -0.013240300118923187, 0.005738750100135803, 0.007643088698387146, 0.03355753421783447, 0.005970358848571777, -0.019558072090148926, -0.0011918200179934502, 0.006871059536933899, 0.02125653624534607, -0.019017651677131653, 0.008826866745948792, 0.02392003685235977, -0.009425189346075058, -0.011117219924926758, -0.00863385945558548, -0.002603990025818348, -0.00903274118900299, -0.029002562165260315, 0.023456819355487823, -0.012712746858596802, -0.0010784282349050045, -0.0010872744023799896, 0.001417799387127161, -0.01574939489364624, 0.002697276882827282, 0.008653160184621811, -0.007102668285369873, 0.010126449167728424, 0.0031942706555128098, -0.028256267309188843, 0.007919732481241226, -0.01802688091993332, 0.02473066747188568, 0.022298775613307953, 0.010460995137691498, -0.021012060344219208, 0.018554434180259705, 0.0028725918382406235, -0.008923370391130447, 0.023006469011306763, -0.022800594568252563, 0.02488507330417633, -0.01604533940553665, 0.0456269234418869, 0.010962814092636108, 0.015723660588264465, -0.04434020817279816, -0.009785469621419907, 0.013343237340450287, -0.0200984925031662, 0.003725040704011917, -0.007836095988750458, -0.004532454535365105, 0.0021713320165872574, 0.0011524143628776073, -0.004847699776291847, 0.012178760021924973, -0.000348820467479527, 0.017550796270370483, 0.010370925068855286, -0.03118997812271118, 0.023212343454360962, 0.03458690643310547, 0.0037346910685300827, 0.014115266501903534, 0.0014998274855315685, -0.0010937079787254333, -0.00716700404882431, -0.003538466989994049, -0.006401408463716507, 0.004899168387055397, 0.013060159981250763, -0.039708033204078674, 0.03615669906139374, -0.0027326615527272224, -0.0005774134770035744, 0.0035095158964395523, -0.004226859658956528, 0.011091485619544983, -0.01042882725596428, -0.04104621708393097, 0.011728409677743912, -0.02766437828540802, -0.025502696633338928, -0.11508381366729736, -0.00759805366396904, 0.0025155283510684967, 0.0196867436170578, 0.014128133654594421, 0.008852601051330566, 0.00217615719884634, 0.008331481367349625, -0.01098211482167244, 0.04266747832298279, -0.0015778345987200737, -0.019763946533203125, -0.01001707836985588, 0.0036414042115211487, 0.021230801939964294, -0.0006345114670693874, 0.020677514374256134, -0.0021504228934645653, -0.05728456377983093, 0.022028565406799316, -0.02180982381105423, -0.015401981770992279, -0.009154979139566422, -0.004175391048192978, -0.01535051316022873, -0.02377849817276001, -0.01231386512517929, 0.01181204617023468, 0.007836095988750458, 0.021282270550727844, 0.0041142720729112625, 0.01868310570716858, -0.008820433169603348, -0.005050357431173325, 0.0006839695852249861, 0.01569792628288269, 0.004307279363274574, -0.014449812471866608, -0.002012101002037525, -0.03118997812271118, -0.008222110569477081, 0.0035738516598939896, -0.010197218507528305, -0.021925628185272217, 0.014025196433067322, -0.008550222963094711, -0.019172057509422302, 0.030160605907440186, -0.02150101214647293, -0.007083367556333542, -0.015968136489391327, 0.02070324867963791, -0.05527728796005249, 0.004651475697755814, 0.02802465856075287, 0.00863385945558548, -0.00038521038368344307, 0.011194422841072083, -0.003776509314775467, -0.00502462312579155, 0.007006164640188217, 0.00015028432244434953, -0.026866614818572998, 0.03044368326663971, 0.005108259618282318, -0.012976523488759995, -0.011290926486253738, 0.00516294501721859, -0.012905754148960114, -0.015131771564483643, -0.02014996111392975, 0.014835827052593231, 0.008813999593257904, 0.0388588011264801, -0.02110213041305542, 0.005713015794754028, -0.001067169476300478, -0.023649826645851135, -0.0056840647011995316, 0.0004688468761742115, -0.005320567637681961, 0.0008074138313531876, -0.017048977315425873, -0.01933933049440384, 0.008781831711530685, 0.0027825217694044113, 0.012867152690887451, 0.009553860872983932, 0.0034097954630851746, -0.032836973667144775, -0.0032007042318582535, 0.05316707491874695, -0.006218051537871361, 0.00999777764081955, 0.011509668081998825, 0.039913907647132874, -0.036568447947502136, -0.014231070876121521, 0.03206494450569153, 0.01942940056324005, -0.01251973956823349, -0.03605376183986664, -0.02987752854824066, 0.025013744831085205, -0.0033937115222215652, -0.02464059740304947, 0.00019009207608178258, 0.008833300322294235, -0.004172174260020256, -0.00032107566948980093, 0.013356104493141174, 0.012198060750961304, -0.010390225797891617, 0.020407304167747498, -0.03821544349193573, -0.034355297684669495, -0.00183035247027874, 0.006359590217471123, 0.008325047791004181, -0.01504170149564743, 0.00999777764081955, -0.004297628998756409, -0.0116383396089077, 0.022594720125198364, -0.0022115418687462807, 0.024113044142723083, -0.019879750907421112, 0.0068581923842430115, -0.01933933049440384, 0.011014282703399658, -0.027123957872390747, 0.005429938435554504, 0.034355297684669495, -0.006710220128297806, -0.02403584122657776, 0.007385745644569397, -0.0082542784512043, -0.015067435801029205, -0.019570939242839813, 0.013459041714668274, 0.002044268883764744, -0.02256898581981659, -0.046012938022613525, -0.004828399047255516, 0.013169530779123306, -0.017151914536952972, -0.0022147586569190025, -0.0017644083127379417, -0.014874428510665894, 0.005159728229045868, 0.0038987472653388977, 0.013317503035068512, 0.035976558923721313, 0.012616243213415146, 0.0007133227773010731, -0.03191053867340088, -0.017627999186515808, -0.0230708047747612, 0.008138474076986313, 0.002719794400036335, -0.007025465369224548, -0.025013744831085205, 0.0018448280170559883, 0.014372609555721283, -0.0025219619274139404, -0.008054837584495544, -0.0024254582822322845, -0.014629952609539032, 0.015556387603282928, -0.022131502628326416, -0.015504918992519379, -0.014346875250339508, 0.017203383147716522, 0.02438325434923172, -0.009380154311656952, 0.021166466176509857, 0.029620185494422913, 0.008595257997512817, -0.026635006070137024, 0.011027149856090546, -0.0033679772168397903, 0.007623787969350815, 0.002668325789272785, 0.0027021020650863647, -0.0020973458886146545, 0.017911076545715332, 0.035719215869903564, 0.0199955552816391, -0.01251973956823349, -0.005889939144253731, -0.010994981974363327, 0.0014033238403499126, 0.0021391641348600388, -0.00809987261891365, -0.004133572801947594, -0.006710220128297806, 0.0077717602252960205, 0.03746914863586426, -0.005124343559145927, 0.002623290754854679, -0.024344652891159058, 0.03710886836051941, 0.010544631630182266, -0.006144065409898758, 0.007893998175859451, 0.006533296778798103, -0.03963083028793335, 0.01932646334171295, -0.024344652891159058, -0.04001684486865997, -0.02640339732170105, -0.0114324651658535, 0.008659593760967255, -0.012719180434942245, -0.008074138313531876, 0.0026409830898046494, 0.0031106341630220413, 0.017100445926189423, -0.016289815306663513, -0.022453181445598602, -0.01443694531917572, 0.031473055481910706, 0.006163366138935089, 0.014346875250339508, 0.02464059740304947, -0.020561710000038147, 0.030520886182785034, -0.009920574724674225, 0.014115266501903534, 0.008292879909276962, 0.01863163709640503, -0.008119173347949982, -0.00571623258292675, -0.0031733615323901176, -0.031112775206565857, -0.007424347102642059, 0.018451496958732605, -0.019879750907421112, 0.028204798698425293, 0.021089263260364532, 0.008106306195259094, 0.04629601538181305, 0.0011781486682593822, -0.027226895093917847, 0.04395419359207153, -0.00309133343398571, 0.01676589995622635, 0.006607282906770706, 0.010203652083873749, 0.008087005466222763, 0.00908420979976654, 0.012577641755342484, -0.01383218914270401, 0.0008596866391599178, -0.03214214742183685, -0.015865199267864227, -0.01191498339176178, 0.01070547103881836, -0.00288385059684515, -0.004088537767529488, -0.020381569862365723, 0.02554129809141159, 0.013793587684631348, -0.016431353986263275, -0.01680450141429901, -0.016598626971244812, -0.03368620574474335, 0.04650188982486725, -0.007520850747823715, -0.013973727822303772, -0.02745850384235382, -0.0019381148740649223, 0.016688697040081024, -0.004287978634238243, 0.0030350396409630775, 0.0005219238810241222, -0.018914714455604553, -0.017859607934951782, -0.01010071486234665, -0.016135409474372864, 0.00699329748749733, -0.016984641551971436, 0.051417142152786255, -0.028101861476898193, -0.05211196839809418, -0.02590157836675644, -0.007784627377986908, -0.0013494426384568214, 0.016598626971244812, -0.00951525941491127]} +{"id": "test:10000061", "text": "\"Sellers who are non-residents (eg. foreigners living abroad and Japanese citizens living abroad).\\nBuyers, both local and overseas residents, who are buying real estate in Japan from a non-resident.\\nIf the seller of Japanese real estate is a non-resident, depending on the situation, the buyer must withhold 10.21% of the sale price and pay it to the tax office, with the remaining 89.79% paid to the seller. The buyer is responsible for paying the 10.21% to the tax office by the 10th of the month following the transaction.\\nIt is your obligation to make this payment to the tax office by the deadline. If you are currently living overseas, you will need to appoint a tax representative or accountant to pay this on your behalf. You will need to appoint a tax representative in Japan to file a final tax return to have the remainder returned to you (less any taxes that may have been owed).\\n*If the payment of deposits and mid-term payments also meet the conditions for withholding tax, the tax must be withheld and paid by the buyer at each payment.\\nThe above information has been provided as a general guide only. For more details on this tax, and for other detailed tax questions, please consult with the tax office or a specialist tax accountant.\"", "vector_field": [-0.0013408978702500463, -0.0007408297969959676, 0.0076287975534796715, -0.012745038606226444, 7.577130600111559e-05, -0.009646653197705746, -0.04382001981139183, -0.017796188592910767, 0.008136516436934471, -0.024357475340366364, 0.012400050647556782, 0.009835421107709408, -0.013643310405313969, 0.010746710002422333, -0.0016191666945815086, -0.015335705131292343, 0.02299053966999054, -0.02273017168045044, 0.009210536256432533, -0.031764958053827286, -0.020712316036224365, 0.014424415305256844, -0.029942378401756287, 0.007453048601746559, -0.016858860850334167, 0.014684784226119518, 0.027885468676686287, -0.02525574527680874, 0.0007367615471594036, -0.0027354967314749956, -0.00515529653057456, -0.0036744505632668734, -0.010518888011574745, -0.016481326892971992, -0.029265420511364937, 0.004178914707154036, 0.014450452290475368, -0.015192503109574318, 0.007498613093048334, -0.013929715380072594, 0.033613573759794235, -0.006187006831169128, -0.007921712473034859, 0.017080174759030342, 0.016585474833846092, 0.01218524668365717, -0.012321939691901207, -0.028640536591410637, -0.031009890139102936, 0.00579319940879941, 0.001143994159065187, 0.028510352596640587, -0.014580637216567993, 0.0074270120821893215, -0.026180054992437363, -0.01109169889241457, -0.016559436917304993, 0.027781320735812187, -0.014580637216567993, -0.022534895688295364, -0.005819236394017935, -0.006763072218745947, -0.02023063413798809, -0.006470157764852047, -0.004074767231941223, 0.0005060912808403373, -0.030723484233021736, -0.0010235736845061183, -0.029473716393113136, 0.005838763900101185, 0.02098570205271244, 0.010395213030278683, 0.02189699187874794, 0.0020764388609677553, 0.030124636366963387, -0.018577294424176216, 0.003830671776086092, -0.005145533010363579, 0.0016329988138750196, 0.005952675361186266, 0.002377490047365427, -0.009783347137272358, -0.0061154053546488285, -0.013174646534025669, 0.024643879383802414, 0.012823149561882019, -0.022534895688295364, 0.02327694557607174, 0.007479085586965084, 0.0003396995598450303, 0.014059899374842644, 0.014827987179160118, -0.010154372081160545, 0.01231543067842722, -0.0009104761411435902, 0.01898086443543434, -0.014072918333113194, 0.026843992993235588, -0.007466067094355822, -0.03796172887086868, 0.026843992993235588, 0.015999644994735718, -0.04150274023413658, -0.014124992303550243, 0.0019006901420652866, 0.013617273420095444, 0.006414829287678003, -0.019618768244981766, -0.0032480971422046423, -0.02043892815709114, -0.012152700684964657, 0.036035001277923584, 0.017952408641576767, -0.029213346540927887, 0.0022456783335655928, 0.012178737670183182, 0.010252010077238083, -0.005484011955559254, -0.00469965161755681, 0.007947748526930809, -0.008162552490830421, 0.006284644827246666, -0.004566212650388479, 0.003440119093284011, 0.007075514178723097, 0.002823370974510908, 0.00010241839481750503, -0.01952763833105564, -0.036295369267463684, -0.015400798059999943, -0.0076287975534796715, 0.01968386024236679, 0.028848830610513687, 0.005617450922727585, 0.00011299586913082749, 0.006261862814426422, -0.012601836584508419, -0.005386373493820429, -0.022560931742191315, -0.04395020380616188, 0.03796172887086868, 0.0007501867949031293, -0.008455467410385609, -0.015557019039988518, 0.022456783801317215, 0.014984208159148693, 0.012484670616686344, 0.001229427638463676, 0.030879706144332886, 0.0039999112486839294, 0.020191578194499016, 0.0017184321768581867, 0.030176710337400436, -0.022899411618709564, 0.01081831194460392, 0.005435192957520485, 0.017080174759030342, -0.005968948360532522, -0.0039022730197757483, -0.006369264796376228, 0.008709326386451721, 0.011840257793664932, 0.011378103867173195, 0.012126663699746132, -0.009470905177295208, 0.01916312240064144, 0.005542594939470291, 0.029682010412216187, -0.016780750826001167, -4.3066324906249065e-06, 0.02392786741256714, -0.007485594600439072, -0.035644449293613434, 0.014085936360061169, 0.016377178952097893, -0.013155119493603706, -0.030332932248711586, 0.0022228960879147053, -0.005324536003172398, -0.015010245144367218, 0.007400975096970797, -0.002260324079543352, -0.005379864480346441, 0.029031088575720787, -0.0054221744649112225, -0.008585651405155659, 0.011436686851084232, 0.007413993589580059, -0.003139067906886339, -0.0028461532201617956, 0.010518888011574745, 0.031947217881679535, 0.028536388650536537, 0.00021460061543621123, -0.6377987265586853, -0.031504590064287186, 0.008715836331248283, -0.019228214398026466, 0.02150643989443779, -0.00011492829071357846, 0.008312264457345009, 0.02351127751171589, -0.013116063550114632, 0.011026606895029545, -0.016715658828616142, 0.007199189625680447, -0.008110479451715946, -0.021363236010074615, -0.0023726080544292927, -0.020191578194499016, -0.00520737050101161, -0.014619692228734493, -0.02424030937254429, 0.012959842570126057, 0.011462723836302757, 0.02377164550125599, -0.029499752447009087, 0.02299053966999054, 0.0005667083314619958, -0.023315999656915665, 0.008455467410385609, -0.002097593853250146, -0.004484847653657198, 0.03968016058206558, -0.028979016467928886, 0.03509767726063728, -0.008514050394296646, 0.007134097162634134, 0.031712885946035385, 0.003586576320230961, -0.004055239725857973, 0.018967846408486366, 0.009939568117260933, 0.03793569281697273, 0.0024002722930163145, -0.026687772944569588, 0.023836737498641014, -0.007459558080881834, 0.005728107411414385, -0.01986611820757389, 0.007706908043473959, -0.017431672662496567, 0.033952053636312485, -0.0038567085284739733, -0.026661735028028488, 0.014762895181775093, -0.022118305787444115, -0.016038700938224792, 0.028302056714892387, -0.006287899799644947, -0.005565376952290535, 0.014320268295705318, 0.005031621549278498, 0.017705058678984642, -0.010369176045060158, -0.008325283415615559, 0.01167752780020237, -0.011280465871095657, -0.028588462620973587, -0.00972476415336132, -0.016611510887742043, 0.014072918333113194, 0.01235448569059372, -0.012152700684964657, -0.02028270810842514, 0.026817956939339638, -0.018759552389383316, 0.00032485040719620883, 0.01003720611333847, 0.030879706144332886, 0.00011228392395423725, 0.004367681685835123, -0.006483175791800022, 0.01308351755142212, -0.013630291447043419, -0.004309098701924086, -0.02194906584918499, -0.023263927549123764, -0.007127588149160147, -0.01900690235197544, 0.01754883863031864, -0.016819806769490242, 0.029473716393113136, -0.018069574609398842, -0.028354130685329437, 0.030306894332170486, 0.01900690235197544, -0.018186740577220917, -0.016051718965172768, 0.007882656529545784, -0.005457974970340729, -0.008090951479971409, 0.000547180708963424, -0.011801202781498432, -0.035436153411865234, -0.0031081491615623236, 0.020477984100580215, 0.006307427305728197, 0.003729779040440917, 0.0015329195884987712, -0.04098200425505638, 0.0025727665051817894, 0.04111218824982643, -0.026479477062821388, 0.023628443479537964, -0.020686278119683266, 0.0038697270210832357, 0.008780928328633308, -0.002240796573460102, -0.012784094549715519, 0.024409547448158264, 0.013207192532718182, 0.027859430760145187, -0.000549214833881706, -0.015452871099114418, 0.029343532398343086, 0.034342605620622635, -0.029708048328757286, -0.006606851238757372, 0.029421642422676086, 0.027625098824501038, -0.010590489022433758, 0.005047894548624754, 0.006600341759622097, -0.004172405228018761, 0.014059899374842644, 0.02069929614663124, -0.025320837274193764, 0.0009535996941849589, -0.00502511253580451, -0.006323700305074453, 0.005698815919458866, 0.0036711960565298796, -0.028458278626203537, -0.024956321343779564, -0.005614195950329304, 0.015413816086947918, -0.016233976930379868, 0.011605926789343357, -0.025607243180274963, -0.025424985215067863, -0.005513303447514772, -0.019202178344130516, 0.024149179458618164, -0.003178123151883483, -0.004367681685835123, -0.005565376952290535, 0.011150281876325607, -0.018447108566761017, -0.009640144184231758, -0.0024116632994264364, -0.017340542748570442, -0.017262432724237442, 0.004559703636914492, 0.004533666651695967, 0.014268194325268269, -0.020451946184039116, -0.0009926549391821027, -0.01887671649456024, -0.009041296318173409, 0.017705058678984642, 0.028692610561847687, 0.00615771533921361, -0.036399517208337784, -0.006750053726136684, 0.012302412651479244, -0.015544000081717968, 0.009184499271214008, 0.01754883863031864, -0.005815981887280941, 0.02106381207704544, 0.007270790636539459, 0.00468012411147356, -0.028614500537514687, -0.006274881307035685, -0.013630291447043419, -0.01809561252593994, 2.317000053153606e-06, 0.010616526007652283, 0.00237586279399693, 0.006945330183953047, 0.009425340220332146, -0.0041105677373707294, 0.021050794050097466, -0.02059515006840229, 0.04894928261637688, 0.01182073075324297, 0.02179284393787384, 0.0027273602318018675, -0.0002312804717803374, 0.03861265257000923, 0.010577470995485783, -0.00279570696875453, 0.014932134188711643, 0.007134097162634134, 0.02369353547692299, -0.002296124817803502, -0.01887671649456024, 0.020634204149246216, -0.00645062979310751, 0.003785107284784317, -0.03671196103096008, 0.021779825910925865, 0.02411012537777424, 0.011527815833687782, -0.031999289989471436, -0.004162641707807779, 0.0032350788824260235, -0.008403393439948559, 0.02145436592400074, -0.011521306820213795, 0.007837092503905296, 0.003622377058491111, -0.012979370541870594, -0.004286316689103842, -0.033587537705898285, 0.028041688725352287, -0.010245501063764095, 0.00952948722988367, 0.013024934567511082, -0.004436028655618429, -0.008377357386052608, -0.006867219693958759, -0.018395036458969116, -0.027677172794938087, -0.027494914829730988, 0.011755638755857944, 0.004995821043848991, 0.033561501652002335, -0.012334958650171757, 0.030775558203458786, -0.005747634917497635, -0.006408320274204016, -0.004191933199763298, 0.014606673270463943, 0.011410649865865707, 0.008403393439948559, -0.023107705637812614, 0.030358968302607536, 0.032415881752967834, 0.030358968302607536, 0.015661166980862617, -0.02611496113240719, 0.034785233438014984, 0.007212207652628422, 0.0034921925980597734, -0.02163662388920784, 0.014255176298320293, 0.017822224646806717, -0.022118305787444115, -0.0038078895304352045, 0.01887671649456024, 0.029239384457468987, 0.027989614754915237, 0.027651136741042137, 0.013656328432261944, 0.026713808998465538, -0.02148040197789669, 0.028952978551387787, 0.018316924571990967, 0.01067510899156332, -0.013617273420095444, 0.030228784307837486, -0.011631963774561882, -0.019696878269314766, -0.013356904499232769, -0.0011407395359128714, 0.004139859229326248, 0.02309468761086464, 0.00261344900354743, -0.029760120436549187, 0.003241587895900011, 0.011976951733231544, 0.008839511312544346, -0.033873945474624634, -0.04363776370882988, 0.029083162546157837, 0.026453441008925438, 0.005685797426849604, -0.001207459019497037, 0.0077589815482497215, -0.005122750531882048, 0.008266700431704521, 0.02317279763519764, 0.008553105406463146, 0.014216121286153793, 0.013369923457503319, -0.017210358753800392, -0.009340720251202583, -0.007967276498675346, 0.014424415305256844, 0.00581272691488266, 0.008025859482586384, -0.014736858196556568, 0.013656328432261944, -0.02215735986828804, -0.012868713587522507, 0.004182169213891029, 0.031140074133872986, 0.01277107559144497, 0.006802127230912447, 0.004374191164970398, -0.013395960442721844, 0.0006932311807759106, -0.027234546840190887, 0.0018600075272843242, -0.033899981528520584, 0.013005407527089119, -0.00044344013440422714, -0.0021106123458594084, -0.022118305787444115, -0.019566694274544716, 0.027911504730582237, -0.027416804805397987, -0.002742005977779627, -0.001419822103343904, -0.04168500006198883, -0.007108060643076897, 0.08857736736536026, -0.017262432724237442, -0.003178123151883483, 0.016481326892971992, -0.014138010330498219, -0.01067510899156332, -0.017848262563347816, -0.008071423508226871, 0.013968770392239094, 0.016663584858179092, -0.0028347622137516737, 0.017887316644191742, -0.020217616111040115, 0.016416234895586967, 0.015101374126970768, -0.004549939651042223, 0.002514183521270752, -0.031764958053827286, -0.007322864606976509, -0.025008395314216614, -0.008924131281673908, 0.004370936658233404, 0.007843601517379284, 0.032077401876449585, 0.003095130668953061, 0.00481030810624361, 0.03947186842560768, 0.035774633288383484, 0.01791335456073284, -0.024253327399492264, 0.014437434263527393, -0.006349737290292978, -0.013487089425325394, 0.022795263677835464, 0.0012676692567765713, 0.010186918079853058, 0.006580814253538847, 0.007466067094355822, 0.0017623694147914648, -0.002012974116951227, -0.003586576320230961, 0.027442840859293938, 0.005402646493166685, -0.024331437423825264, 0.015882479026913643, -0.028588462620973587, 0.003606103826314211, 0.014697802253067493, 0.02530781924724579, 0.009972114115953445, 0.017119230702519417, -0.015101374126970768, -0.03645159304141998, -0.0014938643435016274, -0.025086507201194763, 0.01944952830672264, 0.02098570205271244, -0.007837092503905296, 0.03569652512669563, -0.0006399370031431317, -0.018395036458969116, -0.014098955318331718, -0.007993313483893871, -0.0007583232945762575, -0.015765313059091568, -0.016676602885127068, -0.007980294525623322, 0.014867042191326618, 0.0009544133208692074, -0.022639041766524315, 0.04574674740433693, -0.031374406069517136, -0.03908131271600723, 0.0014596909750252962, 0.02007441222667694, -0.0022538150660693645, 0.014984208159148693, -0.00011665729834930971, 0.008527068421244621, 0.012523725628852844, -0.025333857163786888, -0.008403393439948559, -0.016077755019068718, -0.02218339778482914, -0.01931934431195259, 0.023576369509100914, 0.013011916540563107, -0.0073358830995857716, -0.02379768155515194, 0.018264850601553917, -0.0023986450396478176, -0.003941328264772892, 0.012178737670183182, 0.008754891343414783, -0.008071423508226871, 0.009275628253817558, -0.010486342012882233, 0.029994452372193336, 0.003830671776086092, 0.008364338427782059, -0.012374013662338257, -0.0011407395359128714, -0.02184491790831089, -0.018342962488532066, -0.006154460832476616, 0.0013229975011199713, 0.008826492354273796, -0.02124607004225254, 0.011573380790650845, 0.010199937038123608, 0.026088925078511238, -0.02491726726293564, 0.004940492566674948, 0.005981966853141785, -0.001970664132386446, 0.008800455369055271, 0.024175217375159264, 0.014749876223504543, 0.021285125985741615, 0.027807356789708138, 0.014307250268757343, 0.0046280501410365105, 0.031140074133872986, 0.017093192785978317, -0.010486342012882233, -0.0032627428881824017, 0.015114392153918743, -0.004875400569289923, 0.016715658828616142, -0.019566694274544716, 0.024839157238602638, 0.015921534970402718, -0.008969695307314396, -0.022430747747421265, -0.018707478418946266, 0.017106210812926292, -0.006297663319855928, -0.003404318355023861, -0.025776483118534088, -0.007485594600439072, 0.0038078895304352045, 0.002287988318130374, 0.013096536509692669, -0.015778331086039543, 0.013200683519244194, -0.02239169180393219, 0.0015792978229001164, 0.007915202528238297, 0.0027908249758183956, 0.033092837780714035, 0.009796365164220333, -0.004774507600814104, -0.005415664985775948, -0.01767902262508869, 0.003632140811532736, -0.031973253935575485, -0.017314506694674492, -0.04806402698159218, 0.004540176130831242, 0.02286035567522049, -0.008364338427782059, 0.01913708634674549, -0.0019820553716272116, 0.0019381181336939335, 0.007908693514764309, 0.003095130668953061, 0.013356904499232769, 0.0059396568685770035, -0.008624707348644733, -0.010297575034201145, 0.007511631585657597, 0.04626748338341713, 0.0002916941011790186, 0.0030349204316735268, 0.0048135630786418915, 0.014007826335728168, -0.011605926789343357, 0.011814221739768982, -0.013435015454888344, -0.012048552744090557, 0.024175217375159264, 0.0005931520136073232, -0.02361542358994484, 0.00456946762278676, -0.024409547448158264, -0.02163662388920784, 0.01244561467319727, -0.01997026428580284, 0.029551826417446136, 0.00122535927221179, -0.0067695812322199345, -0.020686278119683266, 0.03960205242037773, 0.004227733705192804, -0.0015321059618145227, 0.017106210812926292, 0.031478554010391235, -0.014867042191326618, -0.000723743112757802, 0.0003856708644889295, 0.004585740622133017, 0.028744684532284737, 2.6062278266181238e-05, -0.00597545737400651, -0.02629721909761429, 0.012842676602303982, 0.002885208697989583, -0.015530982054769993, -0.010388704016804695, 0.010577470995485783, -0.018863698467612267, -0.028979016467928886, 0.027911504730582237, 0.007270790636539459, -0.015335705131292343, 0.01822579652070999, 0.015153447166085243, 0.017236394807696342, 0.016351142898201942, -0.031973253935575485, -0.010772746987640858, -0.027729246765375137, 0.027937540784478188, 0.01277107559144497, 0.014151028357446194, 0.012048552744090557, -0.006652415730059147, -0.01080529298633337, -0.016897916793823242, 0.01189884077757597, -0.005523066967725754, 0.008038877509534359, 0.015726258978247643, -0.010304084047675133, -0.04348154366016388, 0.018993882462382317, -0.00044384694774635136, -0.01076623797416687, -0.01005022507160902, 0.029786158353090286, -0.0016989045543596148, 0.0023449440486729145, -0.009640144184231758, 0.03728477284312248, -0.015661166980862617, 0.024852175265550613, 0.00022863609774503857, -0.021701715886592865, -0.013812549412250519, -0.02161058597266674, -0.01172960177063942, -0.007941239513456821, 0.0033001708798110485, 0.015465890057384968, 0.010147863067686558, 0.00019354737014509737, 0.011137262918055058, -0.027026250958442688, -0.015739277005195618, 0.03751910477876663, 0.004745216108858585, 0.017067156732082367, -0.00248651928268373, 0.025841575115919113, 0.021805863827466965, 0.005464483983814716, -0.032467953860759735, -0.02080344408750534, 0.016077755019068718, 0.02418823540210724, -0.0014637592248618603, -0.028458278626203537, 0.01113075390458107, 0.035514265298843384, -0.010811802931129932, 0.0029356549493968487, -0.0076938895508646965, -0.008038877509534359, -0.016442270949482918, -0.018759552389383316, 0.02090759202837944, -0.011013587936758995, -0.012113644741475582, 0.005620705429464579, -0.013929715380072594, -0.005679288413375616, -0.021180978044867516, 0.0011570125352591276, 0.014502526260912418, 0.030254822224378586, -0.015713239088654518, 0.0063464827835559845, -0.02351127751171589, 0.022352637723088264, -0.014932134188711643, 0.01136508584022522, -0.013474070467054844, 0.033873945474624634, -0.00926911924034357, 0.008728854358196259, 0.01970989629626274, 0.02075137011706829, -0.014854024164378643, -0.002606939757242799, -0.014632710255682468, -0.03660781309008598, 0.04408038780093193, -0.007876147516071796, -0.016403216868638992, -0.01877257041633129, 0.013421996496617794, -0.0037004875484853983, -0.02450067736208439, 0.011195845901966095, -0.01911104843020439, -0.007876147516071796, 0.005998239852488041, -0.015062318183481693, -0.030853668227791786, -0.007016931660473347, 0.007348901126533747, 0.014932134188711643, -0.0018079339060932398, -0.017041118815541267, -0.004449047148227692, -0.008051896467804909, 0.008429430425167084, -0.009666181169450283, -0.015973608940839767, -0.004009675234556198, 0.0010463559301570058, 0.0005048708408139646, -0.01913708634674549, -0.012029025703668594, -0.007160134147852659, 0.0018404799047857523, -0.00586480088531971, 0.032598137855529785, -0.022352637723088264, -0.005737870931625366, -0.006424593273550272, 0.03957601264119148, 0.014528563246130943, 0.0005569445202127099, -0.004035711754113436, 0.0028168619610369205, 0.003703742055222392, -0.009874476119875908, -0.034134313464164734, -0.030619338154792786, -0.017184322699904442, 0.031973253935575485, -0.014046881347894669, -0.014893079176545143, -0.022639041766524315, -0.016090774908661842, -0.04809006303548813, 0.004149623215198517, -0.008084442466497421, 0.030098600313067436, 0.023315999656915665, 0.006235825829207897, -0.005666269920766354, 0.02122003398835659, 0.01113075390458107, 0.031764958053827286, -0.04782969504594803, -0.01023248303681612, 0.0009690590668469667, 0.0067044892348349094, 0.009119407273828983, 0.0009723136899992824, -0.021337199956178665, 0.002108984859660268, 0.02543800324201584, 0.010486342012882233, 0.020035358145833015, 0.014476489275693893, -0.011007078923285007, -0.018368998542428017, 0.014333286322653294, 0.009444868192076683, -0.030228784307837486, 0.010206446051597595, 0.0009780091932043433, -0.033457353711128235, 0.019722914323210716, 0.008357829414308071, 0.0018551256507635117, -0.032598137855529785, 0.014645729213953018, 0.02189699187874794, -0.009900513105094433, 0.004855872597545385, 0.005672778934240341, -0.031061964109539986, 0.0023986450396478176, -0.01968386024236679, 0.026896066963672638, 0.0039055277593433857, -0.011046133935451508, 0.008800455369055271, -0.0053961374796926975, -0.030306894332170486, -0.012035534717142582, -0.02155851386487484, -0.015778331086039543, -0.005672778934240341, 0.00999815110117197, -0.014437434263527393, 0.0017737605376169086, -0.01008928008377552, -0.008943658322095871, -0.030749522149562836, 0.008149534463882446, -0.005545849446207285, 0.02072533406317234, -0.003230196889489889, 0.016858860850334167, 0.017184322699904442, -0.035175785422325134, 0.02348523959517479, 0.006043804343789816, -0.011020096950232983, 0.0003384790616109967, -0.015778331086039543, -0.003038175171241164, -0.01214619167149067, 0.021545493975281715, -0.009171481244266033, -0.013109554536640644, -0.008279718458652496, -0.006170733831822872, -0.04470527544617653, -0.009223554283380508, 0.00654175877571106, 0.18340358138084412, -0.00954250618815422, 0.025021415203809738, 0.028015652671456337, 0.001074833795428276, 0.015635129064321518, 0.031504590064287186, 0.00022334736422635615, -0.01140414085239172, -0.010434268042445183, -0.03983638435602188, 0.01968386024236679, -0.0007916830363683403, 0.010557943023741245, 0.002270088065415621, -0.03941979259252548, -0.020087430253624916, -0.04650181531906128, -0.0022538150660693645, 0.04587693139910698, 0.01008928008377552, 0.014190084300935268, -0.009705236181616783, -0.004891673568636179, -0.011488760821521282, -0.00626837182790041, -0.006873728707432747, 0.009978623129427433, 0.00045116982073523104, 0.005142278037965298, -0.005737870931625366, -0.009041296318173409, 0.008077933453023434, 0.009952586144208908, -0.028562426567077637, 0.012022515758872032, -0.0019299816340208054, 0.010199937038123608, 0.008481504395604134, 0.030020490288734436, -0.002270088065415621, 0.0021561768371611834, -0.02491726726293564, -0.007244754116982222, 0.006330209318548441, 0.015088355168700218, -0.0002984067250508815, -0.021623605862259865, -0.011840257793664932, -0.0009243082022294402, -0.04296080395579338, -0.036165185272693634, -0.0016956499312072992, 0.008201608434319496, 0.0027989614754915237, 0.026088925078511238, 0.028432242572307587, -0.015544000081717968, -0.03749306499958038, 0.017835242673754692, -0.016116810962557793, 0.023263927549123764, -0.02442256733775139, 0.032233621925115585, -0.011521306820213795, 0.011742619797587395, -0.013448033481836319, -0.0012782466365024447, 0.007212207652628422, -0.014033863320946693, 0.01859031245112419, 0.014515544287860394, -0.019280288368463516, -0.013838586397469044, 0.0032057873904705048, -0.012543253600597382, 0.017796188592910767, 0.005868055392056704, 0.02580251917243004, 0.031192148104310036, -0.010941986925899982, -0.002152922097593546, 0.0012985880021005869, 0.015947571024298668, 0.002877072198316455, -0.035149749368429184, 0.0021805863361805677, -0.005269207991659641, 0.024305401369929314, -0.015583056025207043, 0.01965782232582569, 0.005099968519061804, -0.01984008029103279, -0.019202178344130516, -0.0045824856497347355, 0.0027257329784333706, 0.01989215426146984, 0.011723092757165432, -0.04809006303548813, 0.011651490814983845, -0.003996656741946936, 0.04697047919034958, 0.017158284783363342, 0.002896599704399705, -0.00617724284529686, 0.0012546507641673088, 0.018837662413716316, 0.017223376780748367, 0.007322864606976509, 0.007413993589580059, 0.029109200462698936, -0.028588462620973587, -0.0012684828834608197, -0.015062318183481693, 0.009535997174680233, 0.0023384348023682833, -0.033144913613796234, -0.013851605355739594, 0.014932134188711643, -0.032077401876449585, 0.005536085460335016, -0.029317494481801987, 0.007453048601746559, 0.014827987179160118, -0.005871309898793697, -0.008624707348644733, -0.031686849892139435, -0.013929715380072594, 0.014463471248745918, -0.04116426408290863, 0.01244561467319727, -0.02429238334298134, 0.022665079683065414, -0.030957816168665886, -0.014932134188711643, 0.005415664985775948, 0.029395604506134987, -0.0028168619610369205, -0.0055783954448997974, 0.018681440502405167, -0.007511631585657597, -0.00036309202550910413, 0.007127588149160147, 0.005431937985122204, -0.008689799346029758, -0.023524295538663864, 0.04449697956442833, 0.005047894548624754, -0.025216691195964813, 0.004852618090808392, -0.031218184158205986, -0.027520952746272087, -0.013037953525781631, -0.017444690689444542, 0.04345550388097763, -0.005161806009709835, -0.031790994107723236, -0.035253897309303284, 0.034108273684978485, 0.010811802931129932, -0.04082578420639038, -0.019957246258854866, 0.032051365822553635, -0.012413068674504757, -0.02403201349079609, 0.016064736992120743, -0.16267824172973633, 0.016806786879897118, 0.0007758167921565473, -0.029812194406986237, 0.04225780814886093, 0.012705983594059944, -0.012745038606226444, -0.034342605620622635, 0.014567618258297443, 0.013194174505770206, 0.019540656358003616, 0.017809206619858742, -0.0016777495620772243, -0.02332901954650879, -0.016507362946867943, -0.011430177837610245, -0.029760120436549187, 0.02168869785964489, 0.04087785631418228, 0.01071416400372982, 0.03619122505187988, 0.004823326598852873, 0.01034964807331562, -0.026466459035873413, 0.007231735624372959, 0.02457878738641739, -0.017067156732082367, 0.014450452290475368, -0.01118282787501812, 0.006121914833784103, -0.008195099420845509, 0.007745963521301746, 0.016286050900816917, 0.01947556436061859, 0.01812164857983589, -0.019384436309337616, 0.008077933453023434, -0.027078324928879738, 0.0034824288450181484, 0.001403548987582326, -0.005698815919458866, 0.0005097527173347771, -0.030306894332170486, 0.00048168175271712244, -0.0022733425721526146, 0.001011368934996426, 0.009197518229484558, 0.0075637055560946465, -0.010844348929822445, -0.012634382583200932, 0.007706908043473959, -0.02400597743690014, -0.01090944092720747, 0.011397631838917732, 0.018212778493762016, 0.001806306536309421, 0.012575799599289894, 0.019592730328440666, -0.0075181410647928715, 0.003140695160254836, -0.019124068319797516, -0.007453048601746559, 0.032884541898965836, -0.007589742075651884, 0.00456946762278676, -0.015439853072166443, 0.02067326009273529, 0.007667852565646172, -0.008448958396911621, 0.001548378961160779, 0.0034889380913227797, -0.026362312957644463, 0.016416234895586967, 0.007368429098278284, -0.0018665167735889554, -0.020295726135373116, 0.00022944975353311747, -0.005718343425542116, 0.010616526007652283, 0.006860710214823484, -0.02343316562473774, 0.014294231310486794, -0.02072533406317234, -0.0005634537665173411, -0.011573380790650845, -0.012068080715835094, 0.02075137011706829, -0.013226720504462719, 0.004045475739985704, 0.006577559746801853, -0.01140414085239172, -0.014190084300935268, -0.016624528914690018, -0.0077134170569479465, 0.00469965161755681, 0.01937141828238964, 0.021311162039637566, 0.005412410479038954, -0.035201821476221085, 0.0010154371848329902, -0.011072170920670033, -0.014463471248745918, -0.011241410858929157, 0.024175217375159264, 0.004663851112127304, -0.005604432430118322, -0.015218540094792843, 0.02424030937254429, 0.008084442466497421, -0.01911104843020439, -0.028380168601870537, 0.008917621336877346, 0.031764958053827286, 0.011762147769331932, -0.005497030448168516, 0.04353361576795578, 0.004865636583417654, -0.019228214398026466, 0.00926911924034357, 0.014164047315716743, 0.028692610561847687, -0.025763465091586113, -0.0024735007900744677, 0.014619692228734493, -0.005161806009709835, -0.008533578366041183, -0.10800085961818695, -0.010134844109416008, 0.00466059660539031, 0.024018995463848114, 0.001211527269333601, 0.013604254461824894, 0.0076743620447814465, 0.019280288368463516, 0.015960589051246643, -0.002860798966139555, -0.004543430637568235, -0.01877257041633129, -0.006470157764852047, 0.006564541254192591, 0.015530982054769993, 0.0016712404321879148, 0.01799146458506584, -0.0042049516923725605, -0.004868891090154648, 0.007576723583042622, -0.012725511565804482, -0.02002233825623989, 0.006385537795722485, -0.03770136088132858, -0.029005052521824837, -0.032650209963321686, -0.021076831966638565, -0.010701145976781845, 0.0030609574168920517, 0.0026655227411538363, 0.027026250958442688, 0.0021903500892221928, -0.03853454068303108, -0.010564452037215233, 0.017145266756415367, 0.016676602885127068, -0.013929715380072594, -0.030098600313067436, 0.026583625003695488, -0.02384975552558899, 0.005038130562752485, 0.0060958778485655785, -0.0007440844201482832, 0.007752472534775734, 0.0009535996941849589, -0.02127210795879364, -0.013669347390532494, 0.004543430637568235, 0.000755475542973727, -0.005142278037965298, -0.016793768852949142, -0.015661166980862617, 0.0034173368476331234, -0.026505514979362488, 0.029343532398343086, 0.011156790889799595, 0.009106389246881008, 0.013356904499232769, -0.04092993214726448, -0.0005675220163539052, -0.005786690395325422, -0.00020148047769907862, -0.024643879383802414, -0.003882745513692498, 0.025034433230757713, 0.0008885075803846121, -0.008572633378207684, -0.015817387029528618, 0.025906667113304138, -0.019124068319797516, -0.02405805140733719, 0.030176710337400436, -0.007550687063485384, 0.005122750531882048, -0.017757132649421692, -0.004774507600814104, -0.037024401128292084, -0.00015947571955621243, 0.014567618258297443, -0.0014588773483410478, -0.012569289654493332, -0.016819806769490242, -0.014372342266142368, 0.001066697295755148, 0.019644804298877716, 0.008546596392989159, -0.012295902706682682, 0.005907110869884491, 0.026843992993235588, -0.033170949667692184, 0.0008950167684815824, 0.015804369002580643, -0.010206446051597595, 0.004592249635607004, 0.011976951733231544, 0.00039095961255952716, -0.016103792935609818, -0.011651490814983845, 0.019488584250211716, -0.02343316562473774, -0.02567233517765999, -0.008546596392989159, -0.052984993904829025, 0.012191755697131157, -0.022222451865673065, -0.019514620304107666, 0.014997226186096668, 0.02033478021621704, 0.01994422823190689, -0.0003675671177916229, -0.0023660988081246614, -0.013903678394854069, -0.054677389562129974, 0.01895482838153839, -0.013825568370521069, -0.016442270949482918, -0.032155510038137436, -0.0034987018443644047, 0.014020844362676144, 0.007824073545634747, -0.002289615571498871, 0.0030560754239559174, -0.0007977854111231863, -0.017015082761645317, 0.007407484110444784, -0.015478908084332943, -0.015348724089562893, 0.02424030937254429, -0.035019565373659134, 0.022300563752651215, -0.021285125985741615, -0.005965693388134241, -0.00606658635661006, -0.012400050647556782, -0.003178123151883483, 0.012797112576663494, -0.01799146458506584, -0.018017500638961792, 0.007622288074344397, -0.006239080335944891, 0.023602405562996864, -0.007199189625680447, -0.02408408746123314, -0.022873373702168465, 0.010043715126812458, -0.031869105994701385, -0.017457708716392517, -0.016989044845104218, 0.01090944092720747, 0.032936617732048035, 0.0021122395992279053, 0.010160881094634533, 0.03572256118059158, -0.006073095835745335, 0.0033815361093729734, -0.007524650078266859, -0.013903678394854069, -0.031166112050414085, 0.018499182537198067, -0.006990894675254822, -0.028770720586180687, -0.011176318861544132, 0.04116426408290863, 0.0001255057577509433, -0.0013970398576930165, -0.0026297220028936863, 0.0030560754239559174, 0.00230426131747663, -0.04150274023413658, -8.563683513784781e-05, 0.004045475739985704, -0.036894217133522034, -0.030098600313067436, -0.00939930323511362, -0.014242157340049744, 0.008305755443871021, 0.029083162546157837, 0.012413068674504757, 0.008709326386451721, -0.02543800324201584, -0.029395604506134987, 0.028979016467928886, 0.017054136842489243, 0.007609270047396421, -0.053375545889139175, 0.03889905661344528, 0.027078324928879738, 0.009204027242958546, -0.016155866906046867, -0.002250560326501727, -0.025594225153326988, 0.033926017582416534, -0.01892879046499729, 0.018694458529353142, 0.017639966681599617, 0.0002975930692628026, 0.015478908084332943, 0.027078324928879738, -0.0020292471162974834, -0.008344810456037521, 0.020712316036224365, 0.04020090028643608, 0.031608738005161285, 0.0068151457235217094, -0.02356335148215294, -0.018382016569375992, -0.01126093789935112, 0.0006680079968646169, -0.04405435174703598, -0.022248489782214165, 0.013122573494911194, 0.023133741691708565, 0.02301657758653164, 0.005282226018607616, 0.023732589557766914, 0.011124244891107082, -0.019150104373693466, 0.01094849593937397, -0.001025201054289937, -0.030723484233021736, -0.02155851386487484, 0.03983638435602188, -0.0007937171612866223, 0.011592907831072807, -0.005070677027106285, -0.021988121792674065, 0.050042830407619476, -0.023758627474308014, 0.015010245144367218, 0.004234243184328079, 0.012113644741475582, -0.02273017168045044, -0.003876236267387867, 0.014151028357446194, -0.013565199449658394, -0.016806786879897118, -0.02051703818142414, 0.017405634745955467, 0.016689620912075043, 0.016650566831231117, 0.016103792935609818, 0.053948357701301575, 0.004484847653657198, 0.0018486164044588804, -0.004152877721935511, 0.005122750531882048, 0.003249724628403783, 0.008904603309929371, 0.008286228403449059, 0.006977876182645559, -0.0075181410647928715, 0.015179484151303768, -0.007824073545634747, 0.001619980321265757, 0.002087830100208521, -0.032780397683382034, 0.014463471248745918, 0.015452871099114418, 0.031895142048597336, -0.016963008791208267, 0.01846012845635414, 0.015231558121740818, 0.01191185973584652, 0.006525485776364803, 0.002512556267902255, -0.01861634850502014, -0.009015260264277458, 0.02181888185441494, -0.004667105618864298, 0.005220388527959585, -0.014046881347894669, 0.001742841792292893, 0.0005943725118413568, -0.000561012770049274, 0.013148609548807144, 0.0011960678966715932, -0.0037720887921750546, 0.0003586169332265854, 0.008989223279058933, -0.005900601390749216, 0.033196985721588135, 0.026609662920236588, 0.026531551033258438, -0.009366757236421108, -0.008676780387759209, -0.032988689839839935, 0.030098600313067436, -0.0005740312044508755, 0.008123497478663921, -0.04277854785323143]} +{"id": "test:10000062", "text": "\"JPW Osprey Limited specialise in all aspects of fa\u00e7ade glazing, curtain walling and cladding refurbishment, operating under CDM often as the principal contractor.\\nHaving established an excellent reputation for the effective co-ordination and management of suppliers, whilst resolving technical and installation issues.\\nWe have a long history of highly complex projects completed on time and to budget. Projects are often carried out with the building\u2019s occupying tenants in situ, causing minimal disruption.\\nJPW Osprey Limited\u2019s approach to health and safety is extremely important on all projects. The Health and Safety People are retained to carry out regular inspections and audit the business.\"", "vector_field": [-0.0021652451250702143, 0.0017619319260120392, -0.012625754810869694, -0.027179211378097534, 0.0004359969752840698, 0.030269000679254532, -0.005304594524204731, -0.0042245350778102875, 0.002563431626185775, -0.02536088228225708, 0.0056771463714540005, 0.0009629958076402545, -0.007116085849702358, 0.0009604323422536254, -0.014697007834911346, -0.006815310101956129, 0.006565802730619907, -0.02300936169922352, -0.016214558854699135, -0.019796527922153473, -0.0368860699236393, -0.0016781932208687067, -0.0124480240046978, -0.030460402369499207, -0.012133575975894928, -0.004703042563050985, -0.0040741474367678165, -0.0013082046061754227, -0.004265550058335066, 0.008285011164844036, 0.0017243350157514215, -0.007136593572795391, -0.015298559330403805, 0.006094131153076887, -0.002207969082519412, -0.023665599524974823, -0.0047474754974246025, 0.008838712237775326, 0.03242911770939827, 0.0011509808246046305, 0.024731988087296486, 0.0005118318367749453, -4.77706162200775e-05, 0.00681189214810729, -0.01373999286442995, -0.00674011604860425, 0.008811368606984615, 0.009761547669768333, 0.005383206531405449, 0.015421604737639427, 0.003971610218286514, 0.019741840660572052, -0.017349304631352425, -0.014820052310824394, 0.022845301777124405, -0.005759176332503557, 0.0030043418519198895, 0.029476044699549675, 0.0112859345972538, -0.042737532407045364, 0.025087449699640274, 0.008975428529083729, -0.022708585485816002, -0.010349427349865437, -0.016337603330612183, 0.00834653340280056, -0.005995011888444424, -0.009228353388607502, -0.004439863376319408, -0.010171696543693542, 0.03888212889432907, 0.030405716970562935, -0.0064564296044409275, -0.00040416771662421525, 0.026304226368665695, -0.03420643135905266, -0.012304471805691719, -0.018593423068523407, -0.015722380951046944, 0.003349550534039736, -0.0023549392353743315, -0.030022909864783287, -0.001753387157805264, 0.021573839709162712, 0.006507698446512222, -0.009501785971224308, -0.013370859436690807, 0.0218062587082386, -0.011716591194272041, 0.011702919378876686, -0.009358233772218227, 0.016802439466118813, 0.004388594999909401, -0.015421604737639427, -0.015654021874070168, -0.002611282281577587, -0.001040753209963441, -0.010192204266786575, -0.009556473232805729, -0.015722380951046944, -0.020657841116189957, 0.025278853252530098, -0.020575810223817825, -0.005010654218494892, -0.028655746951699257, -0.01112187560647726, 0.012407008558511734, -0.0013483649818226695, 0.04350314289331436, -0.006541877519339323, -0.0041937739588320255, 0.014560291543602943, 0.00846274197101593, -0.02224375121295452, 0.001180032966658473, -0.025347212329506874, 0.01922231912612915, -0.0061351461336016655, -0.004258714150637388, -0.014724350534379482, 0.01395873911678791, 0.014368888922035694, 0.009720532223582268, -0.013808351010084152, 0.029366672039031982, 0.02430816739797592, -0.03705013170838356, -0.012106233276426792, 0.02530619688332081, -0.00882504042237997, 0.008531100116670132, 0.020603153854608536, 0.026358911767601967, -0.012283964082598686, -0.01458763424307108, -0.0034042370971292257, -0.011258591897785664, 0.008954920805990696, -0.021792586892843246, -0.021997660398483276, 0.02175157144665718, 0.01314527727663517, -0.0021447378676384687, 0.008893398568034172, 0.0012885516043752432, 0.026399927213788033, 0.015216530300676823, -0.013678470626473427, 0.0005361844669096172, -0.002423297381028533, 0.007423697970807552, -0.016132529824972153, 0.02783544920384884, -0.005164460279047489, 0.00978889036923647, 0.017349304631352425, 0.02084924280643463, 0.02402106299996376, -0.0157360527664423, -0.028108881786465645, -0.002353230258449912, 0.008045757189393044, 0.00015551484830211848, 0.003776789177209139, 0.011053516529500484, 0.007799667771905661, -0.01003498025238514, -0.00106126070022583, 0.00640516122803092, 0.008312353864312172, 0.01420482899993658, 0.01968715526163578, -0.03524547442793846, 0.011552531272172928, 0.015161843039095402, 0.018866855651140213, -0.027507329359650612, 0.018894199281930923, -0.016296589747071266, -0.030050253495573997, -0.01116972602903843, -0.005396877881139517, 0.01721258834004402, 0.008510592393577099, -0.027876464650034904, -0.024622615426778793, 0.017964528873562813, 0.02190195955336094, 0.018634438514709473, -0.014068112708628178, 0.004904699046164751, 0.03934696689248085, -0.02058948203921318, -0.023159749805927277, -0.6439887285232544, -0.025866733863949776, 0.00015797147352714092, -0.033522848039865494, 0.005290922708809376, 0.030952582135796547, 0.029804164543747902, 0.020480109378695488, 0.0029376926831901073, 7.321801240323111e-05, -0.024595271795988083, 0.011846471577882767, -0.012427516281604767, 0.006432504393160343, 0.028983866795897484, -0.0018695960752665997, 0.006528205703943968, -0.0102537265047431, 0.022517183795571327, -0.011689247563481331, -0.027507329359650612, 0.026960464194417, -0.005113191436976194, 0.017021184787154198, 0.017513364553451538, 0.01121074054390192, 0.017691096290946007, -0.0490264818072319, -0.011696083471179008, 0.007109250407665968, -0.027903806418180466, 0.027315927669405937, -0.0035682967863976955, 0.007827010937035084, 0.05115925893187523, 0.002845409093424678, -0.03565562516450882, 0.02469097264111042, 0.01648799143731594, 0.06261608749628067, -0.008510592393577099, -0.004880773834884167, 0.011080860160291195, 0.004333908203989267, 0.00497989309951663, 0.016187215223908424, 0.029284643009305, -0.007669787388294935, 0.006128310225903988, -0.002826610580086708, 0.030733834952116013, -0.020521124824881554, -0.023802315816283226, -0.013295665383338928, 0.0016687939641997218, -0.00894124899059534, 0.027411628514528275, -0.019755512475967407, 0.003393983468413353, -0.008189309388399124, -0.008175637573003769, 0.02492339164018631, -0.005906146485358477, -0.01983754336833954, 0.007054563611745834, -0.013165784999728203, -0.01328882947564125, 0.0009142906055785716, -0.008107279427349567, 0.005321683827787638, 0.020439093932509422, 0.015435275621712208, -0.004142505582422018, -0.009132652543485165, 0.021218378096818924, 0.025251509621739388, 0.001028790487907827, -0.009747875854372978, 0.00786802638322115, 0.005581445060670376, 0.009317219257354736, -0.010240054689347744, -0.007006713189184666, 0.02445855550467968, 0.003561460878700018, -0.004767982754856348, -0.012837665155529976, -0.0014004880795255303, 0.01994691602885723, -0.027644045650959015, 0.0033222073689103127, -0.025921419262886047, 0.014820052310824394, -0.019318019971251488, -0.04481561854481697, 0.0027274913154542446, -0.008729338645935059, -0.003800714621320367, -0.009952950291335583, -0.015134500339627266, -0.0010219546966254711, 0.0006605108501389623, 0.006883668247610331, -0.020726198330521584, -0.010499815456569195, 0.020398080348968506, -0.01937270723283291, -0.019796527922153473, 0.03056977689266205, -0.012776142917573452, 0.020220348611474037, -0.009351398795843124, 0.0024010809138417244, -0.01201053149998188, -0.010438293218612671, -0.03912821784615517, 0.027056165039539337, 0.010527159087359905, -0.0051883854903280735, -0.014915754087269306, 0.00028518176986835897, -0.010404114611446857, 0.03486267104744911, 0.004928624257445335, 0.011990023776888847, 0.02917526848614216, -0.01259157620370388, -0.025114793330430984, 0.0003255557967349887, 0.009467607364058495, 0.021942974999547005, -0.03486267104744911, 0.001471409690566361, -0.005506251007318497, 0.015859097242355347, 0.019933244213461876, 0.0005088411853648722, 0.009761547669768333, 0.026769060641527176, -0.025702673941850662, -0.010301576927304268, -0.013432381674647331, -0.0005062777199782431, -0.007615100592374802, -0.012516382150352001, -0.014984112232923508, -0.012215605936944485, 0.006275280378758907, -0.0007942365482449532, 0.0014961896231397986, -0.012051546014845371, -0.013644292019307613, 0.0102537265047431, 0.04265550151467323, 0.007608265150338411, 0.011388472281396389, 0.013910888694226742, -0.02771240472793579, -0.007143429480493069, 0.0026864763349294662, 0.007690294645726681, 0.01084844209253788, -0.04063209891319275, -0.01483372412621975, 0.011436322703957558, -0.0006921265157870948, 0.01707587204873562, -0.020411750301718712, -0.007662951480597258, -0.044979680329561234, 0.011149218305945396, 0.006723026745021343, 0.009474443271756172, 0.018019214272499084, 0.012208770029246807, 0.007075071334838867, 0.017431335523724556, -0.010937307961285114, 0.02358357049524784, 0.0010398987215012312, 0.007280145771801472, 0.015202858485281467, -0.019851213321089745, 0.00027535526896826923, 0.0019157378701493144, 0.01054083090275526, -0.007615100592374802, 0.011586710810661316, 0.018948886543512344, -0.009412921033799648, -0.03557359427213669, 0.022585541009902954, -0.007792831864207983, 0.027876464650034904, -0.006986205466091633, -0.004952549934387207, 0.0038759084418416023, 0.028628403320908546, 0.003981863614171743, 0.023419510573148727, 0.037761054933071136, -0.035464219748973846, 0.0350814163684845, 0.0021669541019946337, 0.00834653340280056, -0.013063247315585613, 0.0045629083178937435, -0.022722257301211357, 0.019153960049152374, -0.0010211002081632614, -0.004374923184514046, -0.020808229222893715, -0.027411628514528275, -0.0019994766917079687, 0.019359035417437553, 0.006907593458890915, -0.007717637810856104, 0.033550191670656204, -0.028464343398809433, -0.02242148108780384, 0.013883545063436031, 0.004036550410091877, 0.022175392135977745, -0.017144231125712395, -0.019099274650216103, 0.022995689883828163, 0.014341545291244984, 0.029886193573474884, -0.00037746530142612755, -0.003158147679641843, -0.02765771746635437, 0.00706823542714119, -0.005061923060566187, 0.0019003573106601834, 0.008209817111492157, 0.03778839856386185, 0.028683090582489967, -0.031800221651792526, 0.03021431341767311, -0.007232294883579016, 0.024130435660481453, 0.001462010433897376, 0.004723549820482731, 0.011614054441452026, 0.0050038183107972145, -0.011299606412649155, 0.011060352437198162, -0.0026437523774802685, -0.032702550292015076, -0.012940202839672565, -0.001446629874408245, 0.011525188572704792, 0.006128310225903988, 0.017048528417944908, 0.002759961411356926, 0.021560167893767357, 0.024882376194000244, 0.009884592145681381, 0.01395873911678791, 0.02771240472793579, -0.0008279883768409491, -0.008483249694108963, 0.03912821784615517, 0.0162555743008852, 0.004935460165143013, 0.006435922347009182, -0.019235990941524506, -0.014068112708628178, 0.019058259204030037, -0.017267275601625443, 0.014997784048318863, 0.007006713189184666, 0.02969479188323021, -0.009563309140503407, 0.009392413310706615, 0.0025343792513012886, -0.011292770504951477, 0.02399371936917305, 0.01620088703930378, 0.022271092981100082, -0.01715790294110775, -0.02099963091313839, 0.0045458185486495495, 0.024704644456505775, -0.007355339825153351, -0.015722380951046944, -0.021710556000471115, 0.01367163471877575, -0.03067914955317974, 0.021273063495755196, -0.0011202195892110467, -0.004624430555850267, -0.026331569999456406, -0.0003447815543040633, 0.006347056478261948, -0.01622823067009449, 0.0128991873934865, -0.014738022349774837, -0.004764564801007509, -0.03390565514564514, 0.011566203087568283, 0.0037357741966843605, 0.012174591422080994, -0.007492056116461754, 0.020521124824881554, 0.011347456835210323, -0.006278698332607746, -0.011395308189094067, -0.015011454932391644, -0.01531223114579916, -0.03163616359233856, -0.014341545291244984, -0.023747630417346954, 0.004050221759825945, 0.002886424073949456, -0.02634523995220661, -0.016583694145083427, -0.017554379999637604, 0.03934696689248085, -0.01700751483440399, -0.007758652791380882, -0.010123846121132374, -0.004976475145667791, 0.004433027468621731, 0.09351398050785065, 0.019304348155856133, -0.00729381712153554, 0.011224412359297276, -0.00860629417002201, -0.015667693689465523, -0.011463666334748268, -0.012659934349358082, 0.0021703720558434725, 0.006131728179752827, 0.0016217976808547974, -0.015544649213552475, -0.0030163044575601816, -0.008038921281695366, 0.007341668009757996, 0.014054440893232822, -0.009617995470762253, -0.010773248039186, -0.0023429763969033957, 0.0018576334696263075, -0.01631026156246662, -0.034452520310878754, 0.03382362425327301, 0.02655031532049179, 0.024212466552853584, 0.02840965799987316, 0.01006232388317585, 0.019591452553868294, 0.004692788701504469, 0.013247814029455185, -0.0014773911098018289, 0.009324055165052414, -0.004962803330272436, 0.020480109378695488, -0.007608265150338411, -0.031362731009721756, 0.017280947417020798, -0.03461657837033272, 0.00208663335070014, 0.008285011164844036, 0.00765611557289958, 0.04380391910672188, 0.010944143868982792, -0.007902204990386963, 0.014847395941615105, -0.012400172650814056, -0.008182473480701447, 0.03549156337976456, -0.02058948203921318, -0.019755512475967407, 0.031745538115501404, -0.003534117713570595, 0.0016636671498417854, -0.02399371936917305, 0.0025019091553986073, 0.015954798087477684, 0.0011937046656385064, -0.011798621155321598, 0.0274799857288599, -0.02004261687397957, -0.011887487024068832, -0.039456337690353394, -0.027931150048971176, -0.015079813078045845, -0.032237716019153595, -0.022722257301211357, -0.00432707229629159, -0.022708585485816002, -0.009310383349657059, -0.019960587844252586, 0.007034056354314089, -0.011484173126518726, -0.018169602379202843, 0.008572115562856197, 0.029038552194833755, -0.005376370623707771, 0.020288705825805664, 0.0016064171213656664, -0.0023412674199789762, 0.030351029708981514, -0.009823069907724857, -0.0036298190243542194, -0.04219066724181175, -0.02402106299996376, -0.0248960480093956, 0.01288551650941372, -0.004569744225591421, -0.02097228914499283, -0.015079813078045845, 0.0028710432816296816, 0.0065042804926633835, -0.0061043850146234035, 0.009440263733267784, 0.020247692242264748, -0.0017491148319095373, 0.024144107475876808, 0.040440697222948074, 0.04924523085355759, 0.02326912246644497, -0.012188262306153774, -0.00945393554866314, -0.0036195653956383467, 0.012208770029246807, -0.009467607364058495, 0.018155930563807487, 0.019126618281006813, 0.004744057543575764, 0.016501663252711296, -0.013931396417319775, 0.012502710334956646, 0.0075399065390229225, -0.0233511533588171, 0.015722380951046944, -0.003906669560819864, -0.017294619232416153, 0.02216172032058239, -0.002884715097025037, 0.036858730018138885, 0.031390074640512466, 0.004371505230665207, 0.004340744111686945, -0.025087449699640274, 0.047276515513658524, 0.03286661207675934, -0.01307008322328329, -0.005328519735485315, 0.014409903436899185, -0.03322207182645798, 0.009980293922126293, -0.01162088941782713, -0.01800554431974888, 0.02384333126246929, -0.010793755762279034, -0.022899989038705826, 0.0010877494933083653, -0.013562262058258057, -0.01007599476724863, 0.0010065741371363401, -0.014054440893232822, -0.007560414262115955, -0.03070649318397045, -5.2523646445479244e-05, 0.008558443747460842, 0.021478138864040375, 0.01367163471877575, -0.02300936169922352, 0.01957778073847294, -0.0008895107312127948, -0.011743934825062752, 0.0018747230060398579, -0.014218500815331936, -0.013097426854074001, -0.004214281681925058, 0.002674513729289174, -0.003855401184409857, -0.039073534309864044, -0.008845548145473003, -0.01811491698026657, 0.030979925766587257, 0.02084924280643463, 0.02972213551402092, -0.0061043850146234035, 0.005396877881139517, 0.004163012839853764, -0.00027044201851822436, -0.012270292267203331, -0.006873414386063814, 0.001131327822804451, -0.005772848147898912, 0.018429365009069443, 0.007505727466195822, 0.004368087276816368, 0.008073100820183754, -0.008961756713688374, 0.0029308567754924297, 0.012311307713389397, -0.03464392200112343, -0.018019214272499084, -0.012557396665215492, -0.008326025679707527, -0.004935460165143013, 0.004873937927186489, -0.02739795669913292, 0.016939155757427216, -0.032155685126781464, 0.006569220684468746, 0.015107156708836555, 0.034479863941669464, 0.012181427329778671, -0.016570022329688072, 0.025237837806344032, 0.005263579543679953, 0.023569898679852486, -0.006234265398234129, 0.009214682504534721, 0.011825964786112309, -0.022066019475460052, -0.045471858233213425, -0.0011697792215272784, 0.0006729008164256811, 0.02722022496163845, 0.01940005086362362, -0.01175077073276043, -0.014984112232923508, -0.01893521472811699, 0.03573765233159065, -0.006576056592166424, 0.0014773911098018289, 0.014068112708628178, -0.021860944107174873, -0.020288705825805664, -0.026591330766677856, -0.027329599484801292, -0.032702550292015076, -0.005574609152972698, 0.002536088228225708, -0.007116085849702358, 0.02536088228225708, 0.0007028075051493943, -0.02309139259159565, 0.023952703922986984, -0.01715790294110775, 0.028929179534316063, -0.0005400296067818999, 0.02352888323366642, 0.02620852366089821, 0.012249785475432873, -0.01487473864108324, -0.008408055640757084, 0.0018542155157774687, 0.011395308189094067, 0.00325726717710495, 0.013979246839880943, -0.014355217106640339, -0.017704768106341362, -0.033112701028585434, -0.006261609029024839, -0.014970440417528152, -0.010219546966254711, 0.050530362874269485, -0.019700827077031136, 0.001395361265167594, -0.01689814031124115, 0.011101367883384228, -0.008968592621386051, 0.01150468084961176, -0.012659934349358082, 0.00305731943808496, 0.004265550058335066, -0.009925606660544872, 0.0011287643574178219, 0.015025126747786999, -0.023651929572224617, 0.009515457786619663, 0.010622860863804817, -0.011381636373698711, 0.013890380971133709, -0.02559330128133297, -0.009440263733267784, 0.010937307961285114, -0.010287905111908913, -0.009911935776472092, 0.005420803092420101, -0.02402106299996376, 0.030351029708981514, 0.02495073340833187, 0.002062707906588912, -0.009412921033799648, 0.013938232325017452, 0.03106195479631424, -0.00011524761066539213, 0.011224412359297276, 0.013910888694226742, 0.007334832102060318, 0.0030744089744985104, 0.00194137217476964, -0.004022878594696522, -0.013521247543394566, 0.00904378667473793, -0.02177891507744789, 0.0050824303179979324, 0.020657841116189957, -0.03204631432890892, 5.239013262325898e-05, -0.01640596240758896, -0.003937430679798126, -0.00322137912735343, -0.01167557667940855, 0.007779160514473915, -0.013302501291036606, -0.019851213321089745, 0.008291847072541714, -0.03322207182645798, 0.011108203791081905, 0.02515580877661705, 0.003260684898123145, 0.017663752660155296, 0.024827688932418823, 0.00022408664517570287, 0.014902082271873951, 0.0023857003543525934, 0.003334169974550605, -0.025237837806344032, -0.014779037795960903, -0.0032589759211987257, 0.0036845055874437094, 0.02634523995220661, -0.018866855651140213, -0.005772848147898912, -0.00865414459258318, 0.003436707193031907, 0.029339328408241272, -0.011402144096791744, 0.00971369631588459, -0.0053558629006147385, -0.01058868132531643, -0.002994087990373373, -0.006733280140906572, -0.01922231912612915, -0.005837788339704275, -0.006124892737716436, 0.007806503679603338, 0.03601108863949776, -0.02221640758216381, -0.002500200178474188, 0.026386255398392677, -0.019126618281006813, 0.001589327584952116, -0.01765008084475994, -0.03937431052327156, -0.022872645407915115, -0.011156054213643074, -0.0066888476721942425, -0.03661263734102249, -0.028136225417256355, -0.009631667286157608, -0.006511116400361061, 0.004231370985507965, 0.013548590242862701, 0.0014030515449121594, 0.0067948028445243835, -0.004791907966136932, 0.01097832340747118, 0.02608547918498516, -0.024444883689284325, -0.019085602834820747, -0.016802439466118813, -0.030761178582906723, -0.008968592621386051, -0.024390196427702904, -0.02146446704864502, 0.030351029708981514, 0.04240941256284714, -0.04388594999909401, -0.02268124185502529, -0.023884346708655357, -0.0526357963681221, 0.01256423257291317, -0.03426111862063408, 0.004316818900406361, 0.03828057646751404, -0.000361016602255404, 0.006285534240305424, -0.0047474754974246025, 0.024417540058493614, 0.007423697970807552, 0.0011757606407627463, -0.017896169796586037, -0.0014423575485125184, -0.044077351689338684, 0.024991748854517937, 0.0002695875591598451, -0.046811677515506744, -0.00439884839579463, 0.01367163471877575, 0.003711848985403776, 0.015271216630935669, -0.012065217830240726, -0.011292770504951477, -0.011449994519352913, -0.0020951780024915934, -0.0011065480066463351, 0.028546374291181564, 0.028901835903525352, 0.01762273721396923, -0.030815865844488144, -0.013083755038678646, 0.024362854659557343, -0.004846594762057066, 0.0010185368591919541, 0.017349304631352425, 0.010752741247415543, -0.0048978631384670734, 0.010739069432020187, -0.006709354929625988, -0.0006028336356393993, 0.00575234042480588, -0.009583815932273865, 0.01989222876727581, -0.0004347152716945857, -0.003670834004878998, -0.006008683703839779, 0.010533994995057583, -0.003272647736594081, -0.015161843039095402, -0.0005225128261372447, -0.019235990941524506, -0.022899989038705826, 0.0025702673010528088, -0.013446053490042686, -0.019700827077031136, -0.0035922222305089235, 0.022093363106250763, -0.011402144096791744, -0.015681365504860878, -0.009187338873744011, 0.003012886503711343, -0.013076919130980968, 0.01937270723283291, 0.006586309988051653, -0.010089666582643986, -0.01960512436926365, -0.02536088228225708, 0.012400172650814056, 0.008544771932065487, -0.00995978619903326, -0.00018873265071306378, -0.010185368359088898, -0.016474319621920586, 0.01557199191302061, -0.015462619252502918, -0.0031205506529659033, -0.0012176299933344126, 0.021573839709162712, -0.0036776699125766754, 0.0016764842439442873, 0.21382437646389008, 0.010219546966254711, 0.014232171699404716, 0.03056977689266205, 0.012133575975894928, 0.0206441693007946, 0.019810199737548828, -0.00424504280090332, 0.0014312493149191141, 0.029503388330340385, -0.003800714621320367, 0.010055487975478172, -0.0029599089175462723, 0.00512686325237155, 0.012352322228252888, -0.02474565990269184, -0.03710481896996498, -0.015011454932391644, -0.024991748854517937, 0.03297598287463188, 0.022066019475460052, -0.01628291793167591, 0.02032972127199173, -0.007423697970807552, 0.011197068728506565, -0.002551468787714839, -0.0008267067023552954, 0.01369897834956646, 0.03975711390376091, 0.021218378096818924, -0.0063197133131325245, 0.011976351961493492, 0.027794433757662773, -0.012919695116579533, 0.028710434213280678, 0.0033888565376400948, 0.01391772460192442, -0.029038552194833755, -0.00034136362955905497, 0.003318789415061474, 0.008353369310498238, -0.001149271847680211, -0.0034725952427834272, 0.0012432642979547381, 0.005793355405330658, 0.010561337694525719, -0.0018798498203977942, -0.005728415213525295, 0.010363099165260792, 0.003026558319106698, -0.05955364182591438, 0.004197191912680864, 0.034097056835889816, 0.019564108923077583, 0.0028624986298382282, -0.004822669085115194, 0.008312353864312172, -0.004952549934387207, 0.0009988838573917747, 0.006924683228135109, -0.018306318670511246, 0.007471548393368721, -0.01203787513077259, -0.016775095835328102, -0.00824399571865797, 0.030788522213697433, 0.01940005086362362, -0.0056634750217199326, -0.02311873435974121, -0.018716467544436455, 0.005413967650383711, -0.010424621403217316, -0.012475366704165936, -0.004484296310693026, -0.0015124246710911393, -0.024417540058493614, 0.0077244737185537815, 0.012324978597462177, 0.020903930068016052, 0.025866733863949776, 0.00044390090624801815, -0.0002845409035217017, -0.016146201640367508, 0.01852506585419178, -0.031745538115501404, -0.034944698214530945, 0.0384446382522583, -0.001753387157805264, -0.00020357919856905937, -0.014341545291244984, -0.012714620679616928, 0.005581445060670376, -0.0242534801363945, 0.011990023776888847, 0.032319746911525726, -0.002247275086119771, 0.019071931019425392, 0.01367163471877575, 0.0034589236602187157, 0.002298543695360422, -0.018456706777215004, 0.04030397906899452, 0.02254452556371689, -0.003272647736594081, -0.005912982393056154, -0.002153282519429922, 0.006193250883370638, 0.006131728179752827, 0.004504803568124771, -0.005499415099620819, -0.010137517005205154, -0.023911690339446068, 0.0024369689635932446, -0.010349427349865437, 0.009187338873744011, 0.00900277215987444, -0.0038451473228633404, -0.0150934848934412, 0.0276030320674181, -0.017321961000561714, 0.010410950519144535, -0.013610112480819225, -0.0004242479335516691, -0.0017841483931988478, -0.008305517956614494, -0.0022882898338139057, -0.031171327456831932, 0.013808351010084152, -0.0027292000595480204, -0.0469483956694603, -0.007676623295992613, 0.002645461354404688, 0.012167755514383316, -0.019755512475967407, -0.00038045598194003105, 0.002380573423579335, -0.004145923536270857, -0.006374399643391371, 0.014792709611356258, 0.01785515621304512, 0.004238206893205643, -0.019878556951880455, -0.0013005143264308572, 0.020630497485399246, 0.011101367883384228, -0.016993843019008636, -0.016679394990205765, -0.0004981601960025728, 0.018607094883918762, -0.003510192269459367, -0.04169848561286926, -0.01073223352432251, -0.010431457310914993, 0.0003834466333501041, 0.02076721377670765, 0.01292653102427721, -0.020316049456596375, -0.026167510077357292, 0.00225069303996861, -0.010110174305737019, -0.03510875999927521, 0.02035706490278244, 0.024007391184568405, -0.030077597126364708, -0.009864084422588348, -0.0025104540400207043, -0.1754344254732132, 0.02701515145599842, -2.942605897260364e-05, -0.00644959369674325, 0.023228108882904053, -0.009488115087151527, -0.0050209080800414085, -0.004145923536270857, -0.011149218305945396, -0.008175637573003769, 0.021847272291779518, 0.0018576334696263075, -0.02149181067943573, -0.032183028757572174, 0.0007771470118314028, -0.009638503193855286, -0.029831508174538612, -0.0050345794297754765, 0.014081784524023533, 0.015284888446331024, 0.0216695424169302, -0.0001783721090760082, 0.005044833291321993, -1.2356541446933988e-05, -0.010376770980656147, 0.007204951718449593, -0.006039444822818041, -0.017321961000561714, -0.008544771932065487, -0.0330306701362133, -0.004990146961063147, 0.017294619232416153, 0.040905531495809555, -0.008079936727881432, -0.009132652543485165, -0.02920261211693287, -0.006210340186953545, -0.018196946009993553, -0.015585663728415966, 0.03644857928156853, 0.030050253495573997, 0.011422650888562202, -0.0010544249089434743, 0.004938878118991852, -0.010609189048409462, 0.01051348727196455, 0.02780810557305813, 0.012700948864221573, 0.009946114383637905, 0.002896677702665329, -0.004767982754856348, -0.03713216260075569, 0.014929425902664661, -0.003947684541344643, -0.010964651592075825, 0.007950055412948132, 0.015804409980773926, -0.00820298120379448, 0.010383606888353825, -0.01075957715511322, -0.01602315716445446, 0.01486106775701046, 0.016802439466118813, -0.0174176637083292, -0.004569744225591421, -0.018894199281930923, 0.00038964158738963306, 0.014054440893232822, -0.0006848634802736342, 0.017280947417020798, 0.005287504754960537, -0.0189625583589077, 0.008961756713688374, -0.020657841116189957, 0.01893521472811699, 0.01413647085428238, -0.001972133293747902, 0.007163936737924814, -0.00025313885998912156, -0.023802315816283226, -0.01937270723283291, 0.011408979073166847, -0.02332380972802639, -0.016146201640367508, 0.002568558556959033, 0.019263334572315216, 0.008524264208972454, 0.007136593572795391, 0.01904458738863468, 0.01167557667940855, 0.038608696311712265, -0.022899989038705826, -0.006422250531613827, -0.0074988920241594315, 0.02314607799053192, 0.02713819593191147, 0.015011454932391644, 0.010451965034008026, 0.0007498037302866578, 0.00995978619903326, 0.017062200233340263, -0.02128673531115055, -0.017896169796586037, 0.008141458965837955, 0.015544649213552475, 0.014437247067689896, 0.022899989038705826, 0.025456584990024567, 0.013111097738146782, -0.028190910816192627, -0.0010304994648322463, 0.01942739263176918, 0.025251509621739388, -0.007423697970807552, 0.0006656377227045596, 0.00399211747571826, 0.01927700638771057, 0.004962803330272436, -0.001898648333735764, -0.02440386824309826, 0.047714006155729294, 0.00901644304394722, -0.015517305582761765, -0.01189432293176651, -0.007690294645726681, 0.0024215884041041136, -0.10002168267965317, -0.007621936500072479, 0.024267151951789856, -0.009782054461538792, -0.004938878118991852, 0.025278853252530098, 0.001000592834316194, -0.01003498025238514, -0.021191034466028214, 0.013152113184332848, -0.015517305582761765, -0.027056165039539337, 0.010110174305737019, -0.014779037795960903, 0.028655746951699257, -0.001683320035226643, 0.020425422117114067, -0.0020849243737757206, -0.038608696311712265, 0.00998712982982397, -0.005191803444176912, 0.0023224689066410065, -0.007738145533949137, -0.008695160038769245, -0.03322207182645798, -0.006053116638213396, -0.022325780242681503, 0.011156054213643074, 0.006753787863999605, -0.006463265512138605, 0.018976230174303055, 0.020862914621829987, 0.005147370509803295, 0.0035443713422864676, -0.00878402590751648, -0.011272262781858444, -0.03967508673667908, -0.008120951242744923, 0.028054194524884224, -0.022271092981100082, 0.005294340662658215, 0.005892474669963121, 0.009535965509712696, -0.03467126563191414, -0.026454614475369453, 0.009659009985625744, -0.007410026155412197, 0.03056977689266205, 0.017144231125712395, -0.004901281092315912, -0.030843209475278854, 0.002047327347099781, 0.0015722380485385656, -0.022353123873472214, 0.023433182388544083, -0.0016687939641997218, -0.018894199281930923, 0.026017121970653534, -0.007963727228343487, 0.0042723859660327435, 0.005636131390929222, 0.00795689132064581, -0.0006963988998904824, -0.008572115562856197, 0.024390196427702904, -0.0030521925073117018, -0.026099151000380516, -0.006965698208659887, 0.0038041325751692057, -0.02268124185502529, -0.010561337694525719, -0.004781654570251703, -0.007710802368819714, 0.0011091113556176424, 0.0015337865334004164, -0.02402106299996376, -0.003233341732993722, -0.017499692738056183, -0.008421727456152439, -0.006938355043530464, -0.012584740296006203, -0.016857126727700233, 0.014724350534379482, -0.012502710334956646, 0.003930595237761736, 0.0034349982161074877, 0.0233511533588171, -0.01145683042705059, 0.03609311580657959, -0.012502710334956646, -0.003026558319106698, 0.027876464650034904, 0.02789013646543026, -0.028054194524884224, 0.0019823871552944183, 0.010308412835001945, 0.0012663352536037564, -0.014314201660454273, -0.018566081300377846, 0.043175023049116135, -0.02556595765054226, -0.006555548869073391, -0.07645178586244583, 0.024827688932418823, -0.009761547669768333, -0.024472227320075035, 0.01203787513077259, 0.004938878118991852, 0.012078889645636082, 0.009611159563064575, 0.007861190475523472, -0.004415938165038824, -0.036339204758405685, 0.02698780782520771, -0.010575009509921074, -0.013008560985326767, -0.026946792379021645, -0.003385438583791256, 0.012249785475432873, -0.003551207249984145, -0.002117394469678402, 0.0103220846503973, -0.0061043850146234035, -0.01666572317481041, 0.011497844941914082, 0.011415814980864525, -0.02974947728216648, 0.010602353140711784, -0.024472227320075035, 0.036311861127614975, -0.010021308436989784, -0.01237282995134592, 0.005680564325302839, -0.007929548621177673, -0.00765611557289958, 0.007820175029337406, -0.002148155588656664, -0.02009730413556099, -0.0007301507866941392, 0.00729381712153554, 0.012919695116579533, 0.0019584617111831903, -0.025784702971577644, -0.012830829247832298, 0.007122921757400036, -0.004402266349643469, -0.004856848157942295, -0.008907070383429527, -0.026687031611800194, -0.0038964159321039915, 0.02789013646543026, -0.015298559330403805, 0.014232171699404716, 0.003012886503711343, -0.0053524449467659, -0.007492056116461754, 0.004262132104486227, -0.01940005086362362, 0.014546619728207588, -0.007813339121639729, 0.00011439313675509766, -0.025196824222803116, 0.05405764654278755, -0.009358233772218227, 0.0025941927451640368, -0.007485220208764076, 0.04221801087260246, -0.0010698054684326053, -0.018894199281930923, -0.004180102609097958, 0.010356263257563114, 0.01707587204873562, -0.02265390008687973, 0.0109031293541193, 0.0019311184296384454, 0.036831386387348175, 0.02329646609723568, -0.002720655407756567, 0.006757205817848444, 0.00158334628213197, -0.0353548489511013, 0.025497598573565483, -0.020138317719101906, 0.00820298120379448, -0.021710556000471115, 0.023200765252113342, 0.012844501063227654, -0.01718524470925331, -0.03210099786520004, -0.0006365855224430561, -0.00145602913107723, 0.026509299874305725, -0.008599458262324333, 0.007642444223165512, -0.029093239456415176, -0.0014491933397948742, 0.0014201410813257098, -0.0021396109368652105, 0.013774172402918339, -0.0011637979187071323, 0.0025668495800346136, 0.032292403280735016, 0.00611463887616992, 0.011326950043439865, -0.008524264208972454, -0.012297635897994041, -0.04019460827112198, -0.0031308045145124197, 0.015394261106848717, -0.027124524116516113, -0.008866054937243462, 0.031280700117349625, 0.024294495582580566, -0.0017516782972961664, 0.015476291067898273, 0.005297758616507053, -0.00945393554866314, 0.0023925360292196274, 0.0055985343642532825, -0.028163569048047066, -0.02556595765054226, 0.022845301777124405, -0.005882220808416605, -0.014478261582553387, 0.0029496552888303995, 0.004586833529174328, 0.015654021874070168, 0.004050221759825945, -0.007396354805678129, -0.03207365423440933, -0.0019516259199008346, -0.0005015780916437507, 0.03245646134018898, -0.004918370861560106, 0.008585786446928978, -0.006090713664889336, -0.027179211378097534, -0.015777066349983215, -0.030897894874215126, 0.015148171223700047, 0.004600505344569683, 0.04744057357311249, 0.018921542912721634, -0.025989778339862823, -0.00834653340280056, -0.005407131742686033, 0.01940005086362362, -0.017171572893857956, 0.020780885592103004, -0.006750369910150766, -0.012181427329778671, -0.0038451473228633404, -0.009091637097299099, -0.005953996907919645, -0.0036742519587278366, -0.01651533506810665, -0.024185122922062874, -0.018825842067599297, 0.042710188776254654, -0.010964651592075825, 0.018388349562883377, 0.03428846225142479, 0.015859097242355347, 0.017116887494921684, -0.0071297576650977135, -0.04998349770903587, -0.021860944107174873, 0.01362378429621458, -0.0040638935752213, -0.00011417951463954523, -0.007307488936930895, 0.004128833767026663, -0.00010926627146545798, -0.021737899631261826, -0.022353123873472214, 0.020890258252620697, 0.006418832577764988, -0.013418709859251976, -0.012906023301184177, -0.009317219257354736, 0.013685306534171104, -0.0003330324834678322, 0.030460402369499207, -0.021067989990115166, -0.016542678698897362, 0.010718561708927155, -0.0277397483587265, -0.030269000679254532, -0.00633338512852788, 0.0028642076067626476]} +{"id": "test:10000063", "text": "\"Yesterday [12 Nov], at the Shoreditch Town Hall, Dr. Macdonald, M.P., the coroner for the North- Eastern District of Middlesex, opened his inquiry relative to the death of Marie Jeanette Kelly, the woman whose body was discovered on Friday morning, terribly mutilated, in a room on the ground floor of 26, Dorset-street, entrance to which was by a side door in Miller's-court.\\nSuperintendent T. Arnold, H Division; Inspector Abberline, of the Criminal Investigation Department, and Inspector Nairn represented the police. The deputy coroner, Mr. Hodgkinson, was present during the proceedings.\\nThe Coroner (to the juror, severely): Do you think that we do not know what we are doing here, and that we do not know our own district? The jury are summoned in the ordinary way, and they have no business to object. If they persist in their objection I shall know how to deal with them. Does any juror persist in objecting ?\\nThe Juror: We are summoned for the Shoreditch district. This affair happened in Spitalfields. The Coroner: It happened within my district.\\nThe jury having made no further objection, they were duly sworn, and were conducted by Inspector Abberline to view the body, which, decently coffined, was at the mortuary adjoining Shoreditch Church, and subsequently the jury inspected the room, in Miller's-court, Dorset- street, where the murder was committed. The apartment, a plan of which was given in yesterday's Daily Telegraph, is poorly furnished, and uncarpeted. The position of the two tables was not altered. One of them was placed near the bed, behind the door, and the other next to the largest of the two windows which look upon the yard in which the dustbin and water-tap are situated.\\nJoseph Barnett deposed : I was a fish-porter, and I work as a labourer and fruit- porter. Until Saturday last I lived at 24, New-street, Bishopsgate, and have since stayed at my sister's, 21, Portpool-lane, Gray's Inn-road. I have lived with the deceased one year and eight months. Her name was Marie Jeanette Kelly with the French spelling as described to me. Kelly was her maiden name. I have seen the body, and I identify it by the ear and eyes, which are all that I can recognise; but I am positive it is the same woman I knew. I lived with her in No. 13 room, at Miller's-court for eight months. I separated from her on Oct. 30.\\n[Coroner] Why did you leave her ? - Because she had a woman of bad character there, whom she took in out of compassion, and I objected to it. That was the only reason. I left her on the Tuesday between five and six p.m. I last saw her alive between half-past seven and a quarter to eight on Thursday night last, when I called upon her. I stayed there for a quarter of an hour.\\n[Coroner] Were you on good terms ? - Yes, on friendly terms; but when we parted I told her I had no work, and had nothing to give her, for which I was very sorry.\\n[Coroner] Did you drink together ? - No, sir. She was quite sober.\\n[Coroner] Was she, generally speaking, of sober habits ? - When she was with me I found her of sober habits, but she has been drunk several times in my presence.\\n[Coroner] Was there any one else there on the Thursday evening ? - Yes, a woman who lives in the court. She left first, and I followed shortly afterwards.\\n[Coroner] Have you had conversation with deceased about her parents ? - Yes, frequently. She said she was born in Limerick, and went when very young to Wales. She did not say how long she lived there, but that she came to London about four years ago. Her father's name was John Kelly, a \\\"gaffer\\\" or foreman in an iron works in Carnarvonshire, or Carmarthen. She said she had one sister, who was respectable, who travelled from market place to market place. This sister was very fond of her. There were six brothers living in London, and one was in the army. One of them was named Henry. I never saw the brothers to my knowledge. She said she was married when very young in Wales to a collier. I think the name was Davis or Davies. She said she had lived with him until he was killed in an explosion, but I cannot say how many years since that was. Her age was, I believe, 16 when she married. After her husband's death deceased went to Cardiff to a cousin.\\n[Coroner] Did she live there long ? - Yes, she was in an infirmary there for eight or nine months. She was following a bad life with her cousin, who, as I reckon, and as I often told her, was the cause of her downfall.\\n[Coroner] After she left Cardiff did she come direct to London ? - Yes. She was in a gay house in the West-end, but in what part she did not say. A gentleman came there to her and asked her if she would like to go to France.\\n[Coroner] Did she go to France ? - Yes; but she did not remain long. She said she did not like the part, but whether it was the part or purpose I cannot say. She was not there more than a fortnight, and she returned to England, and went to Ratcliffe-highway. She must have lived there for some time. Afterwards she lived with a man opposite the Commercial Gas Works, Stepney. The man's name was Morganstone.\\n[Coroner] Have you seen that man ? - Never. I don't know how long she lived with him.\\n[Coroner] Was Morganstone the last man she lived with ? - I cannot answer that question, but she described a man named Joseph Fleming, who came to Pennington-street, a bad house, where she stayed. I don't know when this was. She was very fond of him. He was a mason's plasterer, and lodged in the Bethnal-green-road.\\n[Coroner] Was that all you knew of her history when you lived with her? - Yes. After she lived with Morganstone or Fleming - I don't know which one was the last - she lived with me.\\n[Coroner] Where did you pick up with her first ? - In Commercial-street. We then had a drink together, and I made arrangements to see her on the following day - a Saturday. On that day we both of us agreed that we should remain together. I took lodgings in George-street, Commercial-street, where I was known. I lived with her, until I left her, on very friendly terms.\\n[Coroner] Did she express fear of any particular individual ? - No, sir. Our own quarrels were very soon over.\\nThomas Bowyer stated: I live at 37, Dorset-street, and am employed by Mr. McCarthy. I serve in his chandler's shop, 27, Dorset-street. At a quarter to eleven a.m., on Friday morning, I was ordered by McCarthy to go to Mary Jane's room, No. 13. I did not know the deceased by the name of Kelly. I went for rent, which was in arrears. Knocking at the door, I got no answer, and I knocked again and again. Receiving no reply, I passed round the corner by the gutter spout where there is a broken window - it is the smallest window.\\nCharles Ledger, an inspector of police, G Division, produced a plan of the premises. Bowyer pointed out the window, which was the one nearest the entrance.\\nHe [Bowyer] continued: There was a curtain. I put my hand through the broken pane and lifted the curtain. I saw two pieces of flesh lying on the table.\\n[Coroner] Where was this table ? - In front of the bed, close to it. The second time I looked I saw a body on this bed, and blood on the floor. I at once went very quietly to Mr. McCarthy. We then stood in the shop, and I told him what I had seen. We both went to the police-station, but first of all we went to the window, and McCarthy looked in to satisfy himself. We told the inspector at the police-station of what we had seen. Nobody else knew of the matter. The inspector returned with us.\\n[Coroner] Did you see the deceased constantly ? - I have often seen her. I knew the last witness, Barnett. I have seen the deceased drunk once.\\nJohn McCarthy, grocer and lodging-house keeper, testified: I live at 27, Dorset- street. On Friday morning, about a quarter to eleven, I sent my man Bowyer to Room 13 to call for rent. He came back in five minutes, saying, \\\"Guv'nor, I knocked at the door, and could not make any one answer; I looked through the window and saw a lot of blood.\\\" I accompanied him, and looked through the window myself, saw the blood and the woman. For a moment I could not say anything, and I then said: \\\"You had better fetch the police.\\\" I knew the deceased as Mary Jane Kelly, and had no doubt at all about her identity. I followed Bowyer to Commercial-street Police-station, where I saw Inspector Beck. I inquired at first for Inspector Reid. Inspector Beck returned with me at once.\\n[Coroner] How long had the deceased lived in the room ? - Ten months. She lived with Barnett. I did not know whether they were married or not; they lived comfortably together, but they had a row when the window was broken. The bedstead, bed-clothes, table, and every article of furniture belonged to me.\\n[Coroner] What rent was paid for this room ? - It was supposed to be 4s 6d a week. Deceased was in arrears 29s. I was to be paid the rent weekly. Arrears are got as best you can. I frequently saw the deceased the worse for drink. When sober she was an exceptionally quiet woman, but when in drink she had more to say. She was able to walk about, and was not helpless.\\n[Coroner] Where was this ? - In Dorset-street. She went up the court, a few steps in front of me.\\n[Coroner] Was anybody with her ? - A short, stout man, shabbily dressed. He had on a longish coat, very shabby, and carried a pot of ale in his hand.\\n[Coroner] What was the colour of the coat ? - A dark coat.\\n[Coroner] What hat had he ? - A round hard billycock.\\n[Coroner] Long or short hair ? - I did not notice. He had a blotchy face, and full carrotty moustache.\\n[Coroner] The chin was shaven ? - Yes. A lamp faced the door.\\n[Coroner] Did you see them go into her room ? - Yes; I said \\\"Good night, Mary,\\\" and she turned round and banged the door.\\n[Coroner] Had he anything in his hands but the can ? - No.\\n[Coroner] Did she say anything ? - She said \\\"Good night, I am going to have a song.\\\" As I went in she sang \\\"A violet I plucked from my mother's grave when a boy.\\\" I remained a quarter of an hour in my room and went out. Deceased was still singing at one o'clock when I returned. I remained in the room for a minute to warm my hands as it was raining, and went out again. She was singing still, and I returned to my room at three o'clock. The light was then out and there was no noise.\\n[Coroner] Did you go to sleep ? - No; I was upset. I did not undress at all. I did not sleep at all. I must have heard what went on in the court. I heard no noise or cry of \\\"Murder,\\\" but men went out to work in the market.\\n[Coroner] How many men live in the court who work in Spitalfields Market ? - One. At a quarter- past six I heard a man go down the court. That was too late for the market.\\n[Coroner] From what house did he go ? - I don't know.\\n[Coroner] Did you hear the door bang after him ? - No.\\n[Coroner] Then he must have walked up the court and back again? - Yes.\\n[Coroner] It might have been a policeman ? - It might have been.\\n[Coroner] What would you take the stout man's age to be ? - Six-and-thirty.\\n[Coroner] Did you notice the colour of his trousers ? - All his clothes were dark.\\n[Coroner] Did his boots sound as if the heels were heavy ? - There was no sound as he went up the court.\\n[Coroner] Then you think that his boots were down at heels ? - He made no noise.\\n[Coroner] What clothes had Mary Jane on ? - She had no hat; a red pelerine and a shabby skirt.\\n[Coroner] You say she was drunk ? - I did not notice she was drunk until she said good night. The man closed the door. By the Jury: There was a light in the window, but I saw nothing, as the blinds were down. I should know the man again, if I saw him.\\n[Coroner] Do you often hear cries of \\\"Murder?\\\" - It is nothing unusual in the street. I did not take particular notice.\\n[Coroner] Did you hear it a second time? - No.\\n[Coroner] Did you hear beds or tables being pulled about? - None whatever. I went asleep, and was awake again at five a.m. I passed down the stairs, and saw some men harnessing horses. At a quarter to six I was in the Ten Bells.\\n[Coroner] Could the witness, Mary Ann Cox, have come down the entry between one and half-past one o'clock without your knowledge ? - Yes, she could have done so.\\n[Coroner] Did you see any strangers at the Ten Bells ? - No. I went back to bed and slept until eleven.\\n[Coroner] You heard no singing downstairs ? - None whatever. I should have heard the singing distinctly. It was quite quiet at half-past one o'clock.\\nCaroline Maxewell, 14, Dorset-street, said: My husband is a lodging-house deputy. I knew the deceased for about four months. I believe she was an unfortunate. On two occasions I spoke to her.\\n[Coroner] Did you speak to her ? - Yes; it was an unusual thing to see her up. She was a young woman who never associated with any one. I spoke across the street, \\\"What, Mary, brings you up so early ?\\\" She said, \\\"Oh, Carrie, I do feel so bad.\\\"\\n[Coroner] And yet you say you had only spoken to her twice previously; you knew her name and she knew yours ? - Oh, yes; by being about in the lodging-house.\\n[Coroner] What did she say ? - She said, \\\"I've had a glass of beer, and I've brought it up again\\\"; and it was in the road. I imagined she had been in the Britannia beer-shop at the corner of the street. I left her, saying that I could pity her feelings. I went to Bishopsgate-street to get my husband's breakfast. Returning I saw her outside the Britannia public-house, talking to a man.\\n[Coroner] This would be about what time ? - Between eight and nine o'clock. I was absent about half-an-hour. It was about a quarter to nine.\\n[Coroner] What description can you give of this man ? - I could not give you any, as they were at some distance.\\n[Coroner] What sort of dress had the deceased ? - A dark skirt, a velvet body, a maroon shawl, and no hat.\\n[Coroner] Have you ever seen her the worse for drink ? - I have seen her in drink, but she was not a notorious character.\\nSarah Lewis deposed: I live at 24, Great Pearl-street, and am a laundress. I know Mrs. Keyler, in Miller's-court, and went to her house at 2, Miller's-court, at 2.30a.m. on Friday. It is the first house. I noticed the time by the Spitalfields' Church clock. When I went into the court, opposite the lodging-house I saw a man with a wideawake. There was no one talking to him. He was a stout-looking man, and not very tall. The hat was black. I did not take any notice of his clothes. The man was looking up the court; he seemed to be waiting or looking for some one. Further on there was a man and woman - the later being in drink. There was nobody in the court. I dozed in a chair at Mrs. Keyler's, and woke at about half- past three. I heard the clock strike.\\n[Coroner] What woke you up ? - I could not sleep. I sat awake until nearly four, when I heard a female's voice shouting \\\"Murder\\\" loudly. It seemed like the voice of a young woman. It sounded at our door. There was only one scream.\\n[Coroner] Were you afraid ? Did you wake anybody up ? - No, I took no notice, as I only heard the one scream.\\n[Coroner] You stayed at Keyler's house until what time ? - Half-past five p.m. on Friday. The police would not let us out of the court.\\n[Coroner] Have you seen any suspicious persons in the district ? - On Wednesday night I was going along the Bethnal-green-road, with a woman, about eight o'clock, when a gentleman passed us. He followed us and spoke to us, and wanted us to follow him into an entry. He had a shiny leather bag with him.\\n[Coroner] Did he want both of you ? - No; only one. I refused. He went away and came back again, saying he would treat us. He put down his bag and picked it up again, saying, \\\"What are you frightened about ? Do you think I've got anything in the bag ?\\\" We then ran away, as we were frightened.\\n[Coroner] Was he a tall man ? - He was short, pale-faced, with a black moustache, rather small. His age was about forty.\\n[Coroner] Was it a large bag ? - No, about 6in to 9in long. His hat was a high round hat. He had a brownish overcoat, with a black short coat underneath. His trousers were a dark pepper-and- salt.\\n[Coroner] After he left you what did you do ? - We ran away.\\n[Coroner] Have you seen him since ? - On Friday morning, about half-past two a.m., when I was going to Miller's-court, I met the same man with a woman in Commercial-street, near Mr. Ringer's public-house (the Britannia). He had no overcoat on.\\n[Coroner] Had he the black bag ? - Yes.\\n[Coroner] Were the man and woman quarrelling ? - No; they were talking. As I passed he looked at me. I don't know whether he recognised me. There was no policeman about.\\nMr. George Bagster Phillips, divisional surgeon of police, said: I was called by the police on Friday morning at eleven o'clock, and on proceeding to Miller's-court, which I entered at 11.15, I found a room, the door of which led out of the passage at the side of 26, Dorset-street, photographs of which I produce. It had two windows in the court. Two panes in the lesser window were broken, and as the door was locked I looked through the lower of the broken panes and satisfied myself that the mutilated corpse lying on the bed was not in need of any immediate attention from me, and I also came to the conclusion that there was nobody else upon the bed, or within view, to whom I could render any professional assistance. Having ascertained that probably it was advisable that no entrance should be made into the room at that time, I remained until about 1.30p.m., when the door was broken open by McCarthy, under the direction of Superintendent Arnold. On the door being opened it knocked against a table which was close to the left-hand side of the bedstead, and the bedstead was close against the wooden partition. The mutilated remains of a woman were lying two- thirds over, towards the edge of the bedstead, nearest the door. Deceased had only an under- linen garment upon her, and by subsequent examination I am sure the body had been removed, after the injury which caused death, from that side of the bedstead which was nearest to the wooden partition previously mentioned. The large quantity of blood under the bedstead, the saturated condition of the palliasse, pillow, and sheet at the top corner of the bedstead nearest to the partition leads me to the conclusion that the severance of the right carotid artery, which was the immediate cause of death, was inflicted while the deceased was lying at the right side of the bedstead and her head and neck in the top right-hand corner.\\nthe coroner said: It has come to my ears that somebody has been making a statement to some of the jury as to their right and duty of being here. Has any one during the interval spoken to the jury, saying that they should not be here to-day ?\\nJulia Vanturney [Van Turney], 1, Miller's-court, a charwoman, living with Harry Owen, said: I knew the deceased for some time as Kelly, and I knew Joe Barnett, who lived with her. He would not allow her to go on the streets. Deceased often got drunk. She said she was fond of another man, also named Joe. I never saw this man. I believe he was a costermonger.\\n[Coroner] When did you last see the deceased alive ? - On Thursday morning, at about ten o'clock. I slept in the court on Thursday night, and went to bed about eight. I could not rest at all during the night.\\n[Coroner] Did you hear any noises in the court ? - I did not. I heard no screams of \\\"Murder,\\\" nor any one singing.\\n[Coroner] Were you in the house when Joe Barnett called ? - Yes. I said, \\\"Well, Mary Jane, I shall not see you this evening again,\\\" and I left with her two men's dirty shirts, a little boy's shirt, a black overcoat, a black crepe bonnet with black satin strings, a pawn-ticket for a grey shawl, upon which 2s had been lent, and a little girls white petticoat.\\n[Coroner] Have you seen any of these articles since? - Yes; I saw the black overcoat in a room in the court on Friday afternoon.\\n[Coroner] Did the deceased ever speak to you about being afraid of any man ? - She did not.\\nInspector Beck, H Division, deposed that, having sent for the doctor, he gave orders to prevent any persons leaving the court, and he directed officers to make a search. He had not been aware that the deceased was known to the police.\\n[Coroner] Was it by your orders that the door was forced ? - No; I had an intimation from Inspector Beck that the bloodhounds had been sent for, and the reply had been received that they were on the way. Dr. Phillips was unwilling to force the door, as it would be very much better to test the dogs, if they were coming. We remained until about 1.30 p.m., when Superintendent Arnold arrived, and he informed me that the order in regard to the dogs had been countermanded, and he gave orders for the door to be forced. I agree with the medical evidence as to the condition of the room. I subsequently took an inventory of the contents of the room. There were traces of a large fire having been kept up in the grate, so much so that it had melted the spout of a kettle off. We have since gone through the ashes in the fireplace; there were remnants of clothing, a portion of a brim of a hat, and a skirt, and it appeared as if a large quantity of women's clothing had been burnt.\\n[Coroner] Can you give any reason why they were burnt ? - I can only imagine that it was to make a light for the man to see what he was doing. There was only one small candle in the room, on the top of a broken wine-glass. An impression has gone abroad that the murderer took away the key of the room. Barnett informs me that it has been missing some time, and since it has been lost they have put their hand through the broken window, and moved back the catch. It is quite easy. There was a man's clay pipe in the room, and Barnett informed me that he smoked it.\\n[Coroner] Is there anything further the jury ought to know ? - No; if there should be I can communicate with you, sir.\"", "vector_field": [-0.0008467165753245354, 0.011611142195761204, 0.016767876222729683, -0.02305922657251358, -0.018262581899762154, 0.02462187223136425, 0.010809437371790409, -0.011400525458157063, -0.01993393339216709, -0.012820495292544365, 0.007466732524335384, 0.02516540139913559, -0.0005486247828230262, 0.008023849688470364, -0.017691874876618385, 0.017066817730665207, 0.04013963043689728, 0.030383283272385597, 0.01898275688290596, -0.02823634259402752, -0.0016645581927150488, 0.03122575208544731, 0.008662496693432331, 0.0064612035639584064, -0.0014479957753792405, 0.02557304874062538, 0.01744728721678257, -0.02261081524193287, -0.006868850439786911, -0.007847202941775322, -0.007167791482061148, 0.009430231526494026, -0.025804048404097557, -0.025532284751534462, -0.03451410308480263, -0.005985615309327841, 0.015232405625283718, -0.013289288617670536, 0.016115641221404076, 0.0030420650728046894, 0.015490582212805748, 5.578605487244204e-05, -0.00975634902715683, -0.015164464712142944, -0.0003180495114065707, 0.008791584521532059, 1.3150328413757961e-05, -0.007167791482061148, -0.013425171375274658, 0.02611657790839672, 0.008302408270537853, 0.017746228724718094, -0.011393730528652668, 0.002924866508692503, 0.004759277682751417, -0.01842563971877098, 0.022243931889533997, 0.02301846258342266, -0.00732405623421073, -0.006848467979580164, 0.012018789537250996, -0.018738169223070145, -0.027516165748238564, -0.011108377948403358, -0.0036926015745848417, -0.012949583120644093, -0.027787931263446808, -0.003451410448178649, -0.028508106246590614, 0.0017528816824778914, 0.017189111560583115, 0.008125761523842812, 0.015667229890823364, 0.00502085080370307, 0.030029987916350365, -0.022896166890859604, -0.011767406947910786, 0.0005575420800596476, -0.0071813794784247875, -0.0007350383093580604, -0.0004231459752190858, -0.007752085104584694, -0.021686814725399017, 0.010632789693772793, 0.0112850246950984, -0.00894105527549982, -0.007160997483879328, 0.02364351972937584, 0.027448225766420364, 0.005761409644037485, 0.007847202941775322, 0.04674351215362549, 0.01596616953611374, 0.014906288124620914, -0.00847905594855547, 0.03372598811984062, -0.02247493341565132, 0.05421703681349754, -0.003926998469978571, -0.008805173449218273, -0.030899634584784508, -0.0028807048220187426, -0.006651438772678375, -0.008404320105910301, -0.020395932719111443, -0.03323680907487869, 0.010109643451869488, -0.008737231604754925, 0.0044059837237000465, -0.007215350400656462, 0.0035499250516295433, 0.013710523955523968, 0.02509746141731739, -0.00973596703261137, -0.009280760772526264, -0.013982288539409637, 0.011312201619148254, -0.032095398753881454, -0.02050463855266571, -0.0013172090984880924, 0.013982288539409637, 0.02259722724556923, -0.010592025704681873, -0.025776872411370277, 0.008010261692106724, 0.012820495292544365, -0.016631994396448135, 0.0061826445162296295, 0.00895464327186346, -0.01689017005264759, 0.016074875369668007, -0.008900290355086327, 0.02724440209567547, 0.014716053381562233, -0.011882907710969448, 0.008961438201367855, -0.02352122589945793, -0.011373348534107208, -0.013282494619488716, -0.05060257017612457, -0.0030165871139615774, 0.013200965709984303, -0.012304142117500305, -0.020803580060601234, 0.015286758542060852, 0.03734045475721359, 0.011223877780139446, 0.014457876794040203, 0.002688771113753319, 0.0072561148554086685, 0.014444288797676563, -0.012976760044693947, -0.0027023593429476023, -0.013649377040565014, 0.036226220428943634, -0.004558851011097431, 0.00919243786484003, 0.005608541890978813, -0.013527083210647106, -0.011183113791048527, -0.02006981521844864, 0.029540812596678734, 0.016754288226366043, 0.005611938890069723, 0.03524786978960037, 0.018126698210835457, -0.00037282705307006836, -0.011359760537743568, 0.0029877121560275555, -0.018235404044389725, 0.013431965373456478, 0.015531347133219242, -0.06217974051833153, 0.028942929580807686, 0.027937401086091995, 0.003210219321772456, 0.009382672607898712, 0.006926600355654955, -0.007996673695743084, 0.0005715549341402948, -0.03426951542496681, 0.012569112703204155, 0.003135484177619219, 0.036661043763160706, -0.026959048584103584, -0.030627870932221413, 0.0042327335104346275, 0.002417006529867649, 0.01631946489214897, -0.014607347548007965, 0.011733436957001686, 0.020708462223410606, 0.0031813443638384342, -0.01583028770983219, -0.6570181250572205, -0.03223128244280815, 0.008818761445581913, -0.004800042137503624, -0.0003106184594798833, 0.021904226392507553, 0.006539335940033197, 0.013533877208828926, -0.006600482854992151, 0.023888109251856804, -0.02921469509601593, 0.0360088087618351, 0.01536828838288784, 0.002060315338894725, -0.026456283405423164, -0.015259582549333572, -0.017202699556946754, 0.007507496979087591, 0.021224815398454666, 0.005883703473955393, -0.02255646139383316, 0.019390404224395752, 0.002846734132617712, 0.023792991414666176, -0.008533408865332603, -0.002535903360694647, 0.027081342414021492, -0.01843922771513462, 0.0033545943442732096, 0.007249320857226849, -0.03326398879289627, 0.0031473739072680473, 0.015653641894459724, -0.0018293154425919056, 0.04204198345541954, -0.013642583042383194, -0.026959048584103584, 0.0282635185867548, 0.0042565129697322845, 0.01953987404704094, -0.002950344467535615, -0.017827758565545082, 0.020219286903738976, -0.013642583042383194, 0.009396261535584927, -0.016074875369668007, 0.008458673022687435, 0.007602614816278219, 0.004552057012915611, 0.0012628561817109585, 1.0443298378959298e-05, 0.0016857897862792015, -0.006220012437552214, -0.002107874257490039, 0.005183910019695759, -0.012494377791881561, 0.03136163577437401, -0.033508576452732086, 0.0035193515941500664, 0.0026225284673273563, 0.010340643115341663, 0.009015791118144989, -0.003950777929276228, -0.014254053123295307, -0.027366695925593376, 0.0045384690165519714, -0.006005997769534588, -0.006090924143791199, 0.005034439265727997, -0.006389865186065435, 0.011604348197579384, 0.02045028656721115, -0.016672758385539055, -0.014036641456186771, 0.00035435555037111044, 0.017827758565545082, 0.02210805006325245, 0.0014581868890672922, 0.02098022773861885, -0.018085934221744537, 0.002872212091460824, -0.008689673617482185, -0.03992221876978874, 0.016115641221404076, 0.01634664088487625, -0.006400056183338165, -0.008017055690288544, 0.013445553369820118, 0.00020637124544009566, -0.01483834721148014, -0.0011388635030016303, 0.015993347391486168, -0.0009282459504902363, -0.006824688520282507, -0.008336379192769527, 0.04524880647659302, -0.029540812596678734, -0.016278699040412903, 0.022733109071850777, -0.02869834192097187, 0.0141317592933774, -0.013907553628087044, 0.014688876457512379, -0.009131290949881077, 0.0061554680578410625, -0.004946115892380476, 0.007025115191936493, -0.00125096645206213, 0.027991754934191704, -0.01941758021712303, -0.003940586932003498, -0.0072629088535904884, 0.020355168730020523, -0.003188138594850898, -0.010442554950714111, -0.018860463052988052, 0.03519351780414581, 0.013479524292051792, 0.008628525771200657, -0.020151345059275627, 0.015789523720741272, -0.009409849531948566, 0.014349170960485935, -0.020694874227046967, 0.01182175986468792, 0.023956049233675003, 0.004022116307169199, 0.0013936428586021066, -0.0027261385694146156, -0.005788586102426052, -0.013506700284779072, -0.003505763364955783, 0.02414628490805626, -0.008730437606573105, 0.03435104712843895, -0.007222144398838282, 0.02512463741004467, -0.014865524135529995, 0.008886702358722687, -0.02467622607946396, 0.0003106184594798833, -0.004188572056591511, -2.9007685952819884e-05, -0.006070541683584452, -0.006753350142389536, -0.0236299317330122, -0.01638740487396717, -0.0036552338860929012, -0.03590010479092598, 0.020749228075146675, -0.03168775141239166, -0.0022743300069123507, -0.01280011236667633, -0.0007201762055046856, -0.0036654251161962748, 0.009865054860711098, -0.025532284751534462, -0.009443819522857666, -0.026442695409059525, -0.007928731851279736, 0.014090994372963905, 0.01150923129171133, -0.04356386512517929, -0.004365218803286552, 0.0015218817861750722, -0.015476994216442108, 0.02615734376013279, -0.01124426070600748, -0.01027270220220089, -0.020830756053328514, 0.008526614867150784, 0.0016654074424877763, -0.002890896052122116, -0.001423367066308856, -0.0025647785514593124, -0.0023490653838962317, -0.019689345732331276, 0.012684612534940243, 0.01279331836849451, 0.0030896237585693598, 0.009131290949881077, -0.002816160675138235, -0.007439556065946817, 0.007249320857226849, 0.021292757242918015, 0.004820424597710371, -0.009389466606080532, 0.0030998149886727333, -0.008411114104092121, 0.021863462403416634, 0.007670555729418993, -0.009987348690629005, -0.007031909190118313, 0.003162660636007786, 0.0030709400307387114, 0.005027645267546177, 0.023222284391522408, -0.0014522421406581998, -0.008655702695250511, 0.02670087292790413, 0.017854934558272362, 0.017719052731990814, 0.01458017062395811, 0.006862056441605091, 0.0028263519052416086, -0.019865991547703743, 0.020124169066548347, -0.02716287225484848, 0.03644363209605217, -0.008513025939464569, -0.010897760279476643, -0.01943116821348667, -0.015164464712142944, -0.012813701294362545, -0.02262440323829651, 0.022447755560278893, -0.0012959775049239397, 0.02618451975286007, 0.005469262599945068, -0.014947053045034409, 0.003138881176710129, 0.007738497108221054, 0.027855871245265007, 0.024377284571528435, -0.02153734490275383, -0.013268906623125076, 0.018303345888853073, -0.015069346874952316, -0.004083263222128153, -0.012881642207503319, -0.019132228568196297, -0.00131805834826082, 0.01737934723496437, 0.008519819937646389, 0.01746087521314621, 0.0062607768923044205, 0.0021282564848661423, -0.0036993955727666616, 0.022692345082759857, -0.01414534728974104, -0.008295614272356033, 0.00769773218780756, 0.01997469738125801, -0.01992034539580345, 0.004555454012006521, -0.012487583793699741, 0.008852732367813587, 0.005299409851431847, -0.00795590877532959, 0.022841814905405045, 0.021428639069199562, -0.0019108448177576065, 0.00919923186302185, 0.0020925875287503004, 0.006671820767223835, -0.012372083961963654, -0.009647643193602562, 0.007826820947229862, 0.031497515738010406, 0.02262440323829651, 0.0022335653193295, -0.006366085726767778, 0.0082412613555789, -0.013112641870975494, -0.009369084611535072, 0.01182175986468792, -0.003763939719647169, 0.0011371650034561753, 0.003791116178035736, -0.013798847794532776, -0.019145816564559937, -0.023290226235985756, 0.011495642364025116, -0.00741237960755825, 0.028372224420309067, -0.014240465126931667, 0.004884968511760235, 0.016604816541075706, 0.012399259954690933, 0.01790928654372692, -0.049950335174798965, -0.028317872434854507, 0.02040952257812023, 0.011957642622292042, -0.00847226195037365, -0.018778933212161064, -0.010748290456831455, 0.004942718893289566, -0.022760285064578056, 0.019662169739603996, 0.005299409851431847, -0.0013197568478062749, -0.022936932742595673, -0.01258949562907219, 0.012643848545849323, -0.008981820195913315, -0.0002772848238237202, -0.013941524550318718, 0.007140615023672581, 0.017026051878929138, -0.001185573055408895, -0.013173788785934448, 0.0021673226729035378, -0.012942789122462273, 0.039704807102680206, -0.014593759551644325, -0.01483834721148014, -0.013635789044201374, 0.014444288797676563, -0.016210759058594704, -0.004915542434900999, 0.015748757869005203, -0.025953520089387894, 0.014240465126931667, -0.008152938447892666, 0.03475869446992874, -0.00037792263901792467, 0.00794232077896595, 0.01207993645220995, -0.004049292765557766, 0.003211917821317911, -0.0205182284116745, -0.021822698414325714, -0.00821408536285162, 0.118598073720932, 0.047966454178094864, -0.01000773161649704, 0.014077406376600266, 0.009165260940790176, 0.00016412034165114164, 0.002372844610363245, -0.0010225143050774932, 0.0317692831158638, 0.01207314245402813, 0.021455815061926842, 0.00045308255357667804, -0.0036212634295225143, 0.007772467564791441, 0.017868522554636, -0.022977696731686592, -0.005713850725442171, 0.012759348377585411, 0.004619998391717672, -0.009980554692447186, -0.010184378363192081, -0.01850716955959797, 0.00384886609390378, 0.020762816071510315, -0.029595164582133293, -0.013513495214283466, -0.006631056312471628, 0.011713054031133652, 0.00769773218780756, 0.008961438201367855, -0.02157810889184475, 0.018643051385879517, 0.001356275170110166, -7.995027999641025e-07, -0.020694874227046967, 0.0026293224655091763, 0.025926342234015465, -0.013812435790896416, 0.008302408270537853, -0.015001405961811543, 0.009138084948062897, -0.0017460875678807497, 0.008030643686652184, -0.010938525199890137, 0.02725799009203911, -0.0044535426422953606, 0.0035159545950591564, 0.007439556065946817, -0.004134219139814377, -0.013472730293869972, 0.02314075641334057, -0.0010522385127842426, -0.0112850246950984, -0.008282026275992393, -0.00974955502897501, -0.006138483062386513, -0.007820026017725468, 0.009083732031285763, -0.010218349285423756, 0.05598350986838341, -0.0025206166319549084, -0.02622528374195099, 0.006216615438461304, 0.007915143854916096, -0.016822228208184242, -0.00972917303442955, -0.025301285088062286, 0.014620935544371605, -0.007514291442930698, 0.0018530949018895626, 0.01538187637925148, -0.02352122589945793, -0.007724908646196127, 0.022243931889533997, -0.006033174227923155, 0.028426578268408775, 0.03079092875123024, -0.0136018181219697, 0.013730906881392002, 0.00976314302533865, -0.017773404717445374, 0.00794911477714777, -0.019580639898777008, -0.010829819366335869, -0.025736108422279358, 0.016292287036776543, 0.010428966954350471, -0.0219721682369709, -0.0307094007730484, 0.028073282912373543, 0.019852403551340103, 0.003763939719647169, -0.014485053718090057, -0.015694405883550644, -0.021713992580771446, -0.007847202941775322, -0.007622997276484966, 0.03443257510662079, -0.0037809249479323626, -0.017854934558272362, 0.018670227378606796, -0.023181520402431488, 0.0053639537654817104, -0.016047699376940727, 0.006046762224286795, -0.025437166914343834, -0.01437634788453579, 0.019336052238941193, -0.004871380515396595, 0.005717247724533081, -0.001380054629407823, -0.023235874250531197, 0.01941758021712303, 0.014226877130568027, -0.011896495707333088, 0.011889701709151268, 0.006692203227430582, 0.03628057613968849, 0.0021197639871388674, 0.008886702358722687, 0.006593688856810331, -0.004623395390808582, 0.029948459938168526, 0.010911349207162857, -0.007915143854916096, 0.030002811923623085, -0.014158936217427254, -0.03437822312116623, -0.012263378128409386, -0.013995877467095852, -0.014620935544371605, -0.006026380229741335, -0.004647174850106239, -0.0195262860506773, -0.0307094007730484, -0.018085934221744537, 0.01792287640273571, 0.014444288797676563, -0.005992409307509661, 0.006838276982307434, -0.018765345215797424, -0.014947053045034409, -0.011142348870635033, -0.01792287640273571, 0.01180817186832428, -0.024866461753845215, -0.010401790030300617, 0.007840408943593502, -0.002029741881415248, -0.0029316607397049665, -0.03334551677107811, -0.0030488590709865093, -0.012922407127916813, 0.016074875369668007, 0.003208520822227001, -0.014281230047345161, 0.0011787789408117533, -0.012501171790063381, 0.02818198874592781, 0.007996673695743084, 0.03171492740511894, 0.015150876715779305, 0.026768812909722328, -0.00024352656328119338, -0.003560116281732917, 0.01458017062395811, -0.010877378284931183, -0.009797113947570324, -0.028942929580807686, 0.0063321152701973915, 0.030057165771722794, 0.003913410473614931, 0.00945061445236206, -0.004626792389899492, -0.017569581046700478, 0.028399400413036346, -0.006396659184247255, 0.0072357323952019215, -0.0036484398879110813, -0.022393403574824333, -0.020096993073821068, -0.016536876559257507, -0.005554188974201679, -0.013744494877755642, -0.02210805006325245, -0.007663761731237173, 0.024214226752519608, -0.014539406634867191, -0.008961438201367855, -0.0390525721013546, 0.005109174642711878, -0.04375410079956055, 0.0036892045754939318, -0.0011074407957494259, -0.008431497029960155, 0.011882907710969448, -0.006535938475281, -0.00031231698812916875, 0.000494696490932256, -0.012630259618163109, -0.012378877960145473, 0.0011465068673714995, 0.002418705029413104, -0.024336520582437515, -0.010367820039391518, 0.010422172956168652, 0.0034310282208025455, -0.024282166734337807, 0.019662169739603996, -0.0062302034348249435, -0.02159169688820839, -0.005737630184739828, -0.01691734604537487, -0.029486458748579025, 0.0021911021322011948, 0.02360275574028492, 9.288829460274428e-05, 0.01028629019856453, -0.008574172854423523, -0.01004170160740614, -0.012745759449899197, -0.017555993050336838, 0.03894386813044548, 0.008873114362359047, 0.014063818380236626, -0.016767876222729683, 0.006882438436150551, -0.00705229165032506, 0.020722050219774246, 0.014865524135529995, 0.0015133891720324755, 0.006226806435734034, 0.02252928540110588, 0.009321525692939758, 0.004660762846469879, 0.006464600563049316, 0.011420907452702522, -0.0164825227111578, -0.013676553964614868, 0.029975635930895805, 0.014457876794040203, -0.0034344252198934555, -0.01280011236667633, -0.005720644723623991, -0.006471394561231136, -0.007222144398838282, -0.0199882872402668, -0.026429107412695885, -0.017066817730665207, -0.012297348119318485, -0.009144878946244717, 0.012412847951054573, -0.024418050423264503, 0.020219286903738976, 0.021917814388871193, -0.028508106246590614, -0.022773873060941696, -0.023181520402431488, -0.013200965709984303, 0.02924187108874321, -0.0012263377429917455, 0.006124894600361586, -0.047422923147678375, -0.0017987419851124287, 0.0019295286620035768, 0.01581669971346855, -0.008546996861696243, 0.0017002272652462125, -0.009131290949881077, 0.027964577078819275, 0.0011006466811522841, -0.006315129809081554, -0.018221816048026085, 0.0020671095699071884, 0.006033174227923155, -0.006715982686728239, 0.0006913011893630028, -0.027013400569558144, 0.004881571512669325, 0.004324454348534346, 0.005037836264818907, 0.01801799237728119, 0.00013290987408254296, -0.0072357323952019215, -0.01686299405992031, -0.03671539947390556, -0.0021452419459819794, -0.007127026561647654, -0.0009562717168591917, -0.019295286387205124, -0.030926812440156937, -0.03557398542761803, -0.004983483348041773, 0.004884968511760235, -0.023385344073176384, -0.005761409644037485, -0.004630189388990402, 0.030899634584784508, -0.01638740487396717, 0.004925733432173729, -0.022434167563915253, -5.7643821492092684e-05, -0.04049292579293251, -0.006291350349783897, -0.0016620104433968663, -0.003757145721465349, 0.0360088087618351, -0.008968232199549675, -0.022964108735322952, -0.040791865438222885, 0.02159169688820839, 0.013214553706347942, 0.006247188895940781, -0.02978540025651455, 0.01099967211484909, -0.0004027636314276606, 0.0029146752785891294, -0.001997469924390316, -0.023874521255493164, -5.398136636358686e-05, -0.027502577751874924, 0.0004912994336336851, -0.008920673280954361, -0.030084341764450073, -0.002478153444826603, 0.006063747685402632, -0.007670555729418993, -0.0008772900910116732, -0.03293786942958832, 0.0022709330078214407, 0.012378877960145473, 0.025817636400461197, -0.005880306474864483, 0.0052790273912250996, -0.006505365017801523, -0.014090994372963905, -0.01796364039182663, 0.019567051902413368, -0.01208673045039177, 0.009511761367321014, 0.0246898140758276, -0.008669290691614151, -0.016523288562893867, -0.008322791196405888, 0.005703659728169441, 0.00030085191247053444, 0.0033630868420004845, -0.003256079740822315, -0.013649377040565014, -0.008560584858059883, -0.025817636400461197, 0.032041046768426895, 0.013391200453042984, -0.012976760044693947, -0.036199044436216354, 0.0054556746035814285, -0.0503036268055439, -0.010931731201708317, -0.00797629076987505, 0.012195436283946037, 0.01850716955959797, 0.013744494877755642, 0.02509746141731739, 0.011896495707333088, 0.0016153008909896016, 0.02463546209037304, -0.02921469509601593, -0.019240934401750565, 0.020735640078783035, -0.01592540554702282, 0.030029987916350365, -3.2245505281025544e-05, -0.019716521725058556, 0.006569909397512674, 0.011869318783283234, -0.00680430606007576, 0.009572908282279968, 0.010388202033936977, -0.019689345732331276, -0.031606223434209824, -0.014675288461148739, -0.007269703317433596, 0.012222613207995892, 0.019580639898777008, 0.0141317592933774, -0.014539406634867191, -0.011624731123447418, 0.0028807048220187426, 0.01278652437031269, -0.0020976830273866653, 0.0022709330078214407, 0.037204574793577194, 0.006243791896849871, 0.010578436776995659, -0.002316793194040656, 0.009239996783435345, 0.03850904479622841, -0.012406053952872753, 0.047341395169496536, 0.009409849531948566, -0.005377542227506638, 0.018235404044389725, 0.020749228075146675, -0.009552525356411934, 0.01124426070600748, 0.00473889522254467, -0.04522163048386574, -0.017759816721081734, -0.01640099287033081, -0.022977696731686592, 0.01509652379900217, -0.030111517757177353, 0.010761878453195095, -0.008608143776655197, -0.0038590573240071535, -0.02149658091366291, -0.010442554950714111, -0.019227346405386925, 0.01795005239546299, -0.004592821933329105, 0.00020318650058470666, 0.006138483062386513, 0.013194171711802483, -0.028888577595353127, 0.008302408270537853, -0.02674163691699505, -0.008017055690288544, -0.016020523384213448, -0.028018930926918983, 0.0043550278060138226, 0.007772467564791441, -0.013221347704529762, 0.0042327335104346275, 0.01125784870237112, -0.009355496615171432, -0.008526614867150784, 0.21849873661994934, -0.010680348612368107, -0.008017055690288544, 0.02829069457948208, -0.030084341764450073, -0.02463546209037304, 0.005445483140647411, 0.011937260627746582, -0.0013664664002135396, 0.02407834306359291, -0.007439556065946817, -0.0027431240305304527, -0.011692672036588192, 0.0153547003865242, 0.024934401735663414, -0.013099053874611855, -0.020803580060601234, -0.05351044982671738, 0.0062539828941226006, -0.02666010707616806, 0.020857933908700943, -0.0005558435223065317, -0.009321525692939758, -0.005700262729078531, 0.03424233943223953, 0.014906288124620914, -0.0020688080694526434, -0.0019329257775098085, 0.019077874720096588, -0.006478188559412956, -0.01905069872736931, 0.0021520359441637993, 0.010952113196253777, -0.020219286903738976, -0.009871848858892918, 0.0032526825089007616, -0.01508293580263853, -0.010870584286749363, 0.020667698234319687, 0.005751218181103468, -0.005190704017877579, -0.014729641377925873, 0.018738169223070145, -0.018683817237615585, 0.004096851218491793, 0.0003178371989633888, -0.00945061445236206, -0.011916877701878548, -0.008886702358722687, -1.6122752640512772e-05, -0.03704151511192322, 0.0037537484895437956, 0.01993393339216709, 0.018806111067533493, 0.013533877208828926, -0.02875269576907158, -0.000526543939486146, 0.011366554535925388, 0.004524880554527044, -0.013268906623125076, -0.01693093404173851, 0.009688408114016056, -0.016237935051321983, 0.03829163312911987, -0.01697169989347458, 0.043319277465343475, -0.007595820818096399, 0.0031422781758010387, -0.004667556844651699, -0.007915143854916096, -0.009233202785253525, 0.012018789537250996, -0.029866930097341537, 0.005319792311638594, 0.0026429106947034597, -0.02411910891532898, 0.024852873757481575, 0.001866683131083846, 0.023725049570202827, 0.008288820274174213, -0.0045350720174610615, 0.007704526651650667, 0.007215350400656462, 0.0033579913433641195, -0.02608940191566944, -0.02725799009203911, -0.002929962007328868, 0.00059193727793172, 0.020232874900102615, -0.013187377713620663, 0.0011286723893135786, 0.020830756053328514, 0.004983483348041773, 0.0004522332747001201, 0.019105050712823868, 0.0002972425427287817, 0.029975635930895805, 0.028997283428907394, -0.01303111296147108, 0.00705908564850688, -0.04706962779164314, -0.006372879724949598, 0.021442227065563202, -0.011230671778321266, -0.01996110938489437, 0.003733366262167692, -0.013533877208828926, 0.0219721682369709, 0.021333521232008934, -0.016672758385539055, 0.0009809003677219152, -0.027271578088402748, -0.003910013008862734, 0.010632789693772793, 0.02248852141201496, 0.02155093289911747, 0.013730906881392002, -0.010062084533274174, -0.007881173864006996, -0.006851864978671074, -0.011380142532289028, -0.008200496435165405, 0.0072357323952019215, 0.01483834721148014, 0.023792991414666176, -0.030383283272385597, -0.010625995695590973, -0.005061615724116564, 0.0020773005671799183, -0.026945460587739944, 0.02155093289911747, -0.015042170882225037, 0.01126464270055294, 0.004045895300805569, -0.002843337133526802, 0.025953520089387894, 0.030627870932221413, -0.02045028656721115, -0.012127495370805264, 0.005221277475357056, 0.033508576452732086, 0.001657764078117907, 0.0100281136110425, -0.008676084689795971, 0.002892594551667571, -0.01903711073100567, 0.017229875549674034, -0.017134757712483406, -0.009525349363684654, -0.01593899354338646, -0.025396402925252914, 0.0023015064653009176, -0.01800440438091755, -0.044297631829977036, 0.009321525692939758, -0.0164145827293396, -0.0053062038496136665, -0.019621403887867928, 0.00154141488019377, 0.005238262936472893, -0.031551871448755264, -0.015409053303301334, 0.026401931419968605, -0.009892231784760952, -0.0052620419301092625, 0.0011685878271237016, -0.17490769922733307, 0.018833287060260773, 0.020844345912337303, -0.018085934221744537, 0.0399765744805336, 0.0003216588811483234, -0.010096054524183273, 0.004185175057500601, -0.029133165255188942, -0.01856152154505253, 0.006420438643544912, -0.001619547139853239, -0.005696865264326334, -0.004022116307169199, -0.015626464039087296, -0.01795005239546299, -0.02411910891532898, 0.015014993958175182, 0.02872551791369915, 0.018765345215797424, 0.036199044436216354, 0.006325321272015572, 0.020681286230683327, -0.012440024875104427, -0.0045350720174610615, 0.0003832305665127933, 0.01054446678608656, 0.015803111717104912, -0.009423437528312206, -0.02674163691699505, -0.02978540025651455, -0.010361025109887123, -0.007371614687144756, 0.015572112053632736, 0.01583028770983219, 0.008764408528804779, 0.003893028013408184, -0.022216755896806717, -0.006834879983216524, 0.011740230955183506, 0.022991284728050232, 0.039134103804826736, -0.0164825227111578, -0.01795005239546299, -0.030682222917675972, 0.025396402925252914, 0.02877987176179886, 0.00947779044508934, 0.004915542434900999, -0.0006097718141973019, 0.012154672294855118, -0.021809108555316925, 0.018330521881580353, 0.030437635257840157, 0.0026242269668728113, 0.00034649987355805933, -0.007704526651650667, 0.0018547934014350176, -0.009831084869801998, 0.01179458387196064, -0.012630259618163109, -0.007609408814460039, 0.014199700206518173, 0.0022743300069123507, -0.013200965709984303, -0.013153406791388988, -0.010381408035755157, -0.011169524863362312, -0.025763284415006638, -0.0010140216909348965, -0.02402399107813835, -0.006107909604907036, 0.02918751910328865, -0.014050230383872986, 0.01150923129171133, 0.006821291521191597, -0.002532506361603737, -0.008601349778473377, 0.00617585051804781, -0.0118149658665061, 0.011624731123447418, 0.047504451125860214, -0.04242245480418205, -0.010469730943441391, -0.001632286119274795, -0.0148247592151165, 0.025953520089387894, -0.009090526029467583, 0.013676553964614868, -0.010870584286749363, 0.011740230955183506, -0.016006935387849808, 0.005316394846886396, -0.002213183091953397, 0.0007617901428602636, 0.007792850024998188, 0.00411383667960763, -0.00326457223854959, -0.012222613207995892, -0.01843922771513462, 0.008302408270537853, -0.009579702280461788, -0.005608541890978813, 0.03185081109404564, 0.03590010479092598, 0.0025172196328639984, -0.0094642024487257, 0.011128760874271393, 0.023480461910367012, -0.012732171453535557, -0.002580065280199051, -0.01205955445766449, 0.033644456416368484, 0.026782400906085968, -0.009878642857074738, 0.029975635930895805, 0.024418050423264503, -0.021238403394818306, 0.026021460071206093, -0.007120232563465834, 0.05329303815960884, -0.011230671778321266, 0.0018208228284493089, 0.012623465619981289, -0.01124426070600748, -0.015803111717104912, -0.11338019371032715, -0.012997142039239407, 0.007269703317433596, 0.03027457743883133, 0.001267102430574596, 0.00975634902715683, -0.009606878273189068, 0.020232874900102615, 0.0014794185990467668, 0.019105050712823868, -0.012127495370805264, -0.021809108555316925, 0.0015745362034067512, 0.0032798589672893286, 0.01511011179536581, -0.0042599099688231945, 0.023208696395158768, -0.02455393224954605, -0.02160528674721718, 0.025029519572854042, 0.008098585531115532, 0.013105847872793674, -0.004582630470395088, -0.03247587010264397, -0.007290085311979055, -0.019675757735967636, -0.03630775213241577, 0.03185081109404564, 0.008118967525660992, 0.025518696755170822, 0.012256583198904991, -0.0022505505476146936, 8.301559137180448e-05, -0.016740700230002403, -0.005554188974201679, -0.012147877365350723, -0.018045170232653618, -0.006702394690364599, 0.025043107569217682, -0.012766142375767231, 0.0007222993299365044, 0.02248852141201496, 0.03492175042629242, 0.003872645553201437, 0.016156405210494995, 0.00679411506280303, -0.04424327611923218, 0.04416174814105034, -0.008445085026323795, 0.0009571209666319191, 0.003634851658716798, 0.01638740487396717, 0.013805641792714596, -0.0054624686017632484, 0.02615734376013279, -0.011380142532289028, 0.02212163805961609, 0.0037809249479323626, -0.017705464735627174, 0.002525712363421917, -0.007745291106402874, 0.010958907194435596, -0.0009681613883003592, 0.00945740845054388, 0.016795052215456963, -0.00918564386665821, -0.01208673045039177, -0.005802174098789692, -0.00501745380461216, -0.015191641636192799, 0.013160200789570808, 0.012562318705022335, -0.009661231189966202, -0.001948212506249547, -0.016727110370993614, -0.017569581046700478, -0.030383283272385597, 0.010320261120796204, -0.00017707161896396428, -0.011121965944766998, -0.007351232692599297, -0.005296012852340937, 0.0005808968562632799, -0.020789992064237595, 0.016686346381902695, 0.004616601392626762, -0.004884968511760235, -0.0013010730035603046, 0.013955112546682358, -0.015313935466110706, 0.012630259618163109, 0.026863930746912956, 0.0028501313645392656, -0.029051635414361954, -0.018126698210835457, 0.037639398127794266, -0.013194171711802483, -0.006651438772678375, -0.008424703031778336, -0.005394527222961187, -0.02712210826575756, 0.004630189388990402, -0.05989691987633705, 0.008458673022687435, 0.006977556273341179, -0.011393730528652668, -0.012840877287089825, -0.00339535903185606, 0.009124496951699257, 0.025342049077153206, -0.004752483684569597, 0.018683817237615585, -0.01947193406522274, 0.036199044436216354, -0.011889701709151268, -0.0028841018211096525, -0.03853622078895569, -0.007344438228756189, 0.0021554329432547092, 0.00652574747800827, 0.009117702022194862, 0.02301846258342266, 0.009647643193602562, 0.00591427693143487, 0.0035906897392123938, 0.025871990248560905, -0.010884172283113003, 0.004935924429446459, -0.014797582291066647, 0.02610298991203308, 0.004813630599528551, -0.023344580084085464, 0.018208228051662445, -0.03804704546928406, -0.007493908982723951, 0.007772467564791441, -0.001955006504431367, -0.018235404044389725, -0.015803111717104912, 0.023806579411029816, 0.00705908564850688, 0.027556931599974632, -0.019743697717785835, -0.015572112053632736, 0.0005380089860409498, 0.014308406040072441, 0.022352637723088264, -0.0029588371980935335, 0.0015388670144602656, 0.01634664088487625, -0.011577172204852104, -0.021673226729035378, 0.0036993955727666616, 0.026809578761458397, -0.019594227895140648, -0.01698528788983822, -0.02769281342625618, 0.03557398542761803, 0.01258949562907219, -0.003733366262167692, 0.00411383667960763, -0.04008527845144272, 0.026306813582777977, -0.0038522633258253336, 0.0015694405883550644, 0.00169767951592803, 0.04109080880880356, -0.013724112883210182, -0.018262581899762154, -0.006423835642635822, 0.019784463569521904, -0.023167932406067848, -0.01842563971877098, -0.015748757869005203, 0.01438993588089943, 0.038672104477882385, 0.006199629977345467, 0.0064102476462721825, 0.003425932489335537, 0.010442554950714111, -0.0099941436201334, 0.024716990068554878, 0.01206634845584631, 0.016074875369668007, -0.008078202605247498, 0.0011133856605738401, 0.02714928425848484, -0.024445226415991783, -0.02160528674721718, 0.016645582392811775, 0.00036390978493727744, 0.02411910891532898, 0.015979759395122528, 0.012541936710476875, 0.014308406040072441, -0.004327851347625256, -0.00022930138220544904, 0.009688408114016056, 0.001267102430574596, -0.0035906897392123938, 0.030084341764450073, -0.009362290613353252, 0.013377612456679344, 0.003685807343572378, -0.017596758902072906, -0.012433230876922607, -0.021211227402091026, 0.0017350470880046487, -0.007215350400656462, -0.02002905122935772, -0.019227346405386925, -6.800749815738527e-06, 0.009851466864347458, -0.009423437528312206, 0.005839542020112276, 0.03187798708677292, -0.01592540554702282, 0.008152938447892666, 0.0266736950725317, -0.0014556391397491097, -0.025545872747898102, 0.02195858024060726, -0.006427232641726732, -0.028915753588080406, 0.029921282082796097, 0.003505763364955783, 0.030546341091394424, 0.006417041644454002, 0.016781464219093323, -0.015218817628920078, 0.009586496278643608, -0.033100929111242294, 0.01026590820401907, -0.006773732602596283, -0.03342704474925995, 0.001054786378517747, 0.0063626887276768684, -0.023738637566566467, 0.019186580553650856, 0.026999812573194504, 0.0021197639871388674, 0.03451410308480263, 0.005975424312055111, -0.03331834077835083, 0.00383867509663105, -0.00589389493688941, 0.019336052238941193, 0.01953987404704094, -0.0013282494619488716, -0.020205698907375336, -0.014172524213790894, 0.009063349105417728, 0.00012473569950088859, 0.012521553784608841, -0.025844814255833626, -0.02061334438621998, 0.005758012644946575, 0.018276169896125793, 0.01414534728974104, 0.014036641456186771, 0.00732405623421073, 0.008295614272356033, -0.004664159845560789, -0.008601349778473377, -0.00307943276129663, -0.0277335774153471, -0.008186908438801765, 0.025437166914343834, 0.0015677420888096094, -0.01233131904155016, -0.012732171453535557, 0.010775466449558735, 0.001594069297425449, -0.017243463546037674, -0.013180582784116268, 0.012297348119318485, -0.002661594655364752, -0.030600694939494133, -0.004956306889653206, 0.015178052708506584, -0.0003870522486977279, -0.00947779044508934, 0.025776872411370277, -0.01636022888123989, -0.010632789693772793, -0.017773404717445374, -0.007344438228756189, -0.003957571927458048, -0.027883049100637436, -0.007099850103259087]} +{"id": "test:10000064", "text": "\"I did a bit of work on andinfinity.de in regards to style and function. Also rss now works!\\nThere\u2019s now an rss feed. I also did a lot of work on the stylesheet. I hope the readability has increased and most of the bugs are gone. There\u2019s a bit more in the pipeline, but first I\u2019d like to work on my habit on writing posts more regular. The work- in-progress folder is bursting, but nothing\u2019s really finished.\"", "vector_field": [0.003713481593877077, 0.013934135437011719, 0.004945588763803244, -0.026879839599132538, -0.001054498367011547, 0.011689572595059872, -0.023379145190119743, -0.02893907204270363, -0.013824310153722763, -0.015663890168070793, 0.0407453328371048, -0.009767623618245125, -0.0054432363249361515, -0.01656995341181755, 0.02313203737139702, 0.002785111078992486, 0.0190410315990448, 0.01968625746667385, 0.001178910257294774, -0.013288909569382668, -0.007351457607001066, 0.009795079939067364, -0.01118162926286459, -0.014634274877607822, -0.028829246759414673, -0.016020823270082474, 0.0034011646639555693, -0.022610366344451904, 0.0030682554934173822, -0.003699753200635314, 0.02188277058303356, -0.015581521205604076, -0.0007760730222798884, -0.005487852729856968, -0.018285978585481644, -0.018629183992743492, 0.008943930268287659, -0.027305414900183678, 0.00849776342511177, -0.019521517679095268, 0.012616227380931377, -0.013501697219908237, -0.0029052330646663904, -0.042282894253730774, -0.00969898235052824, 0.019150856882333755, -0.006146807223558426, -0.002582619898021221, 0.005065710283815861, 0.002628952730447054, 0.006565517745912075, -0.014565633609890938, -0.025438377633690834, -0.015622706152498722, -0.005559925921261311, -0.01118162926286459, 0.02342033013701439, 0.032178930938243866, -0.0042282892391085625, 0.0013470808044075966, 0.007845673710107803, 0.014648002572357655, -0.007351457607001066, 0.007783896289765835, -0.006146807223558426, 0.006448827683925629, -0.005130919627845287, -0.002409301232546568, -0.0063561624847352505, 0.0072416323237121105, 0.040306031703948975, 0.005958044435828924, -0.029350917786359787, 0.004039526451379061, 0.014702915214002132, -0.0012200948549434543, 0.003861059667542577, -0.014881382696330547, 0.022253433242440224, 0.009602884761989117, 0.0162679310888052, 0.00040927232475951314, -0.012005321681499481, 0.027058307081460953, 0.024683326482772827, 0.024518586695194244, 0.0036002236884087324, 0.02129245735704899, -0.010708006098866463, -0.01383117400109768, -0.003610519925132394, 0.033798858523368835, 0.020509948953986168, 0.028046738356351852, -0.008717414923012257, 0.0008009554003365338, -0.039894185960292816, 0.015842357650399208, -0.007859401404857635, -0.007804488763213158, -0.004190536681562662, 0.018574271351099014, -0.014950023032724857, -0.014963751658797264, -0.027277957648038864, 0.0037786904722452164, 0.020688416436314583, -0.005130919627845287, 0.03349683806300163, -0.005371163133531809, -0.009987274184823036, 0.021992595866322517, 0.019480332732200623, -0.014950023032724857, 0.0009052040404640138, -0.006558653432875872, 0.033661577850580215, -0.01606200821697712, 0.0017134525114670396, -0.00870368629693985, 0.01504612062126398, -0.002670137444511056, 0.026632731780409813, 0.0006653892924077809, -0.004139055963605642, 0.026989664882421494, -0.015554064884781837, -0.008360481821000576, 0.010460898280143738, -0.01372821256518364, 0.02108653448522091, 0.006706231739372015, 0.027978096157312393, 0.005494717042893171, 0.005628567188978195, 0.016473855823278427, -0.009644069708883762, 0.0021930818911641836, -0.006675343494862318, -0.024010643362998962, 0.007502468302845955, 0.03555607050657272, -0.005741824861615896, -0.011112987995147705, 0.005103462841361761, 0.027154404670000076, 0.024175381287932396, -0.01372821256518364, 0.025712942704558372, -0.013357550837099552, -0.01924695447087288, -0.024806879460811615, 0.01692688651382923, -0.004173376597464085, 0.008854697458446026, 0.01419497188180685, -0.008806648664176464, 0.005048550199717283, 0.00534370681270957, -0.0002820718800649047, 0.011545427143573761, 0.029268549755215645, 0.012986889109015465, 0.0018412965582683682, 0.030202068388462067, 0.025246182456612587, 0.03121795505285263, -0.011559154838323593, -0.03286534175276756, 0.004451373126357794, 0.01341246347874403, 0.011421873234212399, -0.029598025605082512, -0.0025002507027238607, 0.003452645381912589, 0.003186661284416914, 0.010145149193704128, 0.02639935351908207, -0.012073962949216366, 0.012190653011202812, -0.006874402519315481, 0.0027902591973543167, 0.02103162184357643, 0.023694895207881927, 0.00033419617102481425, -0.011298319324851036, 0.03495202958583832, 0.020496221259236336, -0.007708391174674034, -0.01242403220385313, 0.023090854287147522, 0.006565517745912075, -0.0028657643124461174, 0.011881767772138119, -0.6321567296981812, -0.00860072486102581, 0.026783742010593414, -0.01128459069877863, 0.028829246759414673, 0.0081339655444026, -0.010756054893136024, -0.0032638825941830873, -0.02009810321033001, -0.0006096184370107949, -0.01987845078110695, 0.006541493348777294, 0.0174073725938797, -0.00863504596054554, 0.0047808499075472355, -0.014126330614089966, 0.017009254544973373, -0.01019319798797369, -0.02285747416317463, 0.012005321681499481, -0.01500493660569191, 0.008298704400658607, -0.03201419115066528, -0.014661731198430061, -0.018354620784521103, 0.004176808521151543, 0.019548974931240082, 0.00037495180731639266, -0.008779192343354225, 0.014098874293267727, -0.038960665464401245, 0.024600956588983536, -0.02484806440770626, 0.01755838468670845, 0.06216134503483772, -0.0017357608303427696, 0.005467260722070932, 0.023310504853725433, -0.009630341082811356, 0.0390704944729805, -0.029048897325992584, -0.013954727910459042, 0.027552522718906403, 0.013810581527650356, 0.009108669124543667, 0.019343052059412003, 0.04154157266020775, 0.027223045006394386, 0.02088061161339283, -0.00966466125100851, -0.00203349138610065, -0.0006529481033794582, 0.004262609872967005, 0.022102423012256622, 0.005494717042893171, -0.009911769069731236, 0.02979022078216076, 0.005048550199717283, 0.014428351074457169, 0.003090563928708434, -0.003162636887282133, 0.013776261359453201, -0.04022366181015968, -0.04274965450167656, -0.018080055713653564, 0.00034706637961789966, 0.003253586357459426, -0.015691347420215607, 0.0025053988210856915, 0.01691315695643425, 0.005357434973120689, 0.010831560008227825, -0.020976709201931953, 0.01727009192109108, 0.0054741245694458485, 0.004873515572398901, 0.017215179279446602, 0.011263998225331306, 0.005065710283815861, 0.019082214683294296, -0.01213573943823576, -0.01079037506133318, -0.0095205157995224, 0.0032415741588920355, 0.05101403594017029, 0.021470924839377403, -0.029460744932293892, -0.025493290275335312, 0.004657295998185873, -0.007289680652320385, 0.013419327326118946, 0.003169500967487693, 0.007996683940291405, -0.022775104269385338, 0.019782355055212975, 0.022102423012256622, 0.008147694170475006, 0.029076354578137398, 0.00909494049847126, -0.016295388340950012, -0.011813126504421234, -0.018739009276032448, 0.02660527639091015, -0.014469536021351814, 0.002807419514283538, 0.0116483885794878, -0.010769782587885857, -0.023749807849526405, 0.007598565425723791, -0.020262841135263443, -0.02123754471540451, -0.013254589401185513, -0.02208869345486164, -0.008490899577736855, -0.015210859477519989, -0.03077865205705166, 0.014304797165095806, -0.016899429261684418, -0.008449714630842209, -0.0005268201348371804, 0.009994138963520527, -0.00444450881332159, -0.0011737621389329433, -0.015059849247336388, -0.011195356957614422, -0.006840081885457039, 0.030915934592485428, -0.008044732734560966, -0.025301095098257065, -0.015265772119164467, 0.024875521659851074, -0.007018548436462879, 0.009328320622444153, -0.0033513999078422785, 0.009156717918813229, -0.0013221984263509512, 0.030915934592485428, -0.013563473708927631, -0.02583649568259716, -0.03077865205705166, 0.004331251140683889, 0.012369119562208652, -0.005213288590312004, -3.3301639632554725e-05, -0.0048941075801849365, -0.00338743650354445, -0.023159494623541832, 0.014263613149523735, 0.009527379646897316, 0.020619774237275124, 0.008092781528830528, 0.00597863644361496, -0.027085762470960617, 0.024861792102456093, -0.002184501849114895, -0.02263782173395157, 0.017681937664747238, -0.036407221108675, -0.028582138940691948, -0.010934521444141865, 0.0006276367348618805, 0.031684715300798416, -0.018299708142876625, -0.004818602465093136, -0.017311276867985725, -0.017009254544973373, -0.018944934010505676, 0.033798858523368835, -0.014675458893179893, -0.03338701277971268, 0.008291840553283691, 0.013693892396986485, -0.015581521205604076, -0.03335955739021301, -0.01832716353237629, -0.012197516858577728, -0.0261110607534647, -0.006448827683925629, -0.013892951421439648, -0.00597863644361496, -0.0021518974099308252, 0.026207158342003822, -0.0048941075801849365, -0.012829015031456947, 0.03292025253176689, 0.026083603501319885, 0.01577371545135975, 0.023241864517331123, 0.0022617229260504246, 0.0008073904900811613, 0.011298319324851036, -0.01812124066054821, -0.00774957612156868, -0.010756054893136024, -0.015252044424414635, 0.0036860250402241945, -0.012945704162120819, 0.00023638266429770738, 0.007221039850264788, 0.001606200821697712, 0.025863952934741974, 0.012575042434036732, 0.011064939200878143, -0.0193842351436615, 0.00396402133628726, -0.04107481241226196, 0.028856702148914337, -0.01361838635057211, 0.006713096052408218, -0.008408530615270138, 0.011765077710151672, -0.008291840553283691, -0.017736850306391716, -0.00016806335770525038, 0.003902244381606579, 0.0072690886445343494, -0.006709663663059473, 0.002407585270702839, -0.00672339228913188, 0.0011188492644578218, -0.01593845523893833, 0.0021021326538175344, -0.007797624915838242, -0.003339387709274888, -0.004166512284427881, 0.004121895879507065, 0.01797023043036461, 0.03149252012372017, 0.009506787173449993, -0.014812741428613663, 0.0013324946630746126, 0.015403054654598236, 0.027827085927128792, 0.0033788562286645174, 0.024477403610944748, 0.013577202335000038, 0.028829246759414673, -0.02738778479397297, 0.029488200321793556, -0.014455807395279408, 0.041019897907972336, -0.0009369504987262189, 0.02313203737139702, 0.001801827922463417, 0.023324232548475266, 0.000663673272356391, 0.06792719662189484, 0.0010871028061956167, -0.039097949862480164, -0.0011257134610787034, 0.0008399949874728918, 0.015032392926514149, -0.01747601479291916, -0.02583649568259716, 0.008847832679748535, -0.004207697231322527, 0.00548442080616951, -0.013563473708927631, 0.0348147451877594, 0.012396575883030891, -0.0058859712444245815, -0.0005602826713584363, 0.009987274184823036, 0.001327346544712782, -0.011106124147772789, 0.010769782587885857, -0.018739009276032448, -0.027277957648038864, -0.01889001950621605, -0.011504242196679115, -0.0033067832700908184, -0.038823384791612625, 0.0019236657535657287, -0.008271248079836369, 0.01711908169090748, -0.01733873225748539, 0.0004491699510253966, -0.01854681596159935, 0.02407928556203842, 0.022418171167373657, -0.033167362213134766, -0.007982955314218998, -0.006599838379770517, 0.01333695836365223, -0.004622975364327431, -0.02009810321033001, -0.003057959256693721, 0.026783742010593414, 0.015018664300441742, -0.0039056765381246805, -0.011916088871657848, 0.027223045006394386, 0.012588771060109138, 0.007138670422136784, -0.004924996290355921, -0.019850995391607285, 0.012616227380931377, -0.015265772119164467, -0.018148696050047874, -0.0027885432355105877, 0.021759217604994774, 0.001892777276225388, 0.024614684283733368, -0.006846946198493242, 0.03750547766685486, -0.0033719921484589577, 0.0009429565980099142, -0.011888631619513035, -0.01834089122712612, 0.002069527981802821, -0.001057930407114327, 0.006541493348777294, -0.022198518738150597, 0.02667391672730446, 0.011943545192480087, -0.006826353725045919, 0.008127101697027683, -0.02173176035284996, 0.026660189032554626, 0.0005813040188513696, -0.015416782349348068, -0.010357935912907124, -0.01400964055210352, 0.029680395498871803, 0.06501681357622147, 0.009088076651096344, 0.004993637092411518, 0.034155793488025665, 0.004605815280228853, -0.010605043731629848, -0.03440289944410324, -0.004224857315421104, 0.003562471130862832, -0.005985500756651163, 0.019521517679095268, 0.010927657596766949, 0.0002698451862670481, -0.002673569368198514, 0.022747648879885674, -0.02343405783176422, 0.0050039333291351795, -0.008573268540203571, -0.006321841850876808, -0.025946320965886116, 0.022418171167373657, 0.002435041591525078, 0.008648773655295372, 0.013680163770914078, 0.04148665815591812, 0.0183958038687706, 0.01394100021570921, -0.0019528382690623403, -0.002467646263539791, -0.014744100160896778, 0.007516196463257074, 0.01692688651382923, 0.02391454577445984, -0.005148079711943865, -0.01677587628364563, 0.04480888694524765, 0.012334798462688923, 0.012355390936136246, -0.0008837537025101483, -0.02287120185792446, -0.015334413386881351, 0.0007061449578031898, -0.0034543615765869617, -0.0051103271543979645, -0.018944934010505676, -0.01437343843281269, -0.02711321972310543, 0.02987259067595005, -0.022267160937190056, -0.016871973872184753, 0.01549915224313736, 0.00305281113833189, -0.014291069470345974, -0.01305553037673235, 0.026701373979449272, 0.017297547310590744, 0.01167584490031004, 0.005360866896808147, -0.022624094039201736, -0.005501581355929375, -0.0016285092569887638, -0.032728057354688644, 0.027140676975250244, 0.006143375299870968, -0.019782355055212975, -0.017668209969997406, -0.0005924581782892346, -0.011236541904509068, -0.03594046086072922, 0.0050828708335757256, 0.009136125445365906, -0.035226594656705856, -0.02469705417752266, 0.010907065123319626, 0.01713280938565731, 0.011250270530581474, 0.02336541749536991, 0.021045349538326263, 0.00013599509838968515, -0.0040875752456486225, -0.025012802332639694, -0.009266543202102184, 0.0006387908942997456, -0.02568548545241356, -0.0009995854925364256, -0.009348913095891476, 0.0006246336852200329, 0.007866266183555126, -0.0018876292742788792, 0.0283899437636137, 0.023104581981897354, 0.01876646652817726, 0.014318525791168213, -0.012005321681499481, 0.005872243084013462, 0.01755838468670845, 0.020551133900880814, 0.010934521444141865, 0.005789873655885458, 0.0028331598732620478, 0.020496221259236336, 0.03297516703605652, -0.025314824655652046, -0.011703301221132278, 0.004695048555731773, -0.00041248989873565733, 0.014359709806740284, 0.017022984102368355, -0.02490297704935074, -0.01079037506133318, 0.029927503317594528, -0.00445823697373271, -0.004684752319008112, 0.02335168980062008, -0.0008138255798257887, 0.029405830428004265, 0.01299375295639038, 0.03404596820473671, 0.017517199739813805, -0.016597408801317215, -0.022184791043400764, -0.025795310735702515, 0.024312663823366165, 0.014675458893179893, 0.015032392926514149, 0.008456578478217125, 0.009293999522924423, -0.0001973430480575189, -0.02561684511601925, 0.012767237611114979, -0.0005156659753993154, 0.017530927434563637, -0.01486765407025814, -0.025232454761862755, -0.011888631619513035, -0.03187691047787666, 0.0068023293279111385, -0.0047774179838597775, 0.005072574596852064, -0.020839426666498184, -0.012273021973669529, -0.027305414900183678, 9.840339771471918e-05, 0.004372435621917248, -0.007598565425723791, -0.03618756681680679, -0.0005542765720747411, -0.017956502735614777, -0.014126330614089966, 0.031602345407009125, -0.0028331598732620478, -0.014648002572357655, -0.01437343843281269, 0.0021313049364835024, -0.015320684760808945, -0.03239858150482178, -0.024738239124417305, -0.013275181874632835, -0.004506285768002272, 0.03022952377796173, 0.016611136496067047, 0.009293999522924423, -0.00610562227666378, 0.005299089942127466, 0.011586611159145832, -0.006826353725045919, 0.020276570692658424, -0.01422242820262909, -0.016789603978395462, 0.0008395659970119596, 0.01834089122712612, 0.005017661489546299, -0.01882137916982174, -0.01643267087638378, -0.02251426875591278, 0.011243405751883984, -0.00863504596054554, -0.004983341321349144, -0.018505631014704704, -0.037121087312698364, -0.015320684760808945, 0.006181127857416868, -0.007804488763213158, 0.003281042678281665, -0.005182400345802307, -0.0006383619038388133, 0.013302638195455074, -0.010728597640991211, 0.02513635717332363, -0.0035590389743447304, 0.015265772119164467, 0.003895380301401019, 0.0024436218664050102, 0.012904520146548748, 0.001314476365223527, -0.01755838468670845, -0.0004577500803861767, -0.03327718749642372, -0.005800169892609119, 0.00948619470000267, 0.016954341903328896, 0.01677587628364563, -0.02753879502415657, 0.01677587628364563, 0.004231721628457308, 0.010007866658270359, 0.030696284025907516, 0.0033702761866152287, -0.00502795772626996, -0.0020369235426187515, 0.0002966580796055496, -0.01747601479291916, -0.0351167693734169, -0.022967299446463585, 0.008257519453763962, -0.013117306865751743, -0.022898659110069275, 0.013165355660021305, -0.008998842909932137, -0.026495449244976044, -0.0028880727477371693, -0.011126716621220112, 0.023804720491170883, 0.011236541904509068, 0.021676847711205482, 0.05230449140071869, 0.02307712472975254, -0.05699953809380531, -0.0018584567587822676, -0.0019699984695762396, -0.0073789143934845924, 0.014208699576556683, 0.035281505435705185, -0.004616111516952515, -0.017379917204380035, 0.009081212803721428, 0.03434798866510391, -0.018848836421966553, -0.01818988099694252, -0.00014436073251999915, 0.001671409816481173, 0.030202068388462067, -0.02660527639091015, 0.003076835535466671, -0.0021210089325904846, 0.028719419613480568, -0.004478829447180033, -0.006551789585500956, 0.0008623033645562828, 0.01221124455332756, -0.024669596925377846, 0.02973530814051628, -0.0014165799366310239, 0.017160264775156975, 0.001489511108957231, -0.014208699576556683, -0.01711908169090748, -0.024532316252589226, -0.006840081885457039, -0.011490513570606709, -0.007838808931410313, 0.031053217127919197, -0.015910997986793518, -0.0037306416779756546, 0.013069258071482182, 0.0017083043931052089, -0.00400177389383316, -0.026358168572187424, -0.022280888631939888, 0.026014963164925575, -0.0017983958823606372, -0.010323615744709969, -0.03547370061278343, -0.005340274889022112, -0.013666435144841671, -0.011696437373757362, -0.007186719216406345, -0.002580903936177492, -0.030806109309196472, -0.007468147668987513, 0.0009206482791341841, 0.0008717415039427578, -0.01592472568154335, -0.012568178586661816, -0.008085916750133038, -0.0164189413189888, 0.0005199560546316206, -0.010701141320168972, 0.02236325852572918, -0.056093476712703705, -0.025177542120218277, -0.019699985161423683, -0.020894339308142662, 0.012732917442917824, -0.001035622088238597, -0.03130032494664192, 0.02079824171960354, 0.01727009192109108, -0.027483880519866943, 0.0072690886445343494, 0.009774487465620041, -0.02973530814051628, -0.011387552134692669, 0.013508561067283154, 5.252650225884281e-05, -0.030806109309196472, 0.011668981052935123, -0.002543151378631592, -0.03399105370044708, -0.000823692767880857, 0.00860072486102581, -0.009122396819293499, 0.009932361543178558, -0.018313435837626457, 0.008367345668375492, 0.004969612695276737, -0.007111214101314545, -0.007186719216406345, -0.033579207956790924, -0.00596490828320384, -0.011847447603940964, -0.0005341132637113333, 0.011586611159145832, 0.0031900934409350157, 0.0033548318315297365, -0.0021536133717745543, 0.009650933556258678, 0.01398218423128128, -0.035638440400362015, -0.026426808908581734, 0.002622088650241494, -0.0016508175758644938, -0.021141447126865387, 0.014991207979619503, -0.006373322568833828, -0.020619774237275124, -0.02195141091942787, 0.03220638632774353, -0.008429122157394886, -0.014098874293267727, -0.056093476712703705, -1.2796464034181554e-05, -0.011600339785218239, 0.01619929075241089, -0.004646999761462212, -0.01245835330337286, -0.014098874293267727, 0.00548442080616951, -0.004581790883094072, -0.0061399429105222225, -0.010124556720256805, 0.04063550755381584, 0.05216720700263977, -0.0006816915702074766, -0.001593330642208457, -0.005196128506213427, -0.0232967771589756, -0.015087305568158627, -0.017146537080407143, 0.01232107076793909, 0.0229947566986084, -0.002781679155305028, 0.01536186970770359, 0.03127286955714226, -0.00024453378864564, 0.0025568795390427113, -0.006225744262337685, -0.012973160482943058, -0.01846444606781006, 0.02519126981496811, 0.008751735091209412, -0.023255592212080956, -0.0328378826379776, 0.006126214750111103, 0.010042187757790089, -0.007365186233073473, -0.004708776716142893, 0.004125327803194523, 0.005175536032766104, 0.00834675319492817, -0.0005456964718177915, -0.005408915691077709, 0.027017122134566307, 0.02471078187227249, 0.03821934387087822, -0.004018933977931738, -0.012547586113214493, 0.03950979560613632, -0.006387050729244947, -0.009046891704201698, -0.009891177527606487, -0.011277726851403713, 0.02166312001645565, 0.005529037676751614, -0.0005619986914098263, -0.0413493774831295, -0.013995912857353687, -0.03885084018111229, 0.026234613731503487, -0.03467746451497078, -0.008353617042303085, 0.004423916339874268, -0.00038181591662578285, 0.00022694451035931706, 0.01195727288722992, 0.006085030268877745, -0.027223045006394386, 0.010337344370782375, 0.018162425607442856, 0.003207253757864237, 0.010248110629618168, 0.009390097111463547, 0.016666049137711525, 0.0017657913267612457, -0.007392642553895712, -0.03616011142730713, 0.005014229565858841, -0.01713280938565731, -0.015238315798342228, 0.0047499616630375385, -0.012444624677300453, 0.0021656255703419447, -0.0026855815667659044, 0.03437544405460358, -0.009252815507352352, -0.03377140313386917, -0.022459356114268303, -0.01890374906361103, -0.01818988099694252, 0.015526608563959599, 0.015128490515053272, -0.03000987321138382, -0.003631112165749073, 0.007969227619469166, -0.011861175298690796, 0.00938323326408863, 0.21372081339359283, -0.017668209969997406, 0.011888631619513035, 0.008648773655295372, -0.021704303100705147, 0.00672339228913188, 0.015622706152498722, -0.013158491812646389, 0.006273793056607246, 0.02053740620613098, 0.00888901762664318, 0.013652707450091839, -0.009554835967719555, -0.013995912857353687, 0.015101034194231033, 0.0010347639909014106, -0.021567022427916527, 0.014414623379707336, -0.02576785534620285, 0.0020403554663062096, 0.019837267696857452, -0.0006053283577784896, -0.022418171167373657, 0.00174777302891016, 0.02696220949292183, -0.015265772119164467, -0.020015733316540718, 0.032947711646556854, 0.021690575405955315, 0.011332639493048191, -0.01284274272620678, 0.006050709635019302, 0.012293614447116852, 0.0013633831404149532, -0.00888901762664318, -0.002647829009220004, 0.022967299446463585, -0.0019408260704949498, 0.007193583529442549, -0.0037478019949048758, 0.007639750372618437, -0.021059077233076096, -0.015608977526426315, -0.010515810921788216, -0.014140059240162373, 0.014853926375508308, 0.01518340315669775, -0.013879222795367241, -0.009822536259889603, 0.031053217127919197, -0.01578744500875473, -0.0006623862427659333, 0.016473855823278427, 0.01440089475363493, 0.01755838468670845, 0.005405483767390251, 0.0013316365657374263, 0.02307712472975254, 0.0036208161618560553, 0.014744100160896778, 0.0010081656510010362, 0.027827085927128792, -0.021278729662299156, 0.040800247341394424, -0.016830788925290108, 0.019068486988544464, -0.0008005264098756015, 0.014538177289068699, 0.022665278986096382, -0.01104434672743082, -0.003720345674082637, -0.025712942704558372, -0.007008252665400505, -0.011737621389329433, -0.03190436586737633, -0.01791531778872013, 0.029598025605082512, 0.0041047353297472, 0.03948234021663666, 0.025095172226428986, -0.005388323217630386, -0.0009369504987262189, -0.0038885162211954594, -0.017530927434563637, 0.008017276413738728, -0.06067869812250137, 0.040031466633081436, -0.006225744262337685, -0.003246722277253866, -0.025671757757663727, -0.00855954084545374, 0.013954727910459042, -0.020441308617591858, 0.002692445646971464, 0.018285978585481644, 0.002294327598065138, -0.009554835967719555, 0.019796082749962807, -0.005958044435828924, 0.018231065943837166, -0.0026684212498366833, 0.047307420521974564, 0.01061190851032734, 0.006153671070933342, -0.014730372466146946, 0.01100316271185875, -0.005882538855075836, 0.006153671070933342, -0.01323399692773819, -0.016679778695106506, 0.012808422558009624, -0.019219497218728065, 0.009527379646897316, 0.017064169049263, 0.01029615942388773, 0.006795465014874935, -0.021388554945588112, -0.0220063254237175, -0.007516196463257074, 0.000834846927318722, -0.010385393165051937, 0.003826739266514778, 0.00018436560640111566, -0.009211630560457706, -0.010131420567631721, 0.007090621627867222, -0.04626407474279404, 0.003607087768614292, -0.019988277927041054, -0.038603734225034714, 0.025026531890034676, -0.030339349061250687, 0.008518355898559093, -0.012238701805472374, -0.016611136496067047, -0.014345982111990452, -0.0070219808258116245, -0.010460898280143738, 0.022610366344451904, 0.018368348479270935, -0.006061005871742964, 0.01747601479291916, -0.01754465512931347, -0.029186179861426353, -0.012581907212734222, -0.042365264147520065, 0.03338701277971268, 0.011010026559233665, -0.030806109309196472, 0.009822536259889603, -0.0064659882336854935, 0.012609363533556461, -0.005728096701204777, 0.020866883918642998, 0.027511337772011757, -0.01840953342616558, -0.020866883918642998, -0.049092087894678116, 0.024038100615143776, 0.00022329796047415584, -0.026097331196069717, 0.004187104757875204, 0.03130032494664192, -0.025809040293097496, -0.03162980079650879, -0.009932361543178558, -0.17605060338974, 0.03898812457919121, 0.02711321972310543, -0.01846444606781006, 0.05076692998409271, -0.0004719073185697198, 0.032453496009111404, -0.007248496171087027, -0.02265155129134655, 0.01390667911618948, 0.01440089475363493, -0.0006332137854769826, -0.036764152348041534, -0.013378143310546875, -0.001204650616273284, 0.006888130679726601, -0.012073962949216366, 0.0021038486156612635, 0.02888415940105915, 0.001156601938419044, 0.03786240890622139, -0.028362486511468887, 0.039317600429058075, 0.013570338487625122, 0.015320684760808945, 0.017022984102368355, -0.0005572796217165887, 0.01776430755853653, -0.004804874304682016, -0.021429739892482758, -0.008044732734560966, 0.0008266958175227046, 0.04780163615942001, 0.003631112165749073, 0.01676214672625065, 0.006534629035741091, -0.011037482880055904, -0.023873360827565193, 0.007886857725679874, 0.028719419613480568, 0.03673669695854187, 0.018587999045848846, 0.005944315809756517, -0.0040052058175206184, -0.024257751181721687, -0.011339503340423107, 0.04535801336169243, -0.00704257283359766, -0.014771556481719017, -0.013282045722007751, -0.0016276511596515775, -0.011634659953415394, 0.004993637092411518, -0.007028844673186541, 0.00962347723543644, 0.007248496171087027, -0.013789989054203033, 0.012012185528874397, -0.006009525153785944, 0.01252699363976717, -0.015265772119164467, 0.0024436218664050102, -0.010330479592084885, 0.000823692767880857, 0.028417399153113365, -0.006586109753698111, -0.012575042434036732, -0.01606200821697712, -0.008463443256914616, 0.011861175298690796, -0.010165741667151451, -0.003409744706004858, 0.003950293175876141, -0.00097813515458256, 0.01440089475363493, -0.006568949669599533, -0.026948481798171997, -0.007035708986222744, -0.006198287941515446, -0.01783294789493084, -0.0061365109868347645, 0.022733919322490692, 0.001502381288446486, 0.0032209819182753563, -0.009191038087010384, 0.004520013928413391, 0.0070494371466338634, 0.007076893467456102, 0.005278497934341431, -0.007557380944490433, 0.010385393165051937, -0.0073857782408595085, -0.021484652534127235, -0.016597408801317215, -0.011620932258665562, 0.01245148852467537, 0.0007215891964733601, 0.009252815507352352, -0.003498978214338422, -0.017036711797118187, 0.015416782349348068, 0.0014337401371449232, -0.04154157266020775, 0.020345211029052734, 0.029076354578137398, -0.013158491812646389, -0.0035761992912739515, 0.025507017970085144, 0.03583063557744026, -0.006088462192565203, -0.028993984684348106, 0.033579207956790924, 0.0036928891204297543, 0.002692445646971464, 0.0026169405318796635, 0.02731914259493351, 0.00202662730589509, -0.004626407753676176, 0.013398735783994198, -0.02256918139755726, 0.05293598771095276, -0.005635431036353111, -0.027566250413656235, 0.006740552373230457, -0.009582292288541794, -0.024449946358799934, -0.1044442430138588, -0.020400123670697212, 0.019782355055212975, 0.013144763186573982, 0.010234382003545761, 0.02263782173395157, -0.018354620784521103, 0.00855954084545374, -0.03637976199388504, 0.015293228439986706, -0.010481489822268486, -0.0200294628739357, -0.005882538855075836, 0.009863720275461674, -0.001690286211669445, -0.004763689823448658, -0.0020540838595479727, -0.015265772119164467, 0.0040669827722013, 0.019562702625989914, 0.008401665836572647, -0.014538177289068699, -0.010591316036880016, 0.0014663446927443147, -0.012218109332025051, 0.0229947566986084, -0.026385623961687088, 0.004365571774542332, 0.004921564366668463, 0.00023616815451532602, -0.01585608534514904, -0.006846946198493242, 0.01691315695643425, -0.025932593271136284, -0.018217338249087334, -0.010948249138891697, -0.00352128641679883, 0.002627236768603325, 0.019782355055212975, -0.03292025253176689, 0.00792804267257452, 0.0074887401424348354, 0.02250054106116295, -0.017750578001141548, -0.022129878401756287, 0.0017563531873747706, 0.001359093002974987, 0.04782909154891968, 0.012122011743485928, -0.01875273883342743, -0.03237112611532211, 0.0019957388285547495, -0.0026186564937233925, -0.024175381287932396, 0.0410473570227623, -0.019123399630188942, 0.0183958038687706, 0.013192811980843544, -0.020976709201931953, 0.011888631619513035, -0.0162679310888052, -0.02074332907795906, -0.003627680242061615, 0.02074332907795906, 0.008669366128742695, -0.0007391784456558526, -0.00015787444135639817, 0.00074732955545187, 0.003675728803500533, -0.011998457834124565, -0.011593475006520748, 0.01039912085980177, -0.023585069924592972, 0.020894339308142662, -0.013282045722007751, -0.0183958038687706, -0.02306339703500271, -0.03508931025862694, 0.024614684283733368, -0.0031935253646224737, -0.01742110215127468, -0.018436988815665245, 0.004962748847901821, -0.013007481582462788, 0.015334413386881351, 0.010783511213958263, -0.009300864301621914, 0.014483264647424221, 0.017736850306391716, -0.01500493660569191, -0.012897655367851257, 0.019507789984345436, 0.019919635728001595, -0.006126214750111103, 0.007701527327299118, 0.018382076174020767, 0.023022212088108063, -0.019343052059412003, -0.027360327541828156, 0.023118309676647186, -0.03459509462118149, -0.03187691047787666, -0.04601696878671646, 0.01902730204164982, 0.0018464445602148771, -0.035858090966939926, -0.015581521205604076, -0.0034423493780195713, -0.003675728803500533, 0.0016593977343291044, -0.00502795772626996, 0.0069430433213710785, -0.016048280522227287, -0.007612294051796198, 0.004386163782328367, -0.02357134036719799, -0.011236541904509068, 4.9335762014379725e-05, 0.02181413024663925, 0.006627294700592756, 0.004176808521151543, -0.014455807395279408, -0.00597863644361496, 0.009033164009451866, 0.017929045483469963, 0.0081339655444026, -0.012403439730405807, 0.037697672843933105, 0.012231837026774883, -0.018670368939638138, -0.003929700702428818, -0.01489511039108038, 0.0061673992313444614, -0.031602345407009125, -0.00700482027605176, -0.004986773245036602, 0.015416782349348068, -0.01047462597489357, -0.022102423012256622, 0.04486379772424698, -0.007330865599215031, 0.04351843520998955, -0.010042187757790089, -0.02789572812616825, 0.01599336788058281, -0.026069875806570053, 0.009321455843746662, 0.005130919627845287, -0.01755838468670845, -0.009630341082811356, 0.00193739403039217, 0.011366959661245346, 0.05491285026073456, 0.03151997551321983, -0.030339349061250687, -0.0048941075801849365, 0.019933365285396576, -0.034732379019260406, 0.031684715300798416, 0.012231837026774883, -0.0040052058175206184, -0.006078165955841541, 0.03000987321138382, -0.009966682642698288, -0.0074956039898097515, -0.03390868380665779, 0.0013848334783688188, -0.010220654308795929, 0.006596405990421772, -0.01812124066054821, 0.013556609861552715, -0.004478829447180033, -0.007303409278392792, -0.005625135265290737, 0.011188493110239506, 0.04804874211549759, -0.0002775673056021333, 0.03335955739021301, 0.005034822039306164, -0.011051211506128311, -0.010488354600965977, 0.015965910628437996, -0.002725050086155534, -0.018313435837626457, -0.0010098816128447652, 0.010543267242610455, 0.01323399692773819, -0.011085531674325466, -0.008662502281367779, -0.006462555844336748, -0.010948249138891697, 0.017860405147075653, -0.016116920858621597, -0.003967453259974718, 0.020619774237275124, 0.013295773416757584, 0.029817678034305573, 0.0040566870011389256, 0.012945704162120819, 0.009191038087010384, -0.004983341321349144, 0.00645912392064929, 0.011305183172225952, 0.007227903697639704, -0.013323230668902397, -0.03432052955031395, -0.012259293347597122, 0.018162425607442856, -0.0073857782408595085, -0.0232967771589756, 0.004609247203916311, 0.018725281581282616, 0.002563743619248271, -0.010076507925987244, 0.010248110629618168, 0.020633503794670105, -0.00873800739645958, 0.006366458721458912, -0.008374209515750408, 0.012128875590860844, -0.025108899921178818, 0.03270060196518898, 0.012437760829925537, 0.0008974818629212677, 0.010344208218157291, -0.009966682642698288, -0.0024865225423127413, 0.004156216513365507, 0.017215179279446602, -0.01419497188180685, 0.0006649603019468486, -0.010124556720256805, -0.021251272410154343, -0.002579187974333763, -0.01107866782695055, -0.010110828094184399, -0.002366400556638837, -0.01518340315669775, -0.02081196941435337, 0.03113558515906334, -0.008010411635041237, 0.07473638653755188, 0.017393644899129868, -0.017942773178219795, 0.03050408884882927, -0.002870912430807948, -0.019480332732200623, 0.00902630016207695, 0.009808807633817196, -0.00021300179651007056, -0.022253433242440224, -0.0036242480855435133, -0.016034552827477455, 0.008998842909932137, -0.026426808908581734, -0.04225543886423111, 0.016185563057661057, -0.025163814425468445, -0.003953725099563599, -0.041925959289073944, -0.00547069264575839, -0.0029481337405741215, -0.0009892892558127642, 0.017599567770957947, 0.0009223642991855741, -0.013714483939111233, -0.006822921801358461, 0.02773098833858967, 0.004794578067958355, 0.012087690643966198, -0.03121795505285263, 0.00870368629693985, -0.001803543884307146, -0.02893907204270363, -0.02306339703500271, -0.012877062894403934, -0.0021518974099308252, -0.005710936617106199, -0.014071417972445488, 0.004478829447180033, 0.022253433242440224, 0.00291896122507751, 0.002961861900985241, -0.025603115558624268, -0.03179454058408737, 0.004911268129944801, -0.017723122611641884, -0.00495245261117816, -0.006064437795430422, -0.0293783750385046]} +{"id": "test:10000065", "text": "\"Weekend fun isn\u2019t just for humans. Dogs love to get out and enjoy some time away from the day-to-day grind. But where can you take your dogs on weekends?\\n#3: Farmers Markets \u2013 Orlando boasts many dog friendly farmers markets every weekend. From Windermere\u2019s Friday Farmers Market to the big Saturday markets in Winter Garden and Winter Park \u2013 you are hard pressed to find a locale without a marketplace of fresh produce. Dogs at these events have become so prominent that many vendors now cater exclusively to dogs selling homemade biscuits and treats just for your pooch.\\n#2: The beach \u2013 Contrary to popular belief there ARE several dog friendly beaches within a short drive of Orlando. The closest is the Smyrna Dunes Park. Dogs can romp and play in the sun alongside your family.\\nBe sure to take poop bags and keep the beach as nice as you found it!\\n#1: The dog park \u2013 What better place to spend some time outdoors and close to home than in the dog park. Central Florida can boast one of the nations best dog friendly parks in Winter Park. Fleet Peeples Park alongside the shores of Lake Baldwin has to be the nicest, most popular and respected dog parks around. It\u2019s huge trees supply lots of shade and dogs love to run in the surf of the lakes sandy shores. There\u2019s even a bathing area for our dog once they\u2019ve had enough fun in the sun.\\nWell, there you have it. The things pet sitters in Orlando like to do with their dogs. Did we miss any? Tell us in the comments.\"", "vector_field": [0.010376278311014175, -0.009754748083651066, -0.006195671856403351, -0.005387682933360338, -0.0210273377597332, 0.027033278718590736, -0.028263254091143608, -0.005574141629040241, -0.014537257142364979, -0.021956361830234528, -0.004664745647460222, 0.037344131618738174, -0.0039123669266700745, 0.009643526747822762, 0.00489372992888093, 0.009813630022108555, 0.04113873839378357, -0.002847588388249278, 0.008315416052937508, -0.001089312951080501, -0.016735510900616646, 0.003765162779018283, -0.003137090476229787, -0.006395215634256601, -0.016408389434218407, 0.03595713898539543, 0.005544701125472784, -0.018344946205615997, -0.026012660935521126, 0.0030520388390868902, 0.015544789843261242, 0.010664144530892372, -0.02842027321457863, -0.016264455392956734, -0.023212505504488945, -0.009800544939935207, -0.008662164211273193, -0.006496623158454895, 0.004687644075602293, -0.034543976187705994, 0.008747215382754803, -0.003994147293269634, -0.00140580243896693, -0.008577113039791584, 0.009767833165824413, 0.027739860117435455, 0.010925841517746449, 0.009689323604106903, -0.027530502527952194, -0.009748205542564392, -0.0105136688798666, 0.03354952856898308, -0.02194327674806118, 0.01153428666293621, -0.008544400334358215, -0.008747215382754803, -0.012129646725952625, 0.001382086193189025, 0.014550342224538326, -0.036532871425151825, -0.009990274906158447, -0.011501573957502842, -0.013490470126271248, 0.01952258124947548, 0.003088022116571665, -0.014118541963398457, -0.005845652427524328, -0.007451816461980343, -0.005325529724359512, -4.9630387366050854e-05, 0.04155744984745979, 0.007281713653355837, -0.01575414650142193, 0.0017435549525544047, 0.017167309299111366, -0.010768823325634003, 0.009617357514798641, 0.008806097321212292, 0.0033186424989253283, 0.010363193228840828, 0.012541819363832474, -0.017703788354992867, -0.03511970862746239, 0.03391590341925621, 0.03352335840463638, 0.008106058463454247, -0.003761891508474946, 0.015453196130692959, -0.026431376114487648, 0.005273190326988697, 0.011462319642305374, 0.029074514284729958, 0.003460940206423402, -0.005711532663553953, 0.006005941424518824, 6.460639997385442e-05, -0.0059143477119505405, 0.02745199389755726, -0.009578103199601173, 0.01024542935192585, 0.0013869929825887084, 0.026078086346387863, 0.017782296985387802, -0.010696856305003166, -0.013137179426848888, 0.028263254091143608, 0.004959154408425093, -0.03470099717378616, 0.03511970862746239, -0.0016846731305122375, -0.01601584441959858, 0.013104467652738094, -0.002993157133460045, -0.046346504241228104, -0.04982706904411316, -0.003382431110367179, -0.001645418582484126, -0.01864589750766754, -0.013621318154036999, -0.012738091871142387, 0.03315698355436325, 0.012842770665884018, 0.023814408108592033, -0.014118541963398457, 0.008511688560247421, -0.0005957691464573145, -0.0023912545293569565, 0.01035010814666748, -0.006051738280802965, -0.016866357997059822, 0.013425045646727085, 0.02662764862179756, 0.01812250353395939, -0.028551120311021805, -0.024272378534078598, 0.015950419008731842, -0.01197262853384018, 0.006251282058656216, 0.023003147915005684, -0.018737491220235825, 0.023906001821160316, 0.02334335446357727, -0.011298758909106255, -0.02080489508807659, 0.0226498581469059, 0.036035649478435516, 0.013411960564553738, -0.02097499929368496, 0.024769602343440056, -0.024285463616251945, -0.009591187350451946, -0.012973618693649769, 0.0038175019435584545, 0.004854475613683462, 0.016185946762561798, 0.002829596633091569, 0.0004980417434126139, -0.006856455933302641, 0.004825034644454718, -0.004115181975066662, 0.029100684449076653, 0.011095943860709667, -0.0012872210936620831, 0.04354634881019592, 0.01089312881231308, 0.011861407198011875, 0.016944868490099907, 0.034099094569683075, 0.011566998437047005, 0.002741273958235979, 0.008341585285961628, 0.03878346458077431, -0.018201012164354324, 0.03213636577129364, 0.007759310305118561, 0.012685752473771572, -0.01372599694877863, -0.016800934448838234, -0.027242636308073997, -0.013091382570564747, 0.0019921669736504555, -0.00995102059096098, 0.03357569873332977, 0.03894048556685448, -0.010265056975185871, -0.014707360416650772, 0.010284684598445892, -0.009682781994342804, 0.0044881002977490425, -0.014785869047045708, 0.0038731126114726067, -0.0018449624767526984, 0.021210525184869766, -0.00036514882231131196, -0.6389589309692383, 0.007752767764031887, 0.005655921995639801, -0.01796548441052437, 0.04001343995332718, 0.003922180738300085, -0.0037357218097895384, -0.007288255728781223, -0.00665691215544939, 0.0037030098028481007, -0.0221133790910244, 0.010206175036728382, 0.010376278311014175, -0.000747471465729177, 0.0047694239765405655, -0.0013878108002245426, 0.0063330624252557755, -0.003050403203815222, 0.013156806118786335, 0.010991265997290611, -0.018999187275767326, 0.014170881360769272, -0.01795240119099617, -0.016526153311133385, -0.0016617747023701668, -0.0021557274740189314, -0.014537257142364979, -0.03548608720302582, -0.004766153171658516, 0.030068961903452873, -0.005786770489066839, 0.028053896501660347, -0.011645507998764515, 0.018253352493047714, 0.05411889776587486, 0.017978569492697716, -0.0045993211679160595, 0.010952010750770569, 0.009264066815376282, 0.06599993258714676, -0.006869541015475988, 0.005799855105578899, 0.027085619047284126, 0.022335821762681007, -0.0036473991349339485, -0.010402447544038296, 0.011566998437047005, -0.0014794047456234694, -0.025031298398971558, -0.0009437440894544125, 0.017114970833063126, 0.02232273668050766, -0.010742653161287308, -0.00819765217602253, -0.010356650687754154, -0.005125985946506262, 0.025541607290506363, -0.03428228199481964, 0.022885385900735855, 0.002854130696505308, 0.01575414650142193, 0.036742229014635086, -0.028315594419836998, -0.00913976039737463, -0.03132510557770729, 0.03904516249895096, -0.022126464173197746, -0.015832656994462013, 0.02469109371304512, -0.0028148761484771967, 0.002819783054292202, 0.028551120311021805, 0.0004526536795310676, 0.004720355849713087, 0.010880044661462307, -0.01464193593710661, 0.011985713616013527, 0.015139159746468067, -0.010912756435573101, 0.010278142057359219, 0.01005569938570261, -0.012901652604341507, -0.0355907641351223, -0.013366163708269596, 0.012339004315435886, 0.012554903514683247, 0.01523075345903635, 0.008243449032306671, 0.028551120311021805, -0.024403225630521774, 0.03158680349588394, 0.056631188839673996, -0.015924250707030296, -0.011678219772875309, 0.0006002670270390809, -0.009237896651029587, 4.221905328449793e-05, 0.02173391915857792, -0.007785480003803968, -0.036035649478435516, -0.0019741752184927464, -0.017337413504719734, -0.0105136688798666, -0.0032384980004280806, 0.009754748083651066, 0.0023536356166005135, -0.009506136178970337, 0.023421863093972206, 0.024076106026768684, -0.029022175818681717, 0.0036539414431899786, 0.01002953015267849, -0.012731549330055714, -0.0010467872489243746, -0.014955972321331501, -0.01390918530523777, -4.464690573513508e-05, -0.009192099794745445, 0.031900838017463684, -0.021275950595736504, -0.012895110063254833, 0.010670687071979046, 0.0031550819985568523, -0.03446546941995621, 0.010271599516272545, 0.006614386569708586, 0.0011661864118650556, -0.021642325446009636, 0.03551225736737251, -0.04087704047560692, -0.010317396372556686, -0.010225802659988403, 0.011050147004425526, -0.008930402807891369, -0.006856455933302641, 0.016526153311133385, 0.009355660527944565, 0.0021900751162320375, 0.02420695312321186, -0.02075255662202835, -0.015780316665768623, -0.0032581251580268145, 0.0030569457449018955, -0.018070163205266, -0.024036850780248642, -0.004759610630571842, -0.005004951264709234, 0.004170792642980814, -0.03548608720302582, -0.010520211420953274, -0.015505535528063774, -0.010068784467875957, 0.002018336672335863, -0.011017435230314732, 0.00018083657778333873, 0.0002733504807110876, 0.011926831677556038, -0.02204795554280281, -0.0175206009298563, 0.0004260751011315733, 0.021053507924079895, 0.002778892870992422, -0.026928599923849106, 0.00840700976550579, -0.016094353049993515, -0.020189907401800156, 0.027399655431509018, 0.00423294585198164, 0.02469109371304512, -0.017219649627804756, -0.00309292902238667, -0.0027674436569213867, -0.017429007217288017, 0.02862962894141674, -0.00013248399773146957, 0.01172401662915945, -0.023526541888713837, -0.02285921573638916, -0.015989674255251884, 0.01649998314678669, 0.004566608928143978, -0.005332072265446186, -0.0009388372418470681, 0.005930703599005938, 0.007451816461980343, -0.011495032347738743, 0.04048449546098709, 0.027739860117435455, -0.01769070327281952, 0.0110566895455122, 0.027216468006372452, 0.0009494686964899302, 0.017298158258199692, -0.012960533611476421, -0.0033562614116817713, -0.028106236830353737, -0.0003919318551197648, -0.0014205229235813022, 0.004053029231727123, 0.006300350651144981, 0.015165329910814762, -0.010415532626211643, 0.022676028311252594, -0.035067372024059296, 0.02042543515563011, -0.03150829300284386, -0.006745235063135624, -0.03051384724676609, 0.015610214322805405, 0.004478286486119032, -0.006022297777235508, -0.019941296428442, -0.00756303733214736, -0.005881635472178459, -0.009185557253658772, 0.037553489208221436, -0.018737491220235825, 0.004026859533041716, -0.0026676717679947615, 0.02425929345190525, 0.019339393824338913, -0.0077069709077477455, 0.01902535744011402, 0.004936255980283022, -0.016264455392956734, 0.006218570284545422, -0.0007143504917621613, 0.013359621167182922, -0.007850904017686844, -0.005283004138618708, -0.030225981026887894, 0.002109930384904146, 0.0231863372027874, 0.007916328497231007, 0.026732327416539192, -0.003451126627624035, 0.010121123865246773, -0.016513068228960037, 0.026980940252542496, -0.004900272469967604, 0.03148212656378746, 0.04932984709739685, 0.0015202948125079274, -0.039176009595394135, 0.029257701709866524, 0.0035950597375631332, 0.01172401662915945, 0.021328289061784744, 0.0025008399970829487, -0.018776744604110718, -0.032319553196430206, -0.008982742205262184, 8.091133349807933e-05, 0.027137957513332367, -0.004053029231727123, -0.009257524274289608, 0.006758319679647684, -0.04509035870432854, 0.025031298398971558, -0.004321268294006586, 0.008348127827048302, -0.016146693378686905, 0.029938112944364548, -0.022388162091374397, 0.007837818935513496, 0.008662164211273193, 0.007118152920156717, -0.009617357514798641, 0.010690314695239067, -0.018318776041269302, -0.007785480003803968, -0.01153428666293621, -0.022872300818562508, -0.006002670153975487, -0.0036473991349339485, 0.009826715104281902, -0.046503521502017975, -0.010108038783073425, 0.00035083727561868727, 0.04058917239308357, -0.007294798269867897, -0.02011139877140522, 0.02054319903254509, 0.026300529018044472, -0.007118152920156717, 0.012836228124797344, -0.008995827287435532, 0.017442092299461365, -0.0009118497837334871, 0.008649079129099846, -0.009080879390239716, 0.030749373137950897, -0.03184850141406059, -0.00672560790553689, 0.013392333872616291, 0.002615332370623946, 0.03700392693281174, -0.020961914211511612, 0.02685009129345417, 0.002785435412079096, 0.002396161202341318, 0.0028524950612336397, -0.026169680058956146, -0.04707925394177437, 0.0037160946521908045, -0.0074191042222082615, -0.014092372730374336, -0.0007437913445755839, -0.011881034821271896, 0.005783499218523502, -0.005479276645928621, -0.01418396644294262, -0.005960144568234682, 0.002809969475492835, -0.012711921706795692, 0.0017811738653108478, -0.00044897355837747455, 0.014131627045571804, 0.023055488243699074, 0.010736111551523209, 0.009479966945946217, -0.039280690252780914, -0.029964283108711243, 0.008806097321212292, 0.10771439969539642, -0.020674047991633415, -0.0007131237653084099, 0.04587544873356819, 0.018737491220235825, -0.0024157885927706957, -0.012672667391598225, -0.0006779582472518086, 0.009728578850626945, 0.003601602278649807, -0.02432471700012684, -0.03436078876256943, 0.01584574207663536, 0.0036801111418753862, 0.01634296588599682, 0.0027527231723070145, -0.018829084932804108, -0.017651449888944626, 0.0009020361467264593, -0.0028884783387184143, 0.019653430208563805, -0.010297768749296665, -0.01464193593710661, 0.036532871425151825, 0.015832656994462013, 0.010729569010436535, 0.004510998725891113, 0.03391590341925621, 0.008204194717109203, -0.017285073176026344, -0.014903632923960686, 0.018789829686284065, 0.0032172349747270346, -0.007602292113006115, -0.0006292990292422473, -0.002270219847559929, -0.005044205579906702, 0.025960322469472885, 0.021262865513563156, -0.010448244400322437, 0.012469852343201637, 0.03258125111460686, 0.0067321499809622765, -0.03245040401816368, 0.0038534854538738728, -0.00878646969795227, -0.004900272469967604, 0.02534533478319645, 0.002324194647371769, -0.0032613964285701513, 0.0158980805426836, 0.013163348659873009, 0.009996817447245121, -0.017442092299461365, 0.006398486904799938, 0.0043343533761799335, 0.01762527972459793, -0.0004972239257767797, -0.013686742633581161, -0.021001167595386505, -0.0005601947195827961, -0.011010892689228058, -0.010310853831470013, 0.013529724441468716, -0.016840189695358276, -0.027582842856645584, -0.01758602447807789, 0.0037128233816474676, -0.02080489508807659, 0.0018138859886676073, -0.008014464750885963, 0.00012481710291467607, 0.00738639198243618, -0.009708951227366924, 0.024625668302178383, 0.008308873511850834, 0.00497551029548049, 0.007785480003803968, 0.018266435712575912, 0.002253863727673888, 0.010297768749296665, -0.0040464866906404495, 0.010317396372556686, -0.016578491777181625, -0.012482937425374985, 0.006257824599742889, 0.01897301711142063, 0.004268928896635771, -0.026483716443181038, -0.008714503608644009, 0.009643526747822762, 0.01617286168038845, 0.003058581380173564, 0.0018645897507667542, -0.002150820568203926, 0.018397284671664238, 0.006270909681916237, 0.03778901696205139, 0.02124978043138981, -0.0221133790910244, 0.02847261168062687, 0.007909785956144333, -0.016369134187698364, -0.01719347946345806, 0.0037128233816474676, -0.011050147004425526, 0.006166230887174606, -0.011089402250945568, 0.013058669865131378, 0.0008038998348638415, 0.018318776041269302, -0.006437741219997406, -0.002952266950160265, -0.0008578748092986643, 0.0045175408013165, 0.017612194642424583, 0.0008198469877243042, 0.002389618894085288, 0.005014765076339245, -0.021917106583714485, -0.036087989807128906, -0.032214876264333725, 0.024612583220005035, 0.008694875985383987, 0.0024043393786996603, -0.016133608296513557, -0.00821727979928255, -0.00011173226812388748, -0.011547370813786983, -0.00967623945325613, -0.003325185040012002, 0.0069611347280442715, -0.012417512945830822, -0.02107967808842659, -0.01687944307923317, -0.002109930384904146, 0.03203168883919716, 0.01596350409090519, -0.02847261168062687, 0.0051161726005375385, -0.021275950595736504, 0.007137780077755451, 0.02075255662202835, -0.03334017097949982, 0.020831065252423286, -0.02469109371304512, -0.008047176524996758, -0.005158698186278343, 0.035721611231565475, 0.02432471700012684, -0.005217580124735832, 0.006032111123204231, -0.0334448516368866, 0.013634403236210346, 0.01291473675519228, -0.009630442596971989, 0.0026447733398526907, -0.0031763447914272547, -0.000209050762350671, 0.03723945468664169, 0.02980726584792137, -0.00415116548538208, 0.0038796551525592804, 0.007641546428203583, -0.0020804896485060453, -0.006771404761821032, -0.03239806368947029, -0.00031546730315312743, -0.004978781566023827, -0.007759310305118561, 0.007033101283013821, 0.012253952212631702, -0.008459349162876606, 0.00738639198243618, -0.013104467652738094, 0.01555787492543459, 0.007072356063872576, -0.044409945607185364, -0.020687131211161613, -0.0022980249486863613, 0.010389362461864948, -0.01135109830647707, -0.010637975297868252, 0.03360186889767647, 0.011292217299342155, -0.006215299014002085, 0.024612583220005035, 0.0008374297758564353, 0.027687521651387215, -0.029519399628043175, -0.000843154382891953, -0.006689624395221472, 0.017873890697956085, -0.005574141629040241, -0.018737491220235825, 0.015924250707030296, -0.01153428666293621, -0.03590480238199234, -0.029257701709866524, 0.013281112536787987, 0.03768434002995491, -0.014890547841787338, 0.009905223734676838, 0.021537646651268005, 0.0056264810264110565, 0.011030520312488079, -0.0079490402713418, -0.0048446618020534515, 0.02081798017024994, -0.021262865513563156, -0.0034118720795959234, -0.015544789843261242, -0.007471443619579077, -0.015256923623383045, -0.022519009187817574, -0.000607627269346267, -0.025149062275886536, -0.0012953991536051035, -0.016473812982439995, -0.016251370310783386, -0.0027052906807512045, -0.010559465736150742, 0.020189907401800156, 0.012038053013384342, 0.01812250353395939, -0.004102097358554602, 0.0229115542024374, -0.020124483853578568, 0.010049156844615936, -0.010166920721530914, 0.0018187927780672908, 0.015531704761087894, 0.01282968558371067, -0.003385702380910516, -0.04820454865694046, 0.01843653991818428, -0.011220250278711319, -0.01466810517013073, -0.04679138585925102, 0.03634968400001526, 0.018266435712575912, -0.01134455669671297, -0.006378859281539917, -0.0033104645553976297, 0.004079198930412531, 0.002018336672335863, 0.0030700305942445993, -0.025410758331418037, -0.01876366138458252, -0.0012962169712409377, -0.019012272357940674, -0.0035034657921642065, -0.025332249701023102, 0.019548751413822174, -0.0039352658204734325, -0.0031289122998714447, -0.013163348659873009, 0.0026365951634943485, 0.016918698325753212, -0.019182374700903893, 0.015427025966346264, -0.00765463151037693, -0.0025008399970829487, -0.0037324505392462015, 0.024180784821510315, 0.015073735266923904, -0.008734130300581455, 0.014838208444416523, 0.0013125729747116566, 0.0231863372027874, -0.016735510900616646, 0.012227782979607582, -0.020530113950371742, -0.01606818288564682, -0.028446441516280174, -0.004089012276381254, -0.006663454696536064, -0.013542809523642063, 0.0036572127137333155, 0.0068499138578772545, 0.0479690246284008, 0.006876083556562662, 0.012214697897434235, -0.0009355660295113921, 0.0015284728724509478, 0.0016028928803279996, 0.00180243665818125, -0.013359621167182922, -0.00225877040065825, -0.0015440110582858324, -0.02032075636088848, 0.002688934560865164, -7.314221147680655e-05, -0.023251760751008987, -0.012816600501537323, -0.029990453273057938, 0.03035682812333107, 0.025436928495764732, -0.028655799105763435, -0.0054563782177865505, -0.02135445922613144, 0.013948439620435238, -0.04383421316742897, -0.01005569938570261, 0.02771369181573391, -0.011835237964987755, -0.009486508555710316, 0.012325919233262539, 0.00048250347026623785, 0.00460913497954607, -0.009237896651029587, 0.0024419582914561033, -0.006928422953933477, -0.02183859795331955, 0.023644305765628815, -0.012175443582236767, 0.005917618982493877, -0.012083849869668484, -0.0067910319194197655, 0.0186066422611475, -0.017101885750889778, -0.03470099717378616, 0.017180394381284714, -0.009519221261143684, 0.022715281695127487, 0.00690225325524807, 0.009722036309540272, 0.0014090737095102668, 0.012221240438520908, -0.013071754947304726, -0.039542388170957565, -0.016212116926908493, 0.0007560584344901145, -0.00028357302653603256, -0.01838419958949089, 0.0022244227584451437, -0.024717262014746666, 0.027582842856645584, 0.0031828873325139284, -0.022440500557422638, 0.03268593177199364, 0.02647063136100769, 0.010009902529418468, -0.0022653129417449236, -0.0016617747023701668, -0.014785869047045708, -0.025057468563318253, -0.009996817447245121, -0.0167224258184433, 0.0008971293573267758, -0.022257313132286072, 0.03381122648715973, 0.010147293098270893, -0.013987693935632706, -0.001921835821121931, 0.017926231026649475, 0.0006137607851997018, -0.016630832105875015, 0.011527744121849537, 0.0010157106444239616, 0.020844150334596634, -0.01326802745461464, 0.021001167595386505, 0.054380595684051514, -0.01151465903967619, 0.036035649478435516, -0.01725890301167965, -0.01533543225377798, 0.005603582598268986, -0.010637975297868252, 0.009172473102807999, -0.008191109634935856, 0.008917318657040596, 0.009636984206736088, -0.010520211420953274, 0.008701418526470661, -0.003349719103425741, 0.027844538912177086, -0.012842770665884018, 0.009061251766979694, -0.02975492551922798, 0.023330269381403923, -0.02292463928461075, -0.00892386119812727, 0.018737491220235825, -0.023212505504488945, -0.006843371316790581, 0.009656611829996109, -0.024115359410643578, 0.02711178921163082, 0.002608790062367916, 0.001508845598436892, -0.007778937462717295, 0.019339393824338913, -0.0064083002507686615, -0.019116951152682304, -0.01116136834025383, -0.012986703775823116, -0.009211727418005466, -0.020202992483973503, -0.022741451859474182, 0.024874281138181686, -0.0031174630858004093, -0.029571738094091415, -0.017049547284841537, 0.0023029318545013666, -0.013169891200959682, 0.009264066815376282, 0.005413852632045746, 0.007301340810954571, 0.011658592149615288, -0.015191499143838882, 0.018240267410874367, -0.019273968413472176, -0.021315203979611397, -0.015008311718702316, 0.0018940306035801768, -0.007353680208325386, 0.02458641491830349, -0.010834247805178165, -0.007942497730255127, -0.015387771651148796, -0.022505924105644226, -0.002468127990141511, -0.013281112536787987, -0.03514587879180908, -0.015505535528063774, -0.0022980249486863613, -0.019548751413822174, 0.014340984635055065, -0.03320932388305664, 0.0007139415829442441, -0.015976589173078537, 0.013595148921012878, -0.02690243162214756, 0.011652049608528614, 0.2082059681415558, -0.015348517335951328, -0.002093574497848749, 0.04058917239308357, -0.005158698186278343, 0.03224104642868042, 0.02842027321457863, -0.00756303733214736, -0.009957563132047653, -0.02620893530547619, -0.010376278311014175, 0.014681190252304077, -0.008472434245049953, -0.00034981503267772496, 0.017285073176026344, 0.00025924338842742145, -0.017599109560251236, -0.02393217198550701, -0.0018187927780672908, -0.004098826088011265, -5.670310088135011e-07, -0.018292605876922607, -0.04998409003019333, 0.021275950595736504, 0.03365420922636986, 0.021825512871146202, -0.008858436718583107, 0.009460339322686195, -0.0017566398018971086, 0.005776956677436829, -0.006862998474389315, -0.00021303755056578666, -0.006454097107052803, -0.012960533611476421, 0.018475793302059174, -0.030278319492936134, 0.01930013857781887, -0.014484917744994164, 0.018292605876922607, 0.022283483296632767, 0.02242741547524929, -0.01328765507787466, 0.0014499637763947248, -0.002590798307210207, -0.005390954203903675, 0.046451181173324585, 0.005453106947243214, -0.0017877162899821997, 0.01827952079474926, 0.02323867566883564, -0.010114581324160099, -0.02868196927011013, 0.011979171074926853, -0.0006014937534928322, -0.004537168424576521, -0.007307882886379957, 0.0016020750626921654, -0.011481947265565395, -0.004625490866601467, 0.024193869903683662, 0.003827315755188465, 0.00865562167018652, -0.01897301711142063, 0.02647063136100769, -0.027687521651387215, 2.726860111579299e-05, -0.023317184299230576, 0.03407292440533638, 0.01871132105588913, 0.017062630504369736, 0.019116951152682304, -0.009329491294920444, -0.0031959721818566322, -0.011756728403270245, -0.012594158761203289, 0.0033022863790392876, 0.0012577802408486605, 0.02372281439602375, 0.024337802082300186, 0.026248188689351082, -0.017075715586543083, -0.006264367140829563, 0.0015489179641008377, 0.0034151431173086166, 0.022178804501891136, -0.026758497580885887, 0.014157797209918499, -0.02130211889743805, 0.0067910319194197655, -0.02043852023780346, 0.005969958379864693, -0.016735510900616646, 0.014615765772759914, -0.02836793288588524, -0.007131238002330065, -0.006843371316790581, 0.011612795293331146, 0.021380629390478134, -0.01967959851026535, -0.003909096121788025, 0.006071365904062986, 0.011920289136469364, 0.02118435502052307, 0.009669696912169456, -0.02565937116742134, 0.010166920721530914, 0.004762881901115179, 0.01383067574352026, 0.003325185040012002, -0.013143721967935562, 0.029938112944364548, -0.04875411465764046, 0.0047334409318864346, 0.012005340307950974, 0.007517240475863218, 0.024612583220005035, -0.005881635472178459, -0.028917497023940086, 0.0004902725922875106, -0.0065031652338802814, 0.009898681193590164, -0.044619303196668625, 0.0029064700938761234, -0.01909078098833561, -0.013235315680503845, -0.020412350073456764, -0.028263254091143608, 0.004919899627566338, 0.0014867649879306555, -0.03891431540250778, 0.010114581324160099, -0.005999399349093437, 0.010081869550049305, -0.00876030046492815, 0.0012651404831558466, -0.005754058249294758, 0.01736358180642128, -0.04922516644001007, 0.002098481170833111, -0.006512979045510292, 0.031141920015215874, 0.004762881901115179, -0.013124094344675541, -0.00989213865250349, 0.01865898258984089, -0.028289424255490303, 0.019993634894490242, 0.00225877040065825, -0.003124005626887083, -0.030906392261385918, -0.005260105710476637, 0.004003961104899645, -0.017599109560251236, -0.010906213894486427, 0.012417512945830822, -0.012306291610002518, -0.03595713898539543, 0.0038665703032165766, 0.006669997237622738, -0.013071754947304726, -0.00543347978964448, 0.019130036234855652, 0.014628850854933262, 0.002358542289584875, -0.010624890215694904, 0.008727588690817356, -0.1639268696308136, 0.03265976160764694, 0.0194048173725605, -0.01116136834025383, 0.017180394381284714, 0.021053507924079895, 0.031900838017463684, -0.001508845598436892, 0.006650370080024004, -0.006153145805001259, 0.013568978756666183, -0.012509106658399105, -0.03339251130819321, -0.025646286085247993, 0.011004350148141384, 0.006401758175343275, -0.011030520312488079, 0.042080845683813095, 0.019313223659992218, -0.0010124394902959466, 0.025423843413591385, -0.007026559207588434, 0.019692683592438698, -0.03025214932858944, -0.003608144586905837, 0.029205363243818283, 0.006784489378333092, -0.021655410528182983, -0.00938183069229126, -0.014903632923960686, -0.021145101636648178, 0.003689924953505397, 0.007242458872497082, 0.009617357514798641, 0.00561993895098567, -0.022008700296282768, -0.01151465903967619, -0.021576901897788048, -0.010441701859235764, 0.025855643674731255, 0.011220250278711319, 0.02399759739637375, 0.009316406212747097, -0.0037259082309901714, 0.0026954771019518375, -0.013621318154036999, -0.009401457384228706, -0.009237896651029587, -0.014367153868079185, -0.025907984003424644, 0.01355589460581541, 0.0180570799857378, 0.015073735266923904, 0.01947024278342724, -0.018214097246527672, 0.019103866070508957, -0.016473812982439995, 0.01693178340792656, 0.004762881901115179, -0.009447254240512848, -0.02679775282740593, -0.025737879797816277, -0.01957491971552372, 0.0013804505579173565, -0.014223220758140087, -0.019535666331648827, -0.038417089730501175, 0.01180906780064106, -0.05568907782435417, 0.0071901194751262665, -0.01487746275961399, -0.03213636577129364, 0.015597129240632057, 0.011488489806652069, 0.015597129240632057, 0.00041871488792821765, -0.022466670721769333, 0.017271988093852997, 0.009211727418005466, -0.00792941264808178, 0.00580966891720891, 0.044671643525362015, -0.04137426242232323, -0.002600611886009574, -0.029074514284729958, -0.010376278311014175, -0.015243838541209698, 0.008289245888590813, -0.0033955159597098827, 0.011357640847563744, 0.014236305840313435, 0.0032352267298847437, -0.004226403310894966, -0.012253952212631702, 0.006993846967816353, 0.013582063838839531, 0.04074619337916374, -0.003447855357080698, 0.01591116562485695, -0.014301730319857597, 0.012325919233262539, -0.0020870319567620754, -0.017782296985387802, -0.002379805315285921, 0.019287053495645523, 0.029938112944364548, 0.028289424255490303, 0.006856455933302641, 0.006248011253774166, -0.014720444567501545, -0.01124641951173544, 0.01084079034626484, 0.016185946762561798, 0.007608834654092789, -0.006771404761821032, -0.0018351487815380096, 0.02980726584792137, -0.021498391404747963, 0.025633201003074646, -0.005044205579906702, -0.000924934633076191, 0.010520211420953274, 0.009990274906158447, 0.002553179394453764, -0.0180570799857378, 0.008858436718583107, -0.10577784478664398, -0.02280687540769577, -0.01197262853384018, 0.017873890697956085, -0.042342543601989746, 0.016473812982439995, -0.005757329519838095, 0.02028150111436844, 0.017926231026649475, 0.015427025966346264, -0.010212717577815056, 0.025214485824108124, -0.010212717577815056, -0.0005552878719754517, 0.029414720833301544, -0.015178414061665535, 0.005250291898846626, -0.008943487890064716, -0.016185946762561798, 0.005341886077076197, 0.027896879240870476, 0.015165329910814762, 0.011475404724478722, 0.007222831714898348, -0.01245676726102829, -0.003253218252211809, -0.016369134187698364, 0.023696644231677055, -0.0017860806547105312, 0.030775543302297592, -0.01326802745461464, -0.0014008956495672464, 0.009682781994342804, -0.02033384144306183, -0.001679766341112554, 0.024599500000476837, 0.012273579835891724, -0.01833186112344265, 0.023971427232027054, -0.014079287648200989, 0.013222230598330498, 0.00836121290922165, 0.021367544308304787, -0.048544757068157196, 0.0019578190986067057, -0.026928599923849106, -0.01180906780064106, 0.003287566127255559, -0.0011416522320359945, -0.02685009129345417, 0.002304567489773035, -0.010291226208209991, -0.04291827604174614, 0.0038044173270463943, 0.0226498581469059, 0.0012929457006976008, 0.011481947265565395, 0.003696467261761427, 0.006869541015475988, -0.014681190252304077, -0.008125685155391693, 0.00019821488240268081, -0.00216390541754663, 0.00989213865250349, -0.007510697934776545, 0.007595749571919441, -0.01622520200908184, -0.009591187350451946, -0.026130424812436104, -0.02285921573638916, -0.037396471947431564, -0.007124695461243391, 0.009610814973711967, 0.0019234714563935995, -0.021001167595386505, -0.002191710751503706, -0.0021524562034755945, -0.008590197190642357, -0.007307882886379957, 0.012312834151089191, -0.0065784030593931675, -0.015937335789203644, 0.00010917663166765124, -0.02873430773615837, 0.030906392261385918, 0.020464690402150154, 0.015780316665768623, 0.020687131211161613, -0.004932984709739685, -0.021393712610006332, 4.845479634241201e-05, 0.023526541888713837, 0.022126464173197746, -0.011861407198011875, -0.002659493824467063, -0.00881918240338564, -0.006954592652618885, 0.005148884374648333, 0.034910354763269424, 0.0074191042222082615, -0.007334052585065365, 0.00217044772580266, -0.06076599657535553, 0.011861407198011875, 0.0075499527156353, -0.007294798269867897, 0.01547936536371708, -0.009205184876918793, 0.010834247805178165, -0.009198642335832119, 0.024939704686403275, 0.00015691584849264473, -0.032476574182510376, -0.014550342224538326, -0.004618948325514793, -0.008413552306592464, -0.029467059299349785, -0.00886497925966978, 0.010723026469349861, -0.003611415857449174, 0.0030160555616021156, 0.012921279296278954, -0.013241858221590519, 0.016513068228960037, -0.0110566895455122, -0.01326802745461464, -0.0015489179641008377, 0.008989284746348858, -0.043703366070985794, 0.007000389508903027, -0.005649379454553127, 0.025790220126509666, -0.001187449204735458, -0.015047566033899784, -0.028655799105763435, 0.023147081956267357, -0.03891431540250778, 0.01331382431089878, 0.010965095832943916, 0.01850196346640587, -0.00867524929344654, 0.02399759739637375, -0.019156206399202347, -0.03035682812333107, 0.03174382075667381, -0.03381122648715973, -0.03930686041712761, -0.03158680349588394, 0.00029931572498753667, 0.017128055915236473, 0.010709941387176514, -0.001662592520006001, 0.03697775676846504, 0.023709729313850403, -0.0037978747859597206, 0.006876083556562662, -0.007327510509639978, -0.024678008630871773, -0.023696644231677055, -0.014890547841787338, 0.0002772350562736392, 0.014131627045571804, 0.013673657551407814, -0.004988595377653837, 0.037710510194301605, -0.0013567343121394515, 0.00995102059096098, 0.019326308742165565, -0.005214308854192495, 0.012168901041150093, 0.0266930740326643, 0.009165930561721325, -0.035773951560258865, 0.01774304360151291, 0.01876366138458252, 0.01655232161283493, 0.01973193883895874, -0.016486898064613342, -0.022505924105644226, 0.0036343142855912447, -0.005417123902589083, 0.011122114025056362, 0.024560244753956795, -0.008969658054411411, -0.028080066666007042, 0.028551120311021805, 0.024507904425263405, 0.0022064312361180782, -0.020150654017925262, -0.004681101534515619, 0.0016699526458978653, 0.020935744047164917, -0.04469781368970871, 0.006097535602748394, 0.011220250278711319, 0.006934965029358864, -0.010225802659988403, 0.0014041669201105833, -0.030225981026887894, -0.01606818288564682, -0.01580648683011532, 0.023474203422665596, 0.010127666406333447, 0.003290837164968252, -0.0025139248464256525, 0.0044881002977490425, -0.02577713504433632, 0.012783888727426529, -0.006745235063135624, -0.016055097803473473, -0.006055009551346302, 0.022204972803592682, 0.02254517935216427, -0.007301340810954571, 0.017546771094202995, -0.0011866313870996237, -0.01575414650142193, -0.0006975855212658644, 0.013896100223064423, -0.0043703364208340645, -0.013634403236210346, 0.011154825799167156, 0.007334052585065365, 0.01585882529616356, -0.017455177381634712, 0.011835237964987755, 0.003294108435511589, 0.008734130300581455, -0.010651059448719025, -0.00598631426692009, -0.006339604966342449, -0.00562320975586772, -0.012221240438520908, -0.023631220683455467, -0.025921067222952843, 0.01667008548974991, -0.007406019605696201, 0.01817484200000763, 0.009833257645368576, 0.028315594419836998, -0.009388372302055359, 0.050114937126636505, 0.029257701709866524, -0.017167309299111366, 0.006797574460506439, 0.007392934523522854, 0.00737330736592412, 0.008092973381280899, 0.007543410174548626, -0.023605050519108772, 0.003948350436985493, 0.02000671997666359, -0.011220250278711319, 0.03498886153101921, -0.007137780077755451, -0.009152845479547977, -0.011953000910580158, -0.020909573882818222, -0.024128444492816925, 0.003457668935880065, 0.009231355041265488, 0.014812039211392403, -0.03203168883919716, 0.01810941845178604, 0.004111911170184612, -0.012600701302289963, -0.034858014434576035, 0.011501573957502842, -0.02792304754257202, 0.00021242420189082623, -0.02674541249871254, 0.023330269381403923, -0.011115571483969688, -0.02771369181573391, 0.014367153868079185, 0.013843760825693607, 0.006418114062398672, -0.005868550855666399, -0.012921279296278954, -0.006598030682653189, 0.012692295014858246, -0.004877374041825533, -0.01736358180642128, -0.02474343217909336, -0.02928387187421322, -0.013752167113125324, 0.01871132105588913, -0.0060255685821175575, 0.004687644075602293, -0.01827952079474926]} +{"id": "test:10000066", "text": "\"Constant development in Unmanned Aircraft System (UAS) technologies results in evolving aircraft, instrumentation, and control programs of these systems. An open source simulator was examined as a potential method of testing UAS platforms changes. The open source simulator was used to simulate flights flown by an actual platform. Existing flight data was used as a base line of comparison for the telemetry data of the simulated flight. Patterns in parameters such as vehicle attitude and throttle output were compared between the simulated and actual results to determine if the simulator produced an accurate representation of a physical flight. Analysis of the simulated vehicle exhibited similar behavior to that of the actual vehicle in most of the flight data collected. Simplification in the flight simulator model may be responsible for the deviations in magnitude observed between the two sets of data. Three points of interest in each of the three test cases were chosen as checks for the values in the data. These points in the flight were studied in the various graphs of the parameters graphed and the difference between the simulated and actual values was calculated. The values at these points stayed within an average of 0\u00b0 and 30\u00b0 for vehicle attitude and between 0% and 33.3% throttle difference over the three test cases. The average difference in attitude was 5.1\u00b0 over 93 compared values and 8.1% in throttle over 18 compared values. On average the simulator showed good agreement to existing flight data and therefore will be a good tool to simulate flights prior to a mission. The accuracy of the results may be improved with further development of the simulation model.\\nRamirez Duarte, Jaime A. (2018) \\\"Feasibility Study for the Application of Open Source UAS Autopilot Simulator,\\\" McNair Scholars Research Journal: Vol. 5 , Article 4.\"", "vector_field": [-0.002322704764083028, -0.019679581746459007, -0.001124870148487389, -0.020888708531856537, -0.007098408415913582, -0.003919238690286875, -0.009937076829373837, -0.0055800508707761765, -0.016427448019385338, -0.03596804663538933, 0.008304060436785221, 0.02530827559530735, -0.0014870869927108288, 0.0007704708841629326, 0.006719687487930059, -0.004860828630626202, 0.025558440014719963, -0.006712738424539566, -0.0046593076549470425, 0.00024430095800198615, -0.011153152212500572, 0.00864456221461296, -0.031242724508047104, 0.0015895848628133535, -0.023015104234218597, 0.03138170391321182, 0.017928432673215866, -0.03046443685889244, -0.005281243473291397, -0.0012230247957631946, 0.031548481434583664, -0.02371000498533249, -0.022903919219970703, -0.019888050854206085, -0.01584373041987419, 0.0022010973189026117, 0.012786168605089188, -0.01201482955366373, 0.015218320302665234, 0.008227621205151081, 0.004346602596342564, 0.022639857605099678, -0.0010327958734706044, 0.005163110326975584, 0.0018553842091932893, -0.0011153152445331216, 0.018387066200375557, -0.008943368680775166, -0.006785702891647816, 0.038636464625597, -0.0012499522417783737, 0.034050121903419495, -0.018123004585504532, 0.004624562803655863, 0.027087220922112465, 0.030186476185917854, 0.0012264993274584413, -0.004252790939062834, 0.006010889075696468, -0.011667379178106785, -0.012702780775725842, 0.012939047068357468, -0.009568779729306698, 0.023543227463960648, -0.016496937721967697, 0.004725323058664799, 0.0019352978561073542, -0.008165080100297928, 0.01216075848788023, -0.012779220007359982, 0.0219032634049654, 0.026934342458844185, 0.021152770146727562, -0.010180291719734669, 0.020874809473752975, 0.005350733641535044, -0.02074972726404667, -0.016246773302555084, -0.021013790741562843, 0.005576576106250286, 0.023001205176115036, -0.0067996010184288025, -0.016427448019385338, 0.009520136751234531, 0.0019335605902597308, 0.004680154845118523, -0.0031722206622362137, 0.027934998273849487, -0.011535347439348698, -0.004836507141590118, -0.008922521956264973, 0.020999891683459282, 0.037913769483566284, 0.015718648210167885, -0.019207049161195755, -0.000820416898932308, -0.010263679549098015, 0.039581529796123505, 0.006716213189065456, -0.025780808180570602, 0.004165928345173597, -0.027017731219530106, -0.014273256063461304, -0.007609160151332617, -0.026392320170998573, -0.013453273102641106, 0.015440688468515873, -0.00799135584384203, 0.009944025427103043, -0.029130227863788605, -0.00778288533911109, 0.011952288448810577, -0.009971821680665016, 0.005090145859867334, -0.007477128878235817, -0.000929429370444268, 0.008012202568352222, 0.026781463995575905, -0.010590283200144768, 0.010701467283070087, 0.03577347472310066, 0.028546512126922607, 0.022236814722418785, -0.012098217383027077, 0.02591978758573532, 0.022070039063692093, -0.0019874153658747673, -0.0017094551585614681, -0.007518823258578777, -0.01431494951248169, -0.005336835514754057, 0.013453273102641106, 0.03733005374670029, 0.028880063444375992, -0.0255445409566164, 0.018539944663643837, -0.02098599448800087, 0.007817630656063557, -0.008311009965837002, -0.01683048903942108, 0.0034727652091532946, 0.009054552763700485, -0.03669074550271034, 0.0014827438862994313, -0.017928432673215866, -0.005010232329368591, 0.01848435215651989, 0.010221986100077629, 0.004096438176929951, -0.006601554341614246, 0.011681277304887772, -0.02302900142967701, -0.017858942970633507, 0.01666371338069439, 0.0036933959927409887, 0.0066189272329211235, -0.003769834991544485, 0.021027687937021255, -0.008220672607421875, -0.010291475802659988, 0.00021313900651875883, 0.00992317870259285, -0.013015486299991608, 0.002423465484753251, 0.0005463655106723309, 0.02280663326382637, -0.0031913304701447487, -0.011674327775835991, -0.03068680502474308, -0.012376177124679089, -0.016858285292983055, 0.017497593536973, -0.025433357805013657, 0.009332513436675072, -0.003874070243909955, 0.03855307772755623, -0.009582677856087685, 0.014870869927108288, -0.03713548183441162, -0.009645218960940838, 0.023306962102651596, -0.0017502806149423122, 0.021305648609995842, 0.01584373041987419, -0.0013889323454350233, -0.005969195161014795, 0.0110975606366992, 0.008033049292862415, 0.0032034912146627903, 0.003285141894593835, 0.019373824819922447, -0.004756593611091375, -0.00032747184741310775, -0.007157474756240845, -0.638641357421875, -0.03455045074224472, 0.004965063650161028, -0.013300394639372826, 0.006177665200084448, 0.018011819571256638, 0.019874153658747673, -0.014551215805113316, -0.00867930706590414, 0.020791422575712204, -0.017025060951709747, -0.017330817878246307, 0.01302938349545002, -0.025405561551451683, 0.004530751146376133, -0.030936969444155693, -0.001527912449091673, -0.03343861177563667, 0.011312979273498058, -0.012035676278173923, -0.0369965024292469, 0.02031888999044895, -0.0033320477232337, 0.0054167490452528, 0.0007930551655590534, 0.01018724124878645, 0.0073103527538478374, -0.007407639175653458, -0.001276879571378231, 0.012654137797653675, 0.012216350063681602, 0.02083311602473259, 0.024752354249358177, 0.0056634387001395226, 0.043834321200847626, -0.019262641668319702, -0.023126287385821342, 0.007289506029337645, 0.017247429117560387, 0.0219032634049654, -0.0027535432018339634, -0.01007605716586113, 0.04825389012694359, 0.009235227480530739, -0.012327534146606922, 0.014829176478087902, 0.03596804663538933, -0.014224613085389137, -0.008957266807556152, -0.026475708931684494, 0.007338149007409811, -0.0015730808954685926, 0.0219588540494442, -0.0010388762457296252, 0.025183193385601044, -0.007372893858700991, 0.03263252601027489, -0.018901292234659195, -0.0051109930500388145, 0.0022844853810966015, 0.0018692822195589542, -0.008595919236540794, -0.017997922375798225, -0.016844388097524643, -0.006389610003679991, 0.02016601152718067, -0.0054167490452528, -0.009117093868553638, 0.006959428079426289, -0.024891335517168045, 0.021041586995124817, 0.007268658839166164, -0.01322395633906126, -0.0018814430804923177, -0.004120759665966034, 0.028685491532087326, 0.026128258556127548, -0.018220290541648865, -0.012549902312457561, 0.0014914300991222262, 0.013918856158852577, -0.007713395170867443, -0.016093894839286804, -0.005350733641535044, 0.02758754789829254, -0.011549245566129684, -0.010249782353639603, 0.01011080201715231, 0.012209401465952396, 0.012758372351527214, 0.009944025427103043, 0.03585686534643173, -0.022792736068367958, -0.02158360928297043, -0.004117285367101431, 0.04555767402052879, -0.02680926024913788, -0.0036447530146688223, 0.02364051342010498, -0.021805977448821068, -0.01970737800002098, 0.011090611107647419, 0.046252574771642685, 0.01659422367811203, 0.027003832161426544, 0.020860912278294563, 0.00753966998308897, -0.0030106562189757824, 0.026920443400740623, -0.040721166878938675, 0.0074632312171161175, 0.011076713912189007, 0.011069764383137226, 0.005732928868383169, 0.021611405536532402, -0.030659008771181107, 0.006792651955038309, -0.012751423753798008, 0.0072478121146559715, -0.023015104234218597, 0.004940742161124945, -0.025336071848869324, 0.008088641799986362, -0.002319230232387781, -0.0005485370638780296, 0.012952945195138454, 0.01659422367811203, -0.03218778967857361, 0.007063663098961115, -0.01090993732213974, 0.024516088888049126, 0.0045203277841210365, 0.024335414171218872, -0.017622675746679306, -0.006983749568462372, -0.017414206638932228, 0.008304060436785221, -0.01848435215651989, 0.004065167624503374, -0.008616765961050987, -0.0334664061665535, -0.0037767840549349785, 0.013133618980646133, -0.011590939946472645, -0.0004456049355212599, -0.016719305887818336, 0.006949004717171192, 0.002963750623166561, 0.01582983322441578, 0.009110145270824432, -0.0028108723927289248, -0.015037646517157555, -0.007894068956375122, 0.009791147895157337, -0.010305373929440975, -0.0030957816634327173, 0.005593948997557163, -0.03471722826361656, -0.003000232856720686, -0.01652473397552967, 0.0003513590490911156, 0.02697603590786457, -0.012216350063681602, -0.0010666722664609551, -0.005496662575751543, 0.005507086403667927, -0.023765595629811287, 0.015704751014709473, -0.04191639646887779, -0.0193182323127985, 0.004061693325638771, -0.0011561407009139657, 0.025975380092859268, 0.03666294738650322, -0.008658460341393948, 0.020513461902737617, -0.03174305334687233, 0.0015400731936097145, -0.004478633403778076, -0.014509521424770355, 0.013765978626906872, 0.028991248458623886, -0.002647570800036192, -0.021458527073264122, 0.0029394289012998343, 0.011229591444134712, 0.008887777104973793, 0.014565113931894302, -0.015704751014709473, 0.03424469381570816, -0.012174656614661217, 0.02537776529788971, -0.013703437522053719, 0.014259357936680317, -0.01933213137090206, 0.02355712652206421, 0.010020464658737183, -0.00966606568545103, 0.007657803129404783, 0.021555813029408455, 0.008721000514924526, 0.008839134126901627, 0.0134463245049119, -0.012452616356313229, -0.0005029342137277126, -0.022778837010264397, 0.017747757956385612, -0.01637185551226139, 0.020888708531856537, 0.009144890122115612, 0.014036989770829678, -0.00044690785580314696, -0.00787322223186493, -0.0041694026440382, 0.018428759649395943, 0.004805236589163542, -0.009325563907623291, 0.0007700365968048573, 0.008957266807556152, 0.0059518227353692055, -0.014509521424770355, -0.00668494263663888, 0.035328738391399384, -0.007588312961161137, 0.005767673719674349, 0.022987307980656624, -0.0006805681623518467, 0.03357759118080139, 0.024905232712626457, -0.0021628777030855417, 0.005041502881795168, -0.006674519274383783, 0.0020586426835507154, 0.009103195741772652, 0.06782228499650955, 0.018373169004917145, 0.012480412609875202, 0.005649541039019823, 0.034522656351327896, -0.006990698631852865, 0.02976953610777855, 0.011451959609985352, 0.00965911615639925, -0.016496937721967697, 0.02439100667834282, -0.0026510453317314386, 0.03363318368792534, 0.005392427556216717, -0.032159995287656784, -0.013265649788081646, -0.015704751014709473, 0.010555538348853588, 0.0041485559195280075, -0.006254104431718588, -0.005847587250173092, 0.014662399888038635, -0.016483038663864136, 0.0021941482555121183, 0.045724451541900635, 0.01431494951248169, 0.009728606790304184, 0.004579394124448299, -0.017636574804782867, -0.001560920150950551, -0.020708033815026283, 0.008172029629349709, -0.023390349000692368, 0.003177432343363762, -0.000631490780506283, -0.025864195078611374, -0.009026757441461086, -0.004850405268371105, 0.0014254145789891481, 0.0002160706208087504, 0.03457824885845184, 0.006691891700029373, 0.0002825204865075648, 0.01238312665373087, 0.04961589351296425, 0.0024582103360444307, -0.023001205176115036, -0.036329396069049835, 0.02311239019036293, 0.012945995666086674, 0.013863264583051205, 9.516879072180018e-05, -0.003825427033007145, -0.0019561448134481907, -0.003665599972009659, 0.001011948799714446, -0.028963452205061913, 0.0018466979963704944, -0.0263645239174366, -0.021500220522284508, -0.0004629774484783411, 0.01523221842944622, 0.013508865609765053, -0.010228934697806835, 0.010777906514704227, 0.012355330400168896, 0.0019352978561073542, 0.0032000166829675436, -0.01322395633906126, -0.008540326729416847, 0.018845701590180397, 0.026044869795441628, -0.0042284694500267506, -0.02158360928297043, 0.020888708531856537, -0.02076362632215023, -0.0005910997278988361, -0.02491913177073002, -0.0198463574051857, 0.007122729904949665, 0.004016524646431208, 0.007963559590280056, -0.008144233375787735, -0.004026948008686304, 0.028393633663654327, 0.012223299592733383, -0.0133281908929348, -0.005104043986648321, -0.015537974424660206, 0.009242176078259945, 0.04461260885000229, -0.011146203614771366, 0.008234570734202862, 0.013689539395272732, -0.021013790741562843, -0.019123660400509834, -0.02031888999044895, -0.02159750647842884, 0.011743818409740925, 0.020096521824598312, -0.04689188301563263, 0.024377109482884407, 0.024585578590631485, -0.029741739854216576, 0.010124700143933296, -0.0005502743297256529, 0.006893412675708532, -0.02023550122976303, -0.008165080100297928, -0.007532720919698477, -0.024113046005368233, -0.0023487636353820562, 0.00276396656408906, 0.015788137912750244, 0.027337385341525078, -0.003436282742768526, 0.019832460209727287, 0.019262641668319702, 0.0046593076549470425, -0.012536004185676575, -0.01007605716586113, 0.02493302896618843, 0.004291010554879904, 0.03090917319059372, -0.01940162107348442, -0.007220015861093998, -0.007150525692850351, 0.005152686964720488, 0.006539013236761093, 0.010249782353639603, -0.02377949468791485, -0.0012569011887535453, 0.0024964299518615007, -0.011264336295425892, -0.006222833879292011, -0.028602102771401405, -0.01200788002461195, 0.01780335046350956, -0.025197090581059456, -0.020888708531856537, -0.008943368680775166, -0.03202101215720177, -0.012146860361099243, 0.00014364895469043404, 0.006167241837829351, -0.008248468860983849, 0.013300394639372826, -0.008297111839056015, -0.02074972726404667, -0.018387066200375557, -0.05498052388429642, -0.006962902843952179, 0.018887395039200783, -0.011764665134251118, -0.047086454927921295, -0.025669623166322708, -0.005166585091501474, -0.011292132548987865, -0.04691968113183975, 0.013091924600303173, 0.00431880634278059, -0.019735174253582954, -0.05000503733754158, 0.0030853580683469772, 0.006632824894040823, -0.025030314922332764, -0.0007179190288297832, 0.006775279529392719, -0.010152495466172695, 0.016483038663864136, -0.020582951605319977, -0.02188936434686184, 0.0016060887137427926, -0.028296347707509995, 0.004297959618270397, 0.004683629143983126, 0.0033598437439650297, 0.007970508188009262, -0.023056797683238983, 0.0062853749841451645, 0.002008262323215604, -0.0027865509036928415, -0.00662935059517622, -0.02849091961979866, -0.004634986165910959, 0.007407639175653458, 0.010812651365995407, 0.020471768453717232, 0.02680926024913788, -0.006010889075696468, 0.010993325151503086, -0.025711316615343094, -0.01527391280978918, -0.005208279006183147, 0.0026423591189086437, -0.0007031524437479675, -0.009235227480530739, 0.006306221708655357, -0.02144462801516056, -0.009374207817018032, -0.0044021946378052235, 0.003121840301901102, -0.010506895370781422, -0.014509521424770355, 0.01208431925624609, -0.0024008811451494694, 0.0060317362658679485, 0.017497593536973, -0.0052881925366818905, 0.0016512572765350342, -0.021416831761598587, 0.0051492126658558846, 0.022903919219970703, 0.02409914880990982, -0.006393084302544594, 0.032993875443935394, -0.006490370258688927, 0.0066189272329211235, -0.04547428712248802, 0.018526047468185425, 0.00890167523175478, 0.017942329868674278, 0.008797439746558666, -0.011125356890261173, -0.004944216925650835, -0.02478015050292015, 0.00108057027682662, 0.014224613085389137, -0.0011674327543005347, -0.003662125440314412, -0.013383783400058746, 0.04586343094706535, 0.019735174253582954, -0.0038045800756663084, 0.03796936199069023, -0.024210331961512566, -0.0074284859001636505, 0.027865508571267128, -0.00281260977499187, 0.03165966644883156, -0.006104700732976198, -0.0022931715939193964, -0.014704094268381596, -0.024585578590631485, -0.012681934051215649, -0.030353251844644547, -0.02266765385866165, -0.009207431226968765, 0.02711501717567444, 0.025586236268281937, 0.04689188301563263, -0.006393084302544594, 0.033077262341976166, 0.00912404339760542, -0.014273256063461304, -0.0017103237332776189, -0.019109763205051422, -0.0053264121524989605, -0.008519480004906654, 0.027031628414988518, 0.02735128253698349, 0.006056057754904032, 0.00031140228384174407, 0.0088669303804636, -0.005934449844062328, 0.028226858004927635, -0.013508865609765053, -0.012577698566019535, -0.014829176478087902, -0.03538433089852333, 0.006730110850185156, 0.00841524451971054, -0.0010223722783848643, 0.014182918705046177, -0.04350076988339424, -0.0015739495866000652, 0.023501534014940262, 0.012952945195138454, 0.006924683228135109, -0.0009224803652614355, 0.026531299576163292, -0.0021854620426893234, -0.003497086698189378, 0.00433965353295207, 0.026767566800117493, -0.0396927148103714, -0.001850172528065741, -0.04044320806860924, 0.005590474233031273, 0.016191180795431137, -0.003637803951278329, 0.004457786679267883, 0.0151210343465209, 0.020110419020056725, -0.026197748258709908, 0.018234187737107277, 0.009075400419533253, -0.007546619046479464, 0.0026041395030915737, -0.02985292486846447, 0.0006076035788282752, -0.017219632863998413, 0.005204804707318544, -0.0032990400213748217, -0.023084593936800957, 0.0242520272731781, 0.022945614531636238, 0.008429142646491528, -0.026559095829725266, 0.003107942407950759, 0.033216241747140884, 0.006163767073303461, 0.023696105927228928, 0.016760999336838722, 0.028768880292773247, 0.04525191709399223, -0.01651083491742611, -0.014509521424770355, -0.01872061938047409, 0.020944301038980484, 0.0008251942927017808, 0.018678924068808556, 0.015051544643938541, -0.013752080500125885, -0.016399651765823364, 0.001034533022902906, -0.007741191424429417, -0.010631977580487728, -0.015579668805003166, -0.015426790341734886, 0.0047357468865811825, 0.008693205192685127, -0.01940162107348442, -0.021389037370681763, -0.04258350282907486, 0.00756051717326045, -0.008053896017372608, -0.009617422707378864, 0.01696947030723095, -0.008498632349073887, 0.011570093221962452, 0.02432151697576046, 0.01703896000981331, 0.013286497443914413, 0.0026962137781083584, 0.007077561225742102, 0.02251477539539337, -0.0020760151091963053, -0.03432808443903923, -0.0017198786372318864, -0.008554224856197834, 0.004019999410957098, -0.01230668742209673, -0.0059518227353692055, -0.00893642008304596, 0.021041586995124817, -0.009026757441461086, -0.0040686423890292645, -0.030408844351768494, 0.029880721122026443, -0.009158788248896599, -0.008498632349073887, 0.013891060836613178, -0.0026614689268171787, 0.0018571214750409126, 0.00981199461966753, -0.002925531007349491, -0.03327183425426483, 0.008595919236540794, -0.0030401896219700575, 0.019193151965737343, 0.003797631012275815, -0.023668309673666954, -0.00965911615639925, -0.004947691224515438, -0.028171265497803688, -0.018595537170767784, -0.0156769547611475, -0.009311666712164879, -0.010089955292642117, -0.012751423753798008, 0.03202101215720177, -0.02568352222442627, 0.0024807946756482124, 0.020721932873129845, -0.01758098229765892, -0.013738182373344898, 0.025280479341745377, -0.027851611375808716, 0.02462727203965187, 0.020443972200155258, 0.005962246097624302, -0.013133618980646133, 0.01637185551226139, -0.023584922775626183, -0.017664369195699692, 0.025252683088183403, -0.01681659184396267, -0.04233333840966225, -0.006493845023214817, 0.02954716794192791, 0.01534340251237154, 0.004089489113539457, 0.009464544244110584, 0.022028345614671707, 0.019276538863778114, -0.012598545290529728, -0.015663057565689087, -0.005468866787850857, 0.02127785235643387, -0.003818478202447295, -0.007532720919698477, 0.0365239679813385, 0.009339462034404278, 0.028907859697937965, -0.006747483741492033, -0.016927774995565414, 0.0031044678762555122, -0.036329396069049835, -0.005229126196354628, -0.011945338919758797, 0.004329230170696974, -0.006865616887807846, -0.007678650319576263, -0.030520029366016388, -0.00802610069513321, -0.01628846675157547, 0.007477128878235817, 0.01872061938047409, -0.004471684340387583, 0.007525772321969271, -0.01719183847308159, -0.02690654620528221, 0.0048538800328969955, 0.0033250986598432064, 0.008039998821914196, -0.008762694895267487, -0.013626998290419579, -0.03332742676138878, 0.00330425170250237, -0.006744008976966143, 0.03616262227296829, 0.009179634973406792, -0.01496815588325262, -0.009540983475744724, -0.017011163756251335, -0.044668201357126236, 0.013939703814685345, -0.020777523517608643, -0.00651469174772501, 0.0026249864604324102, 0.01457901205867529, -0.010972478426992893, 0.017747757956385612, -0.01409258134663105, -0.012855658307671547, -0.0245438851416111, -0.0322989746928215, 0.005750301294028759, -0.013265649788081646, 0.0030123936012387276, 0.0044821081683039665, -0.03480061516165733, -0.020027032122015953, -0.013661743141710758, -0.019151456654071808, 0.018359269946813583, 0.005677336826920509, 0.011201795190572739, -0.0019596193451434374, -0.0037663604598492384, 0.007122729904949665, 0.002484269207343459, 0.011486704461276531, -0.00548971351236105, -0.0318264402449131, -0.015968812629580498, 0.01916535571217537, -0.02068023756146431, 0.012223299592733383, -0.018581638112664223, 0.01644134521484375, 0.006090802606195211, -0.015009850263595581, -0.02697603590786457, -0.010708415880799294, -0.015954915434122086, -0.018387066200375557, 0.009534034878015518, -0.013508865609765053, -0.009408952668309212, 0.025127600878477097, -0.007630007341504097, 0.0021750384476035833, -0.04060998186469078, 0.024794049561023712, 0.00984673947095871, 0.016538631170988083, 0.01011080201715231, -0.01817859709262848, -0.0156769547611475, -0.003912289626896381, 0.004784389864653349, -0.0054028513841331005, 0.017108449712395668, -0.028226858004927635, -0.004787864163517952, -0.02621164545416832, 0.024279821664094925, 0.0044021946378052235, -0.019429417327046394, 0.004155504982918501, 0.008873878978192806, 0.00646257447078824, -0.005785046610981226, -0.01424545980989933, 0.025030314922332764, -0.01659422367811203, 0.0011587465414777398, -0.023960167542099953, -0.009422850795090199, 0.003808054607361555, 0.015426790341734886, 0.02764314040541649, -0.03399452939629555, -0.00020260691235307604, 0.18912410736083984, 0.006816973444074392, 0.014301051385700703, 0.03418910503387451, -0.004690578207373619, 0.009151839651167393, -0.0027396450750529766, 0.004270163364708424, -0.01545458659529686, 0.005298615898936987, -0.009235227480530739, 0.01193839032202959, 0.006160292774438858, -0.004235418513417244, 0.0006492976099252701, 0.02129175141453743, -0.012855658307671547, -0.030103089287877083, 0.005847587250173092, 0.0334664061665535, 0.0015487594064325094, -0.030408844351768494, -0.007178321946412325, -0.015857629477977753, 0.023737799376249313, 0.016941674053668976, -0.004746170248836279, -0.0052881925366818905, 0.029213616624474525, 0.005236075259745121, -0.03877544775605202, -0.004117285367101431, 0.03560670092701912, 0.0019474586006253958, 0.016496937721967697, 0.0014940360561013222, 0.018595537170767784, -0.008616765961050987, 0.027087220922112465, 0.008074743673205376, 0.007317301817238331, -0.0062124100513756275, 0.014030040241777897, -0.016858285292983055, 0.006712738424539566, 0.0017893686890602112, -0.01674710214138031, 0.002135081682354212, 0.01885959878563881, -0.011882797814905643, -0.014773583970963955, 0.0019161880481988192, 0.002197622787207365, 0.016469141468405724, -0.0240157600492239, -0.0034710278268903494, 0.028963452205061913, -0.015941016376018524, 0.03343861177563667, 0.00970081053674221, -0.013529712334275246, -0.0013350775698199868, -0.006994173396378756, 0.016997264698147774, -0.04319501295685768, 0.0036829723976552486, -0.008352703414857388, 0.03196542337536812, 0.006000465713441372, -0.007630007341504097, -0.010590283200144768, -0.005131839774549007, -0.011271285824477673, 0.005239549558609724, -0.022903919219970703, -0.02796279452741146, 0.01245956588536501, 0.011222642846405506, 0.0009737293003126979, 0.004687103908509016, 0.004972012713551521, 0.002935954602435231, -0.010256730951368809, -0.0023400774225592613, -0.006886463612318039, -0.011035019531846046, 0.03721886873245239, -0.018234187737107277, -0.0245438851416111, -0.0075813643634319305, -0.018526047468185425, -0.01193144079297781, -0.008297111839056015, -0.029130227863788605, 0.007400690112262964, 0.00023127156600821763, 0.026350626721978188, 0.01863723061978817, 0.00136461085639894, 0.03054782561957836, -0.009756403043866158, 0.04614138975739479, 0.014148173853754997, 0.007407639175653458, 0.005760725121945143, 0.021305648609995842, 0.0027083745226264, 0.010680620558559895, 0.0012308424338698387, -0.020582951605319977, -0.006021312437951565, -0.053062599152326584, 0.01527391280978918, -0.012529055587947369, 0.010486047714948654, 0.012591596692800522, 0.0006141182966530323, -0.024140842258930206, 0.0365239679813385, 0.007289506029337645, -0.008422194048762321, 0.0035266198683530092, -0.013022434897720814, 0.010354016907513142, 0.014287153258919716, -0.005764199420809746, -0.023320859298110008, 0.001982203684747219, -0.024113046005368233, -0.009895382449030876, 0.015718648210167885, -0.013467171229422092, 0.012355330400168896, 0.00780373252928257, 0.014453929848968983, -0.02901904284954071, -0.011104509234428406, -0.0242520272731781, -0.013821570202708244, -0.0029602760914713144, 0.014773583970963955, 0.007345098070800304, 0.033077262341976166, 0.011257387697696686, 0.00791491661220789, -0.003811529139056802, 0.027629243209958076, 0.005819791462272406, -0.03038104809820652, -0.03266032040119171, -0.009360309690237045, 0.003488400485366583, 0.023515431210398674, -0.0011978347320109606, 0.03357759118080139, 0.01635795645415783, -0.004200673196464777, -0.029713943600654602, 0.001976991770789027, 0.003610007930546999, -0.05278464034199715, 0.007108831778168678, 0.015107136219739914, 0.008818287402391434, -0.015663057565689087, -0.004103387240320444, -0.17822808027267456, 0.014189867302775383, 0.003033240558579564, -0.03449485823512077, 0.01090993732213974, -0.014370542019605637, 0.045946817845106125, 0.011396368034183979, -0.028171265497803688, -0.03607923164963722, 0.02282053232192993, 0.005725979804992676, -0.006320119835436344, -0.02387678064405918, 0.03585686534643173, 0.028143469244241714, -0.0284214299172163, 0.017108449712395668, 0.02462727203965187, 0.01621897704899311, 0.01916535571217537, -0.03160407394170761, 0.006882989313453436, 0.003540517995133996, 0.011125356890261173, 0.025794705376029015, -0.004502954892814159, 0.030964765697717667, -0.027670936658978462, -0.029185820370912552, -0.013015486299991608, -0.004850405268371105, 0.035273149609565735, 0.006167241837829351, -0.0010631977347657084, 0.003825427033007145, -0.013772927224636078, 0.004760068375617266, -0.018428759649395943, 0.03399452939629555, 0.0019978389609605074, 0.028657695278525352, 0.0129181994125247, -0.007935763336718082, 0.0031722206622362137, 0.009360309690237045, 0.016872184351086617, 0.0005650409730151296, 0.007192220073193312, -0.022792736068367958, 0.010124700143933296, 0.0012534267734736204, 0.00019652654009405524, -0.010430456139147282, 0.003055824898183346, 0.015218320302665234, -0.008366601541638374, -0.0035648394841700792, 0.03771919757127762, -0.024752354249358177, 0.013620049692690372, -0.022320203483104706, 0.00890167523175478, -0.008429142646491528, 0.002046481939032674, -0.0260726660490036, -0.017094552516937256, 0.020791422575712204, -0.03569008782505989, -0.0013359461445361376, -0.014759685844182968, 0.015148830600082874, 0.013724284246563911, -0.0024286771658807993, 0.010896039195358753, 0.03480061516165733, -0.0077967834658920765, 0.036107029765844345, 0.009603524580597878, 0.004436939489096403, -0.010576385073363781, 0.022070039063692093, -0.0007157474756240845, 0.004742695949971676, 0.004423041362315416, -0.004103387240320444, -0.004176351707428694, 0.005173534154891968, -0.021152770146727562, -0.011340775527060032, 0.03666294738650322, -0.007393741048872471, -0.012932097539305687, -0.013710386119782925, -0.004124234430491924, 0.010194189846515656, 0.006994173396378756, -0.015134932473301888, 0.0022375795524567366, -0.026878749951720238, -0.009318615309894085, -0.018692823126912117, -0.012397024780511856, -0.019679581746459007, 0.004881675820797682, 0.0001605871511856094, -0.02940818853676319, -0.015315606258809566, 0.006393084302544594, 0.0017120609991252422, -0.03796936199069023, 0.0016095632454380393, 0.014162071980535984, 0.0006353995995596051, 0.007018494885414839, 0.01186195109039545, 0.025822501629590988, -0.003985254094004631, 0.0028803625609725714, -0.029435984790325165, 0.049810465425252914, 0.0025172769092023373, -0.014634603634476662, 0.028963452205061913, 0.0007808944210410118, -0.0203744824975729, -0.11440841108560562, -0.03644058108329773, 0.046474944800138474, 0.0019318233244121075, -0.010576385073363781, 0.0255445409566164, 0.00965911615639925, 0.03302166983485222, -0.03313285484910011, 0.02001313306391239, -0.04491836577653885, -0.039442550390958786, -0.014398337341845036, -0.014189867302775383, 0.02471066080033779, -0.009263023734092712, 0.012056523002684116, -0.0391089990735054, 0.009978771209716797, 0.021180566400289536, 0.01758098229765892, 0.002558971056714654, 0.029213616624474525, 0.01416902057826519, -0.007768987212330103, 0.0011118407128378749, -0.026156052947044373, 0.009026757441461086, 0.0037142429500818253, 0.011980083771049976, 0.02120836265385151, -0.0066189272329211235, 0.009033706039190292, -0.03721886873245239, -0.011750767007470131, -0.006278425920754671, -0.007226964924484491, -0.02712891437113285, 0.02455778233706951, -0.01637185551226139, -0.005805893335491419, 0.010117750614881516, 0.031326115131378174, -0.01809520833194256, -0.008554224856197834, -0.026837056502699852, -0.0221951212733984, 0.021319545805454254, -0.011514500714838505, -0.015051544643938541, -0.026114359498023987, -0.012251094914972782, -0.02304290048778057, 0.008116437122225761, -0.002051693620160222, 0.006744008976966143, 0.0258363988250494, -0.025947583839297295, 0.005260396748781204, -0.0011934915091842413, -0.03599584475159645, 0.024182535707950592, -0.03983169421553612, 0.008046947419643402, -0.007324250880628824, -0.019526703283190727, -0.007470180280506611, -0.024738457053899765, 0.008123386651277542, -0.0013446323573589325, -0.017386410385370255, 0.01705285720527172, -0.011750767007470131, -0.025058111175894737, -0.021861568093299866, -0.004089489113539457, -0.013133618980646133, -0.00810253992676735, 0.0017711275722831488, -0.0006436515832319856, 0.010298425331711769, -0.01901247724890709, 0.011549245566129684, -0.013654794543981552, -0.00045515981037169695, 0.025739112868905067, 0.03302166983485222, -0.020582951605319977, -0.002103811129927635, -0.016622019931674004, -0.00885998085141182, 0.025572337210178375, 0.010472150519490242, -0.02280663326382637, -0.02386288158595562, 0.00973555538803339, 0.015551872551441193, 0.001072752638719976, -0.003974830731749535, -0.009026757441461086, -0.01007605716586113, 0.0018275881884619594, -0.04516853019595146, 0.019582295790314674, 0.026030972599983215, 0.004846930969506502, -0.03318844735622406, 0.0031044678762555122, 0.006903836037963629, -0.022028345614671707, -0.015065441839396954, 0.01072231400758028, -0.01556577067822218, -0.0014254145789891481, -0.02113887295126915, 0.00031465961365029216, -0.032382361590862274, -0.006969851907342672, 0.025252683088183403, 0.0023261792957782745, 0.012355330400168896, 0.014071734622120857, -0.004575919825583696, -0.02537776529788971, 0.007477128878235817, 0.03113154135644436, -0.017469797283411026, 0.0015687377890571952, 0.00023148872423917055, 0.021416831761598587, -0.008811337873339653, -0.012626341544091702, 0.0034762395080178976, 7.589616143377498e-05, -0.028324143961071968, 0.018442658707499504, -0.03188203275203705, 0.0034953493159264326, -0.005927501246333122, 0.013654794543981552, 0.02068023756146431, 0.02251477539539337, 0.009235227480530739, -0.008165080100297928, -0.0017433315515518188, -0.03038104809820652, -0.011882797814905643, 0.002446049591526389, -0.013439374975860119, 0.004746170248836279, 0.011778563261032104, 0.02038837969303131, 0.017970126122236252, -0.00426321430131793, -0.00908929854631424, -0.04197198897600174, 0.007567466236650944, -0.01549628097563982, -0.01105586625635624, 0.0029741739854216576, -0.011570093221962452, -0.015426790341734886, 0.03788597509264946, 0.011590939946472645, 0.00442999042570591, -0.009235227480530739, 0.009381156414747238, 0.008172029629349709, -0.03121493011713028, 0.02001313306391239, 0.01189669594168663, 0.010506895370781422, -0.03332742676138878, -0.004416092298924923, 0.014676298014819622, 0.005600897595286369, 0.01307107787579298, -0.009207431226968765, 0.004231943748891354, 0.03949814289808273, 0.001241265912540257, 0.01223719771951437, 0.031020356342196465, 0.0028247705195099115, -0.027934998273849487, -0.002296646125614643, 0.04158284515142441, -0.009985719807446003, -0.008693205192685127, 0.0162050798535347, -0.011424163356423378, -0.008151182904839516, -0.0243076179176569, 0.015871526673436165, -0.029797332361340523, -0.0004358328878879547, 0.0046975272707641125, 0.013084976002573967, 0.007470180280506611, 0.004019999410957098, 0.006181139498949051, 0.03160407394170761, 0.009902331978082657, 0.015468484722077847, 0.013091924600303173, -0.031048152595758438, -0.02963055670261383, 0.0036933959927409887, -0.038386303931474686, -0.027670936658978462, 0.007630007341504097, -0.013064129278063774, 0.012139910832047462, 0.011980083771049976, 0.0021246580872684717, -0.0005828477442264557, -0.013196160085499287, 0.020485665649175644, -0.005712081678211689, -0.025864195078611374, -0.02074972726404667, 0.019610092043876648, 0.012973791919648647, 0.00549318827688694, 0.010638926178216934, -0.003846274223178625, 0.028518715873360634, -0.013759029097855091, 0.03235456719994545, -0.011048917658627033, -0.015051544643938541, -0.004096438176929951, 0.0221951212733984, 0.0019856782164424658, -0.024891335517168045, -0.025280479341745377, -0.011257387697696686, -0.021013790741562843, -0.0062888492830097675, -0.015732547268271446, -0.008985063061118126, 0.08205384761095047, 0.024516088888049126, 0.011125356890261173, -0.004530751146376133, -0.01999923586845398, -0.0029133702628314495, 0.012494310736656189, 0.0027726527769118547, 0.007449333090335131, -0.02673977054655552, 0.0030679856427013874, 0.007338149007409811, 0.0008816549670882523, -0.029797332361340523, 0.019596192985773087, -0.006222833879292011, -0.0242520272731781, 0.02061074785888195, -0.021263955160975456, 0.007414588239043951, 0.03644058108329773, -0.02234799973666668, 0.0073103527538478374, 0.018081311136484146, -0.018303679302334785, 0.010367915034294128, 0.0245438851416111, 0.0016364905750378966, -0.024960825219750404, -0.02719840407371521, 0.01128518395125866, 0.02774042636156082, -0.04161063954234123, -0.01877621002495289, -0.007734242361038923, 0.018053514882922173, -0.009402003139257431, -0.02993631176650524, 0.03616262227296829, -0.006816973444074392, 0.012000931426882744, 0.016121691092848778, -0.003377216402441263, -0.014426133595407009, -0.0153850968927145, -0.0054445452988147736, 0.00988843385130167, 0.005871909204870462, -0.020582951605319977]} +{"id": "test:10000067", "text": "\"Expand your clientele, meet and network with potential business partners and make new business contacts.\\nLearn from seasoned industry experts sharing their knowledge and experience on scaling up productivity, driving profitability and positioning SMEs for sustainability and global Competitiveness.\\nReach new markets, improve your corporate identity, showcase and promote your products and services to visitors and buyers.\\nExplore potential business deals and commercial partnerships with key trade, commerce and investment representatives from around the Country, the USA and the International Community.\\nRaise brand awareness and generate new business opportunities across multiple market sectors.\\nBooth prices starts from N50,000 for 2 days. Start Booking Now! Booth prices starts from N50,000 for 2 days. Start Booking Now!\\nWe were privilege to meet with Ebeano Supermarket, Shoppers Delite and Just Rite; we took up from there and today we are on the shelf of these malls. The exhibition showed us the way to do business right and to do it effectively and efficiently. The exhibition also made room for us to meet various investors, who have tremendously been of help to the growth of Hyst Global Business Ltd.\\nI exhibited at the African Food and Products Exhibition 2018. The Event was a wonderful one and an eye-opener for not just me but for other exhibitors and attendees. From the conference sessions, I acquired knowledge on exportation of goods. The AFPE 2018 was the door opener to my exquisite Zobo Business. I thought to myself that since the Exhibition was for African Food and Products, Zobo is an African drink so I decided to make and sell for the Exhibition. Just like magic, amidst my other products, the zobo made super sales. Now I have a branded Zobo drink.\\nThe AFPE 2018 was great, it was an opportunity to have a firsthand understanding of the challenges facing SMEs, helping us know what products they needed and how they can take advantage of our service offerings. The AFPE 2019 gave us a chance to partner with many of them.\\nIt was an amazing experience, my business was a month old and I didn\u2019t know anyone. It was the first time I got to be in public and I appreciate the opportunity to be exposed. Coming to the exhibition boosted my sales and created awareness for me when people didn\u2019t know me. I have got contacts from the exhibition and they have formed part of my customer base.\\nLagos is a commercial hub and it was important for us to share with the market. There has always been a great need to take our products and services to our end users, offering them values and benefits and we achieved that at the African Food & Products Exhibition 2018.\"", "vector_field": [-0.005568365100771189, -0.030196234583854675, 0.014065301977097988, -0.012108391150832176, -0.030005978420376778, -0.007596622221171856, -0.006998676806688309, -0.0006646194960922003, -0.014364275150001049, -0.04038848355412483, 0.006859383080154657, 0.011863776482641697, -0.010973653756082058, -0.01053878478705883, -0.019283734261989594, 0.0016817209543660283, 0.020860133692622185, -0.006665730383247137, -0.025752414017915726, -0.01047083642333746, -0.01599503494799137, 0.015492216683924198, 0.018604250624775887, 0.012305441312491894, -0.02186577022075653, 0.0037303627468645573, 0.025004981085658073, -0.017530666664242744, -0.001396337989717722, 0.008772128261625767, -0.0011330381967127323, 0.0059318882413208485, 0.00840520765632391, -0.0011109550250694156, -0.0035740816965699196, -0.017082206904888153, -0.0019433220149949193, 0.010416477918624878, 0.04808023199439049, 0.006030413322150707, 0.014214788563549519, 0.01572324149310589, -0.005327148362994194, 0.01317517925053835, -0.02489626407623291, -0.000243764603510499, -0.001241755555383861, -0.0040667071007192135, -0.0011330381967127323, 0.022871404886245728, 0.013012103736400604, -0.006509449332952499, -0.025806771591305733, 0.0339198000729084, 0.00042467701132409275, -0.009784557856619358, -0.007188932504504919, 0.019283734261989594, 0.006737076211720705, -0.015913495793938637, -0.0015976348659023643, 0.01762579381465912, -0.02092808298766613, -0.006084772292524576, -0.0055955443531274796, -0.021689103916287422, -0.02886444702744484, -0.011945314705371857, -0.010294170118868351, -0.012509285472333431, 0.018454764038324356, 0.031310584396123886, -0.02630958892405033, -0.010402888059616089, 0.002714535454288125, -0.0017836433835327625, -0.013249922543764114, 0.011225062422454357, -0.0012791270855814219, -0.004735997878015041, -0.0025429658126085997, -0.018346047028899193, -0.027301635593175888, 0.003567286767065525, 0.018332457169890404, 0.023238325491547585, -0.010994038544595242, 0.018645018339157104, -0.02234140783548355, -0.012115185149013996, 0.004382666666060686, -0.00412786053493619, -0.01093968003988266, 0.007888800464570522, -0.010620322078466415, 0.012747105211019516, 0.0030763603281229734, 0.03500697389245033, 0.012230698019266129, -0.008235336281359196, -0.0005975205567665398, 0.005687274504452944, -0.010932885110378265, -0.013983764685690403, -0.010640706866979599, 0.0207785964012146, 0.007290854584425688, 0.0012145761866122484, 0.01997680589556694, -0.02363242581486702, -0.008276105858385563, 0.02427113987505436, 0.01826450787484646, -0.02886444702744484, -0.0033651406411081553, -0.022314228117465973, -0.01513888593763113, 0.0051300982013344765, -0.004817536100745201, -0.0032360388431698084, 0.005170867312699556, 0.031147509813308716, 0.006550217978656292, -0.008330464363098145, 0.0013920912751927972, -0.0018974568229168653, -0.028429577127099037, -0.0239721667021513, 0.0024512356612831354, -0.010742629878222942, 0.027328815311193466, 0.012944155372679234, 0.026676511391997337, -0.0023781913332641125, -0.023713963106274605, 0.015886317938566208, -0.014935040846467018, 0.008432386443018913, -0.020588340237736702, -0.018767327070236206, 0.02745112217962742, 0.04060591757297516, 0.006482269614934921, 0.00038178463000804186, -0.008663411252200603, 0.052265848964452744, 0.014187609776854515, 0.01005635131150484, 0.004484589211642742, 0.020370906218886375, 0.012414158321917057, -0.016443492844700813, -0.014867092482745647, 0.01854989118874073, 0.023061659187078476, 0.01728605292737484, 0.007895594462752342, 0.011924929916858673, -0.0049874065443873405, 0.000690524815581739, -0.014921450987458229, 0.00757623789831996, -0.013080052100121975, 0.005463045090436935, -0.00817418284714222, 0.01935168169438839, 0.030495205894112587, 0.01793835684657097, -0.005836760625243187, -0.01134736929088831, 0.013345050625503063, 0.010348529554903507, -0.0175850261002779, 0.030658282339572906, 0.013739150017499924, -0.0010345132322981954, 0.0002531074860598892, 0.008758538402616978, 0.005378109402954578, -0.01868578791618347, -0.00713457353413105, 0.0029591494239866734, 0.02254525199532509, 0.028511114418506622, -0.021186286583542824, -0.013752739876508713, 0.006003234069794416, -0.008194567635655403, 0.014568120241165161, -0.024379856884479523, 0.0031290201004594564, 0.02409447357058525, -0.017734510824084282, -0.0058435555547475815, -0.6431716084480286, -0.008622641675174236, 0.01967783458530903, -0.030984433367848396, -0.006733678746968508, -0.00024907305487431586, 0.016620159149169922, 0.004685036838054657, -0.024053705856204033, 0.018835274502635002, -0.0066589354537427425, 0.005330545827746391, 0.009213792160153389, -0.0037643369287252426, 0.006665730383247137, -0.009003152139484882, 0.03856746852397919, -0.031772635877132416, 0.002186237135902047, 0.004719011019915342, 0.009213792160153389, 0.0207785964012146, 0.0017836433835327625, 0.02219192124903202, -0.0026312987320125103, -0.00649585947394371, 0.002401973120868206, -0.02042526565492153, 0.0011228460352867842, 0.000746582169085741, -0.010932885110378265, 0.02662215195596218, 0.00574842793866992, 0.0074946996755898, 0.052401743829250336, 0.006737076211720705, -0.002636394929140806, 0.03378390520811081, -0.006332783494144678, 0.03106597252190113, -0.003074661595746875, -0.01868578791618347, 0.005592146888375282, 0.006489064544439316, 0.02745112217962742, 0.004630677867680788, 0.024665240198373795, -0.0002053313364740461, -0.01694631204009056, -0.008058670908212662, -0.0006446596817113459, 0.0005147085175849497, -0.026567792519927025, -0.0012986622750759125, 0.01825091801583767, -0.012353004887700081, 0.011517240665853024, -0.027057021856307983, 0.015600934624671936, 0.026268821209669113, 0.038431569933891296, 0.00944481696933508, -0.002288159681484103, -0.041638731956481934, -0.011075575836002827, 0.007501494605094194, 0.005055354908108711, 0.014880682341754436, 0.021634744480252266, 0.019623475149273872, 0.00981853250414133, 0.04661254957318306, -0.0142691470682621, -0.0008285448420792818, 0.017245283350348473, 0.014880682341754436, 0.0017564641311764717, 0.021131927147507668, -0.016756055876612663, 0.01951475813984871, 0.0023527105804532766, -0.0013878444442525506, -0.02345575951039791, 0.0034211978781968355, 0.011693906038999557, -0.004909266252070665, -0.017707332968711853, -0.008147004060447216, 0.010579553432762623, -0.009186613373458385, 0.014649657532572746, 0.024298319593071938, 0.018196560442447662, -0.02331986278295517, -0.002792675979435444, 0.0022711725905537605, -0.007630596403032541, 0.011483266018331051, 0.0020452444441616535, -0.015111706219613552, -0.014595299027860165, -0.03726286068558693, 0.03343057259917259, 0.0038017085753381252, 0.03326749801635742, 0.024706009775400162, -0.013263512402772903, 0.027967529371380806, 0.04517204314470291, -0.016429904848337173, -0.005717851221561432, -0.0021947307977825403, -0.027002662420272827, -0.031011613085865974, -0.007202521897852421, -0.021797820925712585, 0.028674190863966942, 0.0023221338633447886, 0.014405043795704842, -0.017802460119128227, -0.0008616696577519178, 0.013942995108664036, -0.0010786795755848289, -0.015410679392516613, 0.01045724656432867, 0.031799815595149994, 0.003247929736971855, 0.00018823807477019727, -0.005221828352659941, -0.005395096726715565, 0.006393936928361654, -0.03682798892259598, 0.0149893993511796, -0.005021380726248026, 0.030332129448652267, 0.01694631204009056, 0.016212468966841698, 0.0047801644541323185, 0.009037126787006855, -0.03525158762931824, -0.011061986908316612, -0.012393773533403873, 0.0007860771147534251, -0.020262189209461212, -0.010640706866979599, -0.02158038690686226, -0.006635153666138649, 0.014826323837041855, -0.01776169054210186, 0.0034296915400773287, 0.0010752822272479534, 0.004073502030223608, -0.005350930150598288, 0.006546820513904095, 0.010165068320930004, 8.881057874532416e-05, -0.0017870408482849598, -0.02410806342959404, -0.02379550226032734, -0.014133250340819359, 0.0030169053934514523, 0.01776169054210186, -0.02333345264196396, -0.004814138635993004, -0.02474677748978138, 0.016919132322072983, 0.0006055894191376865, 0.0046952287666499615, 0.02443421632051468, -0.016715286299586296, 0.008941998705267906, -0.01316158939152956, -0.015016579069197178, 0.01651144213974476, 0.007671365514397621, 0.012026852928102016, -0.006298809312283993, -0.008439181372523308, 0.0012791270855814219, -0.004467601887881756, 0.03802388161420822, 0.0025888311211019754, -0.010158274322748184, 0.014608888886868954, 0.03364800661802292, 0.0013920912751927972, 0.0006259739166125655, -0.021458080038428307, 0.0018702775705605745, 0.024325499311089516, -0.0012995116412639618, 0.00014056808140594512, 0.00014099276449996978, 0.003665811847895384, -0.027519069612026215, 0.019066298380494118, -0.010715450160205364, 0.008690590038895607, -0.023061659187078476, 0.006060990039259195, 0.005547980312258005, -0.0007860771147534251, 0.016769645735621452, 0.0019993793684989214, 0.0017267366638407111, -0.0015950868837535381, 0.00888084527105093, -0.014092481695115566, 0.028157783672213554, 0.020058345049619675, 0.014663247391581535, -0.03802388161420822, 0.012828642502427101, 0.01077660359442234, 0.006696307100355625, 0.008622641675174236, 0.008581873029470444, 0.02630958892405033, -0.027383172884583473, 0.0157096516340971, -0.008140209130942822, -0.048596639186143875, 0.0006807572208344936, -0.01340620405972004, -0.012577233836054802, 0.008588667958974838, -0.0021267824340611696, 0.024624470621347427, -0.007827646099030972, -0.013875046744942665, -0.0023017493076622486, 0.012556849978864193, 0.021784231066703796, 0.025888310745358467, 0.02900034375488758, 0.01324312761425972, 0.03859464451670647, -0.015356320887804031, 0.03160955756902695, 0.0068899597972631454, 0.03046802617609501, 0.040361303836107254, 0.04144847393035889, 0.017707332968711853, 0.0046068960800766945, 0.017313232645392418, -0.010035966522991657, -0.002597324550151825, -0.018196560442447662, 0.014962220564484596, -0.008554693311452866, 0.03769772872328758, 0.011191088706254959, -0.01428273692727089, 0.0031494046561419964, 0.02236858755350113, -0.014527350664138794, 0.011924929916858673, 0.010864936746656895, 0.017095796763896942, -0.0006026166956871748, -0.01205403171479702, 0.025480620563030243, -0.0037609394639730453, -0.003777926554903388, -0.009064305573701859, -0.005520801059901714, -0.010158274322748184, -0.002638093661516905, -0.01021942775696516, -0.00335155101493001, -0.006166310049593449, 0.0021573591511696577, 0.012978129088878632, -0.00943122711032629, 0.0013657612726092339, -0.027709325775504112, -0.025589337572455406, 0.0027400159742683172, 0.020167062059044838, -0.029897261410951614, -0.007433546241372824, -0.0004425134393386543, 0.0004098133067600429, 0.010783398523926735, -0.005330545827746391, -0.0019212387269362807, -0.007990722544491291, -0.010192248038947582, 0.02220551110804081, -0.011517240665853024, 0.0010557470377534628, -0.016361955553293228, -0.013997353613376617, -0.013596459291875362, -0.005894516594707966, 0.010103914886713028, -0.007637391332536936, 0.016416314989328384, -0.02476036734879017, -0.0057722097262740135, 0.001563660684041679, -0.01497580949217081, -0.0178975872695446, 0.024216782301664352, -0.003910426050424576, -0.021213466301560402, -0.007243291009217501, -0.00975058414041996, 0.007915979251265526, 0.02583395130932331, -0.014405043795704842, -0.015437858179211617, 0.024393446743488312, 0.029380854219198227, -0.0017479705857113004, -0.002945559797808528, 0.010124299675226212, 0.012957744300365448, 0.002967642853036523, -0.008826486766338348, -0.0320444293320179, -0.02492344379425049, 0.0013419793685898185, 0.0980086550116539, 0.010783398523926735, 0.015628114342689514, 0.017163746058940887, 0.001033663866110146, -0.015981445088982582, -0.016688108444213867, -0.002148865722119808, 0.012509285472333431, 0.01649785228073597, -0.008894435130059719, -0.02190653793513775, 0.021797820925712585, -0.01777528040111065, 0.004759779665619135, -0.0024869085755199194, -0.002036751015111804, -0.0028147592674940825, 0.038268495351076126, 0.011918134987354279, 0.010165068320930004, 0.008704179897904396, -0.004443820100277662, 0.027328815311193466, 0.007929569110274315, 0.02962546795606613, 0.027301635593175888, 0.02951675094664097, 0.007983927614986897, -0.03155520185828209, -0.0002031017793342471, 0.015261192806065083, 0.023401401937007904, 0.006393936928361654, -0.010980448685586452, 0.002119987504556775, -0.008289694786071777, 0.007929569110274315, 0.02539908140897751, -0.002524280222132802, 0.01806066371500492, 0.025263184681534767, 0.0009249465074390173, -0.010729040019214153, 0.0026295999996364117, -0.027967529371380806, -0.0032258464489132166, 0.013501331210136414, 0.00100053905043751, -0.0024257551413029432, 0.017204515635967255, 0.0028147592674940825, -0.022069614380598068, 0.005904708988964558, 0.007243291009217501, -0.012638387270271778, -0.01093968003988266, -0.018604250624775887, -0.034626465290784836, -0.04014386609196663, -0.01252287533134222, -0.025412671267986298, 0.023849859833717346, -0.0037303627468645573, -0.052102770656347275, -0.004039527848362923, -0.022939352318644524, 0.0060168239288032055, -0.02076500654220581, 0.01583195850253105, 0.014024533331394196, -0.028891626745462418, -0.015940675511956215, -0.00926815066486597, 0.031337764114141464, 0.012495696544647217, -0.0051606749184429646, 0.011687111109495163, -0.002869117772206664, 0.001739477040246129, 0.02111833728849888, -0.0008816294721327722, -0.014676837250590324, -0.028021886944770813, -0.021036799997091293, 0.003470460418611765, -0.025127289816737175, -0.007521878927946091, -0.02598343789577484, -0.007297649513930082, 0.004719011019915342, -0.027057021856307983, 0.03421877324581146, -0.015967855229973793, -0.007861620746552944, -0.003998758736997843, 0.0013513222802430391, -0.01142890751361847, 0.020574752241373062, 0.008717769756913185, 0.007983927614986897, -0.019161425530910492, -0.005228623282164335, -0.014676837250590324, 0.003910426050424576, 0.01792476698756218, -0.01173467468470335, -0.04549819603562355, -0.021784231066703796, 0.002043545711785555, 0.012128775008022785, -0.025738824158906937, -0.010953268967568874, -0.0011338875629007816, -0.0004040801723022014, 0.01371197123080492, 0.0012434542877599597, 0.015424268320202827, -0.006720088887959719, 0.025725234299898148, 0.003038988681510091, -0.011625957675278187, 0.02840239740908146, -0.009580712765455246, -0.010430066846311092, 0.014704016968607903, -0.002305146772414446, -0.035278767347335815, -0.0074946996755898, 0.0020248598884791136, -0.019120657816529274, 0.020180651918053627, -0.02870137058198452, -0.018359636887907982, -0.03837721049785614, -0.014051713049411774, -0.02632317878305912, -0.004365679807960987, -0.007399572059512138, -0.03307724371552467, -0.014296326786279678, -0.015410679392516613, 0.01712297648191452, -0.024012936279177666, 0.016552211716771126, -0.03389262408018112, -0.007392777130007744, 0.0002981232537422329, -0.01633477583527565, 0.035577740520238876, 0.005442660301923752, -0.004593306686729193, -0.02075141668319702, -0.006305604241788387, -0.040660273283720016, -0.05011868104338646, -0.014241968281567097, -0.01244133710861206, 0.02791316993534565, 0.04411204904317856, 0.0024359473027288914, 0.015261192806065083, 0.01585913822054863, 0.019446808844804764, -0.0031697892118245363, -0.00630220677703619, 0.0062954118475317955, -0.0011797527549788356, -0.009573918767273426, 0.033838264644145966, -0.0053339432924985886, 0.031011613085865974, -0.009485585615038872, -0.0016061284113675356, -0.0063667576760053635, 0.013813893310725689, -0.01188416127115488, -0.03574081510305405, -0.024678830057382584, -0.03093007579445839, 0.0207785964012146, -0.02250448428094387, 0.003217353019863367, 0.017408359795808792, -0.03269673138856888, -0.02775009348988533, 0.01948757842183113, -0.0005087630706839263, -0.0006926481728442013, -0.023537298664450645, 0.02681240811944008, -0.02726086601614952, 0.026581382378935814, 0.0015848946059122682, -0.03979053720831871, 0.005656697787344456, -0.010661091655492783, -0.0368008092045784, -0.005575159564614296, 0.008894435130059719, 0.01854989118874073, -0.0016112246084958315, 0.018984761089086533, 0.001832905923947692, -0.012339415028691292, 0.0034993384033441544, 0.021335773169994354, -0.014921450987458229, 0.0021777437068521976, -0.031772635877132416, -0.003072962863370776, -0.022558841854333878, -0.009879685938358307, -0.01572324149310589, 0.004803946241736412, -0.0024495369289070368, -0.009206997230648994, 0.006125540938228369, -0.005568365100771189, 0.0034262940753251314, 0.0184004046022892, -0.010572758503258228, 0.012556849978864193, -0.0009105075150728226, 0.013365434482693672, 0.028918804600834846, 0.002099603181704879, -0.01967783458530903, -0.0019416232826188207, 0.005704261362552643, 0.00031043888884596527, -0.0042433724738657475, 0.02522241696715355, -0.020099112764000893, -0.0343003123998642, 0.013582869432866573, 0.0049874065443873405, -0.028592653572559357, -0.02823932282626629, 0.02916341833770275, 0.0042569623328745365, 0.014935040846467018, -0.02062910981476307, -0.000710909313056618, 0.003913823049515486, 0.019895268604159355, -0.006859383080154657, 0.02492344379425049, -0.017680153250694275, -0.03106597252190113, -0.015043757855892181, 0.0028402397874742746, -0.017734510824084282, 0.02094167284667492, 0.0049398429691791534, -0.021933717653155327, -0.01428273692727089, 0.0024087680503726006, -0.007059830706566572, 0.016103751957416534, -5.483641871251166e-05, -0.009736994281411171, -0.0017700537573546171, 0.04017104580998421, 0.02741035260260105, -0.011693906038999557, -0.001241755555383861, -0.0038866440299898386, 0.025453440845012665, 0.024855496361851692, -0.014527350664138794, -0.020099112764000893, -0.009838917292654514, 0.017965536564588547, 0.008113029412925243, -0.02393139898777008, -0.012495696544647217, -0.02047962322831154, -0.004858304746448994, -0.017720922827720642, 0.002524280222132802, -0.0029795337468385696, -0.032723911106586456, -7.484931848011911e-05, -0.007202521897852421, -0.009329304099082947, -0.007501494605094194, 0.02156679704785347, 0.021825000643730164, -0.00784123595803976, -0.01483991276472807, 0.012896590866148472, 0.028755730018019676, 0.0038424774538725615, -0.010783398523926735, -0.003072962863370776, -0.005476634483784437, 0.009336099028587341, -0.029924441128969193, -0.0013623638078570366, 0.010368913412094116, -0.007882005535066128, -0.014133250340819359, -0.01663374900817871, -0.036230046302080154, 0.018468353897333145, 0.007243291009217501, 0.002792675979435444, -0.026581382378935814, -0.0014226679923012853, -0.003001617034897208, -0.0007206768495962024, -0.004732600413262844, -0.017476307228207588, 0.03756183013319969, -0.0018515917472541332, 0.0006531532271765172, -0.019541937857866287, -0.026255231350660324, 0.018427584320306778, -0.01584554836153984, -0.015111706219613552, 0.020588340237736702, -0.03777926787734032, 0.018658608198165894, 0.008024696260690689, 0.012944155372679234, 0.007154958322644234, -0.03720850124955177, -0.009723404422402382, 0.005398494191467762, 0.010212632827460766, -0.006247848272323608, -0.016307596117258072, -0.004413243383169174, -0.007637391332536936, -0.010565963573753834, 0.0013869950780645013, -0.005989644676446915, 0.009709814563393593, -0.009879685938358307, 0.004192411433905363, 0.011863776482641697, 0.011958904564380646, 0.006723486352711916, -0.015600934624671936, -0.014106071554124355, 0.0003909151710104197, -0.036692094057798386, 0.00943122711032629, -0.014323505572974682, 0.04691151902079582, 0.005846953019499779, -0.028184963390231133, -0.0374259352684021, -0.010688270442187786, -0.055581726133823395, 0.022708328440785408, -0.003988566342741251, 0.020860133692622185, 0.015342731028795242, 0.011279420927166939, -0.0027994709089398384, 0.029136240482330322, -0.010763013735413551, -0.010253401473164558, -0.039410024881362915, -0.013779919594526291, -0.015437858179211617, -0.018441174179315567, 0.009145843796432018, -0.013052872382104397, -0.03427313268184662, 0.001116900471970439, 0.033539291471242905, 0.004216193221509457, 0.0009351387852802873, 0.020968852564692497, -0.0039953612722456455, -0.0031833788380026817, 0.013379024341702461, 0.026730868965387344, 0.016266828402876854, 0.007698544766753912, 0.007746108341962099, -0.03348493203520775, -0.007012266665697098, 0.0014303121715784073, 0.0001956699270522222, 0.0052320207469165325, 0.012319030240178108, 0.015247602947056293, -0.0035197229590266943, 0.011924929916858673, 0.0009088087826967239, 0.0013351845555007458, 0.014608888886868954, 0.0038662594743072987, 0.026377538219094276, -0.008011107333004475, -0.0169599000364542, 0.0067744473926723, 0.025426261126995087, -0.02205602452158928, -0.012305441312491894, 0.02378191240131855, -0.00864302646368742, -0.016878362745046616, 0.02345575951039791, -0.02519523724913597, -0.015573754906654358, -0.021648334339261055, 0.006533231120556593, -0.022572431713342667, -0.012624798342585564, -0.028619833290576935, -0.031310584396123886, -0.004735997878015041, 0.0028368423227220774, 0.011591983959078789, -0.04150283336639404, -0.014323505572974682, -0.002237198408693075, -0.016742466017603874, -0.017000669613480568, -0.0035502996761351824, -0.005136893130838871, -0.01776169054210186, -0.018196560442447662, 0.010531989857554436, -0.011605572886765003, -0.009968019090592861, -0.008738153614103794, 0.0011355862952768803, -0.006733678746968508, 0.006709896493703127, 0.20634545385837555, -0.0040667071007192135, -0.007712134160101414, 0.015777600929141045, 0.008840076625347137, 0.0074607254937291145, 0.02333345264196396, 0.0069918823428452015, 0.01712297648191452, 0.019270144402980804, -0.030005978420376778, -0.0020605328027158976, -0.03976335749030113, -0.002515786560252309, 0.012787873856723309, -0.0023034480400383472, -0.03886643797159195, -0.021308593451976776, 0.0039817714132368565, 0.03237057849764824, 0.03563209995627403, 0.014445813372731209, 0.005344135221093893, 0.014636068604886532, 0.02106397971510887, 0.002838541055098176, -0.005646505393087864, 0.005782402120530605, 0.03280545026063919, -0.017218103632330894, 0.008982768282294273, -0.005228623282164335, 0.011435702443122864, -0.006421116180717945, 0.001685967668890953, 0.016212468966841698, 0.01663374900817871, -0.021526027470827103, 0.020207829773426056, 0.022749098017811775, 0.015193244442343712, 0.016117341816425323, -0.015546576119959354, -0.010477631352841854, -0.0033243715297430754, 0.008799307979643345, -0.013256717473268509, -0.020968852564692497, -0.0041550397872924805, -0.0003934632404707372, -0.04365000128746033, -0.007814057171344757, 0.0048820869997143745, 0.024379856884479523, -0.003954592160880566, -0.01142890751361847, 0.011517240665853024, 0.014921450987458229, 0.010423272848129272, 3.025823571078945e-05, -0.006393936928361654, 0.0380510613322258, -0.012149159796535969, 0.007990722544491291, -0.0027196314185857773, 0.0017233393155038357, -0.0022796662524342537, 0.007583032362163067, 0.0013869950780645013, 0.028674190863966942, 0.025480620563030243, 0.006784639786928892, -0.037806443870067596, -0.010436861775815487, -0.015791188925504684, -0.027016252279281616, 0.004817536100745201, 0.006404129322618246, 0.044764354825019836, 0.024393446743488312, 0.003489146241918206, -0.008615846745669842, -0.024135243147611618, 0.018780915066599846, -0.008235336281359196, 0.0006484817713499069, 0.029897261410951614, 0.01317517925053835, -0.0078072622418403625, -0.02677163854241371, 0.007983927614986897, 0.003951194696128368, -0.011687111109495163, -0.04223667457699776, -0.0033379611559212208, 0.0056533003225922585, 0.010110709816217422, 0.016728876158595085, -0.011272625997662544, -0.002531074918806553, -0.0023187363985925913, 0.0481889471411705, 0.01070865523070097, -0.010722245089709759, -0.021009620279073715, -0.008303284645080566, -3.418649794184603e-05, 0.009465200826525688, 0.02886444702744484, -0.003110334277153015, 0.01149685587733984, -0.015614524483680725, -0.00464087026193738, -0.008921614848077297, -0.007963542826473713, 0.014608888886868954, 0.012040442787110806, -0.022137563675642014, 0.03745311498641968, -0.018930401653051376, -0.01000199280679226, -0.021444490179419518, 0.020724236965179443, 0.003618248039856553, -0.025738824158906937, -0.005904708988964558, -0.015030168928205967, 0.0003136239538434893, -0.006162912584841251, -0.0314193032681942, 0.01728605292737484, 0.021920127794146538, -0.00840520765632391, -0.015927085652947426, 0.020384496077895164, -0.018019894137978554, 0.011204677633941174, -0.012916975654661655, 0.0003301863616798073, -0.00013122519885655493, -0.002694150898605585, 0.017231693491339684, -0.0034568707924336195, -0.004029335454106331, 0.004651062656193972, -5.616353519144468e-05, 0.004773369524627924, 0.0013768028002232313, 0.008935203775763512, -0.03277827054262161, 0.005150482524186373, 0.0012222203658893704, -0.011252242140471935, -0.03949156403541565, 0.01000199280679226, -0.015587344765663147, -0.027491889894008636, 0.009689430706202984, 0.024610882624983788, -0.011089165695011616, -0.02489626407623291, -0.00038157228846102953, 0.01901194080710411, -0.0010047857649624348, 0.0002607516944408417, 0.008262515999376774, -0.17373026907444, 0.031174689531326294, 0.012488901615142822, 0.006872972473502159, 0.00784123595803976, -0.018672198057174683, 0.0006421116413548589, 0.006390539463609457, -0.02409447357058525, 0.016592979431152344, 0.007352008484303951, -0.015084527432918549, -0.020887313410639763, -0.02505934052169323, 0.03277827054262161, 0.012142364867031574, -0.005415481049567461, 0.01888963393867016, 0.024814726784825325, 0.015573754906654358, 0.015913495793938637, -0.008153798058629036, 0.01727246306836605, 0.005602338816970587, 0.013345050625503063, 0.017788870260119438, -0.00515387998893857, -0.002655080519616604, -0.03658337518572807, -0.011843392625451088, -0.0006595234153792262, -0.0028266501612961292, 0.02079218626022339, 0.01793835684657097, 0.016565799713134766, -0.013399409130215645, -0.018740147352218628, -0.00211828900501132, -0.012244286946952343, 0.03484389930963516, 0.018808094784617424, 0.016212468966841698, -0.008969178423285484, -0.007902389392256737, -0.0161445215344429, 0.023877039551734924, 0.03073981963098049, -0.013875046744942665, 0.007114189211279154, 0.011585189029574394, 0.027519069612026215, -0.014758375473320484, 0.01841399446129799, 0.02473318949341774, 0.031636737287044525, 0.01837322674691677, -0.003859464544802904, -0.01166672632098198, 0.014581709168851376, -0.02539908140897751, -0.007154958322644234, -0.011360959149897099, -0.003123924136161804, -0.018305277451872826, -0.01602221466600895, -0.009859301149845123, -0.002104699146002531, 0.009505970403552055, -0.025793181732296944, 0.017544256523251534, 0.018182970583438873, -0.002952354494482279, 0.014337095431983471, -0.014309916645288467, 0.03171827644109726, -0.004929650574922562, 0.0001313313696300611, 0.0030882512219250202, 0.0013020597398281097, -0.01587272807955742, -0.009743789210915565, 0.04074181243777275, -0.00862943660467863, 0.013440177775919437, -0.014663247391581535, 0.009057511575520039, -0.006230860948562622, 0.018495533615350723, -0.023564478382468224, -0.007909184321761131, 0.02965264767408371, -0.005011188797652721, 0.004290936514735222, 0.006057592574506998, 0.01339261420071125, 0.03881208226084709, 0.001076131477020681, 0.01918860524892807, -0.006241053342819214, 0.017489897087216377, 0.006057592574506998, 0.023183967918157578, -0.023713963106274605, 0.024447806179523468, 0.027994707226753235, 0.011958904564380646, 0.016756055876612663, 0.023265505209565163, 0.03106597252190113, -0.018903221935033798, -0.033349037170410156, 0.016402725130319595, 0.011435702443122864, 0.02902752347290516, 0.0018413994694128633, 0.028130603954195976, 0.017883997410535812, -0.016266828402876854, 0.011021217331290245, -0.0111095504835248, 0.019555525854229927, -0.013514921069145203, -0.002254185499623418, 0.009247766807675362, -0.012529670260846615, -0.0047258054837584496, -0.10746706277132034, -0.030658282339572906, 0.008704179897904396, 0.0016587884165346622, 0.006808421574532986, 0.0007228002650663257, -0.004742792807519436, 0.02713855914771557, 0.008819691836833954, 0.014119661413133144, -0.015940675511956215, -0.002802868140861392, 0.011958904564380646, 0.006546820513904095, 0.06060990318655968, -0.004369076807051897, 0.0047665745951235294, 0.013983764685690403, -0.022069614380598068, 0.023251915350556374, -0.02520882710814476, -0.011591983959078789, 0.018481943756341934, -0.02317037805914879, -0.026703689247369766, -0.01903911866247654, -0.022749098017811775, 0.010919295251369476, 0.005031573120504618, 0.002909886883571744, -0.004705421160906553, -0.02948957122862339, 0.04006233066320419, -0.025602927431464195, 0.011789033189415932, -0.01070865523070097, -0.030359309166669846, 0.01015147939324379, -0.005619326140731573, -0.017883997410535812, 0.007990722544491291, 0.01599503494799137, 0.01528837252408266, -0.044003333896398544, -0.02061551995575428, 0.0004828577511943877, -0.011299805715680122, 0.03424595296382904, 0.009879685938358307, -0.011999673210084438, 0.003536710049957037, 0.003366839373484254, -0.03674645349383354, -0.024665240198373795, 0.00396138709038496, -0.006152720656245947, -0.005174264777451754, 0.0234693493694067, -0.010980448685586452, -0.017802460119128227, 0.0006030413205735385, -0.004814138635993004, -0.013677996583282948, -0.020533982664346695, -0.004705421160906553, 0.016049392521381378, -0.02962546795606613, 0.01220351830124855, -0.0008739852928556502, -0.00010680626292014495, -0.0013368831714615226, -0.030848536640405655, -0.02031654864549637, 0.011911340989172459, -0.034789539873600006, -0.003037289949133992, -0.02804906666278839, -0.01599503494799137, 0.01420119870454073, -0.006006631534546614, -0.012835437431931496, -0.021825000643730164, -0.01872655749320984, -0.011918134987354279, 0.002865720307454467, 0.021322183310985565, -0.0020503406412899494, 0.0027790863532572985, 0.0076509807258844376, -0.023387812077999115, 0.015206834301352501, 0.010185453109443188, 0.048297666013240814, -0.019922448322176933, -0.005846953019499779, 0.022259870544075966, 0.0035502996761351824, -0.0017683550249785185, 0.018427584320306778, 0.016851183027029037, -0.04375872015953064, -0.0011109550250694156, -0.0674590915441513, 0.02250448428094387, 0.011795828118920326, 0.009723404422402382, 0.010212632827460766, 0.005177662242203951, 0.013759534806013107, -0.0002548061893321574, 0.01450017187744379, -0.018767327070236206, -0.04642229154706001, 0.0380510613322258, -0.008609051816165447, -0.01853630132973194, -0.0006620714557357132, -0.02045244537293911, 0.032560836523771286, 0.006071182433515787, 0.01243454311043024, 0.012244286946952343, 0.005673684645444155, 0.001178054022602737, 0.00666233291849494, 0.014540940523147583, -0.024447806179523468, 0.014608888886868954, -0.02965264767408371, -0.011483266018331051, -0.0010676379315555096, 0.004545742645859718, 0.017788870260119438, -0.012509285472333431, -0.006359962746500969, 0.03457210585474968, -0.0025293761864304543, -0.033349037170410156, -0.03093007579445839, 0.018930401653051376, -0.0069918823428452015, -0.0022677751258015633, -0.04153001308441162, -0.0034721591509878635, 0.002847034716978669, -0.021172696724534035, -0.023360632359981537, -0.016402725130319595, 0.015628114342689514, -0.007888800464570522, 0.014867092482745647, -0.017788870260119438, 0.023251915350556374, 0.009240971878170967, -0.013304281048476696, -0.013637227937579155, -0.02424396015703678, 0.010681476444005966, -0.0017615602118894458, 0.006886562332510948, 0.022966532036662102, -0.032615192234516144, 0.051749441772699356, 0.0039647845551371574, 0.011265831999480724, 0.006760857999324799, 0.01743553951382637, -0.02553497813642025, -0.00862943660467863, -0.02378191240131855, 0.0019535142928361893, -0.016620159149169922, 0.0009096581488847733, -0.005986247211694717, 0.009723404422402382, 0.004294333979487419, 0.016932722181081772, -0.0022439933381974697, -0.033349037170410156, -0.0024750176817178726, -0.026037797331809998, 0.04172026738524437, 0.008180977776646614, -0.013297486118972301, -0.01886245422065258, 0.0234693493694067, 0.012719925493001938, 0.0004270127392373979, -0.02791316993534565, -0.005507211200892925, -0.005177662242203951, 0.0036624143831431866, -0.02488267421722412, 0.01728605292737484, -0.01759861595928669, 0.001454943441785872, 0.01999039575457573, 0.009995197877287865, 0.02773650363087654, -0.01869937777519226, -0.004165232181549072, 0.0220288448035717, 0.010008787736296654, 0.0006680169026367366, 0.0007724874885752797, -0.0053339432924985886, -0.010756218805909157, 0.014867092482745647, -0.021838590502738953, -0.024352679029107094, 0.015750421211123466, 0.023197555914521217, -0.0017853421159088612, -0.0055106086656451225, 0.004250167403370142, 0.010572758503258228, -0.020561162382364273, 0.014255557209253311, 0.013331460766494274, 0.006648743059486151, -0.025467030704021454, 0.04411204904317856, 0.010973653756082058, 0.011591983959078789, -0.011999673210084438, -0.02758701890707016, 0.005863939877599478, -0.023116018623113632, -0.004532152786850929, -0.021485259756445885, -0.005085931625217199, -0.007718929089605808, -0.01762579381465912, -0.016443492844700813, -0.015750421211123466, -0.009696225635707378, -0.012597618624567986, -0.010878525674343109, 0.02330627478659153, 0.030957255512475967, 0.0015076033305376768, 0.05427711829543114, 0.005456250160932541, -0.011931724846363068, 0.030658282339572906, -0.0014829720603302121, 0.021988077089190483, 0.0038051060400903225, 0.00975058414041996, -0.005048560444265604, -0.038621824234724045, 0.027070611715316772, -0.019623475149273872, -0.011136730201542377, -0.03348493203520775, -0.016756055876612663, -0.014119661413133144, 0.012020057998597622, 0.009961224161088467, -0.006003234069794416, 0.02442062646150589, 0.027016252279281616, -0.020180651918053627, -0.0016231155022978783, 0.014853502623736858, -0.03726286068558693, -0.01124544721096754, 0.009003152139484882, 0.014948630705475807, -0.0016995574114844203, -0.03587671369314194, -0.004892278928309679, -0.004953432362526655, -0.016266828402876854, 0.003526517888531089, -0.01712297648191452, -0.01245492696762085, -0.013691586442291737, 0.0005873282789252698, -0.021607566624879837, 0.009091485291719437, -0.009193407371640205, 0.014364275150001049, -0.015002989210188389, -0.03381108492612839, -0.00023038727522362024, -0.007949953898787498, -0.024067295715212822, -0.011659931391477585, -0.038458749651908875]} +{"id": "test:10000068", "text": "\"Originally, from Chicago, now living in Hong Kong, Jeanne studied improvisation in Chicago. She holds a B.S. in Anthropology from Loyola University of Chicago and an M.S. in Historic Preservation from the School of the Art Institute.\\nShe has performed, presented and conducted workshops in the US, Europe and Asia. Ms. Lambin\u2019s workshops have been described as \u201camazing,\u201d \u201cpowerful,\u201d and \u201cgreat fun.\u201d Her work has been featured in the Hong Kong\u2019s Top Notch Storytelling Festival, Liar\u2019s League Hong Kong, Literary Death Match, the Art\u2019s House World Voices Presents, the Moth Chicago, Story Lab and This Much is True. Ms. Lambin is a proud to be a founding member of Improv for Humanity and co-creator of The Quest: Improv for Transformation.\"", "vector_field": [0.005429450888186693, -0.00785396434366703, 0.005137580446898937, -0.012218751944601536, -0.03292827680706978, 0.011270173825323582, -0.023415958508849144, 0.0012180609628558159, -0.03510403633117676, -0.038473811000585556, 0.014553714543581009, 0.006364762783050537, 0.03648378700017929, -0.019449176266789436, 0.0025007978547364473, -0.007290123961865902, 0.026241794228553772, 0.00011484101560199633, 0.004945211578160524, -0.02431810274720192, -0.006298428401350975, 0.009479151107370853, -0.004218852613121271, -0.011217106133699417, -0.01442104671150446, -0.01052723079919815, 0.020457454025745392, -0.009319949895143509, -0.02077585831284523, -0.007953465916216373, -0.003044737968593836, 0.01259685680270195, -0.01909097097814083, -0.024888576939702034, -0.009658253751695156, -0.031548526138067245, -0.007900398224592209, -0.0020331419073045254, 0.015389525331556797, -0.0039800493977963924, 0.018878702074289322, 0.0052238148637115955, -0.007263590581715107, -0.02437116950750351, -0.002218877663835883, 0.003053029766306281, -0.020656457170844078, -0.0043946378864347935, -0.026971468701958656, 0.016318203881382942, 0.0359000489115715, -0.0007309195352718234, -0.029372766613960266, -0.009605186060070992, -0.007654962129890919, -0.02852368913590908, 0.00043075744179077446, 0.01139620877802372, -0.009459251537919044, -0.035316307097673416, -0.00031985502573661506, 0.010978303849697113, -0.02524678036570549, 0.017406083643436432, -0.0039833663031458855, -0.01832149550318718, 0.005329949781298637, 0.0027893513906747103, 0.007953465916216373, 0.01979411393404007, 0.022779149934649467, 0.008795909583568573, -0.0012993203708902001, -0.01991351507604122, 0.05420827493071556, -0.003658328903838992, -0.01795002445578575, 0.010699699632823467, 0.015177255496382713, -0.008835709653794765, -0.004291820339858532, -0.017578551545739174, 0.008683141320943832, 0.023137355223298073, 0.009790921583771706, 0.011230372823774815, 0.0021989773958921432, 0.020417654886841774, -0.01483231782913208, 0.0035289772786200047, -0.008384637534618378, 0.02323022298514843, 0.012139150872826576, 0.03690832480788231, -0.01657027378678322, 0.010965036228299141, -0.016450870782136917, 0.003615211695432663, 0.0062320944853127, -0.003678229171782732, 0.004504089243710041, -0.017313215881586075, -0.013094362802803516, -0.01423531025648117, -0.016716208308935165, -0.019130771979689598, -0.004547206684947014, -0.008139201439917088, 0.012278452515602112, 0.01366483699530363, -0.026308128610253334, 0.018281694501638412, 0.017525484785437584, -0.04619510844349861, 0.0006969232927076519, -0.029160495847463608, 0.0200196485966444, -0.009552119299769402, -0.006805884651839733, -0.023150620982050896, 0.026599997654557228, 0.024716107174754143, 0.007794263772666454, -0.027939947322010994, 0.04929954931139946, -0.0021940022706985474, -0.026467328891158104, -0.02356189303100109, 0.014434313401579857, 0.006580348592251539, -0.0023067703004926443, -0.013326533138751984, 0.017711220309138298, 0.01284229289740324, -0.0006977524608373642, 0.021571869030594826, 0.0010447630193084478, 0.0016467454843223095, -0.019714511930942535, -0.010547131299972534, 0.004802593030035496, 0.004192318767309189, -0.011827380396425724, 0.005814189091324806, -0.010202193632721901, 0.02082892693579197, -0.015362991020083427, 0.012795859016478062, 0.009114313870668411, 0.012523889541625977, 0.031601592898368835, 0.003258665557950735, -0.004865610506385565, 0.013067829422652721, 0.008895411156117916, 0.029054362326860428, 0.009936857037246227, 0.04035770148038864, 0.004553840029984713, -0.014620048925280571, -0.00216083531267941, -0.0035621444694697857, -0.0062221442349255085, 0.0013482418144121766, 0.0031740895938128233, 0.029001295566558838, 0.013260198757052422, -0.009996557608246803, -0.011469176039099693, -0.013863839209079742, 0.009028078988194466, 0.029478900134563446, -0.02464977465569973, 0.02023191936314106, 0.009605186060070992, 0.02841755375266075, 0.016291670501232147, -0.005048029590398073, -0.02114732936024666, -0.03199959918856621, 0.01925017312169075, 0.018945036455988884, 0.009983290918171406, 0.016397804021835327, -0.009558752179145813, 0.007290123961865902, -0.008019800297915936, -0.018348028883337975, 0.012278452515602112, -0.02633466199040413, -0.018772566691040993, 0.025671320036053658, 0.040596507489681244, -0.00981082208454609, -0.6542140245437622, -0.015535459853708744, 0.00485566072165966, -0.0001239619596162811, 0.0016624998534098268, -0.021691270172595978, 0.001814239309169352, -0.0030762467067688704, -0.00526029895991087, 0.02547231689095497, -0.007290123961865902, 0.004603590816259384, 0.012616757303476334, -0.005708054639399052, -0.009120946750044823, -0.026918401941657066, -0.020324787124991417, -0.03693486005067825, -0.009830722585320473, -0.023973165079951286, -0.011217106133699417, 0.016835609450936317, 0.016928477212786674, -0.010752767324447632, 0.02333635650575161, 0.02409256622195244, 0.026679599657654762, 0.012331520207226276, 0.008822442963719368, 0.015787530690431595, -0.023482292890548706, 0.010858901776373386, 0.013611769303679466, -0.016663141548633575, 0.053146928548812866, 0.0028656357899308205, -0.022752616554498672, 0.038314610719680786, 0.04102104529738426, 0.017857156693935394, -0.025989724323153496, -0.014646582305431366, 0.008557106368243694, 0.01800309121608734, 0.029346231371164322, 0.01379750482738018, 0.011688078753650188, 0.006945186760276556, 0.01335969939827919, 0.0012155735166743398, -0.02356189303100109, 0.0076947626657783985, 0.008092767558991909, 0.0018407729221507907, 0.0023913464974611998, 0.004474238958209753, 0.015668127685785294, -0.007668228819966316, 0.0022719448897987604, 0.015442592091858387, -0.017299948260188103, 0.012185584753751755, -0.006016508210450411, -0.0036749124992638826, -0.0013905297964811325, 0.0020231918897479773, 0.0013532168231904507, -0.01614573411643505, 0.007575361058115959, -0.02536618337035179, 0.0008412000606767833, 0.03685525804758072, -0.02442423813045025, -0.027966482564806938, 0.015150722116231918, 0.02191680669784546, 0.006361445877701044, 0.0013009787071496248, -0.0013349750079214573, 0.0010555422632023692, 0.018719499930739403, -0.013930173590779305, 0.0036251619458198547, 0.0011699687456712127, 0.03250373899936676, -0.011117605492472649, -0.032424136996269226, -0.006792617961764336, -0.001097830361686647, 0.0018026308389380574, 0.021558601409196854, 0.01418224349617958, 0.015495659783482552, -0.011442642658948898, -0.01728668250143528, 0.04566443711519241, -0.004590323660522699, 0.031150521710515022, -0.007581994403153658, -0.027886880561709404, -0.00414588488638401, 0.0006629269919358194, 0.029797304421663284, -0.011940148659050465, 0.013784238137304783, -0.011840647086501122, -0.03141585737466812, 0.014036308042705059, 0.02754194289445877, -0.005741221364587545, -0.010898702777922153, -0.021943340077996254, -0.012669824063777924, -0.005946857389062643, 0.0032022816594690084, -0.01914403773844242, 0.012643290683627129, 0.016543738543987274, 0.04189665615558624, -0.03393655642867088, 0.0023200372233986855, -0.024185433983802795, -0.003092830302193761, 0.0020182167645543814, -0.014819051139056683, 0.03422842547297478, 0.004726308863610029, -0.008988278917968273, -0.011084438301622868, 0.017406083643436432, -0.013625035993754864, -0.0011011470342054963, 0.019210372120141983, -0.0210544615983963, 0.014407779090106487, 0.022341344505548477, 0.012630023993551731, -0.007263590581715107, -0.0027429175097495317, 0.011130872182548046, -0.03658992424607277, -0.006368079222738743, -0.013691370375454426, -0.008955111727118492, -0.023402690887451172, -0.023216955363750458, -0.010759401135146618, 0.00514089735224843, -0.016265135258436203, -0.011436008848249912, -0.015256856568157673, -0.013160697184503078, -0.021797405555844307, 0.033618152141571045, 0.01814902573823929, -0.0021044511813670397, 0.012384587898850441, -0.018348028883337975, -0.021823938935995102, -0.003459326457232237, 0.008404538035392761, 0.01619880273938179, -0.007509026676416397, 0.02252708002924919, 0.014726183377206326, -0.0023349623661488295, 0.02689186856150627, -0.023362891748547554, -0.02955850213766098, -0.03048717975616455, -0.004102767910808325, -0.011183938942849636, -0.007595261093229055, -0.007900398224592209, 0.00932658277451992, -0.00115587271284312, 0.0010663216235116124, -0.005542219150811434, -0.0037578302435576916, 0.0014361345674842596, 0.006142543163150549, -0.012537156231701374, -0.0085305729880929, -0.006776034366339445, 0.002850710414350033, -0.008404538035392761, 0.008795909583568573, -0.0019402741454541683, -0.02573765441775322, 0.03401615843176842, -0.009883789345622063, 0.03332628309726715, 0.031548526138067245, -0.014447580091655254, -0.015044587664306164, 0.022208677604794502, -0.00864334125071764, -0.016331469640135765, 0.007727929390966892, 0.013525535352528095, -0.0009527243673801422, 0.006494114175438881, 0.0012520572636276484, 0.01647740602493286, 0.01519052218645811, -0.03022184409201145, -0.004530623089522123, -0.023787429556250572, 0.039402492344379425, 0.02000638283789158, 0.0022122443187981844, -0.027674611657857895, -0.009817455895245075, -0.02884209342300892, 0.0009187281248159707, 0.007316657807677984, -0.02066972479224205, 0.004467605613172054, -0.015774263069033623, -0.0023100872058421373, -0.0012744450941681862, 0.01658353954553604, -0.00721052335575223, 0.021027928218245506, -0.02579072117805481, -0.005190648138523102, -0.009512318298220634, -0.005946857389062643, -0.0065737152472138405, -0.0022321445867419243, -0.007475859951227903, 0.00470309192314744, 0.0124641889706254, 0.006640049163252115, -0.012729525566101074, -0.013127529993653297, 0.015309924259781837, -0.01886543445289135, 0.026639798656105995, 0.009120946750044823, 0.023628227412700653, -0.0043681045062839985, 0.03698792681097984, -0.001922032330185175, 0.020205385982990265, 0.009472518227994442, 0.0249283779412508, -0.02023191936314106, 0.010295061394572258, 0.025658052414655685, -0.021027928218245506, 0.01853376440703869, -0.015747729688882828, -0.0010373004479333758, 0.01891850307583809, -0.010726233944296837, 1.1414139407861512e-05, 0.00021247660333756357, 0.05089156702160835, 0.013545435853302479, -0.0170611459761858, 0.01387710589915514, -0.006341545842587948, 0.00029871100559830666, -0.0010663216235116124, 0.00531668309122324, 0.014155710116028786, -0.005465934984385967, -0.017817355692386627, -0.008139201439917088, -0.02431810274720192, -0.03027491085231304, 0.01614573411643505, -0.006623466033488512, 0.028178751468658447, -0.01842762902379036, -0.013233664445579052, 0.0037047627847641706, 0.013611769303679466, 0.00896837841719389, -0.03804927319288254, -0.040543437004089355, 0.03003610670566559, 0.01614573411643505, -0.02421196922659874, -0.005648353602737188, -0.0011840647784993052, -0.006043042056262493, -0.033458951860666275, 0.013678103685379028, 0.003265298902988434, 0.00025144792743958533, -0.002623516134917736, 0.0011567019391804934, 0.03215879946947098, 0.017419351264834404, 0.017790822312235832, -0.001349070924334228, 0.010295061394572258, 0.02110753022134304, -0.001902132062241435, -0.01215905137360096, -0.014275111258029938, -0.01695501059293747, 0.026427529752254486, 0.00737635837867856, -0.006026458460837603, -0.015615060925483704, -0.008152468129992485, 0.0006699750083498657, 0.00904797948896885, -0.006593615282326937, 4.3479969463078305e-05, -0.0059004235081374645, 0.03499790281057358, -0.0018059475114569068, -0.008988278917968273, 0.017658153548836708, 0.016490671783685684, 0.0028954860754311085, -0.0001396126754116267, -0.04324987158179283, -0.032689474523067474, 0.0214392002671957, 0.11282113939523697, 0.010945136658847332, -0.008417804725468159, 0.024742642417550087, 0.002326670568436384, -0.016610072925686836, -0.009970024228096008, -0.022182142361998558, 0.029160495847463608, -0.014129175804555416, 0.0035090770106762648, -0.015323190949857235, 0.010892068967223167, -0.0040165334939956665, 0.004182368982583284, -0.010295061394572258, 0.00152983155567199, -0.0063448622822761536, 0.01991351507604122, 0.011542144231498241, -0.01793675683438778, -0.008848977275192738, 0.008709675632417202, 0.0002143422607332468, -0.005200597923249006, -0.0030662964563816786, 0.03409575670957565, -0.006673216354101896, 0.0224740132689476, -0.010275160893797874, 0.002303453627973795, -0.002560498658567667, -0.0073697250336408615, 0.009373016655445099, -0.004713042173534632, 0.013399500399827957, 0.030938252806663513, -0.004706408828496933, 0.03032797761261463, -0.015694662928581238, 0.021770870313048363, 0.0351305715739727, 0.002318378770723939, -0.005329949781298637, 0.006669899914413691, -0.018573565408587456, -0.007953465916216373, 0.038420744240283966, -0.014195510186254978, -0.013372967019677162, 0.03319361433386803, 0.005727954674512148, -0.024517105892300606, -0.009021446108818054, 0.014593515545129776, -0.00957201886922121, 0.005638403352349997, 0.024079300463199615, 0.013326533138751984, 0.017804088070988655, 0.002258678199723363, -0.052271317690610886, 0.008464238606393337, -0.006825785152614117, -0.0033780671656131744, -0.032477203756570816, -0.025432517752051353, 0.015999799594283104, -0.01668967492878437, 0.005051346495747566, -0.001766146975569427, -0.019223639741539955, -0.016265135258436203, 0.005582019686698914, 0.00905461236834526, -0.005731271579861641, 0.03653685376048088, 0.0060994261875748634, 0.0028822191525250673, 0.01844089664518833, -0.008789275772869587, -0.016596807166934013, 0.01814902573823929, -0.03329974785447121, -0.010898702777922153, 0.027011269703507423, -0.0014087717281654477, -0.014633315615355968, -0.016610072925686836, -0.0037312963977456093, 0.01387710589915514, 0.003248715540394187, -0.0014833976747468114, -0.01940937526524067, 0.005651670508086681, -0.013863839209079742, 0.01691521145403385, 0.02181067131459713, 0.005940224044024944, -0.013585235923528671, 0.0009386283345520496, -0.025339648127555847, -0.014381245709955692, -0.022168876603245735, -0.00464670779183507, 0.0097577553242445, -0.00282915192656219, -0.007031420711427927, -0.009512318298220634, 0.004912044387310743, 0.022208677604794502, -0.01128344051539898, 0.04516029730439186, 0.008305036462843418, -0.0025157229974865913, 0.009313316084444523, 0.0028706106822937727, 0.004033117089420557, 0.014991519972682, -0.018732767552137375, -0.02147900126874447, -0.03369775414466858, 0.0044908225536346436, 0.03767780214548111, -0.02459670603275299, 0.0030845385044813156, -0.005061296280473471, -0.0003397552645765245, -0.0026168825570493937, -0.0028755858074873686, 0.005814189091324806, 0.03778393566608429, 0.005094463471323252, 0.0005497443489730358, -0.03080558404326439, -0.031760796904563904, 0.00825860258191824, -0.01853376440703869, -0.007747829891741276, -0.005031445994973183, -0.0053929672576487064, -0.03401615843176842, 0.007389625534415245, -0.0013299998827278614, 0.01382403913885355, -0.02715720422565937, -0.01991351507604122, 0.006885485723614693, -0.005824139341711998, 0.01251725573092699, -0.002976745367050171, 0.007727929390966892, -0.013366333208978176, 0.012795859016478062, -0.02977077104151249, -0.0067262835800647736, -0.012882093898952007, -0.015482393093407154, 0.004577056970447302, 0.016888676211237907, 0.02398643270134926, 0.014739450067281723, 0.019661445170640945, 0.020815659314393997, 0.014195510186254978, 0.020430920645594597, 0.005207231733947992, 0.01379750482738018, -0.025273315608501434, 0.02028498612344265, 0.0002468874445185065, 0.00284905219450593, -0.014248577877879143, 0.015150722116231918, -0.011707979254424572, 0.015853865072131157, -0.00764832878485322, -0.020709523931145668, 0.00527688255533576, -0.019661445170640945, -0.00018024235032498837, -0.002507431199774146, -0.011568677611649036, -0.004205585923045874, -0.004713042173534632, -0.002723017241805792, 0.019515510648489, -0.023216955363750458, 0.020749324932694435, -0.019727779552340508, 0.03189346566796303, -0.027833813801407814, -0.0041060843504965305, 0.0021492268424481153, -0.016756009310483932, -0.027674611657857895, -0.03292827680706978, -0.032583341002464294, 0.010016458109021187, -0.003887181868776679, -0.008212168700993061, 0.007999899797141552, -0.009333216585218906, -0.020245185121893883, 0.023163888603448868, 0.018241893500089645, -0.006905386224389076, -0.0038274810649454594, -2.5885869035846554e-05, 0.0012653240701183677, 0.013651570305228233, -0.022712815552949905, -0.00709112174808979, -0.020855460315942764, 0.007814164273440838, 0.000499993737321347, -0.0027893513906747103, 0.02644079551100731, -0.02606932446360588, -0.010374662466347218, 0.007993265986442566, -0.017154013738036156, 0.016981543973088264, 0.00267824181355536, 0.01653047278523445, 0.0028407603967934847, -0.010255261324346066, -0.014500647783279419, 0.025936655700206757, 0.010858901776373386, -0.008961744606494904, 0.005664937198162079, 0.024066032841801643, 0.0019767580088227987, -0.0016309911152347922, 0.005810872185975313, 0.017525484785437584, -0.014222043566405773, -0.018732767552137375, 0.03221186622977257, -0.008391271345317364, 0.0036749124992638826, -0.02055032178759575, -0.006318328902125359, -0.0016318203415721655, 0.005631770007312298, -0.008596907369792461, 0.007482493296265602, -0.0004023581277579069, 0.008941845037043095, 0.004998278804123402, 0.010208827443420887, 0.027966482564806938, -0.014925185590982437, -0.020895259454846382, -0.0050712465308606625, -0.0010812467662617564, -0.01322703156620264, -0.01767141930758953, 0.023920098319649696, -0.03027491085231304, 0.01060683187097311, -0.005389650352299213, -0.002737942384555936, 0.011635011993348598, 0.016212068498134613, -0.02077585831284523, 0.01286882720887661, -0.01343930047005415, 0.02754194289445877, -0.007920298725366592, -0.02142593450844288, -0.02589685656130314, -0.015522193163633347, 0.009101047180593014, -0.010096059180796146, -0.0032254985999315977, -0.021293265745043755, -0.0011998191475868225, 0.008132567629218102, 0.002338279038667679, 0.02770114503800869, 0.008676508441567421, -0.015110922046005726, -0.005568752530962229, -0.025764187797904015, 0.004825809970498085, -0.01979411393404007, 0.01844089664518833, -0.014845584519207478, 0.009790921583771706, -0.024158900603652, -0.005120997317135334, -0.013293365947902203, 0.006208877544850111, -0.02109426259994507, -0.0018556980649009347, 0.025604985654354095, -0.005439401138573885, 0.02486204355955124, -0.018852168694138527, -0.01019556075334549, -0.012178951874375343, -0.024689573794603348, 0.025485584512352943, -0.011568677611649036, 0.010129226371645927, -0.0026964836288243532, -0.027727678418159485, -0.004553840029984713, 0.027130670845508575, 0.0015994823770597577, -0.020696258172392845, -0.02693166956305504, 0.008557106368243694, -0.019435908645391464, 0.004650024697184563, -0.005111047066748142, -0.02077585831284523, 0.01838782988488674, -0.03783700615167618, 0.006291795056313276, 0.015774263069033623, -0.013890373520553112, 0.022381145507097244, -0.01382403913885355, 0.0036715958267450333, 0.0058672563172876835, -0.02126673236489296, -0.009989924728870392, 0.0073100244626402855, 1.6415113350376487e-05, -0.020961593836545944, -0.002286870265379548, -0.01810922473669052, -0.029744237661361694, -0.02317715622484684, 0.015402792021632195, -0.016437605023384094, 0.0166764073073864, -0.0040132165886461735, 0.028231818228960037, 0.0012678116327151656, 0.02464977465569973, 0.0018192143179476261, -0.009565385989844799, -0.02296488545835018, -0.001485056011006236, -0.031495459377765656, -0.005790972150862217, -0.004165785387158394, 0.05879859998822212, -0.005200597923249006, -0.005187331233173609, -0.028762491419911385, -0.014076109044253826, -0.02775421179831028, -0.013770971447229385, -0.014964986592531204, 0.016265135258436203, -0.0118870809674263, -0.0018523813923820853, -0.015455858781933784, -0.006590298842638731, 0.005256982054561377, 0.02284548431634903, -0.0054858350194990635, -0.013757704757153988, 0.02551211789250374, -0.024954911321401596, 0.017087679356336594, 0.010905335657298565, -0.02545905113220215, -0.0016964961541816592, 0.010314961895346642, 0.01995331607758999, 0.00820553582161665, 0.01044099684804678, -0.041472118347883224, 0.0013830672251060605, 0.019236905500292778, 0.025233514606952667, 0.0083382036536932, 0.00027963993488810956, 0.003903765231370926, -0.007329924497753382, 0.00093199493130669, 0.007966732606291771, -0.01586713083088398, -0.007893765345215797, 0.0044278050772845745, 0.005784338805824518, 0.01144927553832531, 0.022102542221546173, 0.005018179304897785, -0.007535560522228479, 0.028099149465560913, 0.005502418614923954, 0.03685525804758072, -0.014487380161881447, -0.020629923790693283, 0.022553613409399986, 0.0003660816582851112, -0.01300149504095316, -0.028205284848809242, 0.009426084347069263, -0.005034762900322676, -0.013273465447127819, -0.01303466223180294, -0.02403949946165085, 0.010467530228197575, -0.013220397755503654, 0.01011595968157053, -0.0009958415757864714, 0.004470922518521547, -0.016490671783685684, 0.019117504358291626, -0.01826842688024044, 0.019542044028639793, 0.01346583478152752, 0.010135859251022339, 0.013969973661005497, -0.034918300807476044, -0.023628227412700653, 0.006384662818163633, -0.015986531972885132, -0.000338511512381956, -0.019555309787392616, -0.0010671508498489857, 0.016318203881382942, -0.028762491419911385, -0.01886543445289135, -0.020032916218042374, 0.0009775996441021562, -0.007575361058115959, 0.016013065353035927, 0.2087668627500534, 0.0013648252934217453, 0.016862142831087112, 0.016185535117983818, -0.04123331233859062, 0.028682891279459, 0.026135658845305443, 0.020908527076244354, -0.014566981233656406, 0.006722967140376568, 0.0019187155412510037, -0.01930323988199234, -0.0024808975867927074, 0.006013191770762205, -0.018241893500089645, -0.004875560756772757, -0.04502762854099274, -0.00041541768587194383, -0.012119251303374767, -0.025392716750502586, 0.020152317360043526, -0.00522049842402339, -0.025286581367254257, -0.0038075807970017195, 0.022991420701146126, 0.016225336119532585, 0.008848977275192738, 0.0037014461122453213, 0.004948528483510017, -0.007077855058014393, 0.002384713152423501, 0.005154164042323828, -0.018401095643639565, 0.00034970539854839444, 0.010593565180897713, 0.002449388848617673, 0.004311720374971628, -0.01024862751364708, 0.007635061629116535, 0.009983290918171406, 0.007429426070302725, -0.014023041352629662, 0.0017843889072537422, -0.02109426259994507, -0.007071221247315407, 0.01475271675735712, -0.017817355692386627, 0.01684887707233429, 0.006951820105314255, 0.008132567629218102, -0.005399600602686405, -0.01810922473669052, 0.02847062051296234, 0.028205284848809242, 0.008072867058217525, -0.002482556039467454, 0.040543437004089355, 0.023256756365299225, -0.022394413128495216, -0.00012510207307059318, -0.004802593030035496, 0.02154533565044403, -0.0030314710456877947, 0.015442592091858387, 0.00038909129216335714, 0.013074463233351707, 0.008298403583467007, -0.003601944772526622, -0.014023041352629662, -0.007562094368040562, -0.016702940687537193, -0.007708029355853796, -0.016729475930333138, -0.01635800302028656, -0.020895259454846382, -0.04335600510239601, 0.03802274167537689, 0.007044687867164612, 0.012669824063777924, 0.038792215287685394, -0.00973122101277113, 0.010434363037347794, -0.005469251424074173, -0.01718054711818695, 0.0070513212122023106, -0.04030463472008705, 0.013220397755503654, -0.0016127492999657989, -0.0010646632872521877, -0.0085305729880929, -0.0030712715815752745, -0.020789125934243202, -0.00932658277451992, -0.002926994813606143, 0.032026130706071854, -0.002859002212062478, 0.006533914711326361, -0.0036815458443015814, -0.01837456226348877, 0.0003507418732624501, -0.022726083174347878, 0.017193814739584923, 0.01597326621413231, -0.015906931832432747, -0.008862243965268135, -0.007542193867266178, -0.00426528649404645, -0.018361294642090797, -0.0032719324808567762, -0.0012901994632557034, 0.002132643247023225, -0.03576738014817238, -0.02011251635849476, -0.004928627982735634, 6.400210259016603e-05, 0.028550222516059875, 0.016835609450936317, -0.018069425597786903, 0.004444388672709465, -0.001263665733858943, -0.017525484785437584, -0.010965036228299141, 0.008504039607942104, -0.007847331464290619, 0.00051616266136989, 0.0014543764991685748, -0.02568458579480648, 0.03844727948307991, -0.024835510179400444, -0.015840597450733185, 0.025671320036053658, 0.0011699687456712127, 0.004988329019397497, 0.014805784448981285, -0.0031110721174627542, 0.005708054639399052, 0.012961694970726967, -0.017790822312235832, 0.011243640445172787, -0.005253665614873171, -0.006212193984538317, -0.011382942087948322, -0.00689875241369009, -0.008994911797344685, 0.02394663169980049, -0.014792517758905888, 0.015747729688882828, 0.00825860258191824, 0.0069916206412017345, 0.003565461141988635, -0.00888877734541893, 0.0031409224029630423, -0.023190421983599663, -0.006046358495950699, 0.025697853416204453, -0.01521705649793148, -0.02039112150669098, -0.02689186856150627, -0.016610072925686836, 0.025644786655902863, -0.029187031090259552, 0.024503838270902634, 0.038367677479982376, 0.011973315849900246, -0.01513745542615652, -0.02088199369609356, -0.1694970428943634, 0.019170572981238365, 0.029983039945364, -0.011038004420697689, 0.010188926942646503, 0.010401195846498013, -0.012603490613400936, -0.017764287069439888, -0.021744336932897568, 0.011906981468200684, 0.0012280110968276858, -8.151846122927964e-05, -0.03245067223906517, -0.004467605613172054, -0.008437705226242542, -0.005648353602737188, -0.009167380630970001, 0.029372766613960266, 0.021930072456598282, 0.004868927411735058, 0.021027928218245506, 0.002668291563168168, 0.03937595710158348, -0.0048423935659229755, -0.015097654424607754, 0.025047779083251953, 0.004590323660522699, -0.011250273324549198, 0.0010265210876241326, -0.027197005227208138, -0.01112423837184906, -0.004802593030035496, 0.015853865072131157, 0.007422792259603739, 0.025260047987103462, 0.004888827446848154, -0.009525584988296032, -0.025286581367254257, -0.014129175804555416, 0.0019618328660726547, 0.01952877640724182, 0.037438999861478806, -0.0016591831808909774, -0.0320526659488678, 0.0023681295569986105, 0.01974104531109333, 0.020921794697642326, 0.010142493061721325, -0.014129175804555416, -0.0038606480229645967, 0.0012960036983713508, -0.023309823125600815, 0.02305775322020054, -0.010135859251022339, -0.015177255496382713, 0.005946857389062643, 0.0017495634965598583, 0.01120383944362402, 0.010945136658847332, 0.012013115920126438, 0.009737854823470116, -0.02263321541249752, 0.010228727012872696, 0.0036118950229138136, -0.013120897114276886, -0.028046082705259323, -0.009996557608246803, -0.004858977161347866, -0.02181067131459713, 0.008829076774418354, -0.0029535284265875816, -0.009970024228096008, 0.007727929390966892, -0.025618253275752068, -0.0005136751569807529, 0.012523889541625977, -0.00967815425246954, 0.017154013738036156, 0.003374750493094325, -0.012762692756950855, -0.03223840147256851, 0.002766134450212121, -0.04502762854099274, 0.009658253751695156, -0.009943490847945213, 0.0037047627847641706, -0.004059650469571352, -0.001776097109541297, -0.004845710471272469, -0.019104238599538803, 0.02884209342300892, -0.014845584519207478, -0.023628227412700653, -0.02154533565044403, 0.012941794469952583, 0.009817455895245075, -0.003067954909056425, 0.01733974926173687, -0.004056334029883146, -0.010732866823673248, -0.012013115920126438, -0.006411196663975716, -0.005718004424124956, 0.029638102278113365, 0.021505534648895264, 0.007774363737553358, -0.01967471092939377, 0.0069584534503519535, 0.026573464274406433, -0.0013258539838716388, -0.013678103685379028, 0.0070513212122023106, 0.023628227412700653, 0.01963491179049015, -0.010513964109122753, 0.006553815212100744, 0.009850623086094856, -0.005764438305050135, 0.029133962467312813, 0.0028855360578745604, 0.07211849838495255, 0.0076018949039280415, -0.018891967833042145, 0.0022619948722422123, -0.004510723054409027, -0.0174458846449852, -0.0790703147649765, -0.013067829422652721, -0.007840697653591633, 0.030062641948461533, 0.008172368630766869, 0.04139251634478569, -0.016941744834184647, -0.01347246766090393, -0.006656632758677006, 0.01814902573823929, -0.0076284282840788364, -0.023973165079951286, 0.00981082208454609, 0.017459150403738022, 0.03391002118587494, -0.008305036462843418, 0.010049625299870968, 0.0025986407417804003, -0.00877600908279419, 0.024888576939702034, 0.005001595709472895, -0.012961694970726967, 0.005280198995023966, -0.010772667825222015, 0.005857306532561779, -0.013731171377003193, -0.029956506565213203, 0.03595311567187309, 0.005495785269886255, 0.019011370837688446, -0.016716208308935165, -0.012722891755402088, -0.0011790896533057094, -0.021333064883947372, 0.009313316084444523, -0.016265135258436203, -0.027064336463809013, 0.028046082705259323, 0.027568476274609566, -0.012218751944601536, -0.011761046946048737, 0.020457454025745392, 0.01439451240003109, -0.026586731895804405, 0.01952877640724182, -0.007230423390865326, -0.006530597805976868, 0.02328328974545002, 0.017525484785437584, -0.032583341002464294, -0.024119099602103233, 0.018294962123036385, -0.027117405086755753, -0.020815659314393997, 0.04595630615949631, -0.01028179470449686, 0.001839114585891366, 0.004912044387310743, -0.021452467888593674, 0.015495659783482552, 0.012232018634676933, -0.010666532441973686, -0.034652963280677795, 0.0031094136647880077, 0.0034858600702136755, 0.007741196546703577, -0.012736158445477486, -0.0050049121491611, 0.021027928218245506, -0.030991319566965103, -0.003152530873194337, 0.000168322934769094, 0.001800972386263311, 0.008251969702541828, -0.011502343229949474, -0.010414463467895985, 0.0013258539838716388, -0.008749475702643394, 0.008782642893493176, 0.0002222194307250902, -0.0013374624541029334, -0.011256907135248184, -0.01619880273938179, -0.014898652210831642, 0.020590122789144516, 0.007064587902277708, 0.00913421344012022, -0.0153762586414814, 0.0001816934091039002, 0.004776059649884701, -0.011787580326199532, 0.028391020372509956, 0.006776034366339445, -0.025260047987103462, 0.0059601240791380405, 0.02831142023205757, 0.007489126641303301, -0.023628227412700653, -0.007237056735903025, 0.028550222516059875, -0.030354510992765427, -0.00102237518876791, -0.07848657667636871, 0.013890373520553112, -0.0018457480473443866, -0.01475271675735712, 0.016875410452485085, -0.022606682032346725, 0.004029800184071064, -0.010208827443420887, 0.0032885160762816668, -0.017857156693935394, -0.03908408805727959, 0.03329974785447121, 0.01662334054708481, 0.005170747637748718, -0.024875309318304062, -0.009983290918171406, 0.021850472316145897, 0.00033706045360304415, -0.0015770946629345417, -0.0028855360578745604, 0.007382992189377546, 0.006298428401350975, 0.0021110845264047384, -0.006434413604438305, -0.00569810438901186, 0.0019253490027040243, -0.03802274167537689, 0.02240767888724804, 0.0062818448059260845, 0.007256957236677408, 0.01739281602203846, -0.04123331233859062, 0.007966732606291771, 0.02873595803976059, -0.01679580844938755, -0.032848674803972244, 0.016225336119532585, 0.031150521710515022, -0.006212193984538317, 0.049989424645900726, -0.015628328546881676, 0.0055455355904996395, 0.01749895140528679, 0.0017578551778569818, -0.01374443806707859, -0.012769325636327267, -0.0035190272610634565, 0.03934942185878754, 0.027780747041106224, 0.003903765231370926, 0.014978253282606602, 0.020032916218042374, -0.026865335181355476, -0.011661545373499393, -0.007396258879452944, 0.0038175308145582676, 0.028444087132811546, -0.0059302737936377525, -0.012722891755402088, 0.0027760847005993128, 0.041578251868486404, 5.669497841154225e-05, -0.004308403469622135, 0.0007819139282219112, -0.010898702777922153, -0.032424136996269226, -0.0036550122313201427, 0.0017230297671630979, -0.0035190272610634565, -0.013983241282403469, -0.02585705555975437, -0.0009353116620332003, -0.015110922046005726, 0.03441416099667549, -0.013154064305126667, 0.007111021783202887, 0.023734362795948982, -0.012769325636327267, 0.007356458343565464, 0.022752616554498672, -0.009631720371544361, 0.009286782704293728, -0.0007943515665829182, 0.011568677611649036, 0.01191361527889967, 0.006258627865463495, -0.020749324932694435, -0.014805784448981285, 0.014062841422855854, 0.0021823938004672527, -0.01702134497463703, 0.017047878354787827, 0.006716333795338869, -0.011183938942849636, 0.02149226702749729, 0.01469964999705553, 0.006646682973951101, -0.007097755093127489, 0.010500697419047356, 0.014076109044253826, 0.004902094602584839, 0.0010563714895397425, -0.008470872417092323, -0.013167330995202065, -0.03391002118587494, 0.008026433177292347, -0.01095840334892273, -0.03316707909107208, 0.01565486192703247, -0.007761096581816673, 0.005986657924950123, 0.024503838270902634, 0.00028793170349672437, 0.007489126641303301, -0.021359600126743317, -0.005160797853022814, 0.02017885074019432, -0.0238935649394989, -0.02857675589621067, 0.04269266501069069, 0.033618152141571045, 0.0029104112181812525, 0.020351320505142212, -0.004053017124533653, 0.016437605023384094, 0.019435908645391464, 0.009101047180593014, -0.012603490613400936, -0.02683880180120468, -0.01158857811242342, 0.01653047278523445, 0.009439351037144661, -0.04606243968009949, -0.006719650235027075, -0.02284548431634903, 0.0122718196362257, 0.02258014865219593, 0.005326632875949144, -0.02992997318506241, 0.06559121608734131, -0.018348028883337975, -0.005880523473024368, 0.03385695442557335, -0.00659693218767643, 0.0164110716432333, 0.009830722585320473, 0.007356458343565464, -0.027303140610456467, 0.0013150747399777174, 0.0011376308975741267, -0.013651570305228233, -0.01519052218645811, -0.03990663215517998, -0.015747729688882828, -0.0013208789750933647, -0.009220448322594166, 0.009704687632620335, -0.003863964695483446, 0.015203789807856083, 0.03658992424607277, 0.021014662459492683, 0.033724285662174225, 0.00756872771307826, -0.03059331513941288, 0.019183838739991188, 0.023190421983599663, 0.005339900031685829, -0.020470721647143364, -0.011814113706350327, 0.007734563201665878, 0.014726183377206326, -0.031813863664865494, -0.024238502606749535, -0.010321595706045628, -0.035793911665678024, -0.010467530228197575, -0.006391296163201332, 0.005704737734049559, 0.004709725268185139, 0.0038506980054080486, -0.0006144201615825295, 0.0008142518345266581, -0.029903439804911613, 0.02138613350689411, 0.004746209364384413, -0.015946732833981514, -0.0014029674930498004, -0.029107429087162018]} +{"id": "test:10000069", "text": "\"I grew up with stove top mac and cheese. Luckily my mother often made her own from scratch, but I also had the commercial boxed mac and cheese as well. Personally I\u2019ve always prefered stove top to baked. I know, it is controversial. There is something to that velvety sauce and soft texture. It is simple and concise.\\nHowever, as I grew up, the idea of making it from scratch seemed more and more improbable. So I leaned toward the box. Now I am not saying that something is bad if it is in a box but powdered things often don\u2019t feel super healthy in the food realm. Especially if all you are adding to them is butter and hot sauce and maybe tuna from a can. Wait. What?\\nYes. I would like to take a moment to address something that we call \u201cShame Food.\u201d What is Shame Food? Shame Food is food that you only feel comfortable eating in private. Far from any witnesses. My big Shame Food experience was when I was doing social work. I would work all day in a pretty intense environment and then commute home and my wife worked another 5-6 hours. So I would get home, eat something, and then take a nap. That thing became my Shame Food. It was inspired by a natural food company\u2019s spicy mac and cheese. (It has since been discontinued.) I would make mac and cheese and add a lot of cheap hot sauce (this was before we had a sauce business) and a can of tuna. I would not even plate it. I would eat the whole thing out of the pot and then take a nap. I mean the nap wasn\u2019t a choice. That box of noodles is meant to feed a family of four. Plus the amount of hot sauce I added meant I wasn\u2019t really enjoying the meal as much as surviving it. I had to recoup.\\nThis recipe isn\u2019t that dish. It is tied to that spicy boxed macaroni and cheese, that my wife introduced me to, and we would eat it together at a time that we didn\u2019t eat much. Also the roots are a stove top mac and cheese. The kind where the only powder involved it mustard powder and the spice comes from a complex sauce with a bright balsamic flavor and a nutty essence that pairs well with the gruyere and trust me if you finish it with toasted sesame, you will not regret it. Please enjoy and feel free to share with us your Shame Food.\\nPour the pasta into a strainer and place the empty pot back on the stove.\\nTurn heat to medium and add the butter.\\nOnce the butter has melted, stir in the flour and use a whisk continue for a minute.\\nSlowly add the milk and cream while whisking to combine.\\nAdd the mustard powder and stir.\\nReduce heat to medium-low and add the mascarpone. Continue to whisk.\\nBegin adding the cheese a handful at a time while whisking to combine.\\nAdd the Hatch Sesame Balsamic Sauce.\\nReturn the pasta to the sauce pan and stir to coat.\\nIf the sauce seems a little loose then continue to stir the pasta occasionally on low heat.\\nServe in bowls topped with toasted sesame seeds and Mexican oregano.\"", "vector_field": [0.011657211929559708, -0.025373151525855064, -0.002446410246193409, -0.017779922112822533, -0.0016760585131123662, -0.006019105203449726, 0.013969939202070236, -0.0278864037245512, -0.024637892842292786, -0.02970450185239315, 0.04165581613779068, 0.024103157222270966, -0.009932693094015121, -0.008395331911742687, -0.004140850156545639, 0.04588022083044052, 0.040506139397621155, 0.008308437652885914, -0.0006283130496740341, -0.03983772173523903, -0.022244954481720924, 0.0042277448810637, 0.004361428320407867, -0.013729308731853962, -0.01276678591966629, -0.0029243293683975935, 0.0035158793907612562, 0.0020754383876919746, 0.00041755245183594525, -0.001896636444143951, 0.00682789133861661, -0.009605168364942074, 0.008682751096785069, -0.019103391095995903, -0.012265472672879696, 0.005852000787854195, -0.017245188355445862, -0.018127499148249626, -0.003963719587773085, -2.0835846953559667e-05, -0.008254963904619217, 0.0034958268515765667, 0.009250907227396965, 0.007332547102123499, -0.007419441360980272, 0.01343520451337099, 0.004709005821496248, 0.0025450021494179964, -0.018421603366732597, 0.015721194446086884, 0.041281502693891525, -0.022285060957074165, 0.005537844263017178, -0.02033327892422676, 0.001766294939443469, 0.006299840752035379, 0.001734545105136931, 0.01184436958283186, 0.01433088444173336, -0.011376476846635342, -0.012278840877115726, 0.013007416389882565, -0.01637624390423298, 0.025159258395433426, 0.0009399629780091345, 0.003475774312391877, -0.009845798835158348, -0.02605493925511837, 0.0017846764530986547, 0.004063982050865889, 0.019330652430653572, 0.022298429161310196, -0.008034385740756989, 0.0016501572681590915, 0.012693259865045547, 0.0024831732735037804, -0.025827676057815552, 0.011991421692073345, 0.021482959389686584, -0.018394866958260536, 0.015707826241850853, -0.020386753603816032, 0.008930066600441933, 0.029651029035449028, 0.013662466779351234, -0.00671091815456748, -0.010895215906202793, 0.024263577535748482, 0.012238736264407635, -0.013341626152396202, -0.004184297285974026, 0.03352785483002663, 0.007867281325161457, -0.002434713067486882, -0.007466230541467667, 0.019170232117176056, -0.0007920754724182189, 0.0447305403649807, -0.014157095924019814, -0.021429484710097313, -0.0022793058305978775, 0.00036011027987115085, 0.0034156166948378086, -0.02681693620979786, -0.011657211929559708, 0.017392240464687347, 0.012967311777174473, -0.0011237779399380088, 0.017913606017827988, -0.01798044703900814, -0.024343788623809814, 0.022833162918686867, 0.012800207361578941, -0.025119153782725334, 0.0003974999126512557, -0.0015524011105298996, 0.011349739506840706, -0.04058634862303734, 0.002284318907186389, -0.01692434772849083, -0.01664361171424389, 0.00664741825312376, 0.01640298031270504, -0.008943434804677963, 0.021549800410866737, -0.0038534305058419704, -0.010367165319621563, -0.008194806054234505, -0.0060658943839371204, 0.007726913783699274, 0.05320608243346214, 0.007232284173369408, 0.023073794320225716, 0.0018414919031783938, 0.004150876775383949, 0.01391646545380354, -0.020253069698810577, -0.03991793096065521, -0.007559808902442455, -0.03376848250627518, 0.015226565301418304, 0.023581791669130325, -2.2284955775830895e-05, -0.00664741825312376, -0.040131825953722, 0.014477936550974846, 0.026750093325972557, 0.01276678591966629, -0.005056583322584629, 0.010748163796961308, 0.020212965086102486, -0.0474042147397995, 0.008174753747880459, -0.0036328525748103857, -0.01954454556107521, 0.011042267084121704, 0.0054041603580117226, -0.00715207401663065, -0.012679891660809517, 0.005073293577879667, 0.02244548127055168, 0.014624988660216331, 0.0286082960665226, 0.003133209887892008, 0.00899022351950407, 0.01955791376531124, 0.027672510594129562, -0.016255928203463554, 0.013074258342385292, -0.008134648203849792, -0.005380766000598669, 0.02735166996717453, -0.043393705040216446, 0.02783293090760708, 0.005634764675050974, 0.004361428320407867, 0.010487480089068413, 0.02602820284664631, -0.03200386092066765, 0.002267608419060707, -0.008275016210973263, -0.00795417558401823, 0.028073562309145927, 0.052778296172618866, -0.019143495708703995, -0.028741979971528053, 0.027806194499135017, 0.009832430630922318, -0.014130359515547752, -0.008595856837928295, -0.0034891427494585514, 0.021683484315872192, -0.02783293090760708, -0.013742676936089993, -0.6220031976699829, -0.03874151408672333, -0.009411327540874481, -0.025907887145876884, -0.0010836728615686297, 0.015306775458157063, -0.0020035833586007357, -0.022779690101742744, -0.02211127243936062, 0.01562761515378952, -0.03494489938020706, 0.04462359473109245, 0.012305578216910362, -0.01302078552544117, -0.00911722332239151, -0.01981191337108612, 0.027177881449460983, -0.008021017536520958, -0.019731704145669937, 0.011724053882062435, -0.025159258395433426, 0.022766321897506714, -0.02163001149892807, 0.0007055989117361605, 0.03149586170911789, -0.005176898557692766, 0.022485585883259773, -0.01723182015120983, 0.007265705149620771, 0.002010267460718751, 0.003923614509403706, 0.013187889941036701, -0.013876360841095448, -0.013361678458750248, 0.06021110340952873, -0.02526620589196682, -0.019424231722950935, 0.043393705040216446, 0.026897145435214043, 0.014785408973693848, -0.024637892842292786, -0.03026597388088703, 0.02628220058977604, -0.006206262391060591, -0.01131631899625063, -0.01794034242630005, 0.021242327988147736, 0.00913059152662754, 0.0010218442184850574, -0.0058319480158388615, 0.01851518265902996, -0.011683949269354343, -0.0002801089722197503, -0.0054175290279090405, 0.02034664712846279, -0.014838882721960545, 0.04293918237090111, -0.007726913783699274, 0.0019049916882067919, 0.017325397580862045, 0.008776330389082432, 0.04628127068281174, 0.00989258848130703, -0.01290715392678976, -0.02550683543086052, 0.003121512709185481, -0.05967637151479721, 0.003766536246985197, 0.02160327322781086, -0.016028666868805885, -0.022512322291731834, 0.0027238039765506983, -0.023488212376832962, -0.008642646484076977, 0.011490107513964176, 0.025132521986961365, -0.008569120429456234, -0.00410074507817626, 0.012051578611135483, 0.010647900402545929, 0.011804264038801193, -0.006540471222251654, -0.019624756649136543, -0.023314423859119415, 0.03695015609264374, -0.011897842399775982, -0.013254731893539429, 0.043099600821733475, 0.0034323271829634905, -0.00939127430319786, 0.008789698593318462, 0.023755580186843872, 0.0028658427763730288, -0.038714777678251266, -0.004942952189594507, 0.014638356864452362, -0.010073061101138592, 0.003736457321792841, 0.012345682829618454, -0.038661304861307144, -0.006136078387498856, -0.012392472475767136, 0.039463408291339874, -0.0006408458575606346, 0.007827176712453365, -0.007880649529397488, -0.03144238889217377, 0.030025342479348183, 0.008635962381958961, -0.030827444046735764, 0.01157700177282095, 0.013742676936089993, -0.010407269932329655, -0.02887566387653351, 0.0026987381279468536, -0.017111504450440407, 0.0018030578503385186, -0.006426840554922819, 0.00709860073402524, -0.013548835180699825, 0.00695823272690177, -0.0049997675232589245, -0.010273586958646774, 0.01589498296380043, -0.003273577895015478, 0.029330188408493996, 0.03518553078174591, -0.010240165516734123, -0.016028666868805885, -0.007472914643585682, 0.016068771481513977, 0.012278840877115726, 0.021710220724344254, -0.0057717906311154366, 0.003397235181182623, 0.02919650450348854, -0.0039002196863293648, -0.015761299058794975, 0.009832430630922318, -0.029624290764331818, 0.011236108839511871, 0.006600629072636366, -0.004698979668319225, -0.0130341537296772, -0.0012491063680499792, -0.018608760088682175, 0.0025717387907207012, -0.013729308731853962, -0.007065179757773876, 0.009478169493377209, 0.018354762345552444, -0.004070666618645191, 0.014518042095005512, -0.0030362894758582115, 0.012659839354455471, -0.007880649529397488, -0.0005263792700134218, -0.011851053684949875, -0.006791128311306238, -0.007245652377605438, -0.020761067047715187, 0.04197666049003601, -0.025413258001208305, 0.024143263697624207, -0.0294638704508543, -0.0014228951185941696, -0.00949153769761324, 0.00019060360500589013, -0.002874197904020548, -0.03804636001586914, 0.007546440698206425, -0.016015298664569855, -0.002832421800121665, 0.018033921718597412, -0.01379615068435669, 0.017004556953907013, -0.017552660778164864, -0.01196468435227871, -0.008635962381958961, 0.015427090227603912, 0.02395610511302948, -0.006991653703153133, -0.0002575498656369746, 0.006991653703153133, 0.027993351221084595, -0.01586824655532837, 0.02132253907620907, -0.018328025937080383, -0.01640298031270504, 0.030880916863679886, 0.027966614812612534, -0.01549393218010664, 0.01819434203207493, 0.0022559112403541803, 0.009578431956470013, -0.025947991758584976, 0.008950118906795979, 0.023327792063355446, 0.017512554302811623, -1.9895884179277346e-05, 0.00403390359133482, 0.0060658943839371204, -0.007539756596088409, -0.012312262319028378, 0.022204849869012833, -0.02943713404238224, 0.018274551257491112, -0.005384108051657677, 0.020934855565428734, 0.019985701888799667, -0.00949153769761324, -0.048580631613731384, -0.004317981190979481, -0.0364956296980381, 0.015360248275101185, 0.032137542963027954, 0.0012950601521879435, 0.004815952852368355, -0.012432577088475227, -0.007599913980811834, -0.0032953014597296715, -0.014157095924019814, 0.019958965480327606, 0.011944632045924664, -0.01613561436533928, 0.02759230136871338, 0.01459825225174427, 0.019196968525648117, -0.002347818575799465, -0.02783293090760708, 0.01380951888859272, 0.025480099022388458, 0.010581059381365776, 0.014611620455980301, 0.02477157674729824, -0.019210336729884148, 0.0035960895475000143, -0.012659839354455471, 0.01951780915260315, 0.00016115143080241978, -0.006152789108455181, 0.013702571392059326, 0.027699247002601624, 0.005460976157337427, 0.02914302982389927, 0.014825514517724514, 0.019651493057608604, 0.003243498969823122, -0.0169109795242548, 0.004799242131412029, 0.00709860073402524, -0.011744106188416481, -0.022218218073248863, 0.005995710846036673, 0.023768948391079903, -0.020921487361192703, -0.01564098335802555, -0.01029363926500082, 0.01693771593272686, 0.020694226026535034, -0.0070718638598918915, -0.02970450185239315, -0.014518042095005512, -0.02343473955988884, 0.003519221441820264, -0.0323781743645668, -0.010748163796961308, -0.012325630523264408, -0.013214626349508762, -0.007492967415601015, -0.031041337177157402, -0.003034618217498064, 0.024049684405326843, -0.026669884100556374, -0.0016317757545039058, -0.023060426115989685, -0.0004766656784340739, 0.03914256393909454, 0.01565435342490673, -0.008903329260647297, -0.031094811856746674, -0.01899644359946251, 0.02574746683239937, -0.0066942074336111546, 0.00017838408530224115, 0.018394866958260536, -0.023260951042175293, 0.006363340653479099, 0.0010343770263716578, 0.0019250442273914814, 0.003367156255990267, -0.023488212376832962, 0.0028675138019025326, 0.030640287324786186, -0.01926381140947342, 0.003662931267172098, 0.0006976614240556955, -0.02086801454424858, 0.020493699237704277, 0.007091916166245937, 0.010079745203256607, -0.025480099022388458, -0.021830536425113678, -0.006112684030085802, 0.019384125247597694, -0.016603507101535797, 0.004535216838121414, -0.017298661172389984, -0.007432809565216303, -0.020159490406513214, 0.0203065425157547, 0.0029661054722964764, 0.0030446446035057306, -0.001024350756779313, 0.018434971570968628, -0.004795900080353022, -0.03518553078174591, 0.0030379605013877153, 0.0048593999817967415, -0.004438296426087618, 0.005835290066897869, -0.007653387729078531, -0.014384358190000057, 0.02450420893728733, 0.07668092846870422, 0.01499930303543806, -0.0026903830002993345, 0.0016125587280839682, 0.01640298031270504, 0.01743234507739544, -0.019344020634889603, -0.021670116111636162, 0.026897145435214043, 0.0010903570801019669, 0.015681089833378792, 0.002777277259156108, 0.00018611268023960292, -0.011262845247983932, 0.012018158100545406, -0.019464336335659027, 0.007559808902442455, -0.020199595019221306, 0.02604157105088234, -0.02554694004356861, 0.00949153769761324, 0.00011822646047221497, 0.007867281325161457, 0.0024146605283021927, 0.032164279371500015, 0.01342183630913496, 0.00846885796636343, 0.026629777625203133, 0.017405608668923378, -0.012987364083528519, -0.01326810009777546, 0.00885654054582119, 0.002067083027213812, -0.0006207933183759451, -0.004792558029294014, 0.0026987381279468536, 0.02761903777718544, 0.00963190570473671, 0.004231086932122707, -0.0056414492428302765, 0.007272389251738787, 0.004354744218289852, 0.019090021029114723, -0.00598902627825737, 0.0019150179577991366, -0.0417092926800251, 0.005962289869785309, 0.024116525426506996, -0.024571049958467484, -0.03288617357611656, 0.018702339380979538, 0.0024330420419573784, -0.007439493667334318, 0.011483423411846161, 0.012445945292711258, -0.01977180875837803, 0.0017863474786281586, -0.001831465633586049, -0.018140867352485657, 0.010948688723146915, -0.014103622175753117, -0.02001243829727173, -0.0026201989967375994, -0.014157095924019814, 0.0093645378947258, -0.021402748301625252, -0.007820491679012775, 0.004919557366520166, -0.015761299058794975, 0.011349739506840706, 0.018568655475974083, -0.016042035073041916, -0.032191015779972076, 0.004558611661195755, 0.019638124853372574, 0.006801154464483261, -0.0036996942944824696, -0.01692434772849083, -0.017338765785098076, 0.02549346722662449, 0.01029363926500082, -0.00858917273581028, -0.013702571392059326, -0.027725983411073685, 0.007118653040379286, 0.0075798616744577885, 0.03149586170911789, -0.0011931263143196702, 0.004197665955871344, 0.025172626599669456, 0.003703036345541477, 0.007379336282610893, -0.010561006143689156, 0.0023093847557902336, 0.004521848633885384, -0.00342062977142632, -0.004120797850191593, 0.0027990008238703012, 0.03259206935763359, -0.03459732234477997, 0.007680124137550592, -0.007132021244615316, -0.023113898932933807, -0.020720962435007095, 0.003686326090246439, -0.010754847899079323, -0.007787071168422699, 0.0077603342942893505, -0.0015682760858908296, -0.0026519489474594593, 0.03561331704258919, 0.001955122919753194, -0.013027469627559185, 0.004381480626761913, -0.005668185651302338, 0.031148284673690796, -0.0025115811731666327, 0.03596089407801628, -0.036549102514982224, 0.028260719031095505, 0.004398191347718239, -0.01950444094836712, 0.05566586181521416, 0.03390216827392578, -0.012171894311904907, 0.004261165391653776, 0.025172626599669456, -0.026242095977067947, -0.028955873101949692, -0.013114363886415958, 0.0028909083921462297, 0.0011112451320514083, -0.046495165675878525, -0.04604063928127289, -0.016095507889986038, -0.028554823249578476, 0.008141333237290382, 0.0031449072994291782, -0.000439902680227533, -0.018314655870199203, -0.022605901584029198, 0.011690633371472359, -0.0019668203312903643, 0.016616875305771828, 0.023488212376832962, -0.021830536425113678, -0.010026272386312485, -0.020266437903046608, -0.0008305095252580941, 0.032164279371500015, 0.01748581789433956, -0.020373385399580002, -0.011309634894132614, 0.015400353819131851, 0.00188828120008111, -0.03047986701130867, -0.008622594177722931, 0.009371221996843815, 0.0022475558798760176, 0.0276457741856575, 0.04580001160502434, 0.004839347209781408, 0.030399655923247337, -0.014157095924019814, -0.0010134889744222164, -0.015239933505654335, 0.0015273353783413768, -0.019143495708703995, -0.009912640787661076, 0.03170975670218468, 0.03879498690366745, 0.02687040902674198, 0.001534019596874714, 0.018608760088682175, 0.002864171750843525, 0.034517113119363785, 0.014170464128255844, -0.026656515896320343, -0.009638589806854725, -0.026455989107489586, 0.019664861261844635, 0.004338033497333527, -0.019437599927186966, 0.02027980610728264, 0.0004770834348164499, -0.016777295619249344, -0.003903561970219016, 0.028260719031095505, 0.015935087576508522, 0.02658967301249504, 0.011570317670702934, -0.024317052215337753, -0.010935320518910885, -0.030372919514775276, -0.03021249920129776, -0.0021322539541870356, -0.009344485588371754, -0.023889264091849327, 0.0058319480158388615, 0.0030028684996068478, -0.016336139291524887, -0.008442120626568794, 0.029570817947387695, -0.014892355538904667, -0.006316551472991705, 0.009939377196133137, -0.014384358190000057, -0.003947008866816759, -0.004809268284589052, -0.04515833035111427, -0.032404910773038864, -0.021162118762731552, 0.0007235626108013093, 0.002098832977935672, 0.004795900080353022, -0.009210801683366299, -0.01716497726738453, 0.00939795933663845, -0.035265740007162094, -0.01775318570435047, 0.003410603618249297, -0.015400353819131851, 0.020908119156956673, -0.0014696844154968858, 0.013221310451626778, 0.009946062229573727, -0.0234614759683609, 0.004080692771822214, -0.008388647809624672, 0.010901900008320808, -0.013615677133202553, 0.01645645499229431, -0.0011747449170798063, 0.03564005345106125, 0.003662931267172098, -0.01473193522542715, -0.002920987317338586, -0.027752721682190895, -0.020413490012288094, 0.027431879192590714, 0.0047658211551606655, -0.002164003672078252, -0.04117455706000328, -0.006019105203449726, -0.01931728422641754, 0.03392890468239784, -0.0013100994983687997, 0.0039436668157577515, 0.01614898256957531, -0.004244455136358738, -0.02236527018249035, 0.0375918373465538, 0.005243740044534206, -0.0018281235825270414, 0.01616235077381134, -0.013475309126079082, -0.016042035073041916, -0.029303450137376785, 0.012719997204840183, 0.026215359568595886, 0.010233481414616108, 0.03184343874454498, -0.016349507495760918, 0.01692434772849083, 0.017325397580862045, 0.019357388839125633, -0.034543849527835846, 0.011162582784891129, 0.003203393891453743, 0.03122849576175213, -0.009250907227396965, 0.008288384415209293, 0.0012449288042262197, -0.008829803206026554, 0.034303218126297, -0.037404678761959076, 0.010514217428863049, 0.013375046662986279, 0.007873965427279472, -0.015159723348915577, 0.001943425741046667, 0.011062320321798325, -0.01433088444173336, -0.01261973474174738, 0.013715939596295357, 0.0007440329063683748, -0.017926974222064018, -0.02108190767467022, -0.0006224644021131098, -0.02394273690879345, -0.010086429305374622, -0.033233750611543655, -0.010019587352871895, -0.012472682632505894, 0.009571747854351997, -0.008254963904619217, 9.263857646146789e-05, 0.011804264038801193, -0.015707826241850853, 0.014290779829025269, -0.021763693541288376, 0.02735166996717453, -0.008034385740756989, -0.0018047289922833443, -0.0025450021494179964, -0.0015674405731260777, 0.022017693147063255, 0.006119368132203817, -0.02783293090760708, -0.04352738708257675, 0.014838882721960545, 0.007252336945384741, 0.007365967612713575, -0.0060658943839371204, 0.00514681963250041, 0.017619501799345016, 0.012940575368702412, -0.01930391602218151, -0.011757475323975086, -0.0023595159873366356, 0.003173314966261387, 0.023809053003787994, -0.012085000053048134, -0.0027171196416020393, -0.008502278476953506, -0.0038200095295906067, 8.924426219891757e-05, -0.0037865887861698866, -0.01644308678805828, 0.017392240464687347, -0.006644076202064753, 0.012826943770051003, 1.8538159565650858e-05, -0.0037966149393469095, -0.0037164047826081514, -0.011650527827441692, -0.007894017733633518, -0.009164012968540192, -0.0123657351359725, -0.02553357183933258, 0.021202223375439644, 0.01616235077381134, 0.01132968720048666, 0.009050381369888783, -0.007285757455974817, 0.0036896681413054466, -0.006497024092823267, -0.013114363886415958, -0.018087394535541534, -0.03828699141740799, -0.028447875753045082, 0.03863456845283508, -1.0698606274672784e-05, 0.0032551963813602924, -0.024664629250764847, 6.233782187337056e-06, -0.03569352999329567, 0.0008906671428121626, -0.019704965874552727, -0.0013268099864944816, 0.007967543788254261, -0.006911443546414375, -0.02423684112727642, 0.010688005946576595, -0.00774696609005332, -0.007613282650709152, 0.002194082597270608, -0.002531633712351322, -0.003203393891453743, -0.004966346547007561, -0.004140850156545639, -0.015681089833378792, -0.01898307539522648, 0.004996425472199917, 0.014651725068688393, -0.0005848658620379865, -0.007452862337231636, -0.001291717984713614, 0.013301520608365536, -0.007827176712453365, -0.009732168167829514, 0.01850181445479393, 0.026750093325972557, -0.009591800160706043, 0.044436436146497726, -0.02080117166042328, 0.004421585705131292, 0.017659606412053108, 0.028180507943034172, 0.001945096766576171, -0.03510532155632973, 0.004892820492386818, 0.001130462158471346, 0.01748581789433956, 0.007379336282610893, 0.009799010120332241, 0.002824066672474146, 0.017365502193570137, 0.034303218126297, 0.02554694004356861, -0.024397261440753937, 0.035319212824106216, 0.01550730038434267, -0.01585487835109234, -0.021736957132816315, -0.004558611661195755, -0.03288617357611656, -0.022325165569782257, 0.0005760928615927696, -0.024571049958467484, -0.00391693040728569, 0.011202688328921795, 0.027725983411073685, 0.004170929081737995, -0.0038133254274725914, -0.017900237813591957, -0.012332314625382423, -0.0014329213881865144, 0.03577373921871185, 0.02731156535446644, -0.0009834101656451821, 0.020212965086102486, -0.007546440698206425, -0.019852017983794212, -0.0021122011821717024, -0.01329483650624752, 0.0033537880517542362, -0.01663024351000786, -0.005300555843859911, -0.03221775218844414, -0.013715939596295357, -0.025640519335865974, -0.01329483650624752, -0.03719078376889229, 0.0015791378682479262, 0.004856057930737734, 0.22052450478076935, 0.011663896031677723, -0.007546440698206425, 0.04767158254981041, 0.0187290757894516, -0.03288617357611656, 0.024677997455000877, 0.016563400626182556, 0.009785640984773636, 0.028501348569989204, -0.0003350445767864585, -0.004030561540275812, -0.015948455780744553, -0.010173323564231396, 0.0095182741060853, -0.026977356523275375, -0.009738852269947529, -0.026616409420967102, 0.019063284620642662, -0.02523946948349476, 0.024343788623809814, -0.021175486966967583, -0.006219630595296621, 0.003108144272118807, 0.030346183106303215, 0.034757744520902634, -0.0029243293683975935, -0.005541186314076185, 0.03098786436021328, -0.01261304970830679, -0.008849856443703175, 0.00387348304502666, -0.0038367200177162886, 0.006771075539290905, -0.038126569241285324, 0.016469823196530342, 0.003251854097470641, -0.00937790609896183, 0.022017693147063255, 0.01979854516685009, 0.006029131356626749, 0.01822107844054699, 0.02498546987771988, -0.018274551257491112, -0.04203013330698013, -0.009792326018214226, -0.01538698561489582, 0.004993083421140909, 0.00949153769761324, 0.01592171937227249, -0.009311064146459103, -0.01800718531012535, -0.01470519881695509, 0.016001930460333824, -0.012733365409076214, -0.014665094204246998, -0.0006734312628395855, 0.010099797509610653, -9.222081280313432e-05, 0.008769646286964417, 0.0031582757364958525, 0.05545196682214737, -0.04871431365609169, 0.033982377499341965, -0.021175486966967583, 0.024851785972714424, 0.017365502193570137, -0.025119153782725334, -0.005444265902042389, -0.0004242366412654519, -0.002120556542649865, 0.005180240608751774, -0.009745536372065544, -0.013227994553744793, -0.012312262319028378, -0.02523946948349476, 0.012666523456573486, 0.006931495852768421, 0.016616875305771828, 0.028528084978461266, -0.026148516684770584, -0.001525664352811873, -0.024584418162703514, 0.010079745203256607, -0.00848222617059946, -0.027458617463707924, 0.01614898256957531, -0.000815052364487201, 0.008308437652885914, -0.03160281106829643, -0.0007252336945384741, 0.0038935355842113495, 0.0015181447379291058, -0.011202688328921795, 0.011937947943806648, 0.0008647659560665488, 0.00664741825312376, 0.02470473386347294, 0.006249709520488977, -0.0002953572547994554, -0.012145157903432846, -0.03307332843542099, 0.005103372503072023, -0.014624988660216331, 0.007078547962009907, -0.02244548127055168, 0.012472682632505894, 0.025413258001208305, 0.008682751096785069, -0.008087859489023685, 0.0012766786385327578, -0.02134927548468113, -0.010561006143689156, -0.001130462158471346, -0.0005602179444395006, 0.021496327593922615, -0.014678462408483028, -0.0071119689382612705, 0.011015530675649643, -0.014357621781527996, 0.007245652377605438, -0.028528084978461266, 0.02761903777718544, -0.001074482104741037, -0.021817168220877647, -0.003179999301210046, -0.01187110599130392, 0.0074461777694523335, -0.03355459123849869, -0.034517113119363785, 0.014865619130432606, -0.003656247165054083, -0.024143263697624207, 0.012419208884239197, -0.0007198028033599257, -0.010393901728093624, -0.008762961253523827, 0.008502278476953506, 0.009023644961416721, 0.009451432153582573, -0.010427323170006275, 0.004254481289535761, 0.02681693620979786, 0.012980679981410503, 0.0004146281280554831, -0.0031114863231778145, -0.02056054212152958, -0.023822423070669174, -0.018942968919873238, -0.018836023285984993, -0.024129893630743027, 0.005678211804479361, -0.0012432577786967158, 0.0015365261351689696, -0.0035158793907612562, -0.02058727853000164, -0.03850088268518448, -0.02185727283358574, -0.009885904379189014, -0.0008605883340351284, -0.04352738708257675, 0.013261415995657444, 0.03641542047262192, 0.026723356917500496, 0.0009641931392252445, -0.025359783321619034, -0.1680135875940323, 0.027966614812612534, 0.01368251908570528, -0.008996907621622086, 0.035827212035655975, 0.016857504844665527, 0.02783293090760708, 0.008007649332284927, -0.020761067047715187, 0.017846764996647835, 0.00939795933663845, -0.004067324101924896, -0.010333743877708912, -0.017405608668923378, 0.015935087576508522, -0.005347345024347305, 0.005979000125080347, 0.009150643832981586, 0.03261880576610565, 0.0012850338825955987, 0.05416860431432724, -0.008041069842875004, 0.004211034160107374, -0.0008940092520788312, 0.009057065472006798, 0.018060658127069473, 0.02005254477262497, 0.022071165964007378, -0.010133218951523304, -0.012064946815371513, 0.005150161683559418, -0.015707826241850853, -0.007740281987935305, 0.019972333684563637, 0.017084768041968346, -0.014544778503477573, 0.0013201257679611444, -0.009123907424509525, -0.004709005821496248, -0.0017445713747292757, 0.008669382892549038, -0.0051835826598107815, 0.022833162918686867, -0.009725484065711498, 0.003833377966657281, 0.0008973513613454998, 0.0006721779936924577, -0.010467427782714367, 0.0003302403201814741, -0.006944864522665739, 0.002760566771030426, -0.005654817447066307, -0.003380524693056941, 0.029570817947387695, 0.033420905470848083, 0.0012624746887013316, -0.005621396470814943, 0.029303450137376785, 0.011784211732447147, 0.022779690101742744, -0.004946294240653515, -0.014384358190000057, 0.018140867352485657, -0.0027488695923238993, -0.01773981750011444, -0.0369768925011158, -0.004775847773998976, -0.013582256622612476, -0.003509195288643241, -0.003753167809918523, 0.008669382892549038, 0.00624636746942997, 0.0009558379533700645, -0.013221310451626778, 0.006637392099946737, 0.010948688723146915, -0.01317452173680067, -0.016777295619249344, 0.00042235670844092965, 0.024731470271945, -0.016229191794991493, 0.012051578611135483, -0.04890147224068642, 0.00193674152251333, 0.010347113013267517, 0.029356924816966057, 0.002518265275284648, 0.005270476918667555, -0.005333976820111275, -0.0071721263229846954, -0.02450420893728733, 0.007158758118748665, -0.024677997455000877, 0.00013723460142500699, 0.01433088444173336, 0.00527716102078557, 0.00041024162783287466, 0.021269064396619797, 0.0011831000447273254, -0.02529294230043888, 0.010768216103315353, -0.0018866101745516062, -0.030560076236724854, 0.04769831895828247, 0.044944435358047485, 0.002297687344253063, -0.02211127243936062, 0.014624988660216331, 0.014397726394236088, -0.012151842005550861, -0.005975658074021339, 0.02320747822523117, 0.03203059732913971, -0.0027873036451637745, 0.0007377665024250746, 0.029116293415427208, 0.031121548265218735, -0.004455006681382656, 0.012860365211963654, -0.001958465203642845, -0.0017069728346541524, 9.86438972176984e-05, -0.02160327322781086, -0.004107429645955563, -0.006500366143882275, -0.021563168615102768, -0.11272203177213669, -0.007078547962009907, 0.014411094598472118, 0.02374221198260784, -0.0312819667160511, 0.011583685874938965, -0.01197805255651474, 0.021456221118569374, -0.02761903777718544, 0.008335174061357975, 0.007412757258862257, -0.04839347302913666, -0.0062898145988583565, -0.009498221799731255, 0.00939127430319786, 0.006781102158129215, 0.032939646393060684, -0.009538326412439346, -0.04462359473109245, 0.029490608721971512, -0.0073459153063595295, -0.00201695179566741, -0.02549346722662449, -0.003532589878886938, 0.003179999301210046, -0.004404875449836254, -0.004147534724324942, -0.002864171750843525, 0.0036930101923644543, -0.0018498471472412348, 0.023835791274905205, 0.017245188355445862, -0.005962289869785309, -0.011236108839511871, 0.007533072493970394, -0.017539292573928833, 0.006406787782907486, -0.0031449072994291782, 0.011971368454396725, -0.023581791669130325, -0.001958465203642845, 0.00013305697939358652, -0.011289582587778568, -0.02443736605346203, 0.025854412466287613, -0.016523296013474464, -0.056361015886068344, 0.01225878857076168, -0.008508962579071522, -0.002712106565013528, 0.0022475558798760176, 0.0013811190146952868, -0.018434971570968628, -0.016590138897299767, -0.002687040949240327, -0.013161152601242065, -0.012800207361578941, -0.004271192010492086, -0.005387450102716684, -0.004541901405900717, 0.0037397993728518486, 0.008121279999613762, -0.0046254536136984825, 0.002543330891057849, 0.009037013165652752, 0.008936750702559948, 0.0008831474697217345, -0.0051735565066337585, -0.0036261684726923704, -0.009618536569178104, 0.010721426457166672, -0.023394634947180748, -0.014397726394236088, 0.03489142656326294, -0.02633567526936531, 0.009611852467060089, -0.034249745309352875, -0.010835058055818081, 0.0030479866545647383, -0.02165674790740013, 0.003049657680094242, -0.006911443546414375, -0.012760101817548275, -0.020386753603816032, -0.003243498969823122, 0.009959430433809757, 0.010748163796961308, 0.001226547290571034, -0.007740281987935305, -0.03617478907108307, 0.0029627634212374687, 0.017646238207817078, 0.011937947943806648, 0.00976558867841959, -0.019945597276091576, 0.0022525691892951727, 0.000758654554374516, -0.0057717906311154366, 0.0241833683103323, -0.011182635091245174, -0.0007444507209584117, -0.012218683026731014, -0.04609411582350731, 0.012298893183469772, 0.022539058700203896, -0.010246849618852139, -0.025346415117383003, -0.020212965086102486, -0.038420673459768295, -0.017111504450440407, 0.0047156899236142635, 0.013121047988533974, -0.008127964101731777, 0.02241874486207962, -0.0025633834302425385, -0.020627383142709732, -0.004856057930737734, -0.0025717387907207012, 0.011851053684949875, -0.016496559605002403, -0.016242559999227524, -0.003131538862362504, -0.02161664143204689, 0.010053008794784546, 0.005126766860485077, 0.015039407648146152, -0.006253051571547985, 0.028688505291938782, -0.02264600619673729, 0.04088045284152031, -0.0095182741060853, -0.016576768830418587, 0.010561006143689156, -0.027164513245224953, 0.0028407771605998278, 0.028848925605416298, -0.011216056533157825, 0.004769163206219673, 0.008622594177722931, 0.007546440698206425, 0.0187959186732769, -0.011349739506840706, -0.02189737744629383, -0.01769971288740635, 0.005688238423317671, 0.0025249493774026632, -0.018087394535541534, -0.012412524782121181, -0.012465998530387878, 0.019464336335659027, 0.0080945435911417, 0.0032785909716039896, 0.044115595519542694, 0.0021422801073640585, 0.004575321916490793, -0.028207244351506233, -0.007640019059181213, 0.013575572520494461, 0.005213661585003138, 0.006998337805271149, -0.0022542402148246765, -0.05317934602499008, 0.01184436958283186, 0.018942968919873238, 0.005551212467253208, -0.013261415995657444, -0.0008062793640419841, 0.00016282248543575406, -0.0011563634034246206, 0.01957128196954727, -0.017245188355445862, -0.020974960178136826, -0.01824781484901905, 0.01566772162914276, 0.00534066092222929, 0.01209168415516615, 0.018368130549788475, 0.004277876112610102, -0.009317749179899693, -0.0014588225167244673, 0.012004789896309376, 0.007680124137550592, -0.01300073228776455, 0.015774667263031006, -0.01950444094836712, 0.03446364030241966, 0.0027321591041982174, 0.01775318570435047, -0.018154235556721687, 0.02372884377837181, -0.004170929081737995, -0.005968973971903324, -0.01069469004869461, -0.018889496102929115, 0.022739585489034653, 2.9086631911923178e-05, 0.03160281106829643, 0.0322444923222065, -0.005481028463691473, 0.011149214580655098, -0.014865619130432606, 0.009003592655062675, 0.0004382316255941987, -0.006650760304182768, -0.02213800884783268, -0.018328025937080383, -0.01695108413696289, 0.007232284173369408, -0.04999767616391182, -0.04553264379501343, 0.014812145382165909, 0.013341626152396202, 0.028501348569989204, -0.010647900402545929, 0.008722856640815735, 0.012305578216910362, -0.02138938009738922, 0.014558146707713604, -0.018060658127069473, -0.022218218073248863, 0.014116991311311722, 0.02136264368891716, 0.026148516684770584, 0.0010535940527915955, 0.032431647181510925, -0.0059656319208443165, -0.0070718638598918915, 0.015253301709890366, 0.02634904347360134, -0.03612131625413895, 0.006269762292504311, -0.021041803061962128, -0.0020353333093225956, 0.0011605409672483802, -0.0013827900402247906, -0.0009157328167930245, -0.030052078887820244, 0.0040372456423938274, 0.00749965151771903, 0.05914163589477539, -0.023087162524461746, 0.052190087735652924, 0.019450968131422997, -0.00913059152662754, 0.004237771034240723, 0.02268611080944538, 0.01473193522542715, 0.021001698449254036, -0.0333406962454319, -0.015306775458157063, -0.011202688328921795, 0.013983307406306267, -0.007352599408477545, -0.0012791851768270135, -0.02941039763391018, -0.009745536372065544, 0.014223937876522541, -0.0029978554230183363, -0.0040372456423938274, -0.00676439143717289, 0.01276678591966629, 0.014544778503477573, 0.01612224616110325, 0.02421010471880436, 0.022726217284798622, -0.016349507495760918, 0.00820149015635252, 0.017833394929766655, 0.007078547962009907, -0.014812145382165909, -0.02264600619673729, -0.00488947844132781, 0.015801405534148216, -0.034249745309352875, -0.01199810579419136, 0.002067083027213812, -0.012519471347332, -0.023528318852186203, 0.033447641879320145, 0.019090021029114723, 0.006353314500302076, -0.009585116058588028, 0.01876918040215969, -0.03978424891829491, -0.004622111562639475, 0.008635962381958961, -0.008161385543644428, 0.008335174061357975, -0.020774435251951218, -0.03483795374631882]} +{"id": "test:10000070", "text": "\"Walked around Lake Merritt. Played two hours of the text adventure Firebird. Took BART into SF and went to dinner at Pane e Vino with my father's brother's ex-wife and her new boyfriend, a widower from a 55-year marriage.\\nSaid goodbye; she leaves for Vegas in the morning to visit some college friends.\"", "vector_field": [0.013574485667049885, -0.019034691154956818, 0.008480597287416458, -0.013076846487820148, -0.0024553644470870495, 0.00794840045273304, -0.01946321316063404, -0.012074656784534454, -0.019601445645093918, -0.021177303045988083, -0.012067745439708233, 0.022684043273329735, 0.02318168245255947, -0.00859809573739767, 0.010118658654391766, -0.004620439372956753, 0.01285567320883274, 0.004202284850180149, 0.03276123106479645, -0.03151713311672211, -0.0038808928802609444, -0.0002758181362878531, 0.0028683356940746307, -0.016270030289888382, -0.007934576831758022, -0.013484633527696133, -0.007478408049792051, -0.017942650243639946, -0.002082135295495391, 0.01574474386870861, 0.03184889256954193, -0.005829978734254837, -0.0016708920011296868, -0.013588308356702328, -0.009213232435286045, -0.017693830654025078, -0.015979740768671036, -0.003960376605391502, 0.0160488560795784, -0.00861883070319891, 0.014777112752199173, -0.02414931356906891, -0.02590487338602543, -0.008563537150621414, 0.004928008187562227, 0.0037426596973091364, -0.013512280769646168, -0.005404911935329437, -0.013387870974838734, 0.015869153663516045, 0.012551560997962952, -0.019394095987081528, -0.024840477854013443, -0.0079760467633605, 0.001917983521707356, 0.010160128585994244, -0.0012406415771692991, 0.008715593256056309, -0.00598549097776413, -0.003977655898779631, -0.017735300585627556, 0.010954968631267548, -0.004734481684863567, 0.010671591386198997, -0.023706967011094093, -0.00663173059001565, -0.0018212203867733479, -0.004603160545229912, -0.015205634757876396, 0.02301580272614956, 0.025586936622858047, 0.0005028226878494024, 0.0015499379951506853, -0.004603160545229912, 0.009296172298491001, -0.020679663866758347, -0.017735300585627556, -0.004679188597947359, 0.001822948339395225, -0.0034437307622283697, 0.007443849463015795, -0.03231888636946678, -0.005567335989326239, 0.034143563359975815, 0.03469649329781532, 0.007243411615490913, -0.004803598392754793, 0.02161964774131775, -0.013000817969441414, -0.01171525102108717, -0.0053012375719845295, 0.012745087035000324, -0.0031966392416507006, 0.03989405557513237, -0.004886538255959749, 0.022891392931342125, 0.005204474087804556, 0.03245712071657181, 0.01003571879118681, -0.00419537303969264, 0.0037703062407672405, -0.003073957283049822, 0.0036804547999054193, -0.008909120224416256, -0.015468277968466282, -0.002939180238172412, 0.0036735432222485542, -0.018813516944646835, 0.011196876876056194, -0.005902551114559174, -0.0199193824082613, 0.02554546669125557, 0.017873533070087433, -0.03419885411858559, -0.03237418085336685, -0.0208593662828207, 0.017279131338000298, -0.019380273297429085, -0.005411823745816946, -0.007046429440379143, 0.0154821015894413, 0.010851293802261353, -0.003856702009215951, -0.021702587604522705, 0.011984805576503277, 0.010699237696826458, 0.01351919211447239, -0.01220597792416811, 0.007962223142385483, -0.0015611695125699043, 0.0036804547999054193, 0.001035019988194108, 0.01636679284274578, -0.018343525007367134, -0.021688764914870262, 0.008763975463807583, -0.01795647293329239, 0.0029616430401802063, -0.016684729605913162, -0.018094705417752266, -0.0056364526972174644, 0.011777455918490887, -0.008452950976788998, -0.0073401746340096, -0.0004950470756739378, 0.007982958108186722, 0.011825837194919586, 0.004758672788739204, 0.016905901953577995, -0.02049996145069599, 0.02354108728468418, -0.011653046123683453, 0.011369667947292328, -0.013975361362099648, -0.0020441212691366673, 0.01102408580482006, -0.019573800265789032, 0.012517002411186695, 0.007146648596972227, -0.01177054364234209, 0.0032156461384147406, 0.028614241629838943, 0.028047485277056694, -0.007865460589528084, -0.013014641590416431, 0.03665940463542938, 0.019891735166311264, -0.002356873359531164, -0.00552241038531065, -0.022587280720472336, 0.0018091249512508512, 0.00643820408731699, -0.04066816344857216, 0.009621020406484604, 0.012447886168956757, 0.028476007282733917, 0.02425989881157875, -0.01599356345832348, -0.028268657624721527, -0.017721476033329964, -0.014555939473211765, 0.003967288415879011, 0.02678956463932991, 0.023098742589354515, 0.008376922458410263, -0.016187090426683426, -0.004074418917298317, -0.010830558836460114, -0.0036942781880497932, 0.002854512305930257, -0.00794840045273304, 0.01795647293329239, -0.01550974790006876, -0.002913261530920863, -0.6590951681137085, -0.013664336875081062, -0.021509062498807907, -0.004081330727785826, 0.015039755962789059, 0.015661804005503654, 0.01102408580482006, -0.0049418313428759575, -0.018288232386112213, 0.01740354113280773, 0.006897829007357359, 0.0158553309738636, 0.014763289131224155, 0.011217611841857433, -0.0074507612735033035, 0.0021668029949069023, -0.011639222502708435, -0.01842646487057209, -0.009621020406484604, 0.03024538978934288, -0.033314164727926254, 0.03748880326747894, 0.009745430201292038, -0.012240536510944366, 0.013387870974838734, -0.00019536216859705746, 0.008687946945428848, -0.013553750701248646, -0.01414124108850956, -0.011929512023925781, -0.00836309976875782, 0.005183739122003317, 0.017044134438037872, 0.0034817447885870934, 0.06270250678062439, -0.0202926117926836, -0.010229245759546757, 0.0048208776861429214, 0.01488769892603159, 0.05310913547873497, -0.000762441661208868, -0.005007491912692785, 0.008446038700640202, 0.00890220794826746, -0.00017333128198515624, -0.014569763094186783, 0.030051864683628082, 0.01148716639727354, 0.0011162318987771869, -0.012247447855770588, 0.02322315238416195, 0.021343182772397995, -0.03090890869498253, -0.01750030368566513, -0.014832406304776669, 0.001181028550490737, 0.027079854160547256, 0.0007300432771444321, -0.004292136058211327, -0.008611918427050114, -0.005632996559143066, 0.013885509222745895, 0.0074922312051057816, 0.0018626903183758259, -0.007837813347578049, 0.0005680514150299132, -0.008985147811472416, 0.005391088780015707, 0.03571942076086998, -0.036410585045814514, 0.01158392895013094, 0.03204242140054703, -0.016587965190410614, 0.017223836854100227, 0.0417187325656414, -0.007049885578453541, 0.024688422679901123, 0.018592344596982002, -0.028531301766633987, 0.00776869710534811, 0.02163347229361534, -0.016809139400720596, -0.0364382304251194, -0.0040053026750683784, 0.03359062969684601, 0.0012691521551460028, -0.05451911315321922, 0.017610890790820122, 0.01893792673945427, 0.002453636610880494, 0.01447299961000681, 0.00933764223009348, -0.006299971137195826, 0.003314137225970626, 0.011176141910254955, -0.0033193209674209356, -0.01333948876708746, 0.006894373334944248, 0.013090669177472591, -0.04752452298998833, 0.011176141910254955, -0.016712374985218048, 0.04351576417684555, -0.0012924789916723967, -0.0073263514786958694, -0.010719972662627697, -0.003302041906863451, -0.020790250971913338, 0.03809702768921852, -0.018827341496944427, 0.00014050093886908144, -0.026361042633652687, -0.020099084824323654, 0.0027784842532128096, -0.019490860402584076, -0.022739335894584656, -0.005667555145919323, -0.017610890790820122, 0.026609862223267555, -0.016228560358285904, 0.01169451605528593, -0.016325322911143303, 0.01130746304988861, 0.00014147289039101452, 0.003177632112056017, -0.0011913960333913565, 0.020389374345541, -0.02167494222521782, -0.01925586350262165, -0.022476693615317345, 0.00995969120413065, 0.029609518125653267, 0.014486823230981827, -0.01040203683078289, 0.004368164110928774, 0.0032415648456662893, 0.009925132617354393, -0.01852322742342949, 0.017693830654025078, -0.024024903774261475, -0.03425414860248566, 0.005128446035087109, -0.009095734916627407, -0.012910966761410236, 0.015164164826273918, -0.016919724643230438, 0.0014808215200901031, -0.006960033904761076, -0.03281652554869652, 0.012012451887130737, 3.639416900114156e-05, -0.007046429440379143, -0.01313213910907507, -0.005874904338270426, 0.01153554767370224, -0.025586936622858047, 0.0018696020124480128, -0.024757537990808487, -0.02027878724038601, -0.016035033389925957, -0.007395468186587095, 0.02074878104031086, -0.010623210109770298, -0.0036631757393479347, 0.011784367263317108, -0.03591294586658478, 0.01533004455268383, 0.02322315238416195, -0.020195847377181053, -0.020624371245503426, 0.023983433842658997, -0.022061994299292564, -0.0056779226288199425, -0.00931690726429224, -0.006579893175512552, 0.006040784064680338, -0.0032242857851088047, -0.014085947535932064, -0.009296172298491001, 0.0027007281314581633, -0.0045444113202393055, 0.01042277179658413, 0.01935262605547905, 0.016560319811105728, 0.020389374345541, -0.013360223732888699, 0.025213707238435745, 0.04130403324961662, -0.010339831933379173, 0.03101949580013752, 0.01719619147479534, 0.024453425779938698, 0.007858548313379288, -0.005408368073403835, 0.016560319811105728, 0.02018202468752861, 0.0020458491053432226, -0.009925132617354393, -0.006776875350624323, 0.020223494619131088, -0.005404911935329437, -0.0037841296289116144, 0.014528293162584305, 0.0012579207541421056, 0.011079378426074982, -0.02900129370391369, -0.014348589815199375, -0.027383966371417046, 0.029333053156733513, -0.003267483552917838, 0.023306092247366905, -0.01259303092956543, -0.03148948773741722, -0.016173265874385834, 0.002616060432046652, 0.014030653983354568, 0.005812699440866709, 0.021509062498807907, 0.008045163005590439, 0.00010578068031463772, -0.011646133847534657, -0.0001486005203332752, -0.014224180951714516, 0.0008648204966448247, -0.012261271476745605, -0.021384652704000473, -0.0028942544013261795, 0.003246748587116599, 0.016740022227168083, -0.010042631067335606, 0.024743715301156044, 0.010823647491633892, 0.006324161775410175, 0.014445353299379349, 0.02467459812760353, -0.0199193824082613, 0.010125570930540562, -0.0247298926115036, 0.023499617353081703, -0.013567573390901089, 0.012219801545143127, 0.013415517285466194, 0.031461842358112335, -0.0320977121591568, 0.022573456168174744, 0.009282348677515984, 0.01950468309223652, -0.0016276941169053316, 0.005308148916810751, 0.019076161086559296, -0.0204861368983984, -0.0014065213035792112, -0.021495237946510315, -0.0013279012637212873, 0.012026275508105755, 0.005429103039205074, -0.009869839996099472, -0.005598438438028097, 0.018398817628622055, 0.0462527759373188, 0.02694162167608738, 0.0016579326475039124, -0.009538080543279648, -0.01620091311633587, 0.00031966392998583615, -0.0028251379262655973, 0.005709024611860514, -0.01888263411819935, 0.003652808256447315, 0.01303537655621767, 0.004247210454195738, -0.0320977121591568, -0.004475295078009367, 0.00530469324439764, 0.02204817160964012, -1.3776057130598929e-05, -0.0022635662462562323, 0.001781478407792747, 0.02296050824224949, 0.0021529796067625284, 0.0027663889341056347, -0.03356298431754112, 0.028531301766633987, 0.001506740227341652, -0.00709826685488224, -0.011466431431472301, 0.0013788746437057853, 0.016270030289888382, -0.022006701678037643, -0.011749808676540852, -0.006355264224112034, 0.021329358220100403, 0.013926979154348373, 0.004696467891335487, 0.006458939053118229, 0.03317593038082123, 0.0473862886428833, -0.023762261494994164, 0.001506740227341652, -0.009558815509080887, -0.0102430684491992, 0.012565383687615395, 0.0021909938659518957, -0.01661561243236065, 0.02524135448038578, -0.005484396126121283, -0.004744849167764187, -0.012392592616379261, -0.007893106900155544, -0.0002900733961723745, 0.0012829754268750548, 0.006956578232347965, -0.011784367263317108, -0.02535194158554077, -0.0061824731528759, 0.009510433301329613, -0.015938270837068558, -0.02839306741952896, -0.00011663413170026615, 0.009890574961900711, -0.018550874665379524, -0.03267829120159149, -0.041110508143901825, -0.0017045862041413784, 0.127616748213768, -0.004364708438515663, -0.0013685071608051658, 0.008770886808633804, 0.021688764914870262, -0.018661461770534515, -0.01853705197572708, -0.02131553553044796, 0.0137818343937397, -0.014542116783559322, 0.0021011424250900745, -0.028199542313814163, -0.012116126716136932, -0.008957501500844955, 0.021509062498807907, -0.010111747309565544, -0.01895175129175186, -0.02828248217701912, -0.012309652753174305, -0.036106470972299576, 0.005477484315633774, -0.014638879336416721, -0.004371620249003172, 0.01615944318473339, -0.001578448573127389, -0.005308148916810751, 0.017099427059292793, 0.005588070955127478, 0.017942650243639946, -0.02528282441198826, -0.010595562867820263, 0.018039412796497345, -0.007132825441658497, 0.03453061357140541, -0.018758224323391914, 0.005252855829894543, 0.024660775437951088, 0.0019335347460582852, 0.01729295402765274, 0.006842535920441151, 0.025683701038360596, -0.006147914566099644, -0.0012536009307950735, -0.019947027787566185, 0.02416313625872135, -0.0101946871727705, 0.00019471420091576874, 0.010733796283602715, -0.009738517925143242, -0.01416888739913702, 0.029139526188373566, 0.021868467330932617, -0.005968211684376001, -0.02008526213467121, 0.014694172888994217, 0.018094705417752266, 0.012351122684776783, -0.007132825441658497, -0.015136518515646458, 0.025946343317627907, 0.005601894110441208, -0.041469912976026535, 0.022435223683714867, -0.010249980725347996, 0.001312350039370358, -0.018246762454509735, -0.006548790726810694, 0.0013097581686452031, -0.013788746669888496, 0.01195024698972702, -0.013221991248428822, -0.003198367077857256, -0.028310127556324005, -0.010547181591391563, 0.009786900132894516, 0.007167383562773466, 0.030438916757702827, -0.007616640999913216, -0.00555005669593811, 0.004796686582267284, -0.018302055075764656, -0.0010972247691825032, 0.01580003648996353, -0.015025932341814041, -0.01437623705714941, 0.02471606805920601, 0.006953122094273567, 0.01578621380031109, -0.005899095442146063, 0.007146648596972227, 0.02229699119925499, -0.018053235486149788, 0.006009681615978479, -0.015357691794633865, -0.0025521276984363794, 0.010740707628428936, -0.028863059356808662, 0.01568945124745369, -0.004893450066447258, 0.010726884007453918, 0.011196876876056194, -0.02033408172428608, 0.001938718487508595, -0.030743028968572617, 0.013484633527696133, -0.014721819199621677, -0.004392355214804411, 0.0008916031802073121, 0.01197098195552826, -0.004838156513869762, 0.008452950976788998, -0.011127760633826256, 0.027881605550646782, 0.021066715940833092, 0.011818925850093365, 0.0035456777550280094, 0.015717096626758575, 0.015965916216373444, 0.0069289314560592175, -0.01086511742323637, -0.0058196112513542175, -0.009621020406484604, 0.019394095987081528, 0.015426808036863804, -0.014625056646764278, 0.002890798496082425, -0.0014272561529651284, -0.022158756852149963, 0.006358720362186432, 0.007512966170907021, 0.014500646851956844, 0.02781248837709427, 0.003467921633273363, -0.03292711079120636, -0.0271904394030571, -0.02161964774131775, -8.558569243177772e-05, 0.010540270246565342, -0.01091349869966507, 0.021550532430410385, -0.04321165010333061, -0.008086632937192917, 0.00663173059001565, -0.01708560436964035, -0.013456987217068672, -0.02488194778561592, -0.02911187894642353, -0.016657082363963127, -0.005467116832733154, 0.03679763525724411, -0.01795647293329239, -0.008895296603441238, -0.012378768995404243, 0.01553739421069622, -0.025766639038920403, -0.030798323452472687, 0.0014324398944154382, 0.0005533641669899225, 0.036355290561914444, 0.016740022227168083, 0.011286728084087372, -0.00027668208349496126, 0.01863381452858448, -0.014390059746801853, 0.007519877981394529, 0.015827683731913567, -0.015039755962789059, -0.020251141861081123, -0.011335110291838646, 0.028724826872348785, 0.007969135418534279, 0.00843912735581398, -0.0036182499025017023, 0.003248476656153798, 0.00017711109830997884, 0.04144226759672165, -0.009786900132894516, 0.012931701727211475, -0.014307120814919472, -0.007257234770804644, -0.01086511742323637, 0.015260928310453892, -0.013374047353863716, 0.0046066162176430225, 0.011798190884292126, -0.0034540982451289892, 0.033258870244026184, -0.005384176969528198, 0.011514812707901001, -0.001090313191525638, 0.019394095987081528, -0.028890706598758698, -0.007658110931515694, -0.013698894530534744, -0.007671934086829424, -0.008909120224416256, -0.007692669052630663, -0.02358255721628666, -0.009759252890944481, 0.0036424407735466957, 0.01961527019739151, 0.010540270246565342, -0.004292136058211327, 0.011383491568267345, -0.012786556966602802, 0.009068087674677372, 0.0024812831543385983, 0.0009546719957143068, 0.002082135295495391, -0.019808795303106308, 0.006251589395105839, -0.03030068427324295, -0.015468277968466282, -0.004029493313282728, 0.006403645966202021, -0.0035370381083339453, -0.017320601269602776, 0.03375650942325592, -0.013429340906441212, -0.03267829120159149, -0.020375551655888557, -0.0022411032114177942, 0.03762703388929367, 0.007236499805003405, 0.03077067621052265, -0.0050731529481709, -0.007554436102509499, -0.026250455528497696, 0.029443638399243355, 0.020264964550733566, -0.008072810247540474, 0.0029305405914783478, 0.04196755215525627, 0.0015637613832950592, -0.0032018229831010103, -0.005947476718574762, 0.028503654524683952, -0.014763289131224155, -0.03090890869498253, 0.029858337715268135, 0.005052417982369661, -0.006845991592854261, -0.01347081083804369, 0.011058643460273743, -0.02730102650821209, 0.013816392980515957, -0.004724114201962948, -0.000517509994097054, 0.030328329652547836, 0.00775487395003438, -0.014182711020112038, 0.02261492609977722, -0.005124990362673998, 0.006369087845087051, 0.02101142331957817, -0.010664679110050201, 0.007796343881636858, -0.011777455918490887, 0.01308375783264637, 0.0037703062407672405, 0.0010557548375800252, 0.04495338723063469, -0.01929733343422413, 0.01971203275024891, -0.009275437332689762, 0.00622048694640398, -0.019891735166311264, -0.016187090426683426, 0.007595906034111977, 0.02105289325118065, 0.0154821015894413, -0.012040098197758198, -0.01672619953751564, 0.01733442395925522, 0.00933764223009348, 0.016546495258808136, -0.014320943504571915, -0.02152288518846035, -0.007112090475857258, -0.0006315522477962077, 0.028047485277056694, -0.0049452874809503555, -0.00364589667879045, -0.005729759577661753, -0.016283852979540825, -0.025158414617180824, 0.007796343881636858, -0.003929274156689644, -0.007506054360419512, -0.0487133264541626, -0.01578621380031109, -0.027632785961031914, -0.01257229596376419, -0.005391088780015707, 0.014003007672727108, -0.007181206718087196, 0.0146803492680192, 0.018550874665379524, -0.03154478222131729, 0.01785971038043499, -0.029305405914783478, -0.001247553271241486, -0.030991848558187485, -0.004015670157968998, -0.010616297833621502, -0.01884116418659687, 0.00954499188810587, -0.002438085386529565, -0.025987813249230385, -0.04138697311282158, 0.015053578652441502, 0.004433825146406889, 0.009227056056261063, -0.021923761814832687, 0.020665841177105904, -0.0009745429852046072, -0.014362413436174393, -0.013436252251267433, -0.007374733220785856, 0.007478408049792051, -0.011051732115447521, 0.008211042732000351, 0.023499617353081703, -0.011424961499869823, 0.0182191152125597, -0.001911071827635169, -0.0007369549130089581, -0.006977313198149204, -0.03536001220345497, 0.016491202637553215, -0.01396153774112463, 0.0015646253013983369, -0.018357347697019577, -0.012192154303193092, -0.017251484096050262, 0.007858548313379288, -0.017431186512112617, 0.025932518765330315, -0.010961880907416344, -0.00885382667183876, 0.00823177769780159, 0.024826655164361, 0.026913974434137344, 0.002911533461883664, 0.014113593846559525, 0.010699237696826458, 0.0020873190369457006, -0.0029616430401802063, -0.042769305408000946, -0.01714089699089527, 0.015454454347491264, 0.02828248217701912, 0.006369087845087051, -0.0009667673730291426, -0.027964545413851738, -0.01391315646469593, -0.03475178778171539, 0.007678845897316933, 0.004848523996770382, 0.024218430742621422, 0.04672276973724365, -0.022573456168174744, -0.014182711020112038, 0.02782631292939186, -0.007519877981394529, 0.0061755613423883915, -0.008432216010987759, 0.01714089699089527, 0.02059672400355339, -0.00910955760627985, 0.01042968314141035, -0.013463898561894894, -0.04788392782211304, -0.021509062498807907, -0.006783786695450544, -0.0015335228526964784, 0.003604426747187972, 0.011936423368752003, -0.012136861681938171, -0.005982035305351019, -0.012012451887130737, 0.016684729605913162, 0.01837117224931717, 0.02348579466342926, -0.006289603654295206, -0.022227874025702477, -0.01615944318473339, 0.01929733343422413, -0.015661804005503654, -0.009538080543279648, -0.008266336284577847, 0.008197220042347908, 0.016090326011180878, 0.01961527019739151, 0.005477484315633774, -0.01488769892603159, 0.027522198855876923, -0.025006357580423355, 0.020665841177105904, -0.009296172298491001, 0.00014212085807230324, 0.019518505781888962, 0.01176363229751587, -0.006832168437540531, -0.010381301864981651, 0.015177988447248936, -0.015426808036863804, -0.007651199121028185, -0.01509504858404398, -0.00509734358638525, 0.0011620215373113751, -0.0046480861492455006, 0.018232939764857292, -0.009399847127497196, 0.0024605481885373592, -0.0070118713192641735, -0.005698657128959894, -0.0025676789227873087, 0.014749466441571712, 0.008425304666161537, -0.0028113145381212234, -0.020942306146025658, 0.007934576831758022, -0.0010410676477476954, -0.005836890544742346, -0.013892421498894691, 0.012102303095161915, -0.02952657826244831, 0.0014289841055870056, 0.024854302406311035, -0.018177645280957222, -0.027273379266262054, 0.018232939764857292, 0.00949661061167717, -0.004796686582267284, 0.012095391750335693, 0.22017759084701538, 0.005705568939447403, 0.0022100007627159357, 0.0022411032114177942, -0.0101463058963418, 0.02864188700914383, 0.02952657826244831, 0.015233281999826431, -0.004620439372956753, -0.007312528323382139, -0.029858337715268135, 0.0036700873170048, 0.00575395068153739, 0.0015292030293494463, 0.020140554755926132, -0.009856016375124454, -0.026554567739367485, -0.006161738187074661, -0.032899465411901474, -0.04578278586268425, 0.00775487395003438, -0.012081568129360676, -0.01971203275024891, -0.008190307766199112, 0.024826655164361, 0.013463898561894894, -0.01354683842509985, 0.0031275227665901184, 0.022172581404447556, 0.02348579466342926, 0.0023741526529192924, -0.008542802184820175, -0.03541530668735504, 0.00511116674169898, 0.002192721702158451, -0.010339831933379173, 0.01509504858404398, -0.006610995624214411, 0.007284881547093391, -0.003870525397360325, 0.004281768575310707, 0.0032968581654131413, 0.004395810887217522, -0.016629435122013092, -0.017168544232845306, 0.015979740768671036, -0.005159548483788967, 0.0038739810697734356, 0.006386366672813892, -0.010920410975813866, -0.03787585347890854, 0.004039860796183348, 0.026927797123789787, 0.010830558836460114, -0.011915688402950764, -0.0065315114334225655, 0.012344211339950562, 0.019573800265789032, -0.030217744410037994, -0.0007512102019973099, -0.008086632937192917, 0.007872371934354305, -0.022545810788869858, 0.02745308354496956, -0.0012242264347150922, 0.014694172888994217, -0.0027784842532128096, -0.00171927351038903, 0.015730921179056168, 0.001587088219821453, 0.016836784780025482, -0.0032018229831010103, -0.01580003648996353, -0.014182711020112038, -0.019076161086559296, -0.013325666077435017, 0.03464120253920555, -0.019573800265789032, 0.02105289325118065, 0.004907273221760988, 0.01806706003844738, -0.008971325121819973, 0.007699580863118172, -0.020112907513976097, -0.006047695875167847, -0.040391694754362106, 0.026623684912919998, -0.005764318164438009, -0.015924446284770966, -0.009538080543279648, 0.00818339642137289, -0.016601789742708206, -0.0021149655804038048, -0.00030324875842779875, 0.004039860796183348, 0.00189033686183393, -0.017279131338000298, 0.01553739421069622, -0.009351465851068497, 0.005249400157481432, -0.03101949580013752, -0.005688290111720562, 0.0051215342245996, 0.007678845897316933, -0.025614583864808083, 0.0007473223959095776, 0.012136861681938171, 0.016601789742708206, -0.009144116193056107, -0.01509504858404398, -0.005432558711618185, -0.022020524367690086, -0.007485319394618273, -0.0037564830854535103, -0.008252512663602829, 0.005843801889568567, 0.0020182025618851185, -0.014030653983354568, 0.001287295250222087, -0.004309415351599455, -0.0007075804169289768, -0.002705911872908473, 0.003822143655270338, 0.008936766535043716, -0.02301580272614956, 0.0012933429097756743, 0.004236842971295118, -0.0079276654869318, -0.004039860796183348, -0.027936898171901703, 0.013982272706925869, -0.01620091311633587, 0.010395125485956669, -0.013187432661652565, -0.0030566782224923372, -0.0002801379014272243, 0.01285567320883274, 0.003946553450077772, -0.0176523607224226, 0.007374733220785856, 0.0396452359855175, 0.0146803492680192, -0.010941145941615105, 0.002500290283933282, 0.008148837834596634, -0.014666526578366756, 0.029692457988858223, 0.01215759664773941, -0.00815575011074543, -0.002424261998385191, -0.03815232217311859, 0.013671248219907284, -0.01412050612270832, -0.016919724643230438, 0.03508354723453522, -0.03466884791851044, -0.02999657206237316, -0.022006701678037643, -0.020679663866758347, 0.00025918695610016584, -0.020831720903515816, 0.011853483505547047, 0.033369459211826324, -0.006521143950521946, -0.014134328812360764, -0.019546153023838997, -0.17926061153411865, 0.01703031174838543, 0.015385338105261326, -0.010623210109770298, 0.0343647338449955, -0.0022220963146537542, 0.024799007922410965, -0.02023731730878353, -0.016961194574832916, 0.02225552126765251, 0.014555939473211765, 0.0027871238999068737, -0.02225552126765251, 0.021536707878112793, -0.0032156461384147406, 0.03350768983364105, 0.009088822640478611, 0.017527950927615166, 0.026443982496857643, 0.009759252890944481, 0.036769989877939224, -0.024370485916733742, 0.010457330383360386, -0.013063022866845131, 0.003829055465757847, 0.004271401092410088, -0.011459519155323505, 0.004948743153363466, -0.01130055170506239, -0.014777112752199173, -0.004337061662226915, 0.011286728084087372, 0.01873057708144188, -0.000818166823592037, 0.022103464230895042, 0.0026195163372904062, 0.004354340955615044, -0.00843912735581398, -0.0060338727198541164, 0.026029283180832863, 0.010830558836460114, 0.020513784140348434, 0.008833091706037521, 0.002144340192899108, -0.04017052426934242, 0.014293297193944454, 0.024329015985131264, 0.0016942188376560807, -0.031213022768497467, -0.014106682501733303, 0.0032173742074519396, -0.020168201997876167, 0.0013555478071793914, -0.017099427059292793, 0.005076608620584011, -0.015468277968466282, -0.019394095987081528, 0.010408948175609112, 0.022628750652074814, 0.005256311502307653, -0.01801176555454731, -0.0034489145036786795, 0.006272324360907078, 0.005864536855369806, -0.0013460442423820496, -0.02271169051527977, -0.0009875022806227207, -0.005664099007844925, -0.0061340914107859135, 0.02291903831064701, -0.01950468309223652, -0.017873533070087433, 0.013215078972280025, -0.008038251660764217, -0.0054636611603200436, 0.00149723666254431, -0.01759706623852253, -0.0053012375719845295, 0.0021045980975031853, -0.008141926489770412, -0.01177054364234209, 0.04536808654665947, -0.005888727959245443, -0.011065555736422539, 0.004534043837338686, 0.015661804005503654, 0.007651199121028185, 0.01132819801568985, -0.004765584133565426, -0.01986408792436123, 0.02880776673555374, -0.030162449926137924, 0.003967288415879011, -0.013560662046074867, 0.014486823230981827, 0.007989870384335518, 0.02983069233596325, 0.0137818343937397, -0.00011782207002397627, -0.02012673206627369, 0.011542459018528461, -0.01123143546283245, 0.002643706975504756, -0.006666288711130619, 0.022131111472845078, 0.014320943504571915, 0.002712823450565338, 0.018108529970049858, 0.007485319394618273, -0.006821800954639912, -0.005370353814214468, -0.004326694179326296, 0.019684385508298874, 0.004558234941214323, -0.00035314224078319967, 0.01533004455268383, 0.02084554359316826, -0.024909595027565956, 0.008964412845671177, -0.007678845897316933, 0.03231888636946678, 0.0017451922176405787, 0.000170955405337736, 0.007782520726323128, 0.006431292276829481, -0.0182191152125597, -0.11711103469133377, -0.032291240990161896, -0.010699237696826458, 0.004340517800301313, 0.0016294220695272088, 0.020873190835118294, 0.014459176920354366, 0.02605692856013775, 0.010858206078410149, 0.003008296713232994, 0.007284881547093391, 0.0016562046948820353, 0.005937109235674143, -0.008135015144944191, 0.023651674389839172, -0.021177303045988083, 0.01759706623852253, -0.012102303095161915, -0.006445115897804499, 0.029498932883143425, -0.019518505781888962, 0.0031154272146523, 0.0008674123673699796, -0.009883662685751915, 0.00799678172916174, -0.015164164826273918, -0.006026960909366608, 0.024660775437951088, 0.011065555736422539, 0.019850265234708786, -0.017251484096050262, -0.004758672788739204, 0.008259424939751625, -0.003191455500200391, -0.0225596334785223, 0.00052917335415259, -0.01499828603118658, 0.005391088780015707, 0.009724695235490799, -0.04000464454293251, 0.017279131338000298, -0.00176506326533854, -0.002610876690596342, -0.04451104253530502, 0.007906930521130562, -0.005823066923767328, -0.017154721543192863, 0.02958187274634838, 0.007146648596972227, -0.0137818343937397, -0.012081568129360676, 0.008957501500844955, -0.0024899228010326624, -0.005581159144639969, 0.023140212520956993, 0.007284881547093391, 0.005066241137683392, 0.009710871614515781, -0.008584272116422653, 0.011335110291838646, -0.00753370113670826, -0.02244904637336731, 0.006120268255472183, 0.00465845363214612, 0.014777112752199173, 0.008045163005590439, -0.0061962963081896305, -0.0053599863313138485, -0.02027878724038601, -0.012012451887130737, 0.004910728894174099, 0.005384176969528198, -0.011259081773459911, 0.015385338105261326, -0.016173265874385834, -0.028780119493603706, -0.004337061662226915, -0.021398475393652916, 0.007941488176584244, -0.010726884007453918, -0.022131111472845078, -0.01693354733288288, 0.0008505652076564729, -0.014279473572969437, 0.03359062969684601, 0.015025932341814041, 0.001428120187483728, 0.014956816099584103, -0.0038394229486584663, -0.004174638073891401, 0.0018592345295473933, 0.02101142331957817, -0.0021892657969146967, -0.04346046969294548, -0.02070731110870838, -0.000938256795052439, -0.0056779226288199425, -0.02493724226951599, -0.012551560997962952, 0.002301580272614956, -0.021702587604522705, -0.005169915966689587, -0.0651354119181633, 0.01290405448526144, 0.008812356740236282, -0.0021737145725637674, -0.0007957039633765817, -0.009455140680074692, 0.008632653392851353, 0.006241221912205219, -0.029803045094013214, -0.01284876186400652, -0.026540745049715042, 0.017071781679987907, 0.0016078230692073703, 0.006742316763848066, -0.023859024047851562, 0.005180283449590206, 0.026250455528497696, -0.003590603359043598, 0.00298065016977489, 0.018343525007367134, -0.024481073021888733, 0.009026617743074894, 0.006566069554537535, -0.012675970792770386, -0.007519877981394529, -0.006735405419021845, -0.019449390470981598, 0.012060833163559437, -0.0012674242025241256, -0.03571942076086998, 0.01130055170506239, -0.0011758448090404272, 0.02049996145069599, 0.01646355539560318, -0.00551204290241003, 0.02044466696679592, -0.008888385258615017, 0.02049996145069599, -0.005626085214316845, 0.0339776836335659, -0.011404226534068584, -0.032429471611976624, 0.03430944308638573, -0.012627588585019112, 0.0034420029260218143, -0.002581502078101039, -0.009579550474882126, 0.019781149923801422, 0.017417363822460175, 0.02043084427714348, -0.005408368073403835, 0.007333263289183378, -0.025103121995925903, -0.016283852979540825, -0.011666868813335896, 0.0002719303302001208, 0.002797491382807493, -0.011058643460273743, -0.018606167286634445, -0.005484396126121283, 0.04376458376646042, 0.01893792673945427, 0.0016216464573517442, -0.0024398132227361202, 0.014431529678404331, -0.03972817584872246, 0.008404569700360298, 0.010436595417559147, 0.008743240498006344, 0.004986756946891546, -0.012164507992565632, -0.0013728269841521978, 0.03256770595908165, 0.017832063138484955, 0.010457330383360386, 0.005066241137683392, 0.013097581453621387, -0.010747618973255157, 0.001575856702402234, 0.02394196391105652, 0.028254834935069084, -0.018205292522907257, -0.02471606805920601, -0.0034955681767314672, 0.015468277968466282, 0.03555354103446007, -0.000818166823592037, 0.005055873654782772, -0.0010954969329759479, 0.012517002411186695, -0.0013659152900800109, -0.018205292522907257, 0.006203208118677139, 0.0030238479375839233, 0.023458147421479225, 0.025020182132720947, 0.012095391750335693, -0.021951407194137573, 0.004592793062329292, 0.023817554116249084, -0.014445353299379349, 0.020057614892721176, -0.011342021636664867, -0.01931115612387657, -0.007747962139546871, -0.012040098197758198, -0.01733442395925522, -0.015523571521043777, 0.005076608620584011, 0.004827789030969143, 0.018053235486149788, 0.026416335254907608, -0.017638536170125008, 0.03430944308638573, 0.0021477958653122187, 0.016422085464000702, 0.010802912525832653, -0.028199542313814163, -0.017873533070087433, 0.03834584727883339, 0.025683701038360596, 0.021066715940833092, 0.012150684371590614, -0.0074714962393045425, 0.025047827512025833, 0.0024640040937811136, 0.00794840045273304, -0.00467227678745985, -0.02724573388695717, -0.03359062969684601, 0.015675626695156097, -0.006946210749447346, -0.023706967011094093, -0.00993895623832941, -0.016546495258808136, 0.0070740762166678905, -0.024329015985131264, 0.0343647338449955, -0.018854986876249313, 0.031185375526547432, 0.01419653370976448, -0.006058063358068466, 0.0013002546038478613, -0.006759596057236195, 0.019891735166311264, 0.025310471653938293, 0.009489698335528374, -0.01324272621423006, -0.027522198855876923, 0.033784154802560806, -0.012482443824410439, 0.025517821311950684, -0.03422650322318077, -0.030991848558187485, -0.01914527639746666, -0.010277627035975456, 0.0019715488888323307, 0.00530469324439764, -0.004226475488394499, 0.016035033389925957, 0.009406758472323418, 0.04860273748636246, 0.02373461425304413, -0.01708560436964035, 0.01070614904165268, 0.024591658264398575, -0.006248133722692728, -0.021868467330932617, -0.042714010924100876, 0.014030653983354568, -0.013380958698689938, -0.022310813888907433, -0.010650856420397758, 0.024660775437951088, -0.03237418085336685, -0.008936766535043716, -0.02450871840119362, 0.022642573341727257, 0.020320257171988487, -0.006700846832245588, 0.0033124093897640705, -0.023831376805901527, -0.015025932341814041, 0.02580810897052288, -0.01437623705714941, -0.03876054659485817, -0.01895175129175186, -0.032899465411901474]} +{"id": "test:10000071", "text": "\"Melt margarine in skillet and lightly fry onion, pepper and celery until tender crisp. Add flour and blend in. Add other ingredients and heat through. Serve with rice and grated Parmesan/Romano.\\nHow Do You Cook Creole Skillet?\\nIf you know another way to cook Creole Skillet please make a comment in the form below to help other free recipe users make the best of this free recipie.\"", "vector_field": [0.01922236755490303, 0.005132725462317467, -0.00764838233590126, -0.01893449015915394, 0.009401817806065083, -0.006477244198322296, 0.004174224101006985, -0.020543986931443214, -0.034152742475271225, -0.0427367240190506, 0.04613891616463661, 0.05171326920390129, -0.022964773699641228, 0.006382375489920378, -0.026314621791243553, 0.024901404976844788, 0.027060486376285553, 0.015833264216780663, 0.009015800431370735, -0.02325265109539032, 0.0007074262830428779, 0.01799234375357628, 0.009748579934239388, -0.022127311676740646, -0.02300403080880642, -0.006915602833032608, -0.0039223311468958855, -0.01868586800992489, 0.0008202873286791146, 0.008570898324251175, 0.017416588962078094, 0.01001028623431921, 0.0007049727719277143, -0.01410599797964096, -0.01599028706550598, 0.008407332003116608, 1.2401683306961786e-05, 0.005865504499524832, -0.002167259808629751, 0.006444530561566353, 0.02449575997889042, -0.025450989603996277, 0.006994115188717842, -0.007694181054830551, -0.003598468843847513, 0.027793265879154205, 0.01689317636191845, -0.004095711745321751, -0.013006829656660557, 0.019837377592921257, 0.029101800173521042, -0.00367043842561543, -0.0024600441101938486, -0.019955145195126534, 0.0055023860186338425, 0.0036802522372454405, 0.012738579884171486, 0.04451633244752884, -0.008040942251682281, -0.011338448151946068, -0.0020135068334639072, 0.0049855150282382965, -0.0033400333486497402, -0.006738950964063406, 0.0023471831809729338, -0.013124597258865833, 0.02284700609743595, -0.02270306833088398, -0.028342850506305695, 0.012143196538090706, 0.01971960999071598, 0.015061228536069393, 0.017154883593320847, 0.0025712696369737387, 0.03546127676963806, 0.004390132147818804, -0.020543986931443214, 0.01574166677892208, 0.014158339239656925, -0.01674923673272133, 0.025817379355430603, -0.0008693573763594031, -0.016186567023396492, 0.044830381870269775, 0.008073655888438225, -0.0022572213783860207, -0.014119083993136883, 0.030776724219322205, -0.007883918471634388, -0.01847650296986103, -0.009055056609213352, 0.035644471645355225, 0.011665581725537777, -0.004864475689828396, -0.041951604187488556, -0.002183616394177079, -0.0018532115500420332, 0.019994402304291725, -0.03977943956851959, -0.017913833260536194, -0.0019022815395146608, 0.016526786610484123, 0.0029981788247823715, -0.03069821186363697, -0.011318820528686047, 0.008898031897842884, 0.01045518834143877, -0.018097028136253357, 0.00445883022621274, 0.003716236911714077, -0.013805035501718521, 0.014734094962477684, 0.02747921645641327, -0.04509208723902702, -0.0018548471853137016, -0.004206937272101641, 0.000438767863670364, -0.03184971958398819, -0.012346019968390465, -0.02176092378795147, -0.006470701191574335, 0.020190682262182236, -0.008642868138849735, -0.011868405155837536, 0.039962634444236755, -0.00021754379849880934, 0.023278823122382164, -0.011168339289724827, 0.0007061995565891266, 0.009558841586112976, 0.05684272199869156, 0.005845876410603523, -0.01016076747328043, -0.008204509504139423, 0.0009617726318538189, -0.012424531392753124, -0.0206224974244833, -0.01410599797964096, -0.004707451444119215, -0.017010943964123726, 0.0010615483624860644, 0.023632127791643143, 0.003089776262640953, -0.014642497524619102, -0.02414245530962944, -0.005535099655389786, -0.0034905148204416037, 0.01199925784021616, -0.021826349198818207, -0.021787093952298164, 0.019601842388510704, -0.015715494751930237, 0.022532958537340164, -0.015257508493959904, 0.023187225684523582, 0.004642025101929903, 0.004360690247267485, 0.010919717140495777, -0.0031781024299561977, 0.03349847346544266, -0.002775728004053235, 0.0033760180231183767, 0.0002796991902869195, 0.038130685687065125, 0.010114968754351139, 0.018070856109261513, 0.01708945631980896, -0.008106369525194168, 0.012588098645210266, -0.0027610070537775755, 0.018267136067152023, 0.032661013305187225, -0.012103941291570663, 0.024901404976844788, 0.009244794026017189, 0.0098140062764287, 0.0008370529394596815, -0.007785778027027845, -0.015021972358226776, 0.016631469130516052, -0.017377333715558052, -0.0021623526699841022, 0.027400705963373184, 0.013870461843907833, 0.012391818687319756, -0.0417422391474247, 0.01689317636191845, 0.0029163954313844442, -0.02881392277777195, -0.015218252316117287, 0.012555385008454323, 0.046714670956134796, 0.012588098645210266, -0.01669689640402794, -0.6410770416259766, -0.035094887018203735, 0.008479300886392593, -0.01139733288437128, 0.01648752950131893, 0.004969158675521612, -0.016356676816940308, -0.002342276042327285, -0.008420417085289955, 0.0003009628562722355, -0.010546784847974777, 0.007111883256584406, 0.017874576151371002, -0.005188337992876768, 0.0004076901823282242, -0.010586041025817394, 0.01217591017484665, -0.04551081731915474, -0.01742967590689659, -0.004210208542644978, -0.018646610900759697, 0.012352562509477139, 0.01918311044573784, 0.004187309183180332, -0.0055808983743190765, -0.01038321852684021, 0.02941584773361683, -0.011907660402357578, -0.003941959235817194, 0.007013743277639151, -0.03213759884238243, 0.030959919095039368, -0.019902804866433144, -0.02354053035378456, 0.058203600347042084, 0.009539213962852955, -0.018175538629293442, 0.014367705211043358, 0.03075055219233036, 0.04517059773206711, 0.005937473848462105, -0.03394337743520737, 0.04043370485305786, -0.009702781215310097, -0.006712780334055424, -0.007190395146608353, 0.020112169906497, 0.022964773699641228, 0.019955145195126534, 0.0019137312192469835, 0.008420417085289955, 0.029599042609333992, -0.02067483961582184, -0.0012815456138923764, 0.009696237742900848, -0.014642497524619102, 0.04307694360613823, -0.01443313155323267, -0.0058327908627688885, 0.01763904094696045, -0.008655953221023083, 0.017455846071243286, -0.02980840764939785, -0.0020969260949641466, -0.033262938261032104, -0.0028591470327228308, -0.03540893271565437, -0.0013461544876918197, 0.00528974924236536, -0.027112828567624092, -0.010618754662573338, 0.02181326411664486, -0.029782237485051155, 0.007910088635981083, 0.009918688796460629, 0.010834663175046444, 0.00896345917135477, 0.02325265109539032, 0.02231050655245781, -0.0045504276640713215, 0.009002715349197388, 0.005407517310231924, -0.0012660068459808826, -0.006552484817802906, -0.0055023860186338425, 0.002463315613567829, -0.006778207141906023, -0.0014941823901608586, 0.004563512746244669, -0.009480330161750317, 0.001771428156644106, 0.028185825794935226, -0.010919717140495777, -0.04480420798063278, -0.002967101288959384, 0.012692781165242195, -0.03501637279987335, -0.00490700313821435, 0.014812606386840343, -0.03632490709424019, -0.012169367633759975, -0.027008144184947014, 0.02865689806640148, -0.007412846200168133, 0.014367705211043358, -0.008806434459984303, -0.028499873355031013, 0.03561830148100853, 0.026654841378331184, -0.023239566013216972, 0.01714179664850235, 0.001589869032613933, -0.0004309984506107867, -0.021342191845178604, -0.007661467418074608, -0.027296023443341255, 0.014786436222493649, 0.013333963230252266, 0.01360875554382801, -0.02986074984073639, -0.009506500326097012, -0.014485472813248634, 0.011142168194055557, -0.0032091799657791853, -0.01423685159534216, 0.025307049974799156, 0.013661096803843975, 0.01644827425479889, -0.020543986931443214, -0.02021685242652893, 0.02051781490445137, -0.008073655888438225, 0.001576783717609942, -0.005793535150587559, -0.017154883593320847, 0.03234696388244629, 0.014145254157483578, -0.02802880108356476, 0.01668381132185459, -0.024809807538986206, 0.02494066022336483, 0.010664553381502628, -0.005014956928789616, -0.02488831989467144, 0.004481729585677385, -0.013530243188142776, -0.007785778027027845, -0.007210023235529661, -0.03200674429535866, -0.011135625652968884, 0.014537814073264599, -0.00966352503746748, -0.007811949122697115, 0.0024567728396505117, 0.011017858050763607, -0.007039913907647133, -0.0035493988543748856, -0.01299374457448721, -0.024103200063109398, -0.012404903769493103, 0.013922803103923798, 0.009650439023971558, -0.020727181807160378, 0.01584634929895401, -0.0029262094758450985, -0.010258907452225685, -0.00659501226618886, 0.003107768716290593, 0.007092255167663097, -0.017259566113352776, -0.002056034281849861, -0.027243681252002716, 0.0014761900529265404, 0.01634359173476696, -0.004779420793056488, 0.0402505099773407, -0.02876158058643341, -0.012496501207351685, 0.002800262998789549, -0.006012714467942715, 0.00764838233590126, 0.0028885891661047935, -0.01708945631980896, 0.0029687369242310524, 0.014289192855358124, -0.03344613313674927, 0.014864947646856308, 0.021237509325146675, -0.01445930264890194, 0.008950374089181423, 0.004393403418362141, -0.0030374350026249886, 0.013320878148078918, 0.0069025177508592606, 0.031064601615071297, -0.0017256294377148151, 0.011737551540136337, -0.012287136167287827, -0.004723808262497187, 0.0007098797941580415, 0.03394337743520737, -0.0055023860186338425, -0.01068418100476265, -0.0156500693410635, 0.03352464362978935, -0.024600442498922348, 0.01987663470208645, -0.004488272126764059, 0.026694096624851227, 0.005001871846616268, 0.01053369976580143, -0.008400789462029934, -0.007661467418074608, -0.013032999821007252, 0.01177026517689228, 0.018293308094143867, 0.0029736438300460577, 0.0018842892022803426, -0.007838119752705097, 0.018842892721295357, -0.003470886964350939, 0.007818491198122501, 0.004170952830463648, 0.023043286055326462, -0.031273964792490005, 0.03556595742702484, 0.007838119752705097, 0.01609496958553791, 0.01401440054178238, -0.03444061800837517, -0.004072812385857105, 0.010252364911139011, 0.042893748730421066, 0.003412002930417657, 0.020779522135853767, -0.0094410739839077, 0.006843633484095335, -0.014315363951027393, 0.014302277937531471, -0.007622211240231991, 0.018489588052034378, 0.015231337398290634, 0.031116941943764687, 0.00097485794685781, 0.03255632892251015, 0.0098140062764287, 0.03567064180970192, 0.038235366344451904, 0.0022523144725710154, 0.021747836843132973, 0.013517158105969429, 0.03499020263552666, -0.004661652725189924, 0.0019807936623692513, 0.013281621970236301, 0.004184037912636995, -2.1212565116002224e-05, -0.009683152660727501, 0.011443131603300571, 0.020884204655885696, 0.00722965132445097, 0.008852233178913593, -0.0045438846573233604, -0.007249279413372278, -0.015035057440400124, -0.019641097635030746, 0.017207223922014236, -0.004076084122061729, -0.027138998731970787, -0.01584634929895401, -0.01808394119143486, -0.01799234375357628, 0.007393218111246824, -0.02965138480067253, -0.030070114880800247, -0.015035057440400124, -0.0008873497135937214, 0.0016299429116770625, 0.002873868215829134, -0.008027857169508934, -0.030724382027983665, -0.01897374540567398, 0.06024491414427757, -0.00674549350515008, -0.019654182717204094, -0.025123855099081993, -0.03344613313674927, 0.012542299926280975, 0.008086740970611572, 0.038026001304388046, 0.006320219952613115, 0.020831864327192307, 0.014786436222493649, 0.025751952081918716, 0.0005246404325589538, 0.005806620232760906, 0.0086952093988657, -0.010481358505785465, 0.048913005739450455, 0.004125154111534357, -0.018842892721295357, -0.009002715349197388, -0.01579400710761547, -0.032320793718099594, 0.040564559400081635, -0.01614731177687645, -0.006568841636180878, -0.005757550243288279, 0.012843262404203415, -0.01748201623558998, 0.010540242306888103, -0.004923359956592321, -0.005240679252892733, 0.006545942276716232, 0.009807463735342026, 0.006261336151510477, 0.012476873584091663, -0.0022179654333740473, -0.0010100248036906123, -0.0006996568408794701, -0.002011871198192239, -0.024587357416749, -0.01867278292775154, 0.005286477971822023, 0.06186749413609505, 0.032163769006729126, -0.00355921289883554, -0.002855875762179494, -0.009244794026017189, 0.003932144958525896, -0.006071598269045353, -0.03430976718664169, -0.007543699350208044, -0.0173904187977314, 0.01992897503077984, -0.012280592694878578, 0.005518742837011814, -0.00768763804808259, 0.0030881406273692846, -0.018555013462901115, -0.02503225766122341, -0.03499020263552666, 0.012077770195901394, -0.01468175370246172, 0.016853919252753258, 0.005692123435437679, 0.0063463905826210976, 0.008898031897842884, 0.012581556104123592, 0.014341534115374088, 0.03886346518993378, 0.017900748178362846, 0.009192452766001225, -0.00014424545224756002, -0.005083655007183552, -0.011299192905426025, 0.004390132147818804, -0.007654924876987934, -0.010343962348997593, -0.014223766513168812, 0.023082543164491653, 0.019562585279345512, 0.004390132147818804, -0.00948687270283699, 0.007766150403767824, -0.009630811400711536, -0.013307792134582996, 0.009866347536444664, 0.009179366752505302, -0.027086656540632248, -0.007465187460184097, 0.024220967665314674, -0.016016457229852676, -0.02204880118370056, 0.01867278292775154, 0.014603241346776485, -0.015139739960432053, 0.011423503048717976, 0.0034741582348942757, -0.009356019087135792, 0.007700723595917225, -0.032373134046792984, 0.006477244198322296, 0.009918688796460629, -0.018646610900759697, 0.011711380444467068, 0.0044686440378427505, 0.0033792895264923573, -0.0062351650558412075, 0.005440230946987867, -0.011979630216956139, 0.018764380365610123, -0.014040571637451649, 0.02518928237259388, -0.015636982396245003, -0.03954390063881874, -0.004713994450867176, -0.0005152352969162166, -0.004959344398230314, 0.027897948399186134, -0.0018842892022803426, -0.007641839329153299, -0.007386675104498863, 0.0018123198533430696, 0.012843262404203415, 0.00779886357486248, 0.00889148935675621, -0.028892433270812035, -0.015322934836149216, 0.01207122765481472, 0.008433503098785877, -0.0019055528100579977, -0.010193481110036373, 0.003981214947998524, -0.0036508103366941214, 0.007681095506995916, 0.008250308223068714, -0.024521930143237114, -0.0008280567708425224, -0.016919346526265144, -0.009192452766001225, -0.0005900671239942312, 0.01485186256468296, -0.010147682391107082, -0.012162825092673302, -0.01562389824539423, -0.008309192024171352, -0.033864863216876984, -0.004262549802660942, -0.01648752950131893, 6.92091925884597e-05, 9.929013685905375e-06, -0.0069352309219539165, 0.0028885891661047935, 0.014799521304666996, -0.015061228536069393, -0.004635482095181942, -0.011227223090827465, -0.004835033789277077, 0.028552215546369553, 0.0015628804685547948, 0.007183852605521679, -0.013621840626001358, 0.013752694241702557, 0.015257508493959904, -0.03928219527006149, 0.01623890921473503, 0.016186567023396492, -0.012313306331634521, 0.008001686073839664, 0.023632127791643143, -0.02170858159661293, -0.028238167986273766, -0.01593794673681259, 0.01623890921473503, 0.025097684934735298, -0.014092912897467613, -0.01634359173476696, -0.029101800173521042, -0.00460931146517396, 0.0014524728758260608, 0.0015113569097593427, -0.0034185454715043306, -0.033158253878355026, 0.01882980577647686, -0.006078141275793314, -0.007353961933404207, 0.00667025288566947, 0.006238436792045832, -0.02165623940527439, 0.0010239279363304377, -0.0173904187977314, -0.003768578404560685, 0.01349098701030016, 0.011835691519081593, -0.009748579934239388, -0.010147682391107082, 0.0025483702775090933, -0.003886346472427249, -0.005741193424910307, -0.007465187460184097, 0.01323582325130701, 0.003834004979580641, 0.015636982396245003, 0.01669689640402794, 0.0025483702775090933, 0.04425462335348129, -0.0030227138195186853, -0.015427617356181145, -0.010710352100431919, -0.008269935846328735, -0.014642497524619102, -0.008629783056676388, 0.025699611753225327, 0.03493786230683327, 0.03556595742702484, 0.0016978230560198426, -0.0024731296580284834, 0.007249279413372278, -0.0060585131868720055, 0.00799514353275299, -0.034754667431116104, -0.010618754662573338, -0.03711002692580223, 0.00970932375639677, 0.015349105931818485, -0.00294256629422307, 0.01678849384188652, 0.019117683172225952, -0.027165168896317482, -0.01723339408636093, 0.0044359308667480946, 0.017377333715558052, 0.008839148096740246, 0.024875234812498093, -0.02941584773361683, -0.00460931146517396, -0.025555672124028206, -0.0023504544515162706, -0.0074521019123494625, -0.010494443587958813, -0.011960002593696117, -0.021289851516485214, -0.003575569484382868, -0.015309849753975868, -0.013242365792393684, 0.011685210280120373, 0.004861204419285059, -0.0090681416913867, 0.038235366344451904, 0.014249936677515507, -0.012280592694878578, 0.001907188561744988, -0.010723437182605267, -0.01097860187292099, -0.028133483603596687, -0.03318442776799202, -0.005047670565545559, 0.0010942616499960423, -0.012568471021950245, -0.01996823213994503, 0.030279479920864105, 0.008911117911338806, -0.020596327260136604, 0.006107583176344633, -0.008106369525194168, 0.03831388056278229, -0.000273360958090052, 0.033053573220968246, 0.009421446360647678, -0.012882518582046032, 0.004301805980503559, 0.03705768659710884, 0.01882980577647686, -0.02698197402060032, 0.01199925784021616, 0.0027479217387735844, 0.007962429895997047, -0.00633330550044775, -0.030672039836645126, 0.019366305321455002, -0.006529585458338261, 0.006192638073116541, 0.006323491223156452, 0.017573613673448563, -0.01336667686700821, -0.036351077258586884, -0.02792411856353283, -0.0018712038872763515, 0.015636982396245003, 0.005629968363791704, 0.01390971802175045, -0.017076371237635612, 0.009931773878633976, -0.048363421112298965, 0.04815405607223511, -0.004671467002481222, 0.017953088507056236, 0.0278194360435009, -0.00032283991458825767, -0.011953459121286869, -0.011606697924435139, -0.030986089259386063, 0.022925518453121185, 0.003932144958525896, 0.003398917382583022, -0.0004800684691872448, 0.016670724377036095, 0.0009233343880623579, 0.041114144027233124, -0.0035493988543748856, -0.00959155522286892, 0.01485186256468296, 0.024312565103173256, 0.009951402433216572, 0.015780922025442123, 0.0033302195370197296, -0.013739608228206635, 0.030619699507951736, -0.01259464118629694, -0.013648011721670628, -0.0278194360435009, 0.0008043395937420428, -0.003174830926582217, 0.013131140731275082, -0.004707451444119215, -0.011083284392952919, -0.015676239505410194, -0.00854472815990448, -0.005319191142916679, -0.0005949741462245584, 0.009650439023971558, 0.010828119702637196, -0.02320031076669693, -0.005633239634335041, -0.029284995049238205, -0.020635584369301796, -0.02140761911869049, 0.01623890921473503, 0.004524257034063339, 0.028604555875062943, 0.005610340274870396, -0.005064027383923531, 0.03485935181379318, -0.006676795426756144, 0.010389761067926884, -0.009526128880679607, 0.011273021809756756, -0.008420417085289955, -0.009656982496380806, 0.02364521287381649, -0.02121133916079998, -0.03682215139269829, -0.024652782827615738, 0.00022551768051926047, -0.009055056609213352, -0.00946724507957697, -0.021603899076581, -0.007111883256584406, 0.006765121594071388, 0.007249279413372278, -0.015767836943268776, 0.006081412546336651, 0.008178338408470154, -0.027008144184947014, 0.009990658611059189, 0.022022629156708717, -0.005734650883823633, -0.004082626663148403, -0.012535757385194302, 0.011390789411962032, 0.008237222209572792, -0.007196937687695026, 0.026510901749134064, -0.0017959631513804197, 0.013203109614551067, -0.01833256334066391, -0.001728900708258152, 0.005135996732860804, -0.01075615081936121, -0.0038536330685019493, 0.02016451209783554, -0.005299563519656658, -0.03836622089147568, 0.0059865438379347324, 0.02295168861746788, 0.0030456131789833307, 0.02151230163872242, 0.013477901928126812, -0.010422474704682827, -0.024155540391802788, -0.025895891711115837, -0.002600711537525058, -0.014773350208997726, -0.011436588130891323, 0.03020096756517887, 7.743864262010902e-05, -0.016853919252753258, -0.016932431608438492, -0.022245081141591072, -0.033550817519426346, -0.020858034491539, -0.03273952379822731, 0.001271731685847044, -0.016409019008278847, 0.006981029640883207, -0.009971030056476593, 0.031666528433561325, 0.0037653071340173483, -0.012031971476972103, 0.01326853595674038, -0.006300591863691807, -0.02021685242652893, -0.024652782827615738, -0.008610154502093792, -0.0035788409877568483, -0.010847748257219791, -0.00946070160716772, 0.026497816666960716, -0.011364619247615337, -0.02021685242652893, 0.008178338408470154, -0.021669326350092888, -0.01863352581858635, -0.02378915064036846, 0.030279479920864105, 0.01226750761270523, -0.005080383736640215, 0.0014295735163614154, -0.04004114493727684, 0.001598047325387597, 0.02503225766122341, 0.01579400710761547, -0.008858776651322842, 0.006418359931558371, -0.026013659313321114, -0.017560528591275215, 0.0041480534709990025, -0.005371532868593931, -0.0024993002880364656, 0.004527528304606676, 0.0068305484019219875, 0.01728573627769947, -0.00043958568130619824, 0.0032876920886337757, 0.02339659072458744, 0.00633330550044775, -0.02305637113749981, -0.01423685159534216, -4.1786199290072545e-05, -0.019601842388510704, -0.01838490553200245, 0.0054304166696965694, -0.014446216635406017, -0.008826063014566898, 0.006837090943008661, 0.007576412986963987, 0.003640996292233467, -0.011488930322229862, -0.04723808169364929, 0.018411075696349144, -0.004380317870527506, 0.024181710556149483, 0.018986830487847328, -0.00790354609489441, 0.011299192905426025, -0.0021001973655074835, -0.021237509325146675, -0.02399851568043232, -0.015545385889708996, 0.005741193424910307, -0.014145254157483578, -0.016265079379081726, -0.018607355654239655, -0.01600337214767933, -0.022323593497276306, 0.005080383736640215, -0.02414245530962944, -0.018411075696349144, 0.015479959547519684, 0.22192738950252533, -0.000219383931835182, 0.0005716658779419959, 0.026746438816189766, -0.019392475485801697, -0.00797551590949297, 0.011083284392952919, 0.018698953092098236, -0.005348633509129286, 0.02380223572254181, 0.011528185568749905, -0.018947575241327286, -0.00622207997366786, -0.0014892754843458533, -0.0068632615730166435, 0.0033040486741811037, -0.021603899076581, -0.037188541144132614, 0.014917289838194847, -0.05574355646967888, 0.01824096590280533, -0.0029343878850340843, -0.012575013563036919, -0.011861861683428288, 0.039413049817085266, 0.03234696388244629, 0.0038765324279665947, -0.0060912263579666615, 0.027740923687815666, 0.00149990723002702, -0.014315363951027393, 0.021381448954343796, -0.0060585131868720055, 0.002563091227784753, -0.02721751108765602, -0.0031306680757552385, -0.0027201154734939337, -0.02002057246863842, -0.00327951367944479, 0.025110770016908646, 0.020635584369301796, -0.014092912897467613, 0.02165623940527439, -0.01360875554382801, -0.014969631098210812, 0.015218252316117287, 0.007367047481238842, 0.012319848872721195, 0.007406303193420172, 0.009284050203859806, 0.010193481110036373, -0.042946092784404755, 0.018555013462901115, 0.020491644740104675, 0.027793265879154205, 0.004053184762597084, 0.03679598122835159, -0.00423637917265296, 0.007805406115949154, 0.01822788082063198, 0.0036213682033121586, 0.04038136452436447, -0.03287037834525108, 0.02986074984073639, -0.01996823213994503, 0.028944775462150574, 0.002005328657105565, -0.0392298549413681, 0.024600442498922348, -0.0007356415735557675, 0.008407332003116608, -0.02497991733253002, 0.013948974199593067, 0.004170952830463648, -0.01053369976580143, -0.019745780155062675, 0.02359287068247795, -0.012117026373744011, 0.03119545429944992, 0.011966545134782791, -0.021970288828015327, -0.007471730001270771, -0.0367698110640049, 0.020386962220072746, 0.010972059331834316, -0.011744094081223011, 0.010167310014367104, -0.02240210399031639, -0.006706237327307463, 0.00022572213492821902, 0.0018728395225480199, -0.006666981615126133, 0.0012938131112605333, -0.000978947151452303, 0.008799891918897629, 0.0038111056201159954, 0.013844291679561138, 0.027243681252002716, -0.01948407292366028, -0.011207595467567444, -0.022912433370947838, -0.04629593715071678, 0.010389761067926884, -0.02678569406270981, -0.029075628146529198, -0.004095711745321751, 0.0064608873799443245, 0.020609412342309952, 0.01169829536229372, -0.012718952260911465, -0.0017861491069197655, -0.006683338433504105, 0.009951402433216572, -0.003614825662225485, -0.024678954854607582, 0.012627354823052883, 0.010396303609013557, -0.014393875375390053, 0.010828119702637196, -0.0028673254419118166, 0.0035363135393708944, -0.012378732673823833, -0.004893917590379715, 0.0054304166696965694, -0.030515016987919807, -0.00532573414966464, 0.0026808592956513166, 0.004193852189928293, 0.003030892228707671, -0.04616508632898331, 0.025333222001791, -0.020557072013616562, -0.01778297871351242, 0.01748201623558998, -0.012476873584091663, 0.008008228614926338, 0.005342090502381325, 0.020177597180008888, -0.020897289738059044, -0.025568757206201553, 0.002828069496899843, -0.032713353633880615, 0.02419479750096798, -0.01684083417057991, 0.011868405155837536, 0.0035068716388195753, -0.017547443509101868, -0.0006346390582621098, -0.025673439726233482, -0.01559772714972496, -0.02151230163872242, -0.02126367948949337, -0.011266479268670082, -0.006156653165817261, 0.012339477427303791, -0.03389103338122368, -0.013674181886017323, -0.0028247982263565063, 0.005122911185026169, 0.008682124316692352, -0.014995801262557507, 0.012274050153791904, 0.002877139486372471, 0.008243764750659466, 0.006725865416228771, -0.01833256334066391, -0.16403783857822418, 0.03512105718255043, 0.015584642067551613, -0.028787750750780106, 0.015767836943268776, 0.021944118663668633, 0.029049457982182503, 0.009931773878633976, -0.016853919252753258, 0.0026726811192929745, -0.02067483961582184, 0.008577441796660423, -0.018057771027088165, -0.022585298866033554, 0.00596037320792675, 0.0006370925693772733, 0.004576598294079304, 0.024325650185346603, 0.025320135056972504, 0.016723066568374634, 0.03661278635263443, -0.0038601758424192667, 0.005427145399153233, -0.006179552525281906, 0.017220309004187584, 0.008178338408470154, 0.007105340249836445, -0.008152168244123459, -0.04093094915151596, -0.007720351684838533, 0.01078232191503048, -0.02344893291592598, 0.017796063795685768, 0.03784280642867088, 0.014302277937531471, -0.015414532274007797, 0.005492572207003832, 0.018659697845578194, -0.01982429251074791, 0.007792321033775806, 0.010828119702637196, 0.031116941943764687, 0.01593794673681259, 0.025346307083964348, -0.013975144363939762, 0.027034316211938858, -0.0019758865237236023, -0.016670724377036095, 0.0006788020837120712, 0.0031846449710428715, 0.004334519617259502, -0.01336667686700821, 0.0004857933090534061, 0.023579785600304604, 0.024587357416749, 0.010540242306888103, -0.008727923035621643, 0.008570898324251175, 0.021185167133808136, 0.006895975209772587, -0.001512992661446333, -0.0010329241631552577, 0.008642868138849735, -0.027060486376285553, -0.0017894204938784242, -0.02842136099934578, -0.015192081220448017, 0.003369475482031703, -0.004321434069424868, 0.012306763790547848, -0.008008228614926338, 0.018960660323500633, -0.01537527609616518, 0.013124597258865833, 0.007609126158058643, -0.001126975053921342, -0.02488831989467144, -0.007223108317703009, -0.017377333715558052, -0.011731008999049664, 0.006732408422976732, 0.014904203824698925, -0.02206188626587391, 0.02076643705368042, -0.008152168244123459, 0.014969631098210812, -0.006513228639960289, 0.024234052747488022, -0.01897374540567398, -0.014550900086760521, 0.005011685658246279, -0.011469301767647266, -0.014184510335326195, -0.015152825973927975, 0.015126654878258705, 0.004157867282629013, 0.017102541401982307, 0.009081226773560047, 0.002410974120721221, -0.0004669831250794232, -0.005440230946987867, -0.010553328320384026, -0.018698953092098236, 0.0402505099773407, 0.047866180539131165, 0.013582584448158741, -0.01625199429690838, 0.0014132169308140874, 0.012934859842061996, -0.02394617535173893, -0.010867375880479813, 0.0027642783243209124, 0.01465558260679245, 0.009133568033576012, 0.0027789995074272156, 0.0043247053399682045, 0.003245164640247822, -0.0008669038652442396, 0.03174503892660141, 0.0236059557646513, 0.009120482951402664, 0.00028337942785583436, 0.00434760469943285, 0.010749608278274536, -0.016867006197571754, -0.005266849882900715, -0.10546785593032837, 0.006244979333132505, -0.005973458290100098, 0.04943642020225525, -0.01609496958553791, 0.017298821359872818, -0.00427236407995224, 0.007020285818725824, 0.006725865416228771, 0.021394534036517143, -0.025346307083964348, -0.04540613666176796, -0.006336576770991087, 0.010317792184650898, 0.010016828775405884, 0.01872512325644493, 0.02652398683130741, -0.0018041414441540837, -0.01867278292775154, 0.014511643908917904, 0.006029071286320686, -0.0036344535183161497, -0.018555013462901115, -0.03430976718664169, 0.005862233228981495, 0.030619699507951736, -0.011181424371898174, 5.694168066838756e-05, 0.004125154111534357, 0.01882980577647686, 0.007124968338757753, 0.0007303256425075233, 0.0033253123983740807, 0.0002547552576288581, 0.006490329280495644, -0.02618376910686493, -0.01987663470208645, 0.014079827815294266, -0.009755122475326061, -0.016631469130516052, 0.00911394041031599, -0.011011314578354359, 0.001974250888451934, -0.01907842792570591, 0.008754093199968338, 0.030227139592170715, -0.03156184405088425, 0.00787083338946104, -0.002728293649852276, -0.03184971958398819, -0.0008170159999281168, -0.0036933375522494316, -0.02583046443760395, -0.022990945726633072, 0.019497159868478775, 0.011979630216956139, -0.022611470893025398, -0.007013743277639151, -0.012136653997004032, -0.016382846981287003, -0.03933453559875488, -0.004658381454646587, -0.016657639294862747, -0.00382419116795063, -0.004681280814111233, 0.018895233049988747, -0.01485186256468296, -0.01643518917262554, 0.002021685242652893, 0.01941864751279354, 0.011554356664419174, -0.008878404274582863, -0.00836807582527399, 0.013124597258865833, -0.01913077011704445, 0.002402795944362879, -0.03336762264370918, -0.0030783265829086304, 0.006765121594071388, -0.030096285045146942, -0.011554356664419174, -0.011050570756196976, -0.02310871332883835, -0.019798122346401215, -0.006948316469788551, 0.011168339289724827, -0.0018581184558570385, 0.002046220237389207, -0.017874576151371002, -0.045327622443437576, -0.014982716180384159, -0.00753715680912137, 0.020583242177963257, 0.005342090502381325, -0.01684083417057991, 0.000788391858804971, 0.0038372764829546213, -0.014917289838194847, 0.02602674439549446, 0.0021688954439014196, -0.020988887175917625, 0.00622207997366786, -0.06432753801345825, 0.004115339834243059, -0.012208623811602592, -0.012457245029509068, -0.0004624850407708436, 0.0036050116177648306, -0.01907842792570591, 0.011115998029708862, -0.00685671903192997, 0.016762321814894676, -0.010690724477171898, 0.02991309016942978, 0.004995329305529594, -0.006254793144762516, 0.0056463247165083885, -0.02771475352346897, 0.03323676809668541, 0.00012962667096871883, 0.009624268859624863, 0.01031124871224165, -0.017102541401982307, 0.009055056609213352, 0.019588755443692207, 0.0278194360435009, -0.00936910416930914, 0.042893748730421066, -0.009499957785010338, 0.04679318144917488, 0.0022752138320356607, -0.00974203646183014, 0.00011326999083394185, -0.027688583359122276, -0.0015735123306512833, 0.03030565194785595, -0.009015800431370735, -0.005852418951690197, 0.02776709571480751, 0.04684552177786827, 0.014380790293216705, -0.0037783924490213394, -0.016631469130516052, -0.014276107773184776, 0.013006829656660557, 0.008989629335701466, -0.01349098701030016, -0.026851121336221695, 0.015257508493959904, 0.015231337398290634, -0.0033531186636537313, 0.008911117911338806, 0.029180312529206276, 0.020308449864387512, 0.0004314073594287038, -0.025869719684123993, -0.00453407084569335, 0.004825219511985779, 0.014838777482509613, -0.004857933148741722, 0.012470330111682415, -0.029991602525115013, 0.0447256974875927, 0.012084312736988068, 0.006706237327307463, -0.03666512668132782, -0.0014639225555583835, 0.026340791955590248, -0.003902703057974577, 0.010147682391107082, -0.0022016086149960756, -0.04119265452027321, -0.02682494930922985, 0.023435845971107483, 0.016919346526265144, -0.0037882064934819937, 0.015309849753975868, 0.009388732723891735, -0.012660067528486252, 0.001410763361491263, -0.006437988020479679, 0.03781663626432419, 0.01797925867140293, 0.024063942953944206, -0.006208994425833225, 0.02115899696946144, -0.017468931153416634, 0.02454810030758381, -0.011044028215110302, 0.009696237742900848, -0.004298534709960222, 0.01162632554769516, -0.011168339289724827, -0.009395275264978409, 0.010867375880479813, -0.010265450924634933, 0.011508557945489883, 0.006663710344582796, -0.01507431361824274, 0.021250594407320023, -0.0018286764388903975, 0.007713808678090572, 0.015859434381127357, -0.008649410679936409, -0.02106739953160286, 0.000987125444225967, -0.017455846071243286, -0.02067483961582184, -0.024966832250356674, -0.018855977803468704, -0.010795406997203827, 0.012738579884171486, -0.011122540570795536, -0.004416302777826786, 0.014668667688965797, 0.002868961077183485, -0.013582584448158741, -0.00042486467282287776, -0.010396303609013557, -0.023841492831707, 0.007622211240231991, 0.00026722720940597355, 0.011842234060168266, 0.00318628060631454, 0.0034643441904336214, 0.004059727303683758, 0.0008742643985897303, 0.003238622099161148, 0.004062998574227095, -0.01714179664850235, 0.010658010840415955, -0.01996823213994503, 0.00802131462842226, 0.027845606207847595, -0.003627910977229476, -0.015218252316117287, -0.015061228536069393, 0.012496501207351685, 0.007085712626576424, 0.038235366344451904, -0.012830177322030067, 0.05286477878689766, 0.0058000776916742325, 0.004642025101929903, 0.00716422451660037, 0.008773721754550934, 0.025202367454767227, 0.0014336627209559083, -0.0067193228751420975, -0.03300123289227486, -0.004697637632489204, -0.010514072142541409, -0.0014091277262195945, -0.002746286103501916, -0.008780264295637608, -0.004043370485305786, 0.006104311905801296, 0.012457245029509068, -0.004632210824638605, -0.03114311210811138, 0.000839097541756928, 0.013805035501718521, 0.03561830148100853, 0.006238436792045832, 0.004475187044590712, -0.03444061800837517, -0.015270593576133251, 0.018607355654239655, -0.009676610119640827, -0.0017567070899531245, -0.04600806161761284, 0.0002500526898074895, 0.006837090943008661, -0.016631469130516052, -0.006019257009029388, -0.01977195031940937, -0.019065342843532562, 0.003644267562776804, 0.010926260612905025, 0.004746707621961832, 0.025712696835398674, -0.020988887175917625, 0.009061599150300026, -0.03341996297240257, -0.02007291465997696, -0.02522853948175907, 0.014053656719624996, 0.024116285145282745, -0.03119545429944992, -0.008197966031730175]} +{"id": "test:10000072", "text": "\"Last night a few of us from The Bier Abbey had the opportunity to stop down the Capital Region\u2019s newest brewery, Rare Form Brewing Company, meet owners Kevin Mullen and Jenny Kemp, check out the new space, and of course try some beers they had on tap. It was a breath of fresh air, for both the continuation of the improvement of Troy, but also for the craft beer scene in the Upstate New York.\\nThe taproom itself looks bigger than it really is, given that the brewery is directly behind the bar. The bar area holds about 40 people, with a small room off to the side available for private groups or holding tastings. Stools and standing room are available at the bar, while there are some tables and benches off to each side.\\nThe brew house is a Beast 5 BBL system from Psycho Brew that allows for brewing either a double batch or two beers at the same time, which will be nice to do some smaller experimental batches.\\nThe taps are split between the two sides of the bar, each side with its own glass rinser, which is good to see (nothing worse than putting good beer in a dirty glass). There will be a couple of nitro offerings, as well.\\nWhile looking around I was a little confused exactly where their cooler was, not realizing there was a downstairs not readily visible from the bar. Kevin took me for a quick tour of the cooler they made themselves. It looks like they have plenty of Rare Form branded cooperage hiding out down there.\\nBack at the bar, glass sizes vary depending on the beer and ABV, and as Todd pointed out last night it would be nice to see glass sizes listed on the chalkboard as well. There are still some folks who don\u2019t \u201cget\u201d that not all beers are meant to be served in a 16 ounce glass, and it can be a shock when someone is served a beer in a glass half the size than what they were expecting.\\nAs for the beers themselves, I\u2019ll be honest, whenever I walk into a new brewery opening up in the Capital Region, I have my reservations and things were no different yesterday at Rare Form. I\u2019ve been burned more often than I would like to admit by local brewers who started as homebrewers, were encouraged by friends and family to start brewery, and never really upped their game. Kevin has been brewing in various stages for \u201cmore than half [his] life\u201d at this point, and I gotta say I walked away extremely impressed.\\nSabbatical Session Ale \u2013 a session rye ale coming in under 4% ABV with enough rye to keep things interesting without being over the top by the end of the glass.\\nL\u2019homme Chat Belgian Black Ale \u2013 an odd beer, as \u201csession Belgian black ale\u201d doesn\u2019t necessarily roll off the tongue but this definitely worked surprisingly well. I thought about adding this in Untappd but I couldn\u2019t figure out what style to consider this. I\u2019ll let someone else figure it out. Also below 4%.\\nWee Plaid Scottish Ale \u2013 tons of malty caramel notes without being cloying.\\nCascadia Double IPA \u2013 a DIPA with a \u201cmalty backbone\u201d and some piney, some citrusy hops.\\nKarass Robust Porter \u2013 a classic porter I was unfair to compare side by side with Hill Farmstead\u2019s Everett. Still, Karass held it\u2019s own and had a simple, but great roasty flavor.\\nSatan\u2019s Gut Oak-Aged Imperial Stout \u2013 a big beer, with some coffee notes and oak present from the oak chips but not much from the bourbon the chips were soaked in.\\nFirst off, all of the beers were extremely drinkable. I like the focus on the session beers which are already very popular this year and\u2026 I feel bad that I need to even say this but\u2026 none of them had any glaring flaws or tasted like the dreaded \u201chomebrew on a bigger system\u201d as mentioned above. I\u2019m sure Kevin is going to do some tweaks as he gets used to brewing consequetive batches on this much bigger system and the beers are only going to improve.\\nThe one change I would love to see would be with the Cascadia Double IPA. Just from a demographic perspective, in 2014, in this location, we live where New England\u2019s Gandhi Bot and Coriolis will kick in a matter of hours. We have no problem driving to MA to Tree House or VT to Alchemist/Hill/Lawson\u2019s for IPAs. And Rushing Duck\u2019s War Elephant is one of our most popular local beers. I think it\u2019s pretty clear that malty and piney are not characteristics Capital Region folks are craving for in their IPA, and most of the time any malt presence in an IPA is usually considered \u201cold\u201d to beer geeks these days. I\u2019d love to see Rare Form offer another IPA where the malt steps aside and lets some super juicy, tropical, citrusy notes shine, as that\u2019s more in line with what\u2019s popular in the Northeast today.\\nBeyond the beer, some Rare Form swag is also available including shirts, sweatshirts and aluminum growlers for beer to go, although I did not see pricing for growler fills. No #properglassware yet but I\u2019m sure that\u2019ll happen down the road.\\nAs Rare Form is the beginning of the transformation of the corner of Congress St and 4th, I got the chance to check out the progress with the rest of the space, including The Shop, a beer bar that\u2019s adjacent to the brewery. The opening is still a couple of months out, but some of the detail in the work happening there currently looks insanely good. A delicatessen at the corner of the building is also in the works and overall the building is shaping up nicely.\\nAs some of you know, I\u2019m a little (OK, a lot) harsh on breweries in Upstate NY. Living here, but having such great beer being made in all around us in New England has definitely made me a little jaded at this point. The big breweries in New York seem to be on cruise control, taking little to no risk with new beers. The smaller breweries, while opening up in rapid numbers haven\u2019t done much to shake things up either, and most wouldn\u2019t meet the standards set for the beers poured at The Bier Abbey (the only one I can think of off the top of my head that does would be Rushing Duck).\\nThat said I have high hopes for Rare Form. They have a great brand, a website that looks like those of an established brewery, a small, but well put together tasting room, and brewing equipment that doesn\u2019t look like it is meant for homebrew or cobbled together from used parts from other breweries. Not to mention they are making some beer that\u2019s worthy of my recommendation to friends and fellow beer geeks for a brewery visit, and I would gladly serve to a patron at The Abbey if and when they\u2019re producing enough liquid for distribution. I see a lot of potential for Rare Form, and I wish Kevin and Jenny the best of luck.\\nRare Form\u2019s Grand Opening is tomorrow, Friday May 30th at 4:30PM. Make sure to head down there to check them out!\\nYou lift me up, then you let me down. I\u2019m talking about that Cascadia IPA of course. (I think it\u2019s a bit aspirational to call an 8.8% ABV beer a \u201cdouble\u201d. Imperial, maybe.) War Elephant, now that\u2019s an IPA to my liking. Hope the folks at Rare Form are paying attention to your comments.\\nEven if they keep Cascadia the way it is there are still room for other IPAs in their lineup as well. Not that Cascadia is bad, just not the style I was hoping for.\\nI had the Wee Plaid, Sabbatical and Satan\u2019s Gut when the owners came down for a cheese tasting at The Cheese Traveler. Most breweries in the Upstate area put out mediocre beer. Rare Form is not most breweries. I must say the Sabbatical is an excellent session ale.\\nI think the Sabbatical is my favorite beer they do so far. Such great flavor out of a such a small beer, exactly what I want as these temps warm up.\"", "vector_field": [0.03251700848340988, -0.014903628267347813, 0.010024708695709705, -0.033256031572818756, -0.0039106630720198154, 0.017955519258975983, 0.0007869225228205323, -0.013015015050768852, 0.010558447800576687, -0.017353352159261703, 0.014219348318874836, 0.00507393991574645, -0.0101547222584486, -0.03366659954190254, -0.006179052870720625, 0.019022997468709946, 0.03268123418092728, -0.007930810563266277, 0.009942594915628433, 0.012994485907256603, 0.00349154113791883, -0.008368750102818012, 0.01827028952538967, 0.010900587774813175, -0.004009883385151625, 0.010544762015342712, 0.00945675652474165, 0.0027302789967507124, -0.01924196630716324, 0.015847936272621155, 0.007089145481586456, -0.004974719136953354, -0.012248620390892029, 0.015615280717611313, -0.014862571842968464, -0.03027256764471531, -0.011927008628845215, -0.005135525017976761, 0.03840181976556778, -0.02238965779542923, 0.013801936991512775, -0.009230943396687508, 0.0008083063294179738, -0.013473482802510262, 0.003320470917969942, -0.0046770572662353516, -0.009292528964579105, 0.013192927464842796, 0.012378633953630924, 0.00771184079349041, 0.028575552627444267, -0.028438696637749672, -0.03985249623656273, 0.025674203410744667, 0.0025797374546527863, -0.02999885566532612, 0.012987643480300903, 0.006579356733709574, 0.024209843948483467, -0.004088575951755047, -0.013165555894374847, 0.010989543981850147, -0.01423303410410881, 0.0031870363745838404, -0.015793193131685257, -0.025509975850582123, -0.005768484435975552, -0.004228853154927492, 0.010955329984426498, -0.015560537576675415, 0.02098003961145878, 0.00858087744563818, 0.025140464305877686, -0.007622884586453438, 0.015505795367062092, -0.015368939377367496, -0.022827597334980965, -0.025427863001823425, -0.012440218590199947, 0.015013113617897034, 0.00367458607070148, -0.010052080266177654, -0.015081541612744331, 0.03374871239066124, 0.013473482802510262, 0.011885952204465866, -0.001680763904005289, 0.016367988660931587, -0.024278271943330765, -0.004095418844372034, 0.004827598575502634, 0.01716175489127636, 0.01278235949575901, 0.00873826164752245, -0.020008360967040062, 0.0006971107213757932, 0.0017440598458051682, 0.028110241517424583, -0.004783120471984148, -0.0011564339511096478, -0.007663941476494074, 0.005871126428246498, 0.019543049857020378, -0.02646796964108944, -0.02065158449113369, 0.006449343636631966, -0.009251471608877182, -0.015437367372214794, 0.040564145892858505, 0.009340428747236729, -0.015888992697000504, 0.009360956959426403, 0.015875307843089104, -0.017230182886123657, 0.011010073125362396, 0.009258314967155457, 0.02904086373746395, -0.018981941044330597, 0.003289678366854787, -0.045764677226543427, 0.03081999346613884, -0.002489070175215602, 0.025208892300724983, -0.00668199872598052, 0.014643602073192596, 0.005600835662335157, -0.03550047054886818, -0.00962098315358162, 0.008574034087359905, -0.02560577541589737, 0.017503894865512848, -0.0018390037585049868, 0.04620261862874031, 0.02737121842801571, 0.00043195203761570156, -0.0013257934479042888, -0.02904086373746395, -0.012084392830729485, -0.012577075511217117, -0.032735977321863174, 0.034734077751636505, 0.01106481533497572, -0.0070480890572071075, 0.0035787869710475206, -0.012399162165820599, 0.016942784190177917, 0.02178749069571495, 0.010407906025648117, 0.0012145978398621082, -0.012330734170973301, -0.0009135144064202905, -0.028438696637749672, -0.006264587864279747, 0.005450293887406588, 0.031285304576158524, -0.005737691652029753, 0.0006162801291793585, 0.03311917558312416, -0.01557422336190939, 0.022458085790276527, -0.01861242949962616, 0.0247162114828825, -0.0010401062900200486, -0.014889943413436413, 0.0019279601983726025, 0.029725143685936928, 0.01336399745196104, -0.018434515222907066, -0.001287302584387362, -0.006695684511214495, 0.010736360214650631, 0.0073697008192539215, -0.03933244198560715, 0.006883861497044563, 0.023648733273148537, 0.024647783488035202, -0.008690361864864826, 0.007157573942095041, -0.017558636143803596, -0.037060629576444626, 0.025414176285266876, -0.005234745796769857, 0.02672799490392208, 0.04524462670087814, -0.022307543084025383, -0.017271239310503006, 0.01540999673306942, 0.003200721926987171, -1.4607784578402061e-05, -0.0036780075170099735, -0.022047515958547592, 0.037662796676158905, 0.0007702432340011001, 0.016655387356877327, -0.632822573184967, -0.006883861497044563, -0.0025472340639680624, -0.02555103227496147, 0.01624481752514839, 0.012802887707948685, 0.01365823857486248, 0.007424443028867245, -0.004923398140817881, 0.026043714955449104, -0.0039927763864398, 0.026385854929685593, 0.021021096035838127, -0.00945675652474165, -0.026262683793902397, -0.01712069660425186, 0.028438696637749672, -0.0032725713681429625, 0.004711271263659, -0.011386427097022533, -0.01718912459909916, 0.025441547855734825, 0.013145027682185173, -0.01828397437930107, 0.004971297457814217, -0.015204711817204952, 0.011735410429537296, 0.0018851927015930414, -0.013015015050768852, 0.006589620839804411, -0.041686367243528366, 0.009217257611453533, -0.004608628805726767, -0.03290020674467087, 0.04308229684829712, -0.0035787869710475206, -0.017011212185025215, 0.01527313981205225, 0.009833110496401787, 0.023799274116754532, -0.020022045820951462, -0.010264206677675247, 0.012843944132328033, -0.016737500205636024, -0.006011404097080231, 0.0029304311610758305, 0.010893745347857475, -0.0024582776241004467, -0.01625850424170494, -0.019679905846714973, -0.012180192396044731, 0.0007514255121350288, -0.0009272000170312822, 0.009867324493825436, 0.03629423677921295, -0.021417979151010513, 0.01061319001019001, -0.027508074417710304, 0.0017911040922626853, 0.001931381644681096, 0.00962098315358162, 0.014520431868731976, -0.03142216056585312, -0.007739212363958359, -0.031559016555547714, -0.010818473994731903, -0.014027750119566917, -0.027562817558646202, 0.012118606828153133, -0.010688461363315582, 0.01163276843726635, 0.00901197362691164, -0.032133810222148895, 0.002997148549184203, -0.011475383304059505, 0.012159664183855057, 0.008142937906086445, 0.017887091264128685, -0.0005132103688083589, -0.0020648164208978415, -0.000993917346931994, -0.00719178793951869, -0.020884240046143532, 0.006699105724692345, 0.02010416053235531, -0.007882910780608654, -0.0203094445168972, 0.007465499918907881, 0.019419880583882332, -0.01267287414520979, 0.012433376163244247, 0.0174765232950449, 0.008649305440485477, -0.022047515958547592, -0.00235905684530735, 0.006969396490603685, 0.009066715836524963, 0.03180535510182381, 0.017928147688508034, -0.053018052130937576, -0.016121648252010345, -0.027261734008789062, 0.026878537610173225, -0.01480782963335514, -0.011160614900290966, 0.020241016522049904, -0.008375593461096287, -0.022745482623577118, 0.042918071150779724, -0.003236646531149745, -0.013884050771594048, -0.021938031539320946, -0.02734384685754776, 0.002030602190643549, -0.006541721522808075, -0.017531266435980797, 0.029889371246099472, 0.013021857477724552, 0.010380534455180168, -0.011297470889985561, 0.031531643122434616, 0.0032417788170278072, 0.005111575126647949, -0.013110813684761524, -0.025728946551680565, 0.014657287858426571, 0.004057783167809248, -0.0319695845246315, -0.01393879298120737, -0.012611289508640766, 0.005676106549799442, 0.008909331634640694, 0.047543808817863464, -0.027535445988178253, 0.009894696064293385, 0.023771904408931732, -0.0022803647443652153, -0.009018816985189915, 0.012399162165820599, -0.04280858486890793, -0.032079067081213, -0.007527085021138191, -0.006398022640496492, -0.03404979780316353, -0.022800225764513016, -0.0023658997379243374, -0.026851166039705276, 0.00367116485722363, -0.028931379318237305, 0.01688804291188717, 0.004827598575502634, 0.0037601212970912457, -0.006185895297676325, -0.0014865993289276958, -0.018968254327774048, -0.0060079824179410934, 0.008423492312431335, -0.02207488752901554, -0.011010073125362396, -0.020555784925818443, 0.012878158129751682, 0.011954380199313164, -0.048392314463853836, -0.01862611435353756, -0.006993346381932497, -0.00020400111679919064, 0.002328264294192195, -0.008799847215414047, 0.0028893742710351944, -0.016107961535453796, -0.006151681300252676, -0.003965405281633139, -0.006808590609580278, -0.011557497084140778, 0.004331495612859726, 0.001077741733752191, -0.01321345567703247, -0.0005217638681642711, -0.026522710919380188, -0.017654435709118843, 0.0018030790379270911, 0.023156050592660904, -0.020829496905207634, 0.008115566335618496, 0.021431664004921913, -0.004098840057849884, 0.033885568380355835, 0.02096635289490223, -0.013500853441655636, 0.04223379120230675, -0.010401063598692417, -1.6211566617130302e-05, -0.0013899446930736303, 0.016491159796714783, -0.009121458977460861, 0.0033649492543190718, 0.00719178793951869, 0.007841854356229305, 0.011345370672643185, 0.027097506448626518, -0.0027901537250727415, -0.00901197362691164, 0.00915567297488451, -0.008204522542655468, 0.021664319559931755, -0.021130580455064774, -0.00043259357335045934, -0.023689789697527885, 0.027152249589562416, -0.0014806118560954928, 0.007301272824406624, -0.005925869103521109, -0.031860098242759705, 0.001984413480386138, 0.0012573653366416693, 0.007629727479070425, 0.015519481152296066, 0.01952936500310898, -0.0046360003761947155, 0.02412772923707962, -0.010407906025648117, -0.02704276517033577, 0.003448773641139269, 0.017325982451438904, 0.003972248174250126, -0.012501804158091545, -0.0003352974308654666, 0.009839952923357487, -0.022157002240419388, 0.0009665461839176714, -0.007807640358805656, 0.02613951452076435, 0.018968254327774048, 0.030299939215183258, 0.0189408827573061, 0.00886143185198307, 0.008252422325313091, -0.010873217135667801, 0.0581912100315094, -0.028986120596528053, 0.01831134594976902, -0.003862763289362192, 0.0069351824931800365, 0.017969205975532532, 0.03686903044581413, -0.020145216956734657, 0.01827028952538967, 0.006356965750455856, -0.009634668938815594, -0.005501614883542061, -0.01018893625587225, 0.02728910557925701, -0.020528413355350494, 0.016970155760645866, 0.015601594932377338, -0.0013086864491924644, 0.0015122598269954324, 0.015341567806899548, 0.012098078615963459, 0.027138562873005867, -0.02388138882815838, 0.00465652858838439, 0.00019983128004241735, -0.023457134142518044, 0.0017141225980594754, -0.011523283086717129, -0.0046360003761947155, -0.019830448552966118, -0.010873217135667801, 0.012864473275840282, 0.00901197362691164, 0.0038730273954570293, 0.00034770002821460366, 0.017011212185025215, -0.002903059823438525, -0.007629727479070425, -0.006551985628902912, -0.0173944104462862, -0.0045230938121676445, 0.00813609454780817, -0.005669263657182455, -0.02093898318707943, 0.011270099319517612, 0.016737500205636024, -0.004345180932432413, -0.010996387340128422, 0.009340428747236729, -0.023936130106449127, -0.010524233803153038, 0.04384869337081909, -0.00931305717676878, 0.001714977901428938, -0.034843560308218, 0.01235810574144125, 0.015902677550911903, 0.0010443830396980047, 0.03544572740793228, -0.02879452146589756, -0.00018186891975346953, -0.006374072749167681, -0.021883290261030197, -0.009224100969731808, -0.0033375779166817665, -0.012002279050648212, 0.0377449095249176, -0.01720281131565571, -0.025085723027586937, -0.014616230502724648, -0.03459722176194191, -0.01365823857486248, 0.0016739211278036237, 0.0037943352945148945, -0.013911422342061996, 0.004338338039815426, -0.00958676915615797, -0.0007261926657520235, -0.0033666598610579967, 0.005344230681657791, 0.02496255189180374, 0.002636190503835678, -0.01106481533497572, -0.02705645002424717, -0.013897736556828022, -0.009703096933662891, 0.14473900198936462, 0.008307164534926414, 0.013842994347214699, 0.026002658531069756, -0.006510928738862276, 0.0017517580417916179, -0.01367192342877388, -0.015752136707305908, 0.00552898645401001, 0.0068872831761837006, 0.006695684511214495, -0.005669263657182455, 0.0070070321671664715, -0.010982701554894447, 0.013494011014699936, -0.0021640369668602943, -0.011290627531707287, -0.02066527120769024, 0.019077738747000694, 0.0021862760186195374, 0.013788251206278801, -0.007937653921544552, -0.009750996716320515, 0.032407522201538086, 0.02702907845377922, 0.016121648252010345, 0.022512827068567276, 0.021855918690562248, 0.003801177954301238, -0.0065690926276147366, 0.01092111598700285, 0.014616230502724648, -0.018817713484168053, 0.003903820179402828, -0.007917124778032303, -0.007472342811524868, 0.02416878566145897, 0.001004181569442153, -0.004208324942737818, 0.012234934605658054, 0.02961565926671028, 0.0319695845246315, 0.00931305717676878, -0.00352062308229506, -0.002283786190673709, -0.010572133585810661, -0.016367988660931587, 0.018092375248670578, 0.006493821740150452, -0.04248013347387314, 0.03224329650402069, -0.0005431476165540516, 0.0030364945996552706, 0.0025574981700628996, 0.0057719056494534016, 0.021664319559931755, 0.023347649723291397, -0.004259645938873291, -0.025249948725104332, -0.002670404501259327, -0.0019330923678353429, -0.03462459146976471, 0.018407145515084267, 0.005580307450145483, -0.02359399013221264, -0.009963124059140682, -0.013582967221736908, 0.012652345933020115, -0.028657665476202965, -0.0002230326645076275, -0.0202957596629858, -0.01510891318321228, -0.030409423634409904, 0.005286066792905331, 0.016463788226246834, -0.0027371218893676996, 0.0160121638327837, -0.01033263560384512, 0.002059684135019779, 0.0031613758765161037, -0.0005983177106827497, -0.00786922499537468, 5.725503069697879e-05, -0.017873406410217285, -0.028301840648055077, 0.0030587336514145136, 0.013890893198549747, 0.0079718679189682, -0.0027508074417710304, 0.009210415184497833, 0.0034196916967630386, -0.00813609454780817, -0.002232464961707592, 0.007164416369050741, 0.0009682568488642573, -0.00958676915615797, 0.027097506448626518, -0.01106481533497572, -0.0050773611292243, -0.01569739356637001, 0.007506556808948517, -0.0017568900948390365, -0.001994677586480975, -0.01836608722805977, 0.009928910061717033, -0.002613951452076435, -0.009278843179345131, -0.009839952923357487, -0.004721535369753838, -0.019132481887936592, 0.005638470873236656, -0.013008171692490578, -0.010688461363315582, -0.005706899333745241, -0.012310205958783627, -0.013172399252653122, -0.007759740576148033, 0.01090743113309145, -0.021582206711173058, 0.006764112506061792, 0.003075840650126338, 0.00016497574688401073, 0.022020146250724792, 0.01510891318321228, -0.005607678554952145, 0.031832728534936905, 0.00719863036647439, -0.03227066621184349, -0.01339136902242899, -0.0010657667880877852, -0.005939554423093796, 0.020008360967040062, -0.0039859339594841, -0.008669833652675152, -0.056548938155174255, -0.021363236010074615, 0.006589620839804411, 0.037936508655548096, 0.007164416369050741, 0.005689792335033417, -0.047516435384750366, 0.006842804607003927, -0.008505606092512608, -0.010134194046258926, -0.024825695902109146, -0.021760119125247, 0.0006800037226639688, 0.002335107186809182, -0.011968065053224564, 0.0039106630720198154, -0.001004181569442153, -0.0008630487718619406, -0.007698155473917723, 0.026057399809360504, -0.0033016533125191927, -0.02414141595363617, -0.004786541685461998, -0.00662383483722806, 0.003245200030505657, 0.0027285683900117874, 0.007698155473917723, 0.005987454205751419, 0.01980307698249817, 0.009867324493825436, -0.011324841529130936, 0.005214217118918896, 0.002210225909948349, -0.020514728501439095, -0.025934230536222458, 0.008060824126005173, 0.008190837688744068, -0.0035582585260272026, -0.005559778772294521, -0.013904578983783722, -0.0018236074829474092, 0.023949816823005676, -0.00817030854523182, -0.03801862150430679, -0.02211594581604004, -0.004954190459102392, -0.002836342668160796, -0.0261668860912323, -0.017941834405064583, -0.0029526702128350735, -0.00713704526424408, 0.00189032475464046, 0.0347888208925724, 0.006268009077757597, 0.011817524209618568, -0.006497242953628302, 0.03407716751098633, -0.014164606109261513, 0.03303706273436546, 0.01121535710990429, 0.01540999673306942, 0.012693402357399464, -0.015177341178059578, -0.025811059400439262, -0.010291578248143196, 0.023498190566897392, -0.0007941930089145899, 0.0070549314841628075, -0.003199011320248246, 0.006801747716963291, -0.008047138340771198, 0.025701574981212616, -0.01366508100181818, -0.007465499918907881, 0.004591521807014942, -0.010517390444874763, -0.02122638002038002, -0.006610149517655373, -0.03944192826747894, -0.025811059400439262, -0.025167835876345634, -0.02852080948650837, -0.01423303410410881, 0.030299939215183258, -0.008245579898357391, -0.01599847711622715, -0.001910853199660778, -0.014369890093803406, 0.016422731801867485, 0.0320516973733902, 0.007376543711870909, -0.00147462438326329, -0.01381562277674675, -0.005303173791617155, -0.0073697008192539215, 0.012850787490606308, -0.018448201939463615, -0.004899448249489069, 0.003127161879092455, -0.007479185704141855, -0.01527313981205225, 0.0073697008192539215, 0.004632578697055578, -0.034734077751636505, -0.01718912459909916, 0.030163083225488663, 0.018078690394759178, 0.012248620390892029, -0.02006310410797596, 0.0031630864832550287, -0.013275041244924068, -0.00189032475464046, 0.02214331552386284, -0.004906291142106056, -0.016409045085310936, -0.005662420764565468, -0.03142216056585312, 0.009805738925933838, -0.016764871776103973, 0.006175631191581488, 0.006295380182564259, -0.00945675652474165, -0.003896977286785841, -0.0072123161517083645, 0.007800797466188669, -0.015177341178059578, -0.009046187624335289, 0.02679642289876938, -0.026960650458931923, 0.031257931143045425, -0.004663371481001377, -0.0022068044636398554, -0.02613951452076435, -0.005754798650741577, -0.0032280930317938328, 0.026084771379828453, -0.005573464557528496, 0.001633719657547772, -0.023361334577202797, 0.011899637058377266, 0.017640750855207443, -0.0064117079600691795, -0.017544951289892197, 0.002489070175215602, -0.014424632303416729, 0.017243867740035057, 0.015806879848241806, 0.01175593864172697, -0.02794601395726204, 0.012707088142633438, -0.005689792335033417, -0.040892601013183594, 0.003595893969759345, -0.013535067439079285, -0.008539820089936256, -0.034460365772247314, -0.01803763397037983, -0.022772854194045067, 0.005251852795481682, 0.0038148637395352125, 0.008402964100241661, -0.013151871040463448, -0.006192738190293312, 0.010134194046258926, -0.03432350978255272, 0.0010768864303827286, -0.002851738827303052, 0.024237213656306267, -0.029861999675631523, 0.004984983243048191, -0.00973731093108654, -0.010825317353010178, 0.022909710183739662, 0.010743203572928905, -0.005703477654606104, 0.004588100593537092, 0.013986692763864994, 0.0007629727479070425, 0.004892605356872082, -0.013637709431350231, -0.0004824177303817123, 0.010661089792847633, -0.007581827696412802, -0.003320470917969942, -0.027425961568951607, -0.010229992680251598, 0.019064053893089294, -0.0022341758012771606, 0.017654435709118843, 0.0008373882155865431, 0.011044287122786045, 0.019543049857020378, 0.014014064334332943, 0.0012154531432315707, -0.016737500205636024, -0.004909712355583906, 0.002903059823438525, 0.0027422539424151182, -0.0026088193990290165, 0.0028568708803504705, 0.012433376163244247, -0.005433186888694763, -0.02243071421980858, 0.038264963775873184, 0.021678006276488304, -0.006757269613444805, 0.018735598772764206, 0.021048467606306076, 0.008991445414721966, 0.0008707469096407294, -0.0013497432228177786, -0.01365823857486248, -0.006247480865567923, 0.018147118389606476, -0.04223379120230675, -0.009066715836524963, -0.041440024971961975, 0.026084771379828453, 0.0022136473562568426, -0.00589165510609746, -0.04762592166662216, -0.009265157394111156, -0.03580155596137047, 0.001717544044367969, -0.002393270842730999, 0.017038583755493164, 0.04327389597892761, -0.01148906908929348, -0.0024702525697648525, 0.013001329265534878, -0.008485077880322933, 0.03812810778617859, 0.0024411706253886223, 0.006264587864279747, -0.0019074318697676063, -0.0001150446551037021, 0.00812925212085247, -0.014575174078345299, -0.009744154289364815, 0.006001139525324106, 0.03235277906060219, 0.014493060298264027, 0.016176389530301094, 0.017271239310503006, 0.003375213360413909, -0.0005166317569091916, -0.006962553597986698, 0.006486978847533464, -0.0016576694324612617, 0.018448201939463615, 0.04570993408560753, -0.008690361864864826, -0.03081999346613884, 0.01947462186217308, 0.001986124087125063, 0.0038798702880740166, 0.009648354724049568, 0.018448201939463615, 0.01236494816839695, 0.007444971706718206, 0.008142937906086445, -0.02240334264934063, 0.015779508277773857, -0.010366849601268768, 0.03347500041127205, 0.00388329173438251, -0.01950199343264103, 0.04414977505803108, 0.0160258486866951, -0.027973385527729988, 0.0008668978116475046, -0.006203002296388149, -0.03946929797530174, 0.013138185255229473, -0.003299942472949624, 0.001303554279729724, -0.010202622041106224, -0.007547613698989153, 0.011536968871951103, 0.0012034783139824867, -0.024018244817852974, -0.020829496905207634, 0.004102261271327734, -0.011605396866798401, 0.020870555192232132, 0.005108153913170099, -0.014835200272500515, -0.00889564584940672, -0.03136741742491722, -0.001052936539053917, -0.007588670589029789, -0.004755749367177486, -0.0023231322411447763, -0.025427863001823425, -0.014520431868731976, 0.01366508100181818, -0.033557113260030746, -0.03632160648703575, -0.00287226727232337, 0.025195207446813583, -0.01103060133755207, 0.007109674159437418, 0.20407980680465698, 0.0036335294134914875, 0.0007326077902689576, 0.012317048385739326, -0.015930049121379852, 0.01631324738264084, 0.0026122406125068665, 0.019392509013414383, 0.008683519437909126, 0.009224100969731808, 0.0010478044860064983, 0.018092375248670578, -0.01525945495814085, -0.006025089416652918, -0.012070707976818085, 0.006192738190293312, -0.03659531846642494, -0.013418739661574364, -0.0189135130494833, -0.01251548994332552, 0.010243678465485573, -0.01293974369764328, -0.005611099768429995, 0.005658999551087618, 0.03084736317396164, 0.005703477654606104, 0.005867705214768648, 0.0015618702163919806, 0.034131910651922226, 0.013343469239771366, -0.02873978018760681, -0.0174765232950449, 0.012091236189007759, -0.026248998939990997, -0.01686067134141922, 0.006603306625038385, 0.014588859863579273, -0.022252801805734634, 0.008122408762574196, 0.008081352338194847, -0.009025659412145615, 0.015191026963293552, -0.0016773424576967955, 0.005576885771006346, -0.0069351824931800365, -0.0053202807903289795, -0.0010888612596318126, 0.006545142736285925, 0.013521382585167885, -0.008423492312431335, -0.026686938479542732, -0.0016260214615613222, -0.0004892605356872082, 0.04622998833656311, 0.006394600961357355, -0.0030997905414551497, 0.01974833384156227, 0.015040485188364983, -0.02793232910335064, 0.03399505466222763, -0.01660064421594143, 0.03525412827730179, -0.017367038875818253, 0.014944685623049736, -0.026358483359217644, 0.024853067472577095, 0.005138946231454611, 0.011420641094446182, 0.009956280700862408, -0.016149019822478294, -0.017243867740035057, -0.028986120596528053, -0.027467017993330956, -0.023169737309217453, -0.01770917885005474, -0.036513205617666245, 0.031832728534936905, 0.02553734742105007, 0.027179621160030365, 0.0108800595626235, -0.0014635048573836684, 0.005929290316998959, -0.0008018912049010396, 0.017312295734882355, 0.0037738068494945765, -0.025167835876345634, 0.01004523690789938, 0.0059600831009447575, -0.007294429931789637, -0.030108340084552765, 0.01075688935816288, -0.00843033567070961, -0.022841282188892365, -0.01573844999074936, 0.03547310084104538, -0.009874166920781136, -0.019283024594187737, 0.0013745484175160527, -0.01922828145325184, 0.009066715836524963, -0.014602545648813248, 0.00697966106235981, 0.009183043614029884, 0.02882189303636551, -0.022334914654493332, 0.012173349969089031, 0.0019057211466133595, 0.028438696637749672, 0.017914462834596634, -0.007663941476494074, -0.007506556808948517, -0.025249948725104332, 0.0021281123626977205, 0.010414748452603817, 0.022813910618424416, 0.0031408474314957857, 0.004509408492594957, -0.0066922628320753574, 0.024634096771478653, -0.008252422325313091, -0.02477095276117325, -0.01681961491703987, 0.008793003857135773, -0.012145978398621082, 0.005286066792905331, -0.01412354875355959, -0.019857820123434067, 0.029424060136079788, 0.0016054930165410042, -0.024291956797242165, -0.0016414177371188998, 0.026235314086079597, 0.027480704709887505, -0.008396121673285961, -0.011557497084140778, -0.023073937743902206, 0.00012434659583959728, -0.031504273414611816, -6.607583054574206e-05, 0.011365898884832859, -0.0058437553234398365, 0.033885568380355835, 0.013076599687337875, 0.013630867004394531, 0.01857137121260166, -0.035281501710414886, 0.019939932972192764, -0.00962098315358162, -0.013582967221736908, -0.01977570541203022, -0.005515300668776035, -0.0016422731569036841, -0.017586007714271545, -0.020268388092517853, 0.031011590734124184, 0.010667932219803333, -0.021965403109788895, -0.01628587581217289, -0.01651853136718273, 0.00856034830212593, -0.027754416689276695, 0.005635049659758806, 0.04789963364601135, -0.007725526578724384, -0.021376922726631165, -0.006021668203175068, -0.1747378557920456, 0.0033444208092987537, -0.01918722502887249, -0.01540999673306942, 0.02269074134528637, -0.005559778772294521, 0.015642652288079262, 0.019912561401724815, -0.022526513785123825, 0.014287776313722134, -0.014493060298264027, 0.00928568560630083, -0.013945636339485645, 0.0005816384218633175, 0.00015952288231346756, -0.002584869507700205, 0.00800608191639185, 0.011954380199313164, 0.04179584980010986, -0.0048378631472587585, 0.01946093700826168, -0.03257175162434578, 0.006842804607003927, 0.008211365900933743, -0.01336399745196104, 0.03613000735640526, 0.007663941476494074, 0.028438696637749672, 0.003115186933428049, -0.009716782718896866, -0.018776657059788704, -0.006011404097080231, 0.02846606820821762, -0.01045580580830574, 0.007657098583877087, 0.0002409950247965753, -0.013316097669303417, 0.002873977879062295, -0.007397071924060583, 0.02674168162047863, 0.02999885566532612, -0.003448773641139269, 0.006404865067452192, -0.008108723908662796, -0.0030467587057501078, 0.037909138947725296, 0.0174765232950449, 0.017257554456591606, 0.01918722502887249, 0.003701957408338785, -0.016929099336266518, -0.020870555192232132, 0.004287017043679953, 0.0047968062572181225, 0.015546852722764015, 0.01077057421207428, -0.004779699258506298, 0.011324841529130936, 0.009032501839101315, -0.00610378198325634, -0.014287776313722134, -0.01889982633292675, 0.009846796281635761, 0.013117657043039799, 0.0027148828376084566, -0.05003459006547928, -0.013877208344638348, 0.014780458062887192, 0.0017389277927577496, 0.009689411148428917, 0.013145027682185173, 0.01251548994332552, -0.018817713484168053, 0.007759740576148033, 0.003361527808010578, 0.012289677746593952, 0.0014412658056244254, -0.008970917202532291, 0.022293858230113983, -0.009141987189650536, -0.003968826960772276, 0.04773540422320366, -0.03369396924972534, -0.0065690926276147366, 0.009066715836524963, 0.006281694862991571, -0.010818473994731903, -0.011201671324670315, -0.012234934605658054, -0.02124006673693657, 0.011174299754202366, 0.0009237786289304495, -0.015478424727916718, -0.02208857424557209, -0.0041159470565617085, 0.024045616388320923, -0.029013492166996002, 0.007451814133673906, -0.013336626812815666, -0.02912297658622265, 0.016723815351724625, -0.004584679380059242, -0.0553172342479229, 0.01743546687066555, 0.010791103355586529, 0.0061208889819681644, 0.0031562435906380415, 0.006025089416652918, 0.016053220257163048, -0.02674168162047863, 0.003127161879092455, 0.0008566335891373456, 0.029752515256404877, 0.021390607580542564, -0.008088194765150547, -0.003477855585515499, 0.0059600831009447575, -0.01776392199099064, 0.005398972891271114, -0.011899637058377266, 0.03755331411957741, -0.011276941746473312, -0.007308115251362324, -0.0007197775412350893, -0.004057783167809248, -0.02556471899151802, -0.11802469938993454, -0.01572476513683796, -0.01248127594590187, -0.0034231131430715322, -0.02415510080754757, 0.02069264091551304, -0.005388708785176277, 0.03517201542854309, -0.01827028952538967, 0.018995625898241997, -0.0024223527871072292, -0.026892222464084625, 0.013603495433926582, 0.0013026989763602614, 0.03284546360373497, 0.012994485907256603, 0.005395551677793264, -0.011673824861645699, -0.006237216293811798, 0.01972096413373947, -0.009135144762694836, 0.016710128635168076, -0.010072608478367329, -0.001703003072179854, 0.0012428243644535542, -0.02759018912911415, -0.028602924197912216, 0.01710701175034046, 0.0035719440784305334, 0.0026310584507882595, 0.013801936991512775, -0.0022752326913177967, 0.012419690378010273, -0.0021606155205518007, 0.0009930620435625315, 0.006521192844957113, -0.022334914654493332, -0.007540770806372166, 0.03437824919819832, -0.02675536647439003, 0.0028756887186318636, 0.015519481152296066, 0.025687888264656067, -0.011516440659761429, -0.013795094564557076, -0.008108723908662796, -0.007013875059783459, 0.06043565273284912, 0.03131267428398132, -0.029889371246099472, 0.00827979389578104, -0.016121648252010345, -0.023771904408931732, -0.015492109581828117, 0.02622162736952305, -0.024059301242232323, 0.006897547282278538, 0.013227141462266445, -0.02964303083717823, -0.0232655368745327, -0.003701957408338785, -0.010784259997308254, 0.00486865546554327, -0.014794143848121166, 0.022321229800581932, -0.00946359895169735, -0.01572476513683796, 0.002903059823438525, -0.002063105581328273, -0.017024898901581764, -0.0232655368745327, -0.014096178114414215, -0.002528416458517313, -0.009244629181921482, -0.012255463749170303, 0.004861812572926283, -0.010811631567776203, -0.028849264606833458, 0.004030412063002586, -0.0024616990704089403, -0.03224329650402069, -0.015177341178059578, 0.020733699202537537, -0.014369890093803406, 0.02263599820435047, 0.01018209382891655, 0.008526134304702282, -0.00786238256841898, 0.018557686358690262, -0.03697851672768593, -0.007061774376779795, 0.02586580254137516, 0.005724006332457066, -0.0290956050157547, -0.010496862232685089, 0.019009310752153397, 0.017627064138650894, -0.005833491217344999, -0.00027863046852871776, -0.004974719136953354, -0.034925676882267, -0.025988971814513206, -0.06924918293952942, 0.0006060159066691995, 0.00258315890096128, -0.028876636177301407, -0.015628965571522713, -0.015861621126532555, -0.011858580633997917, 0.0031973004806786776, -0.01628587581217289, 0.004964455030858517, -0.04127579927444458, 0.024565668776631355, -0.007800797466188669, -0.018721913918852806, -0.028137613087892532, -0.024319328367710114, 0.010421591810882092, -0.008409807458519936, 0.005621363874524832, 0.004625735804438591, 0.00771184079349041, 0.0174765232950449, 0.00523132411763072, 0.004663371481001377, -0.015806879848241806, -0.010503705590963364, -0.015355253592133522, 0.02208857424557209, 0.0023419498465955257, -0.008957231417298317, 0.007684469688683748, -0.030409423634409904, 0.008909331634640694, 0.05299067869782448, -0.014301462098956108, -0.004858391359448433, -0.02416878566145897, 0.03522675856947899, 0.02241702750325203, 0.006599884945899248, -0.017339667305350304, -0.0008745960076339543, 0.02824709750711918, -0.008382435888051987, -0.017804978415369987, -0.011995436623692513, -0.007882910780608654, 0.009805738925933838, 0.029424060136079788, 0.001185515895485878, 0.005303173791617155, 0.012713931500911713, -0.01250864751636982, -0.02412772923707962, -0.016723815351724625, 0.016367988660931587, -0.0203778725117445, 0.002016916638240218, -0.007226001936942339, -0.0012530885869637132, 0.04237064719200134, -0.0002362905943300575, 0.026016343384981155, -0.008943545632064342, 0.030628394335508347, 0.00048156239790841937, -0.025400491431355476, 0.021281123161315918, -0.016381675377488136, -0.036157380789518356, -0.01033947803080082, 0.013151871040463448, 0.035938411951065063, 0.03454247862100601, 0.012871315702795982, 0.005142367910593748, -0.002391560235992074, -0.02529100701212883, -0.02240334264934063, 0.018242917954921722, 0.00030920925200916827, -0.020131532102823257, -0.0013454664731398225, 0.024661468341946602, 0.023156050592660904, 0.01335715502500534, -0.0016961601795628667, 0.005282645113766193, -0.009415699169039726, 0.03544572740793228, -0.008950388990342617, 0.009066715836524963, -0.014205662533640862, 0.021144267171621323, 0.023087622597813606, 0.03295494616031647, 0.0026173728983849287, -0.010065766051411629, 0.050144072622060776, 0.01408249232918024, -0.014452003873884678, 0.015560537576675415, -0.02098003961145878, -0.010086294263601303, -0.003736171405762434, -0.004464929923415184, -0.013788251206278801, -0.010866373777389526, -0.008423492312431335, 0.001430146163329482, 0.018106061965227127, 0.019693592563271523, 0.0189408827573061, 0.0159437358379364, -0.0001018936382024549, 0.008512449450790882, -0.007663941476494074, -0.014602545648813248, -0.01805131882429123, 0.03306443244218826, 0.01263866014778614, -0.002477095229551196, 0.005286066792905331, 0.007698155473917723, 0.03024519607424736, 0.0203094445168972, 0.007547613698989153, 0.004256224725395441, -0.010722675360739231, -0.004581257700920105, 0.014150920324027538, -0.004061204381287098, -0.015013113617897034, -0.0025763160083442926, -0.024333013221621513, 0.01468465942889452, 0.021376922726631165, 0.027781786397099495, -0.021896975114941597, 0.057151105254888535, 0.015628965571522713, -0.02236228622496128, 0.02819235622882843, 0.004191217944025993, 0.04907659441232681, 0.010223150253295898, 0.002222200855612755, 0.009326742962002754, -0.0378817655146122, 0.027193306013941765, -0.006497242953628302, 0.03279072046279907, -0.027206990867853165, -0.011653296649456024, 0.01836608722805977, 0.004892605356872082, 0.011578025296330452, 0.002497623674571514, -0.00414331816136837, 0.00888196099549532, -0.0037840709555894136, 0.027248049154877663, 0.0043178098276257515, -0.027781786397099495, -0.0030775514896959066, 0.013624024577438831, 0.003178482875227928, -0.012919215485453606, -0.019707277417182922, 0.0203915573656559, -0.016409045085310936, -0.04108420014381409, -0.004177532158792019, 0.0203778725117445, -0.004892605356872082, -0.002328264294192195, 0.010264206677675247, 0.0013865233631804585, 0.01309028547257185, 0.017846034839749336, 0.007061774376779795, -0.02882189303636551, -0.0290682353079319, 0.03114844672381878, -0.006264587864279747, -0.012741302140057087, -0.015888992697000504, -0.023087622597813606]} +{"id": "test:10000073", "text": "\"Athlone Community Radio will be broadcasting live for 4 hours from Golden Island Shopping Centre in Athlone, the heart of Ireland, to Celebrate World Radio Day 2019. Our live broadcast will begin at 12pm and finish at 4pm. The Theme of World Radio Day is \u201cDialogue, Tolerance and Peace\u201d.\\nThis hour will feature local Musicians from Athlone. We will discuss how music plays a key role in promoting Tolerance , Dialogue and Peace. This programme will be presented by presenter/Sound Engineer with the station and also a musician Cathal McCormack.\\nAthlone Today is our daily current affairs programme. We will be delving into the World Radio Theme 2019 of tolerance, dialogue and peace in our community. We will feature an interview with renowned expert in Media Literacy Irena Djak Cvetkovic on access to media, creation of Media and the critical evaluation of media and how media can play a part in this theme. With over 20 years of experience in training, Deridre Hunt shares her unique perspective into the world of training and radio. Nikki Dube who presented radio in South Africa to over 1 million listeners gives us an insight into the importance of Radio as tool for Community Development. Presenter and Producer of the Arts Programme,Philomena Barry, chats about Radio in Gambia. Writer Aidan Shortall has written a special poem entitled \\\"Dialogue, Tolerance and Peace\u201d to mark the theme.\"", "vector_field": [0.0008850814774632454, 2.845055678335484e-05, 0.0028798929415643215, -0.029971156269311905, -0.0016620614333078265, 0.0025809246581047773, -0.03018735907971859, 0.0030673816800117493, -0.02110682986676693, -0.024593103677034378, -0.003445737063884735, 0.014309943653643131, -0.021052777767181396, -0.02579573355615139, -0.012877598404884338, 0.025579530745744705, 0.009621039032936096, -0.0017059777164831758, -0.007080651819705963, -0.021052777767181396, -0.014553172513842583, 0.017485426738858223, -0.0026772026903927326, 0.02918742038309574, -0.019120462238788605, -0.0010075402678921819, 0.02183651365339756, -0.02039065584540367, 0.014661273919045925, -0.005891534965485334, 0.011938465759158134, -0.01142498292028904, 0.0005375518812797964, 0.015647700056433678, -0.006536765955388546, 0.0036450494080781937, -0.0015091990353539586, -0.014850451610982418, 0.028349634259939194, -0.010729079134762287, 0.0026147065218538046, 0.02274186536669731, 0.01276274025440216, -0.007952220737934113, -0.041862327605485916, 0.027065927162766457, 0.015080166980624199, -0.024417439475655556, 0.012033054605126381, 0.021350057795643806, 0.01391807571053505, 0.024957947432994843, -0.01414779108017683, 0.0067360782995820045, 0.010438556782901287, 0.0002903117856476456, -0.010654759593307972, 0.029727928340435028, 0.025322789326310158, -0.0075265709310770035, -0.02156626060605049, -0.011891171336174011, -0.014499121345579624, 0.0023968142922967672, -0.02141762152314186, 0.0026180846616625786, -0.015890927985310555, 0.015363933518528938, -0.0013614039635285735, 0.014174817129969597, 0.008188692852854729, 0.004259876906871796, 0.021728413179516792, -0.018850209191441536, 0.008168423548340797, -0.009121068753302097, 0.021039266139268875, 0.012620856985449791, 0.036403197795152664, -0.0008876151405274868, 0.017188146710395813, -0.03751124069094658, -0.030457613989710808, 0.024849845096468925, 0.014863964170217514, 0.010188572108745575, -0.00626989034935832, 0.003454182529821992, 0.003932194318622351, -0.0020167697221040726, 0.0036416712682694197, 0.0019154244801029563, 0.04115966707468033, 0.03005223348736763, 0.002648488152772188, 0.008661637082695961, 0.0017667848151177168, 0.016228746622800827, 0.0034896533470600843, -0.02948470041155815, -0.001564094447530806, -0.008310306817293167, -0.002189056482166052, -0.012830303981900215, -0.012141156010329723, 0.010418287478387356, -0.0023799233604222536, 0.03972732275724411, 0.010681784711778164, 0.012560050003230572, -0.027660485357046127, 0.02826855704188347, 0.027322668582201004, -0.047861963510513306, -0.009073774330317974, -0.00849948450922966, -0.010749348439276218, -0.010168302804231644, -0.0004573202459141612, -0.014012664556503296, 0.009654819965362549, 0.02316075935959816, -0.022890504449605942, -0.009857511147856712, 0.022174332290887833, 0.007377931382507086, -0.028214506804943085, -0.0031552142463624477, -0.012472216971218586, -0.0005696445587091148, 0.019390717148780823, -0.015431497246026993, 0.015485548414289951, -0.0034828970674425364, -0.01816106215119362, -0.008884596638381481, -0.007425225805491209, 0.03756529092788696, -0.019242078065872192, -0.00036737637128680944, 0.00557060819119215, 0.01744488812983036, -0.02197164110839367, -0.01290462352335453, 0.005611146334558725, 0.041402895003557205, -0.007215778809040785, -0.01296543050557375, 0.010641247034072876, 0.021052777767181396, 0.03091704472899437, 0.017498940229415894, -0.00937105342745781, -0.0014965309528633952, 0.009681846015155315, 0.023930981755256653, -0.004803762771189213, 0.0017051331233233213, 0.0018512391252443194, 0.0169854573905468, 0.013499181717634201, -0.012080349028110504, -0.003026843536645174, 0.0199582502245903, 0.013337030075490475, 0.014728836715221405, 0.02594437450170517, 0.03302502632141113, -0.013465399853885174, -0.018958309665322304, -0.010411531664431095, 0.019823122769594193, -0.010620977729558945, 0.035835664719343185, 0.017525965347886086, 0.0339709147810936, -0.006874583195894957, 0.023403987288475037, -0.002113047521561384, -0.02009337767958641, -0.03764636814594269, 0.015363933518528938, 0.00916836317628622, 0.013066776096820831, -0.01122229266911745, 0.023768829181790352, -0.00864136777818203, 0.012837059795856476, 0.001053145620971918, -0.011816851794719696, 0.026620008051395416, 0.02870096266269684, 0.0049388897605240345, -0.010789887048304081, -0.6365019679069519, 0.011249318718910217, 0.015863902866840363, -0.014904501847922802, 0.0049456460401415825, -0.00015339019591920078, 0.002537008374929428, -0.004047052003443241, -0.031592678278684616, 0.009330515749752522, 0.008783251978456974, 0.019269103184342384, 0.014863964170217514, -0.021579774096608162, -0.02006635069847107, -0.01003993209451437, -0.009891292080283165, -0.007269829511642456, -0.0049388897605240345, 0.020701447501778603, -0.008830546401441097, 0.018431315198540688, 0.0023934361524879932, 0.013864024542272091, -0.019890686497092247, -0.004607828799635172, -0.005959098227322102, -0.023633703589439392, -0.01683681830763817, -0.0017127339961007237, -0.010289916768670082, -0.01099257729947567, 0.009114312939345837, 0.01933666691184044, 0.0492132343351841, 0.003506544278934598, -0.013674846850335598, 0.020917652174830437, 0.01178982574492693, 0.008506241254508495, 0.00711443368345499, 0.014012664556503296, 0.029673878103494644, 0.002729564206674695, 0.015228806994855404, 0.007999515160918236, 0.02753887139260769, 0.006063821725547314, 0.012141156010329723, -0.02899824269115925, 0.004729443229734898, -0.012289796024560928, 0.004878082778304815, 0.017188146710395813, 0.02182300202548504, 0.01326946634799242, 0.04232176020741463, -0.006367857567965984, 0.009060261771082878, 0.03226831555366516, 0.00396259780973196, 0.015417984686791897, -0.0057023572735488415, 0.026620008051395416, -0.0049321334809064865, 0.002678891643881798, 0.006864449009299278, 0.009039992466568947, 0.019390717148780823, 0.004094346426427364, 0.011837120167911053, 0.04905107989907265, -0.021012241020798683, -0.03021438606083393, 0.0177286546677351, 0.010519633069634438, 0.011323638260364532, 0.004401760175824165, -0.0011299990583211184, 0.04129479452967644, 0.0020201478619128466, -0.007573865354061127, -0.00937105342745781, 0.016607102006673813, 0.04994291812181473, -0.0016781077720224857, 0.012783009558916092, 0.004888217430561781, 0.03972732275724411, 0.004570669028908014, 0.02049875818192959, 0.04615936428308487, 0.015539598651230335, -0.00483754463493824, -0.0007077273912727833, 0.007675210479646921, -0.007215778809040785, -0.012587075121700764, 0.010134520940482616, -0.021877052262425423, -0.007668454200029373, -0.006773238070309162, -0.0018309701699763536, -0.004384869243949652, -0.0188907478004694, -0.01936369203031063, -0.02916039526462555, -0.013674846850335598, 0.0422406829893589, -0.016512513160705566, -0.011249318718910217, 0.003982866648584604, -0.026133552193641663, -0.01056692749261856, -0.008404895663261414, -0.01370187196880579, 0.009810216724872589, 0.0017870538868010044, 0.007884657010436058, -0.027160516008734703, 0.018552929162979126, -0.020877113565802574, 0.020160939544439316, -0.017971884459257126, -0.009060261771082878, 0.008810277096927166, -0.007749530486762524, -7.75924272602424e-05, -9.469442738918588e-05, -0.010722323320806026, -0.00939132273197174, -0.0075400834903120995, 0.01143849641084671, -0.002244796371087432, 0.046564746648073196, 0.017039507627487183, 0.01377619244158268, -0.014728836715221405, 0.011120948009192944, -0.033187177032232285, -0.008202205412089825, -0.011418227106332779, -0.022404048591852188, -0.013708628714084625, -0.011796582490205765, -0.028241531923413277, -0.03034951165318489, -0.017053021118044853, -0.006320563144981861, -0.006063821725547314, 0.0007427759119309485, -0.003922059666365385, -0.0169854573905468, 0.006320563144981861, 0.014012664556503296, -0.008256256580352783, -0.025011997669935226, -0.037754468619823456, -0.03713288530707359, -0.016742229461669922, 0.0049388897605240345, 0.00939132273197174, -0.019133975729346275, -0.00945212971419096, 0.0007465764065273106, 0.011593892239034176, -0.02430933713912964, 0.0018039447022601962, -0.01464776135981083, -0.0331331267952919, 0.009952099993824959, 0.015134218148887157, -0.008458946831524372, 0.0012870841892436147, 0.011195267550647259, -0.0008525665616616607, -0.008837302215397358, -0.008100860752165318, 0.017255710437893867, -0.0028325985185801983, -0.020120402798056602, 0.014053202234208584, -0.016809791326522827, -0.03343040496110916, 0.0025775465182960033, -0.0036382931284606457, 0.011404714547097683, 0.01758001558482647, -0.014134278520941734, 0.011546597816050053, -0.00017893764015752822, 0.028971217572689056, -0.013823486864566803, -0.00319068506360054, -0.0031602815724909306, 0.013836999423801899, 0.009033236652612686, 0.011593892239034176, 0.01787729561328888, 0.02097170241177082, 0.018688056617975235, 0.005925316829234362, 0.020323092117905617, 0.012884354218840599, -0.00319068506360054, -0.005665197037160397, 0.017931345850229263, -0.018809670582413673, 0.04856462404131889, 0.008310306817293167, 0.001485551823861897, -0.025552505627274513, -0.007776555605232716, -0.0030640035402029753, -0.009681846015155315, 0.02889014035463333, -0.022768890485167503, -0.004381491336971521, -0.01890425942838192, -0.0030488017946481705, -0.00556723028421402, 0.011701993644237518, 0.009918318130075932, 0.006644867826253176, -0.023714778944849968, 0.003553838701918721, -0.025620069354772568, 0.027755074203014374, 0.008519753813743591, -0.04070023447275162, -0.007884657010436058, -0.0023731670808047056, 0.0046179634518921375, 0.024417439475655556, 0.005077395122498274, -0.026133552193641663, 0.0019069790141656995, -0.017971884459257126, 0.008871084079146385, 0.007215778809040785, 0.04183530434966087, 0.009296733886003494, 0.024984972551465034, -0.008600830100476742, 0.0011908062733709812, 0.0003445737238507718, 0.01860698126256466, 0.004165288060903549, -0.012708689086139202, 0.007817094214260578, -0.010925013571977615, -0.00229040184058249, -0.04905107989907265, -0.01921505108475685, 0.012053323909640312, 0.00446932390332222, -0.0055402047000825405, -0.011262831278145313, 0.005367917940020561, 0.014634247869253159, 0.017215173691511154, 0.013080288656055927, 0.01744488812983036, -0.0061212508007884026, -0.026011936366558075, 0.017269223928451538, 0.007898169569671154, -0.005867887753993273, -0.004972671624273062, 0.0050638820976018906, 0.005209143739193678, -0.01115472987294197, 0.017377324402332306, -0.026471368968486786, 0.0037261254619807005, 0.007067139260470867, -0.018944798037409782, 0.007810337468981743, 0.01201278530061245, 0.02918742038309574, -0.033322304487228394, -0.018431315198540688, 0.0006908365176059306, 0.02316075935959816, -0.00945212971419096, -0.02052578330039978, -0.03413306549191475, 0.005850996822118759, -0.019728533923625946, 0.0016840195748955011, -0.0035403259098529816, 0.019228564575314522, -0.013485669158399105, -0.0014002530369907618, 0.010019662790000439, -0.024822819977998734, 0.000171125604538247, 0.01656656339764595, 0.0012152979616075754, -0.00407407758757472, 0.002982927253469825, 0.005452372133731842, 0.001176449004560709, -0.033646609634160995, 0.0366194024682045, 0.0032210885547101498, -0.008634611964225769, -0.026552444323897362, -0.02651190757751465, 0.0009104178170673549, -0.00047167748562060297, 0.016161182895302773, -0.0005785122630186379, -0.013755923137068748, 0.020998727530241013, -0.00975616555660963, 0.008377870544791222, 0.0003171260468661785, 0.011100678704679012, -0.02875501476228237, -0.011174998246133327, -0.020620372146368027, -0.0019981898367404938, -0.0055334484204649925, 0.1212899461388588, 0.0075063020922243595, 0.034079015254974365, 0.021795976907014847, -0.0018816427327692509, -0.007276586256921291, -0.04094346612691879, -0.005465885158628225, 0.0039051687344908714, 0.01964745856821537, -0.0006773238419555128, -0.00483754463493824, 0.018444828689098358, 0.015647700056433678, -0.004259876906871796, 0.009634551592171192, 0.010702054016292095, 0.011654699221253395, 0.0010396329453215003, 0.031187299638986588, 0.014377507381141186, -0.009073774330317974, -0.00997912511229515, 0.013397837057709694, 0.0003012908564414829, -0.02886311523616314, 0.03548433631658554, 0.006236108485609293, 0.006952281575649977, -0.018363751471042633, 0.02097170241177082, 0.036295097321271896, 0.006334075704216957, -0.03151160478591919, 0.004915242549031973, 0.019877173006534576, 0.008411652408540249, 0.00747927650809288, 0.0049118646420538425, -0.027619948610663414, 0.01626928523182869, 0.01875562034547329, 0.003513300558552146, -0.028484759852290154, 0.0064556896686553955, -0.007837362587451935, -0.002903540153056383, 0.040511056780815125, -0.0016046324744820595, -0.021512210369110107, 0.02445797808468342, 0.011384445242583752, -0.019417742267251015, -0.012465461157262325, 0.011701993644237518, 0.005705735180526972, -0.00022633763728663325, -0.016201721504330635, 0.011526328511536121, -0.005334136076271534, -0.017350299283862114, -0.00410785898566246, 0.012323577888309956, -0.008884596638381481, -0.017363812774419785, 0.0009171741548925638, -0.0004682993167079985, 0.02272835187613964, -0.009046749211847782, 0.013972125947475433, -0.006881339941173792, -0.01187090203166008, -0.010296673513948917, 0.029403623193502426, 0.03761934116482735, 0.009195388294756413, 0.028241531923413277, 0.023782342672348022, -0.020431194454431534, 0.008830546401441097, -0.005057125817984343, 0.0035369477700442076, -0.013114070519804955, -0.030998121947050095, -0.0303224865347147, 0.030673816800117493, 0.014391019940376282, 0.005584121216088533, -0.022201357409358025, 0.02622814103960991, 0.000710683292709291, 0.008364357985556126, 0.05156444385647774, -0.02667406015098095, 0.0020691314712166786, 0.02255268767476082, 0.017552990466356277, 0.006239486858248711, -0.004432163666933775, 0.0005468418821692467, -0.015715263783931732, -0.0119317090138793, -0.027673998847603798, -0.018944798037409782, -0.005928694736212492, 0.004928755573928356, 0.0047868723049759865, -0.003587620332837105, -0.005560474004596472, -0.024363389238715172, 0.02155274711549282, -0.020255528390407562, 0.008384627290070057, -0.0043443311005830765, 0.012789765372872353, 0.022971581667661667, 0.01181009504944086, 0.02536332793533802, -0.0024018813855946064, -0.00604355288669467, -0.00714145926758647, -0.037889596074819565, 0.00043409530189819634, 0.004293658770620823, -0.0026839589700102806, 0.030160333961248398, -0.017971884459257126, 0.0008529888582415879, 0.019147489219903946, 0.0014374129241332412, 0.01933666691184044, 0.012614100240170956, 0.004371356684714556, -0.006080712657421827, 0.005925316829234362, -0.02024201676249504, 0.019296128302812576, -0.008344088681042194, -0.009094043634831905, 0.0043037934228777885, -0.024701206013560295, 0.004459189251065254, -0.023782342672348022, -0.007499545346945524, 0.014269405975937843, -0.027728049084544182, -0.014701811596751213, -0.006530009675770998, -0.001633347012102604, 0.01655305176973343, -0.012242501601576805, -0.0069320122711360455, -0.008175180293619633, 0.016755741089582443, -0.020890625193715096, -0.012384384870529175, -0.027065927162766457, -0.030835969373583794, 0.008100860752165318, 0.032187238335609436, 0.03251154348254204, -0.0002687759406398982, 0.013100557029247284, 0.02448500320315361, 0.00424974225461483, -0.007999515160918236, 0.012668151408433914, -0.0043375748209655285, 0.002106291241943836, -0.00835760124027729, 0.011756043881177902, 0.018539417535066605, -0.005070638842880726, -0.02445797808468342, -0.008479216136038303, 0.023782342672348022, 0.010702054016292095, -0.01128309965133667, -0.000971224915701896, -0.015580137260258198, -0.0177286546677351, -0.011823607608675957, -0.007202266249805689, -0.0030471126083284616, -0.019971761852502823, 0.011654699221253395, 0.027444282546639442, -0.007188753690570593, 0.03380876034498215, -0.0007520659128203988, 0.013303248211741447, -0.0008065389702096581, 0.023376962170004845, 0.01129661314189434, -0.015066654421389103, -0.013830242678523064, -0.003979488741606474, -0.038484156131744385, 0.007431982085108757, 0.025133611634373665, 0.015755802392959595, -0.014499121345579624, -0.007445494644343853, 0.0031079198233783245, -0.011479034088551998, 0.02097170241177082, -0.01595849171280861, -0.0024593104608356953, -0.020444706082344055, -0.016093619167804718, -0.0016671287594363093, 0.0035200570710003376, -0.02889014035463333, -0.004607828799635172, 0.018512392416596413, 0.008979185484349728, -0.016336847096681595, 0.008776495233178139, 0.021228443831205368, -0.03929491713643074, -0.006763103883713484, -0.011695236898958683, 0.02156626060605049, -0.009073774330317974, 0.014134278520941734, 0.015931466594338417, -0.00953320600092411, -0.012512755580246449, 0.013931588269770145, 0.0029254984110593796, -0.021795976907014847, 0.038916561752557755, 0.013512694276869297, -0.029809003695845604, -0.010330455377697945, -0.005320623517036438, 0.02595788612961769, -0.0005181274027563632, -0.018417803570628166, 0.008371114730834961, 0.00872244406491518, -0.0035234352108091116, -0.012614100240170956, 0.010688541457057, -0.017539476975798607, 0.018958309665322304, -0.016796279698610306, -0.018728595227003098, -0.020741986110806465, 0.01408022828400135, -0.015539598651230335, 0.006614463869482279, -0.021431133151054382, 0.016661152243614197, 0.0020961568225175142, -0.02359316498041153, 0.01833672635257244, -0.01115472987294197, 0.0008407429559156299, 0.013809974305331707, 0.009708871133625507, 0.015499060973525047, -0.03310610353946686, -0.0031264997087419033, 0.010377749800682068, 0.0016544605605304241, 0.005111176520586014, -0.012816790491342545, 0.02536332793533802, 0.029836030676960945, -0.012783009558916092, -0.016188208013772964, 0.012938405387103558, 0.005242925602942705, -0.009330515749752522, -0.009262952022254467, -0.020404169335961342, -0.008621099404990673, -0.025728169828653336, -0.02493092231452465, -0.004769981373101473, -0.0003690654702950269, -0.006307050120085478, -0.00688809622079134, -0.0007866921951062977, -0.03126837685704231, -0.006577304098755121, 0.0006722565740346909, 0.01156686618924141, -0.014809913001954556, -0.02082306332886219, -0.0022059474140405655, -0.024674180895090103, 0.009404835291206837, -0.005445615854114294, 0.012107374146580696, 0.01195197831839323, 0.022985093295574188, -0.014728836715221405, -0.004401760175824165, 0.0010979064973071218, 0.013539720326662064, 0.004172044340521097, -0.00711443368345499, -0.012573562562465668, -0.019120462238788605, 0.02329588495194912, -0.017917832359671593, -0.028349634259939194, -0.018079984933137894, 0.002741387812420726, 0.004628098104149103, -0.031889960169792175, -0.0007638895185664296, 0.00545912841334939, 0.004050430376082659, 0.001934004365466535, -0.005442237947136164, -0.043618977069854736, 0.027728049084544182, -0.02567411959171295, -0.019552869722247124, 0.018363751471042633, -0.0009594013099558651, 0.04091643914580345, -0.0039017905946820974, 0.013012724928557873, -0.012330333702266216, -0.025295764207839966, 0.019579894840717316, 0.007938708178699017, 0.02270132675766945, -0.021160880103707314, -0.01156686618924141, -0.010330455377697945, 0.012674907222390175, -0.007945463992655277, 0.02271484024822712, -0.01670169085264206, -0.0028596241027116776, 0.02479579485952854, 0.031862933188676834, 0.021620310842990875, 0.006844179704785347, 0.012668151408433914, 0.003577485913410783, 0.003935572225600481, -0.00776304304599762, -0.03943004459142685, -0.021323032677173615, 0.00857380498200655, 0.03707883507013321, -0.005033478606492281, -0.002347830682992935, 0.0006198948831297457, -0.01371538545936346, -0.027498332783579826, -0.018134037032723427, 0.013526207767426968, 0.022958068177103996, 0.01428291853517294, 0.027755074203014374, 0.009864266961812973, 0.03148457780480385, -0.02666054666042328, -0.008229231461882591, -0.005124689545482397, 0.0022583091631531715, -0.0011139527196064591, -0.02813342958688736, -0.0009881157893687487, 0.001942449831403792, -0.0342952199280262, 0.016066594049334526, 0.0483754463493824, -0.0012853951193392277, -0.002425528597086668, 0.018201598897576332, -0.025701144710183144, 0.007580621633678675, -0.004776737652719021, -0.00714145926758647, -0.009965612553060055, 0.011411470361053944, 0.015431497246026993, -0.03329528123140335, 0.019161000847816467, -7.479487976524979e-05, 0.0032954083289951086, -0.014742350205779076, 0.007283342536538839, 0.003584242193028331, 0.01598551869392395, 0.00159534253180027, -0.00186981912702322, -0.009972368367016315, 0.01268166396766901, -0.051105011254549026, 0.01570175029337406, -0.004232851788401604, -0.017796218395233154, 0.002781925955787301, 0.012411409988999367, -0.011262831278145313, 0.010127764195203781, -0.03915978968143463, -0.03367363661527634, 0.003614645916968584, -0.01156686618924141, -0.03383578732609749, 0.009729140438139439, -0.005594255402684212, -0.0015320017701014876, -0.022485123947262764, 0.001697532250545919, -0.0196339450776577, -0.03718693554401398, -0.04297037050127983, 0.012451947666704655, 0.011235805228352547, -0.04326764866709709, -6.861915608169511e-05, -0.015728777274489403, -0.013282978907227516, -0.014012664556503296, -0.017809731885790825, -0.0018411045894026756, -0.01833672635257244, -0.031160274520516396, -0.010438556782901287, -0.01599903032183647, -0.016161182895302773, 0.007918438874185085, -0.003969354089349508, -0.021363569423556328, 0.007823850028216839, 0.2114466428756714, -0.03180888295173645, -0.01774216815829277, -0.012188450433313847, -0.035241108387708664, 0.007195509970188141, 0.031889960169792175, 0.005057125817984343, 0.0007507990812882781, 0.02094467729330063, -0.023106707260012627, -0.022187843918800354, -0.006175301503390074, -0.003621402196586132, -0.015566624701023102, -0.008404895663261414, -0.03883548453450203, -0.05029425024986267, -0.03632212430238724, -0.010607465170323849, 0.0075400834903120995, -0.01741786301136017, -0.014012664556503296, 0.02067442238330841, 0.00901972409337759, 0.02567411959171295, 0.01231682114303112, -0.008384627290070057, 0.03897061198949814, -0.005969232879579067, -0.023201296105980873, -0.006641489453613758, -0.005590877495706081, 0.007006332278251648, -0.00666513666510582, -0.0067495908588171005, 0.010283160954713821, -0.018147548660635948, 0.015174755826592445, -0.0023849906865507364, -0.026309216395020485, -0.01290462352335453, -0.008229231461882591, 0.0012304997071623802, 0.00916836317628622, 0.03178185597062111, 0.005871265660971403, 0.0026180846616625786, 0.008567048236727715, 0.005019966047257185, -0.009783190675079823, 0.0069184997119009495, 0.026592982932925224, 0.03148457780480385, 0.022458098828792572, -0.010870962403714657, 0.017931345850229263, 0.012120886705815792, -0.00997912511229515, 0.0013960303040221334, -0.02564709447324276, 0.026174088940024376, -0.017634065821766853, 0.023998545482754707, -0.017823243513703346, 0.017404351383447647, -0.007641428615897894, 0.007722504902631044, 0.034538447856903076, -0.0014010975137352943, 0.023566139861941338, 0.006472580600529909, -0.011850633658468723, -0.004783493932336569, -0.023971520364284515, -0.03221426531672478, 0.025430891662836075, 0.005881400313228369, 0.006019905675202608, 0.03913276270031929, -0.021296007558703423, 0.015080166980624199, 0.001537913572974503, 0.005803702399134636, -0.00689147412776947, -0.020255528390407562, 0.0260930135846138, -0.004888217430561781, -0.02199866622686386, 0.01625577174127102, -0.02478228136897087, -0.031106222420930862, -0.023187784478068352, 0.003743016393855214, -0.002231283811852336, -0.009925073944032192, 0.029430648311972618, -0.0039017905946820974, -0.024133672937750816, -0.01253978069871664, -0.00922917015850544, 0.03191698342561722, 0.0032970975153148174, -0.0005413523176684976, -0.015782827511429787, 0.012114130891859531, 0.014566685073077679, -0.0020269041415303946, 0.01127634383738041, -0.018417803570628166, -0.0020894003100693226, -0.0354573093354702, 0.003577485913410783, -0.012911379337310791, -0.009904805570840836, 0.028403684496879578, -0.01966097019612789, -0.02739023230969906, 0.02490389533340931, -0.0001272093504667282, -0.005465885158628225, -0.0334574319422245, -0.019390717148780823, 0.006280025001615286, -0.04415948688983917, -0.022106768563389778, -0.009864266961812973, 0.013850511983036995, -0.030241411179304123, -0.010897988453507423, 0.0004093924071639776, -0.001989744370803237, 0.0020860221702605486, -0.007958977483212948, -0.022093255072832108, -0.0001532846363261342, 0.01076961774379015, -0.019890686497092247, -0.02141762152314186, -0.009053505025804043, 0.02872798964381218, -0.008371114730834961, -0.002771791536360979, -0.0031636597122997046, 0.016674665734171867, -0.036268074065446854, 0.03337635472416878, -0.0328088216483593, -0.0035301914904266596, 0.0014298120513558388, -0.0055266921408474445, -0.004678770434111357, 0.0034761407878249884, -0.028106404468417168, 0.013154608197510242, -0.004428785759955645, -0.001993122510612011, -0.010235866531729698, -0.005496288649737835, 0.02945767343044281, 0.00436122203245759, 0.007222535088658333, 0.02109331637620926, 0.0031805504113435745, -0.020985214039683342, -0.003018398070707917, -0.1726381927728653, 0.017525965347886086, 0.009141338057816029, -0.025147125124931335, 0.018431315198540688, 0.009012967348098755, 0.022039204835891724, 0.01378294825553894, -0.00472606485709548, 0.0011088855098932981, -0.002163720317184925, -0.009411592036485672, -0.025714658200740814, -0.009566987864673138, 0.017782706767320633, -0.005516557488590479, -0.0236066784709692, 0.027457796037197113, 0.012499242089688778, 0.007864388637244701, 0.0306467916816473, 0.013485669158399105, 0.006219217553734779, -0.020147427916526794, 0.011384445242583752, -0.004381491336971521, -0.009938586503267288, -0.007661697920411825, -0.0018360373796895146, -0.011715506203472614, 0.007952220737934113, 0.003013330977410078, 0.021431133151054382, 0.007107677403837442, 0.009688601829111576, 0.00446932390332222, -0.01568823866546154, -0.021350057795643806, -0.005476019345223904, 0.029673878103494644, 0.03588971868157387, 0.024552566930651665, -0.002486335812136531, 0.016215233132243156, -0.024403925985097885, 0.0177286546677351, 0.0188907478004694, -0.01668817736208439, 0.026187602430582047, 0.0019559625070542097, 0.0029440782964229584, 0.011715506203472614, -0.014120765961706638, -0.007580621633678675, -0.0033477700781077147, 0.007600890938192606, 0.0015818298561498523, 0.017255710437893867, 0.013735653832554817, -0.01760704070329666, -0.018796158954501152, -0.00995885580778122, 0.019701508805155754, -0.03264667093753815, 0.0010463893413543701, -0.002349519869312644, -0.02286347933113575, -0.003016709117218852, -0.015553111210465431, 0.02420123666524887, -0.021458158269524574, -0.019161000847816467, -0.0139991519972682, -0.028836090117692947, -0.010756105184555054, -0.0038950343150645494, -0.04178125038743019, 0.001151957199908793, 0.006739456672221422, 0.013303248211741447, -0.005661819130182266, 0.028079379349946976, 0.012945161201059818, -0.0008225853089243174, -0.012924892827868462, 0.0027042280416935682, 0.0006735233473591506, 0.028214506804943085, -0.020769011229276657, 0.006516497116535902, 0.022660788148641586, -0.016607102006673813, -0.00981697253882885, -0.01156686618924141, 0.023566139861941338, 0.03148457780480385, 0.004753090441226959, 0.017944859340786934, -0.005688844248652458, -0.007681966759264469, -0.009107556194067001, -0.008695418946444988, 0.000797671265900135, 0.01845834031701088, 0.031106222420930862, -0.02009337767958641, -1.8302838725503534e-05, 0.017620554193854332, 0.01716112159192562, 0.011938465759158134, -0.022985093295574188, 0.030484639108181, 0.009715627878904343, 0.028214506804943085, 0.027755074203014374, 0.003440669970586896, 0.040808338671922684, -0.03840307891368866, 0.013972125947475433, -0.0020877113565802574, 0.039592195302248, 0.015201781876385212, -0.026336241513490677, 0.020485244691371918, -0.020633885636925697, 0.014769375324249268, -0.07810337841510773, -0.009762922301888466, 0.017674604430794716, 0.017215173691511154, 0.0008411652524955571, 0.037727441638708115, -0.00030889175832271576, -0.0007410868420265615, -0.01127634383738041, 0.042564988136291504, -0.017769193276762962, -0.012120886705815792, 0.00542534701526165, 0.006732699926942587, 0.025255225598812103, -0.02564709447324276, -0.009411592036485672, -0.0324845165014267, -0.012296551838517189, -0.005469263065606356, 0.006124628707766533, -0.0222824327647686, -0.013796460814774036, -0.01464776135981083, -0.020877113565802574, -0.014485608786344528, -0.023201296105980873, 0.027890201658010483, -0.006334075704216957, 0.0007026601233519614, -0.015917954966425896, -0.002548831980675459, 0.007236048113554716, -0.038754407316446304, 0.007945463992655277, -0.0027329425793141127, -0.0281604565680027, 0.010452069342136383, 0.007742774207144976, -0.03748421370983124, 0.0009450440993532538, 0.04205150529742241, 0.00443892041221261, -0.012749227695167065, 0.019539356231689453, -0.0346195250749588, -0.02009337767958641, 0.004560534376651049, 0.004013270139694214, -0.0035403259098529816, -0.039213839918375015, -0.024282312020659447, -0.016958432272076607, -0.021039266139268875, 0.022904017940163612, 0.007134702987968922, 0.019012361764907837, -0.009398079477250576, -0.02214730717241764, 0.006371235474944115, 0.00922917015850544, -0.0002090244961436838, -0.006847558077424765, 0.016674665734171867, -0.008904865942895412, 0.009722383692860603, -0.025187663733959198, 0.005070638842880726, 0.036889657378196716, 0.012472216971218586, -0.002771791536360979, -0.007310367655009031, -0.013465399853885174, -0.005199009086936712, -0.014404532499611378, -0.016769254580140114, -0.03999757766723633, -0.012924892827868462, 0.0020758877508342266, 0.014215354807674885, -0.01583687774837017, -0.015309883281588554, -0.005077395122498274, -0.019323153421282768, 0.012560050003230572, 0.03488977625966072, -0.0034592498559504747, 0.013451887294650078, -0.005249681882560253, -0.020782524719834328, -0.019823122769594193, 0.034376293420791626, -0.006850935984402895, -0.006722565740346909, -0.011519571766257286, -0.0001910779537865892, 0.021323032677173615, 0.008519753813743591, -0.008283281698822975, -0.014634247869253159, -0.020566321909427643, -0.008371114730834961, -0.08258958905935287, 0.02126898057758808, 0.019674483686685562, -0.0013090423308312893, -0.003749772673472762, -0.019971761852502823, 0.009114312939345837, -0.001991433324292302, 0.0069049871526658535, -0.020728474482893944, -0.010884475894272327, 0.0014044756535440683, -0.013012724928557873, 0.013093801215291023, -0.013836999423801899, -0.012127643451094627, 0.014674786478281021, 0.010391262359917164, 0.0028579349163919687, -0.016634127125144005, -0.010087226517498493, 0.018715081736445427, 0.007756286766380072, 0.020552808418869972, 0.01683681830763817, -0.010303429327905178, -0.023498576134443283, 0.013796460814774036, 0.013546476140618324, 0.0077022360637784, 0.012992455624043941, -0.03324122726917267, 0.0021991911344230175, 0.017661092802882195, -0.019890686497092247, -0.01994473673403263, -0.010472338646650314, 0.021498696878552437, -0.02899824269115925, 0.02549845539033413, -0.020485244691371918, -0.01964745856821537, 0.013282978907227516, 0.0024069487117230892, -0.022458098828792572, -0.006057065445929766, -0.0075198146514594555, 0.014823426492512226, -0.0030927180778235197, -0.0047868723049759865, 0.024944433942437172, 0.007290098816156387, -0.018836695700883865, 0.01194522250443697, -0.01830970123410225, 0.010019662790000439, 0.010431800037622452, 0.004516618326306343, 0.010168302804231644, -0.017552990466356277, 0.033349331468343735, -0.00596585450693965, 0.012647882103919983, -0.003259937511757016, 0.01862049289047718, -0.003702478250488639, 0.0007203955319710076, -0.03697073087096214, -0.001937382621690631, -0.014661273919045925, -0.018809670582413673, 0.01274247094988823, 0.027160516008734703, 0.03529515862464905, 0.013445131480693817, -0.001145200920291245, -0.018107010051608086, 0.008168423548340797, -0.0084927286952734, 0.007127946242690086, 0.023822881281375885, 0.023039143532514572, -0.02433636225759983, 0.025606555864214897, 0.020877113565802574, 0.013958613388240337, -0.007729261182248592, -0.0018056337721645832, -0.020174453034996986, 0.01459371019154787, -0.00027806591242551804, 0.002371477894484997, 0.016012543812394142, 0.025147125124931335, -0.006536765955388546, 0.004033539444208145, 0.01580985262989998, -0.02464715577661991, -0.004959159065037966, 0.03737611323595047, 0.014053202234208584, 0.0026738245505839586, -0.0048071411438286304, -0.030403563752770424, -0.028673937544226646, 0.033781737089157104, -0.02448500320315361, -0.023282373324036598, 0.004810519516468048, -0.0037092347629368305, 0.0048138974234461784, 0.02695782668888569, 0.02082306332886219, -0.0005354405147954822, -0.01860698126256466, -0.00593207310885191, 0.03872738406062126, 0.013181633315980434, -0.01355323288589716, 0.02447148971259594, -0.004077455494552851, -0.014012664556503296, 0.016161182895302773, -0.029646852985024452, -0.003574107773602009, -0.02916039526462555, 0.012755983509123325, 0.0005139046697877347, 0.02214730717241764, -0.010161546058952808, 0.023795856162905693, -0.026944313198328018, -0.037457190454006195, -0.014634247869253159, 0.00857380498200655, 0.006979306694120169, 0.014755862765014172, 0.0419163778424263, -0.049456462264060974, 0.053564321249723434, 0.0007955598994158208, -0.009398079477250576, 0.02257971279323101, -0.005624659359455109, 0.021660849452018738, 0.0024289069697260857, 0.024809306487441063, -0.016593588516116142, -0.036862630397081375, -0.000514326966367662, -0.03734908625483513, -0.005705735180526972, -0.022809429094195366, -0.008425164967775345, -0.004060564562678337, -0.009499424137175083, 0.002175543922930956, -0.01976907253265381, -0.004479458089917898, 0.010958795435726643, -0.014782887883484364, 0.011668211780488491, -0.002591059310361743, -0.012891110964119434, -0.016917893663048744, 0.00785087514668703, -0.010026419535279274, -0.02899824269115925, -0.034538447856903076, -0.006225973833352327, -0.0089454036206007, -0.027295643463730812, -0.024579592049121857, -0.0017296249279752374, -0.021647335961461067, -0.0006612775032408535, 0.0015539599116891623, 0.03267369419336319, 0.022890504449605942, 0.01612064428627491, 0.024363389238715172, -0.03416009247303009, -0.024809306487441063, 0.005290220025926828, -0.003200819483026862, -0.003376484615728259, -0.02417421154677868, -0.025241713970899582]} +{"id": "test:10000074", "text": "\"On the 12th April 2017 Centrica Storage Ltd announced that injections at Rough, UK's biggest storage facility, will not take place until May 2018 at the earliest, following increasing concern about the condition of the majority of the wells. Should Rough continue to work intermittently over the next years, LNG players will want to reassess their regas position in NW Europe. They will be seeking winter access in the UK and summer access in continental Europe. Access to the IUK or BBL will also be an option, however Brexit adds uncertainties. Expect more volatility and trading opportunities as the market adjusts.\\nThis report highlights some of the key market developments in the European energy sector for April 2017.\\nThe rebalancing of LNG market will be driven by strong growth in Asia gas demand, especially in China.\"", "vector_field": [-0.005732362624257803, -0.020456798374652863, 0.013829282484948635, -0.030590616166591644, -0.02392932027578354, 0.008525918237864971, -0.0327795185148716, -0.00436429725959897, -0.01618708297610283, -0.024996748194098473, 0.003898141672834754, 0.008039494976401329, -0.029131345450878143, 0.011633621528744698, 0.013890084810554981, -0.007330127526074648, 0.01744367741048336, 0.0025993240997195244, 0.018321940675377846, -0.012768609449267387, 0.0036346623674035072, 0.00021386575826909393, -0.012951018288731575, -0.008971805684268475, -0.005154734943062067, -0.01578173041343689, 0.024713002145290375, -0.033995579928159714, -0.01366714108735323, -0.016754576936364174, -0.004533194005489349, -0.0028036893345415592, 0.0033711830619722605, 0.0010868519311770797, -0.02290242724120617, -0.039481353014707565, 0.00799220334738493, -0.021199945360422134, 0.02684785984456539, -0.02980693429708481, 0.006357281003147364, 0.026145247742533684, 0.01745718903839588, -0.00764089822769165, -0.0020216964185237885, 0.02083512768149376, 0.015754707157611847, -0.008093541488051414, -0.010295958258211613, 0.029455628246068954, 0.02384824864566326, 0.014444067142903805, -0.017173441126942635, -0.005381057038903236, -0.0005210470990277827, -0.004053527023643255, -0.03226607292890549, 0.006478887051343918, 0.010289202444255352, 0.00248954095877707, -0.014160320162773132, -0.014741325750946999, -0.026158759370446205, 0.0017784849042072892, -0.0015302064130082726, -0.021794462576508522, -0.016646483913064003, 0.015741195529699326, 0.0024490058422088623, -0.0005890281172469258, 0.023091591894626617, 0.03096894547343254, 0.01971365138888359, 0.010390540584921837, 0.01764635369181633, 0.018038194626569748, 0.007803039159625769, -0.011289072223007679, 0.007897621020674706, 0.005864101927727461, 0.011701180599629879, -0.020037934184074402, -0.018956994637846947, 0.03383343666791916, -0.00218045967631042, 0.014160320162773132, -0.017511235550045967, 0.00032153757638297975, -0.002649993170052767, 0.019686628133058548, 0.015565543435513973, 0.0034961667843163013, 0.013795503415167332, 0.015457448549568653, -0.014862931333482265, -0.015200725756585598, -0.005083798430860043, 0.015443936921656132, -0.020497333258390427, -0.03186072036623955, -0.003398206550627947, -0.00851240660995245, -0.010032478719949722, -0.013944132253527641, -0.022159280255436897, 0.007803039159625769, 0.00980277918279171, -0.007890865206718445, 0.0058607240207493305, 0.020389240235090256, -0.005793165415525436, 0.01744367741048336, 0.017281536012887955, -0.0018848900217562914, 0.03318487107753754, -0.03486033156514168, 0.006370793096721172, -0.013903597369790077, -0.009904117323458195, -0.03353617712855339, 0.023186173290014267, 0.006972066126763821, 0.007870597764849663, -0.011951148509979248, 0.002155125141143799, -0.005303364247083664, -0.011647134087979794, 0.00457035144791007, -0.00260439096018672, -0.001642522867769003, -0.017078859731554985, -0.0034623874817043543, 0.008181368000805378, 0.01455216109752655, 0.012653759680688381, 0.023996878415346146, -0.01659243553876877, -0.01659243553876877, -0.011883589439094067, -0.017767958343029022, 0.0019203582778573036, 0.029671816155314445, -0.02630738914012909, -0.0037765358574688435, -0.010390540584921837, 0.029158368706703186, 0.03353617712855339, 0.01910562254488468, 0.003999479580670595, -0.018497593700885773, 0.0024169152602553368, 0.004219045862555504, -0.007451733574271202, -0.008451603353023529, -0.000698388903401792, 0.01620059460401535, 0.00652955612167716, 0.019686628133058548, -0.017578795552253723, -0.01961906999349594, 0.01440353225916624, -0.017376117408275604, -0.0007216122467070818, 0.006813303101807833, 0.0100662587210536, 0.016227619722485542, 0.0024760293308645487, -0.01413329690694809, 0.019835257902741432, -0.016457319259643555, 0.024996748194098473, 0.0016991032753139734, -0.044940099120140076, 0.030509544536471367, -0.0024371829349547625, 0.0422917976975441, 0.007323371712118387, 0.004678445402532816, -0.020578404888510704, -0.030914897099137306, 0.01047161128371954, 0.005354033317416906, 0.016078989952802658, 0.019321810454130173, -0.030185263603925705, -0.026577623561024666, 0.0071409628726542, 0.00020795437740162015, 0.025861501693725586, -0.008343509398400784, 0.010964790359139442, 0.009201506152749062, -0.008917759172618389, -0.01440353225916624, -0.6420786380767822, -0.0016864360077306628, 0.01847057044506073, 0.02319968491792679, 0.015416913665831089, 0.01590333692729473, 0.027618030086159706, -0.004887877963483334, -0.016078989952802658, 0.02528049610555172, 0.0037461344618350267, 0.007803039159625769, -0.028347663581371307, -0.009147458709776402, -0.0259966179728508, -0.022145768627524376, 0.012194359675049782, 0.006164738908410072, 0.04183239862322807, -0.011208001524209976, 0.014011691324412823, 0.03272547200322151, -0.010762114077806473, 0.015916848555207253, 0.027888264507055283, -0.0018848900217562914, 0.028104452416300774, -0.021537739783525467, -0.014092762023210526, 0.019146157428622246, -0.010059501975774765, 0.0013275300152599812, -0.0011248537339270115, 0.00974197592586279, 0.05129062756896019, -0.003148239105939865, -0.023159150034189224, 0.005107443779706955, 0.021308040246367455, 0.024740025401115417, -0.017808495089411736, -0.00033209362300112844, 0.017767958343029022, -0.005847212392836809, 0.00019307032926008105, 0.030914897099137306, 0.019132645800709724, 0.013173962011933327, -0.0011620110599324107, -0.002008184790611267, 0.012619980610907078, 0.012606468051671982, -0.01022164337337017, 0.011653889901936054, 0.0007423021015711129, 0.008863711729645729, -0.007600362878292799, -0.023037543520331383, 0.00799220334738493, 0.009356890805065632, -0.023375337943434715, 0.004516304470598698, -0.003587371204048395, -0.015187214128673077, -0.0322120264172554, -0.010985057801008224, -0.03115811012685299, 0.01440353225916624, 0.015268283896148205, 0.018727293238043785, 0.008850200101733208, 0.03034740500152111, -0.010282446630299091, -0.0066916970536112785, 0.014173831790685654, -0.0035704814363271, 0.0019000907195731997, 0.0007372351828962564, 0.013275300152599812, 0.008586720563471317, 0.007107183802872896, -0.01794361136853695, -0.024010390043258667, 0.019551511853933334, 0.010998569428920746, 0.007620630320161581, 0.0056546698324382305, -0.0177544467151165, 0.013342859223484993, 0.007235545199364424, -9.716852946439758e-05, 0.013917108997702599, 0.02126750349998474, -0.03196881338953972, -0.0070734042674303055, 0.019038064405322075, 0.014092762023210526, 0.0031161487568169832, 0.010079770348966122, -0.017308559268712997, 0.004404832608997822, -0.00360088306479156, 0.018511105328798294, -0.00960010290145874, 0.008944782428443432, 0.04002182185649872, -0.014160320162773132, 0.027104582637548447, -0.0030384561978280544, -0.017376117408275604, -0.005641158204525709, 0.0062897223979234695, 0.0032918015494942665, -0.009302844293415546, -0.004658177960664034, -0.025131866335868835, 0.008390800096094608, 0.014714302495121956, -0.004377808887511492, -0.005600622855126858, 0.023645572364330292, 0.01543042529374361, 0.019456928595900536, -0.012748342007398605, -0.028023382648825645, 0.01713290624320507, 0.01434948481619358, -0.0015783420531079173, -0.0007093672174960375, 0.0039825900457799435, -0.0006975444266572595, -0.024415742605924606, 0.04126490280032158, -0.019997399300336838, 0.028644923120737076, 0.02228088490664959, 0.047020912170410156, -9.679906361270696e-05, -0.014525137841701508, -0.009904117323458195, -0.01590333692729473, -0.007154474966228008, 0.011478236876428127, -0.011539040133357048, -0.009998699650168419, -0.014916978776454926, -0.013687408529222012, 0.015187214128673077, -0.007897621020674706, 0.00048515648813918233, -0.006343769375234842, 0.023969855159521103, -6.370581832015887e-05, 0.01244432758539915, -0.016551900655031204, 0.00673223240301013, -0.02165934443473816, -0.023510456085205078, 0.002660126890987158, -0.007985447533428669, -0.01621410809457302, 0.024699490517377853, -0.027185652405023575, 0.006178250536322594, 0.0028780039865523577, -0.004691957496106625, 0.009836558252573013, 0.008755617775022984, 0.003979212138801813, -0.023915808647871017, -0.002144991187378764, -0.02103780396282673, -0.013228009454905987, 0.002996231894940138, -0.01068779919296503, -0.008174612186849117, -0.011397166177630424, 0.018497593700885773, 0.011066128499805927, 0.00799895916134119, 0.005168246570974588, 0.00032913792529143393, -0.02157827466726303, -0.006380926817655563, 0.02313212677836418, 0.023996878415346146, -0.0020841883961111307, 0.021618809551000595, -0.006066778674721718, -0.004232557490468025, -0.009167726151645184, 0.009066388010978699, 0.008120565675199032, 0.014106273651123047, -0.03364427387714386, -0.02980693429708481, -0.004935169126838446, 0.016457319259643555, -0.0030857473611831665, -0.0043845647014677525, 0.03991372883319855, 0.00368870934471488, 0.005080420058220625, 0.009410938248038292, 0.004239313304424286, 0.004289982374757528, 0.009100168012082577, -0.021956603974103928, 0.04302143305540085, 0.01598440669476986, 0.02301052026450634, -0.043291665613651276, 0.0009382225689478219, 0.01440353225916624, -0.022686239331960678, 0.03096894547343254, -0.0070531368255615234, -0.00260439096018672, 0.00391840934753418, 0.001585097867064178, -0.0010876964079216123, -0.026145247742533684, -0.0023831359576433897, 0.01433597318828106, -0.02880706451833248, 0.0041886442340910435, 0.003540080040693283, 0.012011950835585594, 0.0026111467741429806, 0.00042730927816592157, -0.018186824396252632, 0.009093412198126316, 0.009464984759688377, 0.008032739162445068, 0.027118094265460968, 0.0007076782057993114, 0.032968685030937195, 0.0023662461899220943, 0.003215797943994403, 0.000536247796844691, 0.011255293153226376, -0.012606468051671982, 0.025145377963781357, 0.012194359675049782, 0.01834896393120289, 0.005904637277126312, 0.02929348684847355, 0.02105131559073925, -0.03288761526346207, -0.0018173311837017536, 0.00013849549577571452, 0.022564632818102837, -0.018430035561323166, -0.009573078714311123, 0.004107574000954628, -0.013018577359616756, 0.020700009539723396, 0.009350134991109371, 0.023672597482800484, 0.016443807631731033, 0.001960893627256155, -0.01104586012661457, -0.004202156327664852, -0.0006937442230992019, -0.0009432894876226783, -0.017673376947641373, -0.01516018994152546, -0.0029455628246068954, -0.0003557392046786845, -0.002285175723955035, 0.0012152135604992509, 0.008789397776126862, 0.013917108997702599, -0.02313212677836418, 0.0186732467263937, 0.01022164337337017, 0.017592307180166245, -0.02374015562236309, 0.003225931664928794, 0.01878134161233902, -0.004691957496106625, -0.03323892131447792, 0.0047932956367731094, 0.01156606338918209, -0.011430945247411728, -0.023469921201467514, -0.014579184353351593, 0.026982976123690605, -0.021524228155612946, -0.0008841755334287882, 0.016281666234135628, 0.007417954038828611, -0.027631541714072227, 0.033292967826128006, 0.006330257747322321, -0.005154734943062067, 0.011262048967182636, -0.022537609562277794, -0.00360088306479156, -0.013221253640949726, 0.014633231796324253, -0.0006557424203492701, 0.02869896963238716, 0.015025072731077671, 0.04404832422733307, -0.005614134483039379, -0.023037543520331383, -0.005198648199439049, 0.018038194626569748, 0.009654149413108826, 0.0033830059692263603, -0.015525007620453835, -0.029077298939228058, 0.006441729608923197, -0.018727293238043785, 0.020808104425668716, -0.0005772053264081478, 0.013505000621080399, 0.043399762362241745, -0.014268414117395878, -0.004631154239177704, -0.02313212677836418, -0.003577237483114004, 0.007911133579909801, 0.09577132761478424, -0.0008757307077758014, 0.02888813428580761, 0.027699099853634834, 0.0016864360077306628, -0.010816160589456558, 0.014227879233658314, -0.018754318356513977, -0.006205273792147636, -0.005107443779706955, 0.013457708992064, 0.00851240660995245, 0.003577237483114004, -0.0017565282760187984, 0.01753825880587101, -0.0009559567552059889, -0.0005468038725666702, -0.009985188022255898, 0.01621410809457302, -0.008404312655329704, 0.028320640325546265, -0.034725211560726166, -0.024996748194098473, 0.059505775570869446, 0.02115941047668457, 0.02187553234398365, 0.033779390156269073, 0.008019227534532547, 0.018335452303290367, 0.0006177406175993383, -0.02001091092824936, -0.013119915500283241, -0.0077287242747843266, 0.027496423572301865, -0.015335842967033386, -0.02030816860496998, 0.023294268175959587, -0.023145638406276703, 0.008606988936662674, -0.010606728494167328, 0.009431205689907074, -0.007884109392762184, 0.01663297228515148, 0.003411718411371112, 0.029536698013544083, -0.022740285843610764, -0.0035907491110265255, 0.02880706451833248, -0.019254252314567566, 0.00938391499221325, 0.026874883100390434, -0.01015408430248499, -0.040643360465765, 0.0011983239091932774, -0.02021358720958233, 0.02744237706065178, 0.0285638514906168, 0.00567831564694643, -0.010295958258211613, -0.03580615296959877, -0.002660126890987158, -0.020173052325844765, 0.005404702387750149, -0.017416654154658318, -0.021321551874279976, -0.032752495259046555, -0.030941922217607498, -0.012011950835585594, -0.0002178770664613694, -0.015349354594945908, 0.006772767752408981, -0.042264774441719055, -0.02725321240723133, 0.02972586266696453, 0.029023252427577972, -0.019727163016796112, 0.009768999181687832, 0.0035603477153927088, -0.019362347200512886, 0.01149850431829691, -0.004908145405352116, 0.006610626820474863, -0.007620630320161581, -0.01857866533100605, -0.016268154606223106, 0.019970376044511795, 0.015376378782093525, -0.012005195021629333, -0.014538649469614029, 0.011126930825412273, 0.01940288208425045, -0.015511495992541313, 0.03953539952635765, -0.010917498730123043, -0.004276470746845007, 0.0060262433253228664, 0.04156216233968735, -0.002364557236433029, -0.0027057291008532047, -2.252399099234026e-05, -0.006823436822742224, 0.018038194626569748, -0.006205273792147636, -0.014200855977833271, 0.005350655410438776, 0.015943871811032295, -0.010613484308123589, 0.000864752393681556, -0.003979212138801813, -0.0022868646774441004, 0.012511885724961758, -0.025212936103343964, -0.015552030876278877, -0.01733558252453804, -0.04364297166466713, 0.0030131216626614332, 0.01326178852468729, 0.0015082497848197818, -0.004438611678779125, 0.02815849892795086, -0.01082967221736908, -0.025010259822010994, 0.007323371712118387, 0.012532154098153114, 0.015403402037918568, 0.00995140802115202, -0.0252669844776392, -0.0006012731464579701, -0.031131086871027946, -0.005735740531235933, -0.0003323047421872616, 0.011816030368208885, 0.007830062881112099, 0.0033171360846608877, -0.02137559838593006, -0.017686888575553894, -0.006012731231749058, 0.005462127272039652, -0.01951097510755062, -0.035752106457948685, -0.01805170625448227, 0.0058303228579461575, 0.012464595027267933, -0.015552030876278877, 0.018511105328798294, -0.01621410809457302, -0.012201115489006042, 0.005769520066678524, 0.0008605299517512321, -0.001440690946765244, -0.007755747996270657, 0.0018713781610131264, -0.017997659742832184, -0.010289202444255352, -0.008613744750618935, -0.01651136577129364, 0.018092241138219833, -0.014998049475252628, 0.020916199311614037, 0.03767077624797821, 0.029969075694680214, -0.0161060132086277, 0.002847602590918541, 0.013065868057310581, -0.008911003358662128, 0.011208001524209976, 0.018511105328798294, 0.005722228903323412, -0.02291593886911869, -0.000467000063508749, -0.013863061554729939, -0.01786254160106182, -0.006931530777364969, 0.0008715082658454776, -0.015146678313612938, 0.022834867238998413, -0.011822786182165146, 0.0006793880020268261, -0.012782121077179909, -0.02103780396282673, -0.024402230978012085, -0.014768349006772041, 0.008080029860138893, -0.00764089822769165, -0.02106482721865177, -0.010444587096571922, 0.011525527574121952, -0.0031904634088277817, 0.0226997509598732, -0.02495621331036091, 0.007282836362719536, -0.010505390353500843, 0.020497333258390427, -0.005891125649213791, -0.024294137954711914, -0.00934337917715311, -0.008775885216891766, -0.01609250158071518, -0.020254122093319893, -0.003931920975446701, -0.02310510352253914, -0.002999609801918268, 0.02702351287007332, -0.020159540697932243, 0.00980953499674797, 0.009843314066529274, -0.028617899864912033, 0.012559177353978157, 0.009519032202661037, -0.025618288666009903, -0.012309209443628788, -0.003298557363450527, -0.003553591901436448, -0.014065737836062908, 0.002641548402607441, 0.005658047739416361, -0.0015758085064589977, 0.012741586193442345, -0.015443936921656132, 0.005695205181837082, 0.004148108884692192, -0.004259581211954355, 0.027996359393000603, 0.0044318558648228645, 0.033995579928159714, 0.019254252314567566, 0.029131345450878143, -0.044642843306064606, -0.03931920975446701, 0.008532674051821232, -0.0203216802328825, 0.015376378782093525, 0.00415824307128787, -0.024577884003520012, -0.005718850530683994, 0.0012185914674773812, 0.013971155509352684, -0.017011301591992378, -0.014700790867209435, 0.029428604990243912, -0.018186824396252632, 0.0018139532767236233, -0.009309600107371807, -0.037103280425071716, -0.02074054628610611, 0.00932986754924059, -0.03342808410525322, -0.0009948030347004533, -0.02155125141143799, -0.012741586193442345, -0.008701571263372898, 0.032455239444971085, -0.02332129143178463, 0.010579705238342285, 0.005050018895417452, -0.016903206706047058, -0.01670053042471409, -0.001695725368335843, 0.0009508898365311325, 0.01155930757522583, -0.026564111933112144, -0.003435363993048668, -0.019240740686655045, -0.00037558458279818296, 0.00799895916134119, 0.012147068977355957, 0.012126801535487175, -0.01848408207297325, -0.01196466013789177, 0.022132256999611855, -0.011782251298427582, 0.00740444241091609, 0.014444067142903805, 0.007985447533428669, 0.004293360281735659, -0.00719501031562686, -0.016281666234135628, -0.010113549418747425, -0.009735220111906528, 0.003702221205458045, 0.010572949424386024, 0.009991943836212158, -0.029969075694680214, -0.03196881338953972, -0.005539820063859224, -0.02692892961204052, -0.008221903815865517, -0.009843314066529274, 0.001113875419832766, -0.027604518458247185, -0.008370532654225826, -0.0014567362377420068, -0.02024061046540737, 0.003263089107349515, -0.018605688586831093, 0.0019068465335294604, -0.009532543830573559, 0.03115811012685299, -0.02921241708099842, 0.008829932659864426, 0.014471090398728848, 0.020902685821056366, 0.0021939713042229414, 0.013788746669888496, -0.008289461955428123, -0.004931790754199028, 0.023767178878188133, 0.02011900395154953, -0.022672725841403008, -0.014862931333482265, -0.00476964982226491, 0.018430035561323166, -0.009836558252573013, -0.0008571520447731018, 0.017092371359467506, 0.013383394107222557, 0.001499804900959134, -0.020821616053581238, -0.01609250158071518, 0.01244432758539915, -0.004698713310062885, 0.010377028957009315, 0.0021939713042229414, -0.04610211029648781, 0.023037543520331383, -0.009309600107371807, 0.021213456988334656, -0.0152817964553833, 0.006806547287851572, -0.012025462463498116, 0.0371573306620121, 0.011370142921805382, -0.0004708002379629761, 0.0007140118395909667, 0.01041756384074688, -0.03013121522963047, -0.017173441126942635, 0.042778220027685165, 0.014038714580237865, -0.0006215407629497349, 0.030050145462155342, 0.027293747290968895, 0.031428344547748566, 0.010336493141949177, -0.010181108489632607, -0.021429644897580147, -0.02384824864566326, -0.0060634007677435875, -0.028725992888212204, 0.0063167461194098, -0.0028273349162191153, 0.024510325863957405, -0.002025074325501919, -0.050209686160087585, -0.0209567341953516, 0.011322851292788982, -0.023172661662101746, -0.00520878192037344, -0.005144601222127676, 0.018186824396252632, 0.02538858912885189, 0.02796933427453041, -0.009464984759688377, 0.045453548431396484, 0.009843314066529274, -0.0045973751693964005, 0.00260439096018672, -0.005911393091082573, 0.012869947589933872, -0.012579444795846939, 0.030158240348100662, 0.00412446353584528, -0.028320640325546265, 0.006313368212431669, 0.02363206073641777, 0.023996878415346146, 0.0009263997781090438, 0.04031908139586449, -0.027888264507055283, 0.0002318110637133941, 0.0027479534037411213, 0.009701441042125225, -0.0033863838762044907, 0.026442507281899452, 0.01041756384074688, -0.028969205915927887, 0.013660385273396969, 0.04415641725063324, -0.007330127526074648, -0.011282316409051418, 0.005722228903323412, -0.0025165646802634, -0.007573339156806469, -0.007417954038828611, 0.006958554498851299, 0.005147979129105806, 0.013396905735135078, -0.01455216109752655, 0.022051185369491577, 0.014565672725439072, -0.0026955953799188137, 0.003935298882424831, -0.004820318892598152, -0.023591525852680206, -0.015525007620453835, -0.023496944457292557, -0.02991502732038498, -0.020578404888510704, 0.01940288208425045, -0.010404052212834358, -0.003529946319758892, -0.00039310764987021685, 0.006212030071765184, -0.00014419577200897038, -0.025983106344938278, -0.022348444908857346, 0.000518513610586524, -0.010870208032429218, 0.029563723132014275, -0.0009551122784614563, 0.00025038974126800895, 0.0015901648439466953, -0.010647263377904892, -0.009194750338792801, 0.019335322082042694, -0.009093412198126316, 0.010762114077806473, -0.041021689772605896, -0.027834217995405197, 0.001949070836417377, -0.004506170749664307, -0.020794592797756195, 0.02725321240723133, -0.01351175643503666, 0.01570066064596176, 0.014295438304543495, 0.20116302371025085, -0.032671425491571426, 0.004766271915286779, 0.03694114089012146, 0.018403012305498123, -0.026239831000566483, 0.028834087774157524, 0.0007651032065041363, -0.008911003358662128, 0.027591006830334663, -0.007965180091559887, 0.026996487751603127, -0.03756268322467804, -0.008296217769384384, 0.0044487458653748035, -0.004137975163757801, -0.04448070004582405, -0.023713132366538048, -0.014971025288105011, 0.03480628505349159, 0.008019227534532547, 0.021213456988334656, 0.003124593524262309, -0.012984797358512878, 0.011789007112383842, -0.02465895563364029, 0.002573989564552903, 0.032347142696380615, -0.0010378718143329024, 0.024334672838449478, -0.018105752766132355, 0.010762114077806473, 0.005796543322503567, -0.002754709217697382, 0.003296868409961462, 0.00336780515499413, 0.008593477308750153, 0.020781081169843674, -0.003053656779229641, 0.039373256266117096, -0.000545114919077605, 0.0047865393571555614, -0.009525788016617298, 0.002883070847019553, -0.01764635369181633, 0.0104783670976758, -0.006566713564097881, -0.017308559268712997, -0.024604907259345055, -0.002454072702676058, -0.030887873843312263, -0.0034387419000267982, 0.018713781610131264, 0.013477976433932781, 0.014930490404367447, -0.0074652452021837234, 0.005509418435394764, 0.010073014535009861, -0.0005662270123139024, 0.0005130245117470622, -0.02074054628610611, 0.017916588112711906, 0.008404312655329704, 0.006407950539141893, -0.012565933167934418, 0.011930881068110466, -0.018119264394044876, 0.012606468051671982, -0.008167856372892857, 0.005803299136459827, 0.010863452218472958, 0.011322851292788982, -0.0266316719353199, -0.007458489388227463, -0.03950837627053261, -0.03640067204833031, 0.011775495484471321, 0.016065478324890137, 0.015741195529699326, 0.014849419705569744, 0.018105752766132355, 0.027482911944389343, 0.009478497318923473, -0.0039015195798128843, -0.015970895066857338, -0.006218785885721445, -0.02405092492699623, 0.011545795947313309, -0.02044328674674034, -0.012038975022733212, 0.02426711469888687, -0.0023493566550314426, -0.0029573854990303516, 0.011444457806646824, 0.019456928595900536, -0.008579964749515057, 0.02248356118798256, 0.015579055063426495, -0.017997659742832184, -0.03045549802482128, -0.03034740500152111, 0.04461582005023956, 0.020186563953757286, -0.01413329690694809, -0.03232011944055557, 0.010255422443151474, -0.0008959983242675662, 0.014768349006772041, -0.011160710826516151, 0.010991813614964485, 0.007384174503386021, -0.023780690506100655, 0.015133166685700417, -0.008080029860138893, -0.019767699763178825, -0.01434948481619358, 0.0035096786450594664, -0.02878004126250744, 0.0163897592574358, 0.0006650317227467895, -0.004925034940242767, -0.030941922217607498, 0.019578535109758377, -0.02044328674674034, -0.0037427563220262527, -0.025929059833288193, -0.024077950045466423, -0.00846511498093605, -0.017376117408275604, -0.0259966179728508, -0.0028121343348175287, 0.006066778674721718, -0.005536442156881094, -0.029023252427577972, 0.011579575017094612, 0.013646873645484447, 0.03491437807679176, -0.03783291578292847, 0.022240350022912025, 0.0023814470041543245, -0.0039927237667143345, -0.004502792842686176, 0.00830297451466322, 0.007046381011605263, 0.016268154606223106, -0.029996098950505257, 0.016741065308451653, 0.01305911224335432, -0.010849940590560436, 0.0006443418678827584, -0.010485122911632061, -0.0023240221198648214, 0.00585396820679307, 0.014944002032279968, 0.020821616053581238, -0.01680862531065941, -0.028617899864912033, -0.03261737897992134, 0.01755177043378353, 0.04415641725063324, -0.03248226270079613, -0.0036042609717696905, 0.016268154606223106, -0.02083512768149376, 0.011255293153226376, -0.0010581393726170063, -0.17219382524490356, 0.0169302299618721, 0.02361854910850525, -0.008275950327515602, 0.032860592007637024, 0.016943741589784622, -0.010897231288254261, -0.013687408529222012, -0.0015048718778416514, -0.006850460544228554, 0.003381317015737295, 0.02506430819630623, -0.032671425491571426, -0.00892451498657465, -0.02413199655711651, 0.0032799788750708103, -0.014971025288105011, 0.013417174108326435, 0.009498764760792255, 0.01285643596202135, 0.03653578832745552, -0.036481741815805435, 0.013133427128195763, -0.017186954617500305, -0.0010572948958724737, 0.002467584563419223, -0.024537349119782448, -0.006360659375786781, -0.023064566776156425, 0.01640327274799347, -0.0169302299618721, -0.0060735344886779785, 0.03407664969563484, 0.0003806515014730394, 0.01331583596765995, 0.004901389591395855, 0.002698973286896944, 0.0012886837357655168, -0.04096764326095581, 0.04877743870019913, 0.030104191973805428, 0.03342808410525322, 0.004273092839866877, -0.003945432603359222, 0.005739118438214064, 0.010505390353500843, 0.012295697815716267, -0.02413199655711651, 0.00918123871088028, -0.015065607614815235, 0.004560217726975679, -0.009350134991109371, -0.0022193058393895626, 0.0008009938173927367, -0.010768869891762733, 0.028969205915927887, 0.012741586193442345, 0.024929190054535866, 0.01743016578257084, -0.005928283091634512, -0.023469921201467514, -0.008836688473820686, 0.005762763787060976, -0.032752495259046555, -0.014916978776454926, -0.009681173600256443, 0.01156606338918209, -0.006543067749589682, -0.020510844886302948, 0.01237676851451397, -0.008938026614487171, -0.030374428257346153, 0.012342989444732666, -0.007478756830096245, 0.030698709189891815, -0.018632711842656136, -0.00038424055674113333, -0.000743991055060178, 0.009681173600256443, 0.004404832608997822, -0.004347407724708319, 0.04323761910200119, -0.01836247742176056, 0.008093541488051414, -0.014254902489483356, 0.015470961108803749, 0.0402650348842144, 0.009363646619021893, 0.018308429047465324, 0.00039395212661474943, 0.01814628764986992, -0.011687668971717358, 0.003229309804737568, -0.003817070974037051, -0.0209567341953516, 0.009410938248038292, -0.006482264958322048, 0.0117146922275424, -0.008917759172618389, 0.004445367958396673, 0.0010513835586607456, 0.012113288976252079, -0.01859217695891857, -0.011072884313762188, 0.010093281976878643, 0.015403402037918568, -0.019443416967988014, 0.0074855126440525055, 0.030374428257346153, -0.02022709883749485, -0.018754318356513977, 0.0070734042674303055, -0.0008791086147539318, 0.009019097313284874, -0.03034740500152111, 0.008843444287776947, -0.003404962597414851, -0.002134857466444373, 0.022159280255436897, -0.021105362102389336, 0.00974197592586279, -0.009100168012082577, 0.010464855469763279, 0.0007562360842712224, 4.72647552669514e-05, -0.00907314382493496, -0.09874391555786133, -0.048696368932724, -0.006664673797786236, 0.024996748194098473, -0.006391060538589954, 0.01641678437590599, 0.014146808534860611, 0.021213456988334656, -0.006951798684895039, 0.026266854256391525, -0.028050405904650688, -0.004806807264685631, 0.005658047739416361, 0.027915287762880325, 0.03664388135075569, -0.00960010290145874, 0.002158503048121929, -0.023253731429576874, 0.012525398284196854, 0.015457448549568653, -0.006215407978743315, -0.00830297451466322, -0.01156606338918209, -0.012599712237715721, -0.006188384257256985, -0.023794202134013176, -0.026861371472477913, -0.0054587493650615215, 0.02548317238688469, -0.0031499280594289303, 0.00435754144564271, -0.01724100112915039, 0.022983497008681297, -0.014741325750946999, -0.012282186187803745, -0.02529400773346424, -0.04599401727318764, -0.006468753330409527, -0.006816681008785963, -0.04077848047018051, 0.0036684416700154543, 0.02010549232363701, 0.006347147282212973, -0.01837598904967308, 0.000723301200196147, -0.02755047008395195, -0.025253472849726677, -0.00010740790457930416, -0.01433597318828106, 0.0201325174421072, -0.01599791832268238, -0.007235545199364424, -0.02932051010429859, 0.014038714580237865, 0.011984927579760551, 0.01063375174999237, 0.012106533162295818, 0.021726904436945915, -0.023064566776156425, -0.023348314687609673, -0.0014651810051873326, 0.005026373080909252, -0.03253630921244621, -0.0011898790253326297, 0.005955306347459555, 0.04469688981771469, -0.0044419895857572556, -0.02217279188334942, -0.0015774975763633847, -0.00239833677187562, -0.026780299842357635, 0.01630868948996067, 0.009478497318923473, -0.014930490404367447, -0.034292835742235184, -0.014065737836062908, -0.008093541488051414, -0.019227229058742523, 0.022226838394999504, -0.019132645800709724, -0.0054857730865478516, -0.02961776964366436, 0.008235415443778038, -0.01910562254488468, 0.02383473701775074, 0.033779390156269073, -0.019038064405322075, -0.013498243875801563, 0.019470440223813057, -0.002698973286896944, 0.0039048974867910147, 0.025766918435692787, 0.0044318558648228645, -0.021524228155612946, -0.006924774963408709, 0.04161620885133743, -0.036779001355171204, 2.3652171421417734e-06, 0.002421982353553176, 0.012072754092514515, -0.030806804075837135, -0.0011670779203996062, -0.06534285098314285, 0.04993944987654686, 0.025726383551955223, -0.012079509906470776, -0.01587631367146969, -0.0013309079222381115, 0.023659083992242813, 0.007336883340030909, -0.023051055148243904, 0.014565672725439072, -0.021132387220859528, 0.032239049673080444, -0.018308429047465324, -0.019740676507353783, -0.008681302890181541, -0.01732207089662552, 0.02311861515045166, 0.003594127018004656, -0.007276080548763275, -0.011674157343804836, 0.002712485147640109, -0.012282186187803745, -0.008863711729645729, 0.024834606796503067, -0.01393062062561512, 0.010289202444255352, -0.011228268966078758, 0.042778220027685165, -0.006820058915764093, 0.0009677795460447669, 0.014781861566007137, -0.034509025514125824, -0.008012471720576286, 0.010241910815238953, 0.018132776021957397, -0.020618939772248268, -0.027280235663056374, 0.03250928595662117, 0.02053787000477314, 0.0177544467151165, -0.020821616053581238, -0.016349224373698235, 0.011430945247411728, 0.028320640325546265, -0.024307649582624435, -0.018403012305498123, 0.00326477806083858, 0.007046381011605263, 0.01630868948996067, 0.0009846691973507404, 0.023591525852680206, 0.016795113682746887, -0.006120825652033091, -0.041643232107162476, -0.005465505179017782, 0.012687538750469685, -0.020659474655985832, 0.02053787000477314, 0.00810705404728651, -0.0006988111417740583, 0.024902166798710823, 0.007593607064336538, 0.02240249142050743, -0.015943871811032295, 0.029374558478593826, -0.017889564856886864, -0.015552030876278877, -0.008958294056355953, -0.0032326877117156982, -0.03218500316143036, -0.014998049475252628, 0.01278887689113617, 0.011620109900832176, 0.014254902489483356, -0.006576847285032272, 0.0064755091443657875, 0.012505129911005497, 0.0027631542179733515, -0.01135663129389286, 0.007769259624183178, -0.0104783670976758, -0.01212004479020834, -0.01865973509848118, 0.031509414315223694, 0.00283746886998415, 0.0036988432984799147, -0.02291593886911869, 0.00673223240301013, -0.03261737897992134, 0.0013173961779102683, -0.018024682998657227, 0.022578144446015358, -0.008100297302007675, 0.010586461052298546, 0.011741716414690018, 0.03934623301029205, 0.003122904570773244, 0.00034877221332862973, 0.00872183870524168, 0.0005235805292613804, 0.0035096786450594664, 0.004296738654375076, -0.015308819711208344, -0.028050405904650688, -0.019335322082042694, 0.002423671307042241, -0.00611744774505496, -0.02529400773346424, 0.005499284714460373, -0.017808495089411736, 0.0003631284344010055, -0.010444587096571922, 0.0074855126440525055, 0.011424189433455467, -0.019146157428622246, -0.008694815449416637, -0.007282836362719536, -0.02826659381389618, -0.006461997516453266, 0.004195400048047304, 0.02237546816468239, 0.010316225700080395, 0.018902946263551712, 0.004634532146155834, 0.016146548092365265, 0.001070806640200317, 0.016173571348190308, -0.0038576063234359026, -0.0017210599035024643, -0.012464595027267933, -0.012876703403890133, 0.003915031440556049, -0.02888813428580761, -0.013984667137265205, -0.010336493141949177, -0.0008047939627431333, -0.017213977873325348, 0.01068779919296503, -0.011059372685849667, 0.06437000632286072, 0.016281666234135628, -0.02705053612589836, 0.00831648614257574, 0.007431465666741133, 0.03831934183835983, -0.017686888575553894, 0.013687408529222012, -0.008836688473820686, -0.025875013321638107, 0.004056904930621386, -0.004350785631686449, 0.005772897973656654, -0.0408325269818306, -0.02818552404642105, -0.00412446353584528, 0.02084863930940628, -0.014673766680061817, -0.016889695078134537, 0.011545795947313309, 0.02251058630645275, -0.04548057168722153, 0.026658695191144943, 0.0021145897917449474, -0.03356320038437843, 0.002834090730175376, 0.015038584358990192, 0.0013765101321041584, -0.012599712237715721, -0.008694815449416637, 0.007803039159625769, 0.0342387892305851, -0.021713392809033394, -0.02115941047668457, -0.008606988936662674, 0.0022057942114770412, -0.016457319259643555, -0.015052095986902714, 0.01259295642375946, 0.002351045608520508, 0.005127711221575737, 0.03788696229457855, -0.0056175123900175095, -0.030671685934066772, -0.019659604877233505, -0.01660594902932644, 0.015538519248366356, 0.016538389027118683, 0.0009736909414641559]} +{"id": "test:10000075", "text": "\"Disturbia Clothing coupon code gift !\\nDisturbia Clothing coupon code gift ! ! !\\nDisturbia Clothing is always bringing something different and out of the mainstream, focusing on the dark side of popular culture, subversive iconography, childhood nostalgia, and angry slogans, all made with a strong sense of independence, and a quintessential British punk D.I.Y. ethic.\\nIf you are a fan on Facebook, you\u2019ve already noticed they are running a 20% discount, with the coupon code FACEBOOK20 until Monday the 18th.\\nDisturbia Clothing coupon code! Don\u2019t forget to use it until Monday the 18th ! ! !\\nJust log into your Facebook account, get your coupon and stay tune for our next special discounts ! Keep your eyes on the prize and, of course, on our Facebook page.\\nInspiring graphics of the day \u2013 intense emotions graphics !\"", "vector_field": [-0.02990773878991604, 0.005355913192033768, -0.005909513216465712, -0.03302924335002899, -0.0037184576503932476, 0.00392188923433423, -0.037004489451646805, -0.014126804657280445, 0.013199690729379654, -0.038845378905534744, -0.00895764771848917, 0.0031031614635139704, -0.006076260004192591, -0.020890062674880028, -0.003545040963217616, 0.047169387340545654, 0.03292252495884895, 0.014393599703907967, -0.010098197497427464, -0.015820953994989395, 0.010144886560738087, 0.04359433054924011, -0.024104943498969078, 0.002337792655453086, -0.028627121821045876, 0.008490756154060364, 0.026159266009926796, -0.01002482883632183, -0.01602105051279068, 0.0035383710637688637, 0.002737985458225012, 0.0034349877387285233, 0.0006102939369156957, -0.010931932367384434, -0.019969617947936058, -0.007276838179677725, -0.021103497594594955, -0.0071834600530564785, 0.038205068558454514, 0.0007912144064903259, 0.02163708768785, 0.013132992200553417, 0.0031481829937547445, 0.0025362216401845217, -0.02309112250804901, 0.01495386939495802, 0.015540818683803082, 0.002959758974611759, 0.021316934376955032, 0.008250640705227852, 0.0020443180110305548, 0.01527402363717556, -0.012566052377223969, -0.008764221332967281, 0.016381222754716873, -0.008210621774196625, 0.04631564021110535, 0.009457889012992382, 0.01430022157728672, 0.011552231386303902, -0.004949050489813089, -0.029667623341083527, -0.025985850021243095, 0.019822880625724792, -0.013699932023882866, 0.0023477976210415363, -0.016034390777349472, 0.0008779228664934635, 0.0021610409021377563, 0.017381705343723297, 0.03364287316799164, 0.02198392152786255, 0.018222110345959663, -0.03385630622506142, 0.023037763312458992, 0.0015916000120341778, 0.008103903383016586, -0.0012581059709191322, 0.039539042860269547, 0.005019084084779024, 0.028733840212225914, -0.026492761448025703, -0.025639016181230545, 0.019195912405848503, 0.010451700538396835, 0.044474754482507706, -0.0013506505638360977, 0.021223556250333786, -0.014260201714932919, -0.013293069787323475, 0.0030264577362686396, 0.012552713043987751, 0.006816616747528315, 0.019889580085873604, -0.02211732044816017, 0.010558418929576874, -0.02894727699458599, 0.03313596174120903, 0.01746174320578575, -0.0056160385720431805, 0.01722162775695324, 0.03142847120761871, -0.011612259782850742, -0.01826212927699089, -0.017608480527997017, -0.013132992200553417, 0.005852818954735994, -0.0037451372481882572, 0.014513657428324223, -0.003204877022653818, -0.03177530691027641, 0.03508356586098671, 0.028520403429865837, -0.030067816376686096, 0.04410124197602272, 0.017955314368009567, 0.01125875674188137, 0.010851893573999405, -0.0025312192738056183, -0.0568273700773716, 0.007010043133050203, 0.022010602056980133, 0.021970583125948906, 0.011892395094037056, 0.02647942118346691, 0.021036799997091293, -0.025398900732398033, -0.001348149380646646, 0.0015524143818765879, -0.006192983128130436, 0.019502727314829826, -0.0012239228235557675, -0.016674697399139404, -0.0016799758886918426, -0.013253049924969673, 0.025412240996956825, -0.019075855612754822, 0.0031498505268245935, -0.015780935063958168, -0.0007224312867037952, 0.0006373903597705066, 0.03364287316799164, -0.01854226365685463, -0.010344983078539371, 0.007023382931947708, 0.025292182341217995, 0.0073035177774727345, -0.00042395421769469976, 0.00548597564920783, 0.004745619371533394, 0.007003373466432095, 0.016754737123847008, 0.013019604608416557, -0.003605069825425744, 0.009224442765116692, 0.021517030894756317, 0.013699932023882866, 0.011338794603943825, 0.015420760959386826, 0.0030731467995792627, -0.0007607830921187997, -0.0009079372975975275, 0.02159706875681877, -0.002727980725467205, 0.03332271799445152, 0.012792828492820263, 0.013032943941652775, 0.01636788435280323, 0.002899730112403631, 0.005112462677061558, 0.0068132816813886166, 0.01343980710953474, -0.05333235487341881, -0.0039018793031573296, 0.0027613299898803234, 0.04623560234904289, -0.017555123195052147, 0.011965763755142689, -0.0007391059771180153, -0.025625675916671753, -0.0203297920525074, -0.012592731975018978, 0.020943421870470047, 0.028387006372213364, -0.013273060321807861, -0.0012122505577281117, 0.015887653455138206, 0.0008458240772597492, -0.005646052770316601, -0.004825657699257135, -0.00798384565860033, 0.007790419273078442, 0.019155893474817276, -0.008664173074066639, -0.6275022625923157, 0.00717679038643837, -0.010778524912893772, -0.013106312602758408, -0.0024761927779763937, 0.026732876896858215, 0.000607792753726244, -0.005866158753633499, -0.01883573830127716, 0.02027643285691738, 0.013753291219472885, 0.010451700538396835, -0.00631637591868639, -0.016808096319437027, -0.006703228689730167, -0.004318746738135815, -0.006843296345323324, -0.011618929915130138, 0.008017195388674736, 0.031962063163518906, 0.007323527708649635, 0.03137511387467384, -0.018582284450531006, 0.013379777781665325, 0.0003164024092257023, 0.01808871328830719, 0.0046422360464930534, -0.005802794825285673, -0.010838554240763187, 0.02297106385231018, -0.005986216478049755, 0.029560904949903488, 0.012359286658465862, 0.013353098183870316, 0.04735614359378815, -0.012532703578472137, -0.0029497542418539524, 0.03204210102558136, 0.0056760674342513084, 0.022237377241253853, -0.03124171495437622, -0.006983363535255194, 0.031721945852041245, 0.0031681926921010017, -0.0036517588887363672, 0.004578872118145227, 0.012385966256260872, -0.025425579398870468, 0.01263275183737278, -0.005659392569214106, 0.003898544469848275, 0.02686627395451069, -0.006499797571450472, -0.0006832457729615271, 0.025105426087975502, -0.000719096336979419, 0.011625600047409534, 0.0032865831162780523, 0.0033316046465188265, 0.007943826727569103, -0.01819543167948723, -0.0029147372115403414, 0.0032265540212392807, 0.0024245011154562235, -0.030174534767866135, 0.024024905636906624, -0.00710342125967145, -0.022864345461130142, 0.02130359411239624, 0.013753291219472885, 0.006326380651444197, 0.01116537768393755, -0.033109281212091446, -0.0162478256970644, -0.0073035177774727345, 0.036711014807224274, 0.0399659164249897, -0.01156557071954012, -0.005509320180863142, 0.004095305688679218, -0.016007710248231888, 0.0017791902646422386, -0.0244117584079504, -0.019102534279227257, 0.040339428931474686, -0.006453108042478561, 0.018422206863760948, -0.004899026360362768, -0.0015290698502212763, 0.013653242960572243, -0.006162968464195728, 0.0364975780248642, 0.0069366744719445705, -0.004298737272620201, -0.009657985530793667, 0.01067847665399313, -0.010111536830663681, -0.012979584746062756, 0.01883573830127716, -0.009364510886371136, -0.008997667580842972, 0.008430727757513523, -0.025065407156944275, 0.015140625648200512, 0.01116537768393755, 0.024425098672509193, 0.017141589894890785, 0.0392722487449646, 0.01895579695701599, -0.019142553210258484, 0.006296365987509489, -0.020369810983538628, 0.00548597564920783, -0.031081637367606163, -0.0055526746436953545, -0.022317416965961456, 0.020890062674880028, -0.00979138258844614, -0.008964317850768566, -0.00348167703486979, -0.02750658243894577, -0.011992443352937698, 0.0010738505516201258, -0.023184500634670258, 0.012325936928391457, 0.021343613043427467, 0.008530776016414165, -0.00995812937617302, -0.04524846002459526, -0.01722162775695324, 0.025012047961354256, 0.033509474247694016, 0.013766630552709103, -0.021783825010061264, 0.010324973613023758, 0.019089194014668465, 0.03639086335897446, -0.008910958655178547, 0.006393079180270433, -0.03756476193666458, -0.021263575181365013, -0.00100798555649817, 0.016207806766033173, -0.009164414368569851, -0.0064631132408976555, -0.003925223834812641, -0.009798052720725536, -0.0110786696895957, -0.013819989748299122, -0.00018217107572127134, -0.025825772434473038, 0.0038118360098451376, -0.015447440557181835, 0.007757069543004036, -0.001862563774921, -0.01740838587284088, -0.012272577732801437, -0.005005744285881519, -0.009951460175216198, -0.009591286070644855, 0.011919074691832066, 0.004725609440356493, -0.017648501321673393, -0.02239745482802391, -0.022984404116868973, 0.005255864933133125, 0.01137214433401823, 0.016434581950306892, -0.019422689452767372, -0.030067816376686096, -0.0083840386942029, 0.012059141881763935, -0.005746101029217243, -0.0006682385574094951, 0.002341127721592784, 0.009251122362911701, -0.02257087081670761, -0.020169714465737343, -0.005315893795341253, -0.01826212927699089, -0.009197763167321682, -0.01097195129841566, -0.03663097694516182, -0.004999074619263411, 0.04231371358036995, 0.009944790042936802, 0.00935784075409174, 0.015394081361591816, -0.017475083470344543, 0.026426061987876892, 0.004422130063176155, 0.0051858313381671906, -0.020876722410321236, -0.004588876850903034, -0.01113869808614254, 0.005329233594238758, 0.028120210394263268, 0.005205840803682804, 0.012252568267285824, 0.04017935320734978, 0.04575537145137787, 0.004685590043663979, 0.02079668454825878, 0.007703710813075304, -0.0013306409819051623, -0.024024905636906624, 0.016794756054878235, -0.02583911269903183, 0.035056885331869125, 0.01762182079255581, -0.003156520426273346, -0.015820953994989395, 0.018422206863760948, -0.005372587591409683, -0.027639979496598244, 0.03313596174120903, -0.024278361350297928, 0.02434505894780159, -0.028493724763393402, -0.004302072338759899, -0.026212625205516815, -0.0012389300391077995, -0.006449773442000151, -0.007403566036373377, -0.011632269248366356, -0.016261165961623192, 0.004945715423673391, 0.0021927228663116693, -0.008090564049780369, -0.032415613532066345, 0.005956202279776335, 0.01740838587284088, 0.0235446747392416, 0.0035950648598372936, 0.009784713387489319, -0.02026309259235859, -0.008684182539582253, -0.011098679155111313, 0.030121175572276115, 0.0021326940041035414, 0.011812356300652027, 0.05218513682484627, 0.024331720545887947, -0.00030368793522939086, 0.0024578506126999855, -0.003971913363784552, 0.014887170866131783, 0.00036976143019273877, -0.016034390777349472, 0.00680327694863081, -4.132198591833003e-05, 0.0048089828342199326, -0.025225482881069183, -0.003881869837641716, 0.0030864865984767675, -0.003401638474315405, 0.022544192150235176, 0.01992959901690483, 0.009231112897396088, 0.008797571063041687, -0.007210139650851488, -0.005579354241490364, 0.03150850906968117, -0.007416905835270882, 0.018128732219338417, 0.02106347866356373, 0.00032536505023017526, -0.029134033247828484, -0.01843554712831974, -0.014166823588311672, -0.0049323756247758865, 0.009311151690781116, 0.017008192837238312, -0.009084375575184822, 0.030921559780836105, -0.021236896514892578, -0.009978139773011208, 0.019942939281463623, 0.013653242960572243, 0.012812837958335876, -0.004158669617027044, -0.03836514428257942, 0.008530776016414165, 0.05114463344216347, 0.008697522804141045, 0.019649464637041092, 0.0015715903136879206, -0.0088175805285573, -0.006253011990338564, 0.014807132072746754, -0.010618448257446289, 0.011585580185055733, 0.013699932023882866, 0.005069108214229345, 0.01930263079702854, 0.0004910698626190424, 0.011812356300652027, -0.019075855612754822, 0.0038718648720532656, 0.02027643285691738, 0.02083670347929001, 0.013299738988280296, -0.044421397149562836, -0.008197281509637833, 0.005679402034729719, 0.0009454553946852684, -0.002459517912939191, 0.01019157562404871, 0.0033332721795886755, -0.015607517212629318, -0.023958206176757812, -0.0057160863652825356, -0.021663768216967583, -0.003511691465973854, -0.005219180602580309, 0.012099160812795162, 0.014380259439349174, -0.03054804727435112, -0.0022077299654483795, 0.01664801873266697, -0.008090564049780369, 0.015153965912759304, -0.016234485432505608, -0.003053137334063649, 0.08537445217370987, 0.0021677108015865088, -0.012179199606180191, 0.02066328562796116, 0.0015023903688415885, 0.01464705541729927, -0.005576019175350666, -0.013846669346094131, 0.012986254878342152, -0.014460298232734203, 0.00016309939383063465, -0.014460298232734203, -0.014513657428324223, -0.00034329036134295166, 0.018222110345959663, 0.0026679516304284334, 0.0034449927043169737, -0.01005150843411684, -0.01223255880177021, 0.0003168192633893341, -0.008237301371991634, -0.007750399876385927, -0.004912366159260273, 0.007223479449748993, 0.008790900930762291, -0.025358881801366806, 0.012379296123981476, 0.03433653712272644, -0.010991960763931274, -0.02486531063914299, -0.027319826185703278, -0.002911402378231287, -0.0032582359854131937, 0.011925743892788887, -0.010378331877291203, 0.013913367874920368, 0.014100125059485435, 0.012385966256260872, 0.0038085011765360832, 0.00014611204096581787, 0.028787199407815933, 0.02796013467013836, -0.005786120425909758, -0.020249754190444946, 0.020196394994854927, -0.017488423734903336, -0.012292587198317051, 0.01991625875234604, 0.0012339276727288961, -0.005736096296459436, 0.019729502499103546, -0.007623672019690275, 0.002754660090431571, -0.013419797644019127, -0.0097313541918993, -0.0033666216768324375, -0.023758109658956528, 0.00658984063193202, -0.009404529817402363, -0.020356472581624985, 0.017101570963859558, -0.05522660166025162, 0.009364510886371136, 0.0023194504901766777, -0.009971469640731812, -0.060722582042217255, 0.0006986698717810214, 0.010478380136191845, 0.010718496516346931, -0.02106347866356373, -0.009984808973968029, -0.02990773878991604, 0.001092192716896534, 0.022290736436843872, -0.0017808577977120876, 0.012359286658465862, 0.029534226283431053, 0.004915701225399971, 0.028493724763393402, 0.012892876751720905, -0.03204210102558136, -0.011085339821875095, -0.0038451855070888996, -0.02314448170363903, 0.002766332356259227, 0.013966727070510387, -0.005566014442592859, 0.0008745879167690873, -0.0328424870967865, 0.03593730926513672, 0.024478457868099213, 0.03374958783388138, 0.026386043056845665, -0.007570312824100256, 0.011605589650571346, 0.029640942811965942, 0.0406595841050148, 0.0033649541437625885, 0.006366400048136711, 0.0021593733690679073, 0.0074902744963765144, 0.02481195144355297, -0.016621340066194534, 0.02860044315457344, 0.0023494649212807417, 0.001148052979260683, 0.00275632762350142, 0.0033182650804519653, 0.0060295709408819675, 0.012325936928391457, 0.01568755693733692, 0.0017191614024341106, 0.007456925231963396, 0.009151074104011059, -0.002677956596016884, -0.010905252769589424, -0.017315005883574486, 0.004859006963670254, 0.0067098988220095634, -0.013253049924969673, 0.0031665251590311527, -0.031268395483493805, 0.0027930119540542364, 0.03975247964262962, -0.012926226481795311, -2.570248398114927e-05, -0.019222592934966087, -0.026105906814336777, -0.01099863089621067, -0.0017191614024341106, -0.015207325108349323, 0.027426544576883316, -0.018689000979065895, -0.026746215298771858, -0.011038649827241898, 0.0073101879097521305, 0.014860491268336773, 0.012072481215000153, -0.021570390090346336, -0.03433653712272644, -0.031268395483493805, -0.010498389601707458, -0.01872902177274227, -0.0314551517367363, 0.0020593253429979086, -0.03188202157616615, -0.014180163852870464, -0.032869163900613785, 0.002429503481835127, 0.041433289647102356, -0.045515257865190506, -0.01992959901690483, -0.008110573515295982, 0.008750881999731064, 0.006603180430829525, -0.046022165566682816, -0.05592026934027672, -0.031268395483493805, 0.016181128099560738, 0.011892395094037056, 0.02985437959432602, 0.01711491122841835, 0.008804240263998508, 0.031134996563196182, -0.006923334673047066, -0.006233002059161663, 0.001608274644240737, 0.010578428395092487, -0.032575689256191254, -0.003908549435436726, 0.009971469640731812, 0.007943826727569103, 0.029187392443418503, -0.012526033446192741, 0.016221147030591965, 0.028520403429865837, -0.03831178694963455, -0.01398006733506918, 0.007256828714162111, 0.0016466265078634024, 0.009991479106247425, -0.014473638497292995, -0.008477416820824146, -0.0039018793031573296, -0.03825842961668968, 0.017941975966095924, 0.004832327365875244, 0.014380259439349174, -0.00264794216491282, -0.012626081705093384, 0.024825291708111763, -0.0011955759255215526, 0.03289584442973137, 0.022077299654483795, -0.010458370670676231, 0.009884760715067387, -0.013646572828292847, -0.011605589650571346, -0.013186351396143436, 0.021663768216967583, 0.009411199949681759, 0.036951132118701935, -0.005772780627012253, -0.02653278037905693, -0.019556086510419846, 0.018222110345959663, 0.009317821823060513, -0.01455367635935545, -0.022410795092582703, -0.020943421870470047, -0.030361291021108627, -0.0034983516670763493, -0.026719536632299423, -0.029427507892251015, -0.00941786915063858, 0.04330085590481758, -0.021783825010061264, 0.014180163852870464, 0.021677108481526375, -0.027746697887778282, -0.021610409021377563, -0.015340722166001797, 0.01357987429946661, -0.01532738283276558, 0.00835735909640789, 0.011945754289627075, -0.0028213588520884514, -0.02370475046336651, 0.01357987429946661, 0.002357802353799343, -0.006323045585304499, 0.036257464438676834, 0.04020603373646736, -0.01814207248389721, -0.004548857919871807, -0.00039081324939616024, 0.030921559780836105, -0.011638939380645752, -0.006202987860888243, 0.0044588143937289715, -0.01484715100377798, 0.0063397204503417015, -0.020516548305749893, -0.013546524569392204, 0.003214881755411625, 0.007463594898581505, -0.017128249630331993, 0.022584211081266403, -0.014980548992753029, -0.006713233422487974, -0.005582688841968775, 0.016741396859288216, -0.01392670813947916, 0.013299738988280296, -0.006243007257580757, 0.017995335161685944, -0.028440365567803383, -0.0007799590239301324, 0.031108317896723747, -0.0008612481760792434, -0.007016713265329599, -0.004448809660971165, 0.015380742028355598, 0.012419315055012703, 0.021050138399004936, -0.0070567321963608265, -0.0016674698563292623, -0.009657985530793667, -0.020076336339116096, 0.019476046785712242, -0.00981806218624115, -0.010605107992887497, -0.03828510642051697, -0.022424133494496346, 0.006062920205295086, -0.006059585139155388, -0.024091603234410286, -0.01670137792825699, -0.009117725305259228, -0.020983440801501274, 0.010918592102825642, 0.01007818803191185, -0.00027075540856458247, 0.013026274740695953, -0.014020086266100407, -0.016287844628095627, -0.007536963559687138, -0.042473793029785156, 0.046769194304943085, 0.013593213632702827, -0.03647090122103691, -0.010618448257446289, -0.002462852979078889, 0.01692815311253071, -0.0005865325219929218, 0.02239745482802391, 0.01283284742385149, 0.02726646699011326, -0.036097388714551926, 0.03791159391403198, 0.01987623982131481, 0.023331237956881523, -0.024385079741477966, 0.02990773878991604, 0.01928929053246975, -0.045275140553712845, 0.03553711622953415, 0.0006653204909525812, -0.010031498968601227, -0.014820471405982971, 0.005012414418160915, -0.010204914957284927, -0.0059028430841863155, -0.011225407011806965, 0.024651873856782913, 0.010531739331781864, -4.51519554189872e-05, 0.0006190481944940984, -0.056934088468551636, 0.0018292143940925598, -0.003233223920688033, -0.007650351617485285, 0.000770371058024466, 0.006126284133642912, 0.02193056233227253, -0.017848597839474678, 0.016354544088244438, 0.0015123951015993953, -0.0328424870967865, -0.024425098672509193, -0.01595435105264187, 0.03014785423874855, 0.0004072794981766492, -0.01283284742385149, -0.02515878528356552, 0.01652796007692814, -0.02366473153233528, 0.029640942811965942, 5.181532287679147e-06, -0.019449368119239807, -0.0032232191879302263, 0.03268240764737129, -0.013846669346094131, 0.023758109658956528, 0.0182487890124321, 0.0011355469468981028, -0.0019776192493736744, -0.021543709561228752, -0.009031016379594803, -0.01706155203282833, -0.016634678468108177, 0.019942939281463623, 0.007223479449748993, -0.027933454141020775, -0.011705637909471989, -0.006673214491456747, -0.023678071796894073, -0.013166341930627823, 0.00251287710852921, 0.0005957036046311259, 0.004405455198138952, 0.01692815311253071, -0.020756663754582405, 0.0160744097083807, 0.015140625648200512, 0.024371739476919174, -0.013299738988280296, 0.013686592690646648, 0.013519844971597195, 0.004672250244766474, -0.0007036722963675857, -0.0008737542084418237, -0.03567051514983177, -0.015167305245995522, 0.01928929053246975, 0.023851487785577774, 0.004955720156431198, 0.0477830171585083, -0.04850336164236069, 0.01317968126386404, 0.0022877685260027647, -0.006132954265922308, 0.02801349386572838, -5.778825288871303e-05, -0.002863045781850815, -0.0182487890124321, -0.012252568267285824, -0.018675662577152252, 0.041780125349760056, -0.017555123195052147, -0.005789455026388168, 0.013619893230497837, -0.018862418830394745, 0.011652279645204544, -0.0450083464384079, -0.01614110730588436, 0.010031498968601227, 0.006363064981997013, 0.005005744285881519, 0.0237180907279253, -0.01239930558949709, 0.005339238326996565, 0.008490756154060364, -0.02090340107679367, -0.0001725831243675202, 0.012012452818453312, -0.012526033446192741, -0.00847074668854475, 0.012752809561789036, -0.030227893963456154, 0.019262611865997314, -0.012586061842739582, 0.031054958701133728, 0.0038285106420516968, 0.03334939479827881, -0.03745804354548454, -0.009477898478507996, -0.011031980626285076, -0.0015590842813253403, 0.012145849876105785, 0.0029064000118523836, 0.002216067397966981, -0.0014031758764758706, -0.016581319272518158, -0.016154447570443153, -0.02382480911910534, -0.005592694040387869, -0.030281251296401024, -0.02337125688791275, 0.00979138258844614, 0.0032265540212392807, -0.014060105197131634, 0.0003628831182140857, 0.0034133107401430607, -0.006983363535255194, 0.017728539183735847, 0.20489870011806488, -0.021917223930358887, 0.0012822842691093683, 0.035056885331869125, -0.0004906529793515801, 0.005759440828114748, 0.01911587454378605, -0.004638900980353355, 0.0022277396637946367, 0.008697522804141045, -0.014486977830529213, 0.004612221382558346, -0.019662804901599884, -0.00933116115629673, 0.011865715496242046, -0.011492202058434486, -0.02365139126777649, -0.02830696851015091, -0.04434135928750038, -0.011125358752906322, -0.004822322633117437, -0.006733243353664875, -0.016848115250468254, 0.00019155059999320656, 0.02916071191430092, -0.010351652279496193, -0.013032943941652775, -0.008717532269656658, 0.02550561912357807, 0.008717532269656658, -0.027426544576883316, -0.012539372779428959, 0.012786158360540867, -0.03190870210528374, 0.006499797571450472, 0.01185904536396265, -0.010058177635073662, -0.01906251534819603, 0.018102051690220833, -0.025052066892385483, 0.010111536830663681, -0.017355026677250862, 0.007750399876385927, 0.0045855422504246235, -0.0012939565349370241, 0.007883797399699688, -0.0036517588887363672, -0.002062660176306963, 0.007650351617485285, 0.0031515180598944426, -0.025358881801366806, -0.01699485257267952, 0.015820953994989395, 0.02670619636774063, -0.0018342167604714632, -0.03185534477233887, 0.013279729522764683, -0.0020109685137867928, -0.009491238743066788, 0.010004819370806217, -0.01400674693286419, 0.023958206176757812, -0.022090639919042587, 0.022944385185837746, -0.01492718979716301, 0.028787199407815933, -0.001894245739094913, 0.009517917409539223, 0.0023744769860059023, -0.026199286803603172, -0.01153889112174511, -0.009724684059619904, -0.014593696221709251, 0.022264057770371437, -0.009431209415197372, -0.02003631740808487, -0.004202024079859257, 0.01836884766817093, 0.002184385433793068, 0.031268395483493805, -0.007697040680795908, -0.006889985408633947, 0.021170197054743767, -0.0071834600530564785, 0.01692815311253071, -0.034256499260663986, -0.0014181830920279026, 0.008924298919737339, -0.0040252720937132835, -0.0201563760638237, 0.0020726649090647697, -0.0024678553454577923, 0.01320636086165905, 0.02072998508810997, 0.010004819370806217, 0.00944454874843359, 0.025812432169914246, 0.0048723467625677586, -0.013539855368435383, -0.016567980870604515, -0.01723496802151203, 0.007336867041885853, 0.009184423834085464, 0.006546486634761095, -0.025065407156944275, 0.0033449444454163313, 0.003601734759286046, 0.0014390264404937625, 0.014567016623914242, -0.01390002854168415, -0.0027329830918461084, -0.01390002854168415, 0.01671471819281578, -0.00400526262819767, 0.0008954313234426081, 0.029320789501070976, 0.013012934476137161, -0.026492761448025703, -0.010224925354123116, -0.01179901696741581, 0.008083893917500973, -0.005142476875334978, 0.010298294015228748, -0.007930486463010311, 0.0189424566924572, -0.001985956681892276, 0.0032899179495871067, -0.005279209464788437, 0.008964317850768566, -0.029374148696660995, 0.028813878074288368, -0.014380259439349174, 0.010004819370806217, -0.013699932023882866, -0.024024905636906624, 0.012225888669490814, 0.020516548305749893, -0.021743806079030037, -0.000824563845526427, 0.0014832144370302558, -0.02113017812371254, -0.004915701225399971, 0.014446958899497986, -0.015420760959386826, -0.008223961107432842, -0.024078264832496643, -0.0022911035921424627, -0.009151074104011059, 0.011992443352937698, -0.0044554793275892735, -0.013213030993938446, -0.012966245412826538, 0.008730871602892876, -0.01392670813947916, 0.013426466844975948, -0.01325972005724907, -0.03124171495437622, -0.032655730843544006, 0.0002688795211724937, -0.008397378027439117, -0.01568755693733692, -0.026679517701268196, 0.027746697887778282, 0.012499353848397732, 0.004402120131999254, -0.02709304913878441, -0.16850782930850983, 0.0025078747421503067, -0.00392188923433423, 0.0018759035738185048, 0.03345611318945885, -0.014420279301702976, 0.00023240360314957798, 0.0064030843786895275, -0.026919633150100708, -0.019089194014668465, 0.005872828885912895, 0.011185388080775738, -0.006266351789236069, 0.0026596144307404757, 0.021143518388271332, -0.01427354197949171, -0.010905252769589424, 0.017528442665934563, -0.005069108214229345, 0.009311151690781116, 0.03932560980319977, -0.02274428866803646, 0.01119205728173256, 0.03334939479827881, -0.018395526334643364, 0.0024678553454577923, -0.004382110666483641, 0.008023864589631557, -0.01065179705619812, -0.007703710813075304, -0.012425985187292099, -0.0007937156478874385, 0.007343537174165249, 0.016274506226181984, 0.006423093844205141, 0.011071999557316303, 0.012526033446192741, 0.002064327709376812, -0.0029680964071303606, 0.025972509756684303, -0.0008887614239938557, 0.026319343596696854, -0.0027513252571225166, 0.028493724763393402, 0.011758997105062008, 0.039192210882902145, 0.004825657699257135, -0.01257939264178276, 0.009718013927340508, -0.005919517949223518, -0.006366400048136711, 0.003931893967092037, 0.014393599703907967, 0.0010538409696891904, 0.024585174396634102, 0.01819543167948723, 0.0009221108048222959, -0.009231112897396088, 0.0015140626346692443, -0.015487459488213062, 0.014967209659516811, -0.017501763999462128, 0.00392188923433423, -0.012045801617205143, 0.002464520512148738, -0.025078745558857918, -0.012839517556130886, -0.0009287807042710483, -0.020169714465737343, -0.0038918745703995228, -0.011405494064092636, -0.0014448625734075904, 0.00893096812069416, -0.0029747660737484694, 0.007903806865215302, -0.008730871602892876, -0.03807166963815689, 0.014780452474951744, 0.002020973479375243, -0.03519028425216675, -0.01325972005724907, 0.017648501321673393, -0.017368365079164505, 0.006529811769723892, -0.0013664915459230542, -0.0034349877387285233, 0.022264057770371437, 0.008850930258631706, -0.006389744579792023, -0.002451180713251233, 0.023558013141155243, -0.02309112250804901, 0.0003389132325537503, 0.006836626213043928, 0.01228591799736023, 0.03302924335002899, 0.029560904949903488, -0.005005744285881519, -0.024611854925751686, 0.003369956510141492, 0.0004285397590138018, -0.008170601911842823, 0.004402120131999254, 0.04775633662939072, 0.02273094840347767, 0.002371142152696848, -0.01007818803191185, 0.010244934819638729, 0.0017024866538122296, -0.021730467677116394, -0.018115391954779625, -0.010358322411775589, 0.018355507403612137, 0.02079668454825878, -0.013253049924969673, 0.00858413428068161, 0.000503992778249085, -0.021850524470210075, 0.0024361733812838793, 0.005982881877571344, 0.046475719660520554, -0.00965131539851427, -0.00970467459410429, 0.0021693783346563578, -0.022877685725688934, -0.04546189680695534, -0.10431691259145737, -0.019742842763662338, 0.03209545835852623, -0.0010813542176038027, -0.01838218793272972, 0.014100125059485435, -0.013713272288441658, 0.01532738283276558, 0.001847556559368968, 0.023277878761291504, 0.012792828492820263, -0.024131624028086662, -0.02210398018360138, -0.005042428616434336, -0.007903806865215302, -0.016621340066194534, 0.025332201272249222, -0.01647460274398327, 0.00446548406034708, -0.0024228335823863745, 0.0058428142219781876, -0.001361489063128829, -0.010204914957284927, -0.012219218537211418, -0.039138853549957275, -0.00944454874843359, -0.03262905031442642, -0.001352317980490625, 0.011652279645204544, 0.018969137221574783, -0.023264538496732712, -0.015860972926020622, 0.0117723373696208, -0.022717608138918877, -0.021343613043427467, -0.005899508483707905, -0.0008103903383016586, -0.012159190140664577, 0.006182977929711342, -0.035003527998924255, -0.017074890434741974, -0.015887653455138206, -0.013739950954914093, -0.011585580185055733, 0.010238264687359333, -0.001348149380646646, -0.01053840946406126, 0.013473155908286572, 0.014126804657280445, -0.02043651044368744, -0.021263575181365013, 0.004775633569806814, 0.019022496417164803, -0.031268395483493805, 0.01568755693733692, -0.01631452515721321, 0.021890543401241302, -0.03417646139860153, 0.004845667164772749, 0.009784713387489319, -0.01883573830127716, -0.013966727070510387, -0.026052549481391907, 0.02451847679913044, -0.0028397010173648596, -0.0005298385513015091, -0.03140179067850113, 0.00526920473203063, 0.02394486591219902, -0.0189424566924572, 0.0012180866906419396, -0.008283990435302258, -0.00826398003846407, 0.016434581950306892, -0.023211179301142693, -0.0013273060321807861, 0.0031281732954084873, -0.0027646650560200214, -0.0006607349496334791, 0.018048694357275963, -0.014180163852870464, -0.01228591799736023, 0.02177048660814762, -0.018902437761425972, 0.01854226365685463, 0.004852337297052145, 0.013479826040565968, -0.007156780455261469, -0.009457889012992382, -0.007677031215280294, -0.003551710629835725, 0.02629266493022442, 0.01831548847258091, -0.01452699676156044, -0.02589247189462185, -0.0025495614390820265, 0.01010486762970686, -0.025478938594460487, -0.000839571061078459, 0.01687479391694069, -0.014033426530659199, -0.02349131554365158, -0.06317709386348724, 0.0020126360468566418, 0.03204210102558136, -0.0025929156690835953, 0.003571720328181982, -0.0160744097083807, 0.0189424566924572, 0.025639016181230545, 0.019249271601438522, -0.011065329425036907, -0.015807613730430603, 0.006032906007021666, -0.010985291562974453, 0.011092009022831917, -0.02055656723678112, -0.043487612158060074, 0.031161677092313766, 0.002819691551849246, -0.00044104576227255166, 0.003431652905419469, 0.012826178222894669, 0.007543633691966534, 0.001669971039518714, -0.01495386939495802, 0.008130582980811596, 0.026199286803603172, -0.0074902744963765144, 0.047169387340545654, 0.002000963781028986, -0.0037651469465345144, 0.009064366109669209, -0.028920596465468407, -0.0097313541918993, 0.009811392053961754, -0.017074890434741974, -0.030361291021108627, -0.02481195144355297, 0.026399381458759308, 0.000300144572975114, -0.010991960763931274, -0.016007710248231888, -0.021316934376955032, 0.003835180541500449, -0.04898359254002571, 0.00784377846866846, -0.015700895339250565, -0.004045282024890184, -0.0038418504409492016, 0.020289773121476173, -0.013726611621677876, 0.029187392443418503, 0.007697040680795908, -0.021490350365638733, -0.010818543843925, -0.01314633246511221, -0.020062997937202454, 0.01727498695254326, -0.012432655319571495, 0.01390002854168415, 0.0035683854948729277, 0.025452259927988052, 0.002679624129086733, 0.026719536632299423, 0.0005840312805958092, 0.04210027679800987, -0.002878052880987525, -0.039939235895872116, -0.004185349214822054, 0.0021477011032402515, 0.01484715100377798, -0.010891912505030632, 8.68126517161727e-05, 0.008504096418619156, -0.0018575614085420966, 0.004802313167601824, 0.0038551902398467064, -0.01934264972805977, 0.004602216649800539, -0.028787199407815933, 0.036097388714551926, -0.004972395021468401, -0.00653314683586359, -0.016607999801635742, 0.028573762625455856, 0.020863382145762444, 0.030521368607878685, -0.0033132624812424183, -0.03084152191877365, -0.0054092719219625, 0.008837589994072914, -0.0403127521276474, -0.012932895682752132, -0.014807132072746754, -0.018809059634804726, -0.010811874642968178, 0.008550785481929779, 0.0017958650132641196, 0.0178085770457983, -0.005479305982589722, 0.011792346835136414, 0.03612406551837921, 0.004905696026980877, -0.008970987983047962, -0.007410235702991486, -0.008624154143035412, 0.0410064198076725, 0.0021193542052060366, -0.02830696851015091, -0.010131546296179295, 0.025065407156944275, 0.0005840312805958092, -0.011318785138428211, -0.01524734403938055, 0.014460298232734203, -0.024078264832496643, 0.0003078566223848611, 0.012612741440534592, -0.0042120288126170635, 0.000538592750672251, 0.037004489451646805, 0.010458370670676231, 0.007710380479693413, 0.023691412061452866, -0.009424539282917976, 0.018355507403612137, 0.006849966011941433, 0.00761033222079277, -0.011412163265049458, 0.0033232674468308687, -0.00578278535977006, 0.0005102457944303751, 0.012672770768404007, -0.008617484010756016, -0.0058394791558384895, -0.0036617638543248177, -0.02067662589251995, -0.005549339577555656, 0.027346504852175713, -0.004498833790421486, 0.057627756148576736, 0.026999671012163162, -0.011552231386303902, 0.03313596174120903, -0.032468970865011215, 0.015647536143660545, -0.004185349214822054, -0.0063964142464101315, -0.03020121343433857, -0.0070634023286402225, 0.033669549971818924, -0.01992959901690483, 0.005682737100869417, -0.010518399998545647, -0.03817838802933693, 0.011605589650571346, -0.01225923839956522, 0.01912921480834484, -0.005762775894254446, -0.013793310150504112, 0.029934419319033623, -0.025332201272249222, 0.009091045707464218, -3.1102219963941025e-06, -0.01723496802151203, -0.029267430305480957, -0.009231112897396088, 0.009197763167321682, -0.035216961055994034, -0.03294920548796654, 0.0071167610585689545, -0.020369810983538628, -0.013313079252839088, -0.01274613942950964, -0.02985437959432602, 0.013286399655044079, -0.0314551517367363, 0.010405011475086212, -0.019902920350432396, 0.02198392152786255, -0.00979138258844614, 0.038445185869932175, -0.010525069199502468, -0.006930004805326462, -0.0021360288374125957, -0.020569907501339912, -0.002482862677425146, 0.0003186951798852533, -0.029027314856648445]} +{"id": "test:10000076", "text": "\"Do You Need Residential Roofing Contractor in Hearne Texas?\\nHaving a high-quality roof indicates that only a experienced contractor should be retained to do any repairing or even replacements. The only way to separate amateur roofing companies from the professional bunch is examining our jobs. As a roofing company in Hearne Texas, we help make our customers satisfied by giving them extremely reliable Residential Roofing Contractor.\\nThe expert services our company offers are very reasonable, and this suggests that our customers could have their roofing problem remedied and not having to dig further in their bank account. Our friendly support desk is all set to assist our clients. Call Roofing Bryan to setup an appointment!\\nPrior to doing the actual roof repair, we hold appointments with the clients. This technique makes sure that our specialists are better set at choosing the correct methods to any sort of roofing problem at hand. The result is something that the clients as well as our firm are forever happy with.\\nDo You Want Residential Roofing Contractor in Hearne Texas?\"", "vector_field": [8.236199209932238e-05, 0.0023506113793700933, -0.005910296458750963, -0.011794237419962883, -0.019187049940228462, 0.003185761859640479, 0.010608225129544735, 0.011102396994829178, 0.01582668162882328, -0.0015953518450260162, -0.01862039975821972, 0.016050705686211586, -0.02105831541121006, -0.020359884947538376, -0.00945515651255846, -0.01377092581242323, 0.025169825181365013, -0.006737211253494024, 0.015787146985530853, -0.01276281476020813, -0.026869777590036392, -0.003093516454100609, 0.005337057169526815, 0.00661860965192318, -0.005007609259337187, -0.02232339419424534, 0.030809974297881126, -0.007524591870605946, 0.00565003277733922, -0.004157633520662785, 0.02244199626147747, 0.021150559186935425, -0.028991421684622765, 0.009830727241933346, -0.012973661534488201, -6.511744868475944e-05, -0.007287389133125544, -0.002220479305833578, 0.004408014006912708, 0.006737211253494024, 0.03363005071878433, -0.030361926183104515, 0.005893824156373739, 0.0023967339657247066, -0.00189432583283633, 0.011807415634393692, 0.002049166476354003, -0.0185940433293581, -0.012077562510967255, -0.004388247150927782, 0.013810459524393082, -0.0026471144519746304, -0.021466830745339394, -0.014258508570492268, -0.010193120688199997, -0.005844407249242067, -0.0261581689119339, 0.032813020050525665, -0.010924494825303555, -0.034078098833560944, 0.005399652291089296, -0.01583985798060894, 0.009290432557463646, -0.010034984908998013, -0.00026850009453482926, -0.005284345708787441, -0.003946786746382713, 0.01515460666269064, -0.029861165210604668, 0.00646047480404377, 0.025710120797157288, 0.019569208845496178, 0.010173353366553783, 0.011708580888807774, 0.02476130984723568, -0.009217954240739346, -0.01908162608742714, -0.007023830898106098, 0.0012543731136247516, 0.0006378936232067645, -0.0008182664168998599, 0.006246333476155996, -0.04791491478681564, 0.015325919725000858, 0.019107982516288757, 0.000943044840823859, 0.01692044734954834, 0.02339080534875393, 0.024089235812425613, 0.005738983862102032, -0.0052118669264018536, 0.02062344364821911, -0.000536176550667733, 0.008045119233429432, 0.005155860912054777, 0.008809438906610012, -0.013942237943410873, 0.013863170519471169, -0.02231021597981453, -0.0037425290793180466, -0.0022237738594412804, 0.02427372708916664, -0.011095807887613773, -0.006180443800985813, -0.002194123575463891, 0.0037655902560800314, -0.014337575994431973, -0.012439955957233906, 0.010430322960019112, -0.03945469111204147, -0.004388247150927782, 0.029729384928941727, -0.014956938102841377, 0.0039039584808051586, -0.0031281085684895515, -0.004154338967055082, 0.04027172178030014, -0.018804891034960747, 0.006480241660028696, -0.010483034886419773, 0.011800826527178288, 0.0013523839879781008, 0.027014734223484993, -0.00900051835924387, 0.021888524293899536, 0.010094285942614079, -0.0436716228723526, -0.022112548351287842, 0.030546415597200394, -0.006697677541524172, -0.010160175152122974, 0.02598685584962368, 0.03249674662947655, -0.0010130525333806872, 0.02431325986981392, 0.003508620895445347, -0.003646989120170474, -0.007267622277140617, -0.011234176345169544, -0.03457885980606079, -0.012723281048238277, 0.012795759364962578, -0.019608743488788605, -0.012756225652992725, -0.0022616602946072817, 0.024866733700037003, 0.029571250081062317, 0.0035514491610229015, -0.011794237419962883, -0.019266117364168167, 0.018659932538866997, -0.011293476447463036, 0.010410555638372898, -0.005814756732434034, 0.006463769357651472, 0.009053230285644531, 0.002607580740004778, -0.001984924077987671, -0.017737479880452156, -0.010819071903824806, 0.01645922102034092, 0.0023720255121588707, 0.04000816121697426, 0.037267155945301056, 0.020794756710529327, 0.010226065292954445, 0.015576300211250782, 0.01091790571808815, 0.012077562510967255, 0.0073335119523108006, 0.030072011053562164, 0.014258508570492268, -0.03708266466856003, 0.022020302712917328, 0.011431844905018806, -0.025301603600382805, -0.029254980385303497, 0.015049183741211891, -0.0013935649767518044, -0.01598481647670269, 0.006700971629470587, -0.007966051809489727, 0.0215458981692791, 0.0412205308675766, -0.040535278618335724, -0.0008330915588885546, 0.01706540584564209, 0.018514975905418396, 0.02783176489174366, -0.024379150941967964, -0.017447564750909805, 0.030203789472579956, 0.019793234765529633, -0.012242286466062069, -0.6338052153587341, -0.010931083932518959, -0.013929060660302639, 0.0017296018777415156, 0.014245330356061459, -0.006677910219877958, 0.030651839450001717, 0.0032813018187880516, -0.025103935971856117, -0.0034526148810982704, 0.0010534098837524652, -0.008269144222140312, -0.03692452982068062, -0.007893573492765427, -0.013421710580587387, -0.007386223413050175, -0.0076036592945456505, 0.007590481545776129, 0.014350754208862782, -0.01536545343697071, -0.025486094877123833, 0.01377092581242323, -0.046992458403110504, -0.008993929252028465, 0.031020822003483772, 0.01999090239405632, 0.02415512502193451, -0.0164328645914793, -0.006733916699886322, 0.0018514975672587752, -0.030520061030983925, 0.023614831268787384, -0.008308677934110165, -0.001281552598811686, 0.056190647184848785, 0.010614813305437565, 0.010588457807898521, 0.03123166784644127, 5.0549671868793666e-05, 0.0381896086037159, 0.0018004331504926085, -0.019635099917650223, 0.05134117230772972, 0.01315156277269125, 0.002890906063839793, 0.019305652007460594, 0.013929060660302639, -0.010048163123428822, -0.01968781091272831, 0.001348265795968473, -0.022362928837537766, -0.0018745589768514037, -0.02676435373723507, -0.004134572111070156, 0.0010674113873392344, -7.742027082713321e-05, -0.0009521046304143965, -0.02888599783182144, -0.014429821632802486, -0.010884961113333702, -0.01483833696693182, 0.027462782338261604, -0.01359961275011301, -0.022257504984736443, -0.04430416226387024, 0.0025103935040533543, -0.00891486182808876, -0.03044099360704422, 0.006256217136979103, 0.005281051155179739, -0.006724033039063215, 0.0009776368970051408, -0.027910832315683365, -0.0002732359280344099, -0.016116594895720482, 0.018686288967728615, 0.004968075547367334, 0.010792715474963188, 0.01229499839246273, 0.006256217136979103, 0.02875421941280365, -0.010041574016213417, -0.01346124429255724, -0.009428801015019417, 0.020017258822917938, -0.04585915803909302, -0.015576300211250782, -0.01909480430185795, 0.009890028275549412, 0.0017823135713115335, -0.020662976428866386, 0.033893607556819916, 0.00010532039596000686, -0.0580882653594017, -0.01647239923477173, 0.009553991258144379, -0.024352794513106346, 0.006101376377046108, 0.02907048910856247, -0.012011673301458359, -0.005574259907007217, -0.00798581913113594, 0.020030437037348747, -0.0014923993730917573, 0.017315786331892014, -0.00017398971249349415, 0.00892804004251957, -0.00537988543510437, 0.020676154643297195, -0.004549676552414894, -0.001930565107613802, -0.018422730267047882, 0.0027739519719034433, 0.03133708983659744, 0.01768476702272892, -0.02415512502193451, 0.018725823611021042, 0.016353797167539597, -0.006170560605823994, -0.013705035671591759, -0.0027212402783334255, -0.013336054049432278, 0.026882953941822052, -0.009105942212045193, 0.01566854491829872, 0.030414637178182602, 0.015260030515491962, 0.026948845013976097, 0.006523069925606251, -0.009725304320454597, 0.021743565797805786, -0.033287424594163895, 0.021005602553486824, -0.013270164839923382, 0.01676231250166893, 0.0077947392128407955, 0.008288910612463951, -0.017170827835798264, 0.016037527471780777, -0.02812168002128601, -0.0005085853044874966, 0.0022962524089962244, 0.013810459524393082, 0.018198706209659576, -0.009099353104829788, -0.013520545326173306, -0.03215412050485611, -0.008499757386744022, -0.02000408060848713, 0.030809974297881126, -0.03015107847750187, 0.01106945239007473, 0.014706557616591454, -0.0065362476743757725, 0.012347710318863392, -0.01627472974359989, 0.019714167341589928, -0.025130292400717735, 0.0031346974428743124, -0.020689332857728004, 0.009705536998808384, 0.02754185162484646, -0.02072886750102043, -0.0025598108768463135, 0.001795491436496377, -0.006483536213636398, 0.01724989525973797, 0.009949328377842903, 0.0338672511279583, -0.02412877045571804, 0.022692376747727394, 0.0062661003321409225, -0.0036931117065250874, 0.023601653054356575, 0.001235429896041751, 0.006638376507908106, -0.023931100964546204, 0.006615315563976765, -0.005106443539261818, 0.01709176041185856, 0.00776179414242506, 0.02476130984723568, -0.009053230285644531, 0.00222871545702219, -0.004375068936496973, -0.011477967724204063, 0.015286386013031006, -0.0006411881186068058, -0.00222871545702219, 0.020399417728185654, -0.0314161591231823, 0.013902704231441021, 0.004160928074270487, 0.025262070819735527, 0.006170560605823994, -0.03091539815068245, -0.011952372267842293, 0.01862039975821972, -0.0010179942473769188, 0.018027393147349358, 0.018264595419168472, -0.024787666276097298, 0.02153271995484829, -0.0164328645914793, 0.0019651572220027447, 0.0018119638552889228, 0.01860722154378891, 0.007524591870605946, 0.03671368211507797, -0.027647273615002632, -0.010054752230644226, -0.06193621829152107, -0.028727862983942032, 0.008592003025114536, 0.009646236896514893, 0.014522066339850426, -0.009250898845493793, 0.045569244772195816, 0.010410555638372898, -0.006938174366950989, 0.01891031302511692, -0.008400923572480679, 0.031732428818941116, 0.0019256233936175704, -0.02244199626147747, 0.0026932372711598873, 0.012716691941022873, 0.01570807956159115, -0.015260030515491962, -0.003871013643220067, 1.1993965017609298e-05, 0.029281336814165115, 0.022402461618185043, 0.007998996414244175, 0.027910832315683365, 0.04274917021393776, 0.01154385693371296, -0.017921969294548035, -0.005360118579119444, -0.020504841580986977, 0.0206102654337883, 0.0033537803683429956, -0.01359961275011301, -0.010957439430058002, 0.016103416681289673, 0.04232747480273247, -0.007122665178030729, 0.013520545326173306, -0.010884961113333702, 0.015497232787311077, -0.003643694566562772, 0.01996454782783985, 0.005669799633324146, -0.01676231250166893, 0.022204793989658356, -0.007583892438560724, 0.011412077583372593, 0.0033406023867428303, 0.03088904172182083, 0.01891031302511692, 0.02167767658829689, 0.0007140784873627126, 0.032681237906217575, -0.011412077583372593, 0.003439436899498105, 0.007537769619375467, 0.014548422768712044, -0.01908162608742714, 0.00036239277687855065, 0.01719718426465988, 0.01785608008503914, 0.0003953375562559813, 0.03979731351137161, 0.0015187552198767662, 0.0011300065089017153, 0.003314246656373143, -0.03157429397106171, 0.0041905781254172325, -0.022231148555874825, 0.0035053263418376446, -0.006447297055274248, -0.0173684973269701, 0.018712645396590233, 0.005930063780397177, 0.012057796120643616, 0.0005946535966359079, -0.004790173377841711, -0.0045595597475767136, -0.03355098143219948, 0.02473495341837406, 0.019332006573677063, 0.004141161218285561, -0.011023329570889473, 0.015101894736289978, 0.03162700682878494, -0.010627991519868374, 0.005455658305436373, -0.0025136880576610565, -0.0006658967467956245, -0.03273395076394081, -0.003927019890397787, 0.005594026762992144, -0.013744569383561611, -0.01996454782783985, 0.026487616822123528, 0.013968594372272491, -0.00483959075063467, 0.00023864387185312808, -0.014943759888410568, -0.0038677193224430084, 0.006437413394451141, -0.0044146026484668255, -0.022573774680495262, 0.01077294908463955, 0.027120158076286316, -0.047256018966436386, -0.014535244554281235, -0.010469856671988964, 0.029149556532502174, 0.004246584605425596, 0.007959462702274323, 0.0038677193224430084, 0.001646416261792183, 0.0050669098272919655, 0.11470060795545578, 0.007630015257745981, -0.007175377104431391, -0.0003018566931132227, 0.01084542740136385, -0.027699986472725868, -0.013744569383561611, -0.043144505470991135, 0.006658143363893032, -0.005814756732434034, -0.0038051242008805275, -0.017144473269581795, -0.004912069533020258, 0.017342140898108482, 0.0005757103208452463, -0.0024313260801136494, -0.00642423564568162, 0.008282322436571121, -0.0022369518410414457, -0.010226065292954445, -0.026962021365761757, 0.0019420958124101162, 0.017711123451590538, -0.0012337826192378998, 0.019015736877918243, -0.0018630282720550895, 0.011108986102044582, 0.017012692987918854, 0.013276753015816212, -0.006163971498608589, -0.008111009374260902, -0.009059819392859936, 0.023575296625494957, 0.0016851263353601098, 0.012255464680492878, 0.03181149810552597, 0.007129254285246134, -0.012169808149337769, 0.020715689286589622, -0.018567688763141632, -0.0013968594139441848, 0.03447343781590462, 0.033577337861061096, 0.0069974749349057674, 0.010746592655777931, -0.02212572656571865, -0.00024976275744847953, 0.009890028275549412, -0.023786144331097603, 0.006411057431250811, -0.005729100201278925, 0.003854541340842843, -0.025947323068976402, -0.011188053525984287, -0.006308928597718477, -0.020412595942616463, 0.006457180250436068, 0.004071976989507675, 0.006048664916306734, -0.007023830898106098, -0.014377109706401825, -0.026962021365761757, 0.012281820178031921, -0.0028661973774433136, -0.00047399327741004527, -0.016828201711177826, -0.010008629411458969, -0.01062140241265297, -0.024352794513106346, 0.027647273615002632, 0.017131295055150986, -0.017618877813220024, -0.008242788724601269, 0.0238520335406065, 0.02232339419424534, 0.025090757757425308, 0.003923725336790085, 0.016880914568901062, 0.001235429896041751, -0.002571341348811984, 0.004902185872197151, -0.016235196962952614, 0.033129289746284485, -0.004137866664677858, -0.042037561535835266, -0.027093801647424698, -0.02215208113193512, 0.0031890564132481813, 0.0031824675388634205, 0.014851514250040054, -0.00929702166467905, -0.01153067871928215, 0.022362928837537766, -0.00330930482596159, 0.01407401729375124, 0.010792715474963188, 0.04433051869273186, 0.0002612934331409633, 0.014482532627880573, -0.013270164839923382, -0.004071976989507675, -0.013836815021932125, -0.007122665178030729, -0.016973160207271576, 0.02970302850008011, 0.0008557410910725594, -0.009718715213239193, 0.008829205296933651, -0.024655885994434357, 0.006924996618181467, 0.028938710689544678, -0.023403983563184738, 0.012354299426078796, 0.0027212402783334255, 0.009824139066040516, 0.026935666799545288, -0.021295517683029175, -0.002220479305833578, 0.003419670043513179, 0.00010202592238783836, 0.008941218256950378, -0.037135377526283264, 0.04090426117181778, 0.028833286836743355, -0.018528154119849205, 0.009613292291760445, -0.002049166476354003, -0.03771520406007767, -0.010344666428864002, 0.009600114077329636, -0.011510912328958511, -0.005300818011164665, -0.02246835269033909, -0.021954413503408432, -0.03318199887871742, -0.040825191885232925, -0.004608977120369673, 0.008137364871799946, -0.011899661272764206, -0.018528154119849205, -0.014034483581781387, -0.003834774484857917, 0.0031544642988592386, -0.007300567347556353, 0.009843905456364155, -0.0207552220672369, -0.010693881660699844, 0.0034723817370831966, -0.016366975381970406, -0.011853538453578949, -0.003798535093665123, 0.01130006555467844, -0.014469355344772339, -0.0007206674199551344, -0.012011673301458359, -0.018383197486400604, 0.016722779721021652, 0.018185527995228767, -0.008269144222140312, 0.031152600422501564, 0.01581350341439247, 0.028016256168484688, 0.019556032493710518, -0.016828201711177826, -0.0004369303642306477, 0.020847467705607414, -0.012281820178031921, 0.004793467931449413, -0.029334047809243202, -0.022797800600528717, 0.017421208322048187, 0.014153084717690945, 0.0009619880584068596, 0.03078361786901951, 0.004592504817992449, 0.011893072165548801, -0.015128251165151596, -0.027172869071364403, -0.006944763474166393, -0.024827199056744576, -0.010384200140833855, -0.015734435990452766, -0.017118116840720177, 0.008532702922821045, -0.014403465203940868, 0.009705536998808384, 0.004071976989507675, -0.007695904467254877, -0.00045010828762315214, -0.012598090805113316, 0.015866214409470558, -0.014891048893332481, 0.006005836650729179, -0.0017625467153266072, 0.01935836300253868, -0.0018366724252700806, -0.00837456714361906, -0.029940232634544373, -0.031152600422501564, 0.0012914360268041492, -0.006615315563976765, 0.012057796120643616, -0.022666020318865776, -0.001587939215824008, -0.030546415597200394, 0.020083148032426834, -0.0249326229095459, -0.02600003406405449, 0.014350754208862782, 0.017039049416780472, -0.028490660712122917, -0.04013993963599205, 0.005472131073474884, 0.0013087320839986205, -0.006944763474166393, -0.0070963092148303986, -0.0009282196406275034, 0.010206297971308231, -0.0030473938677459955, -0.021638143807649612, -0.02032035030424595, -0.02754185162484646, 0.021769922226667404, 0.017763834446668625, 0.02460317499935627, 0.028780575841665268, -0.017921969294548035, -0.014218974858522415, -0.011649280786514282, 0.022731909528374672, -0.0035316823050379753, 0.005854290444403887, 0.016182484105229378, 0.0012823762372136116, -0.015629012137651443, 0.03763613849878311, -0.005406241398304701, -0.022059835493564606, -0.02182263322174549, 0.014943759888410568, -0.0006597195751965046, -0.021914878860116005, 0.001966804498806596, -0.011359366588294506, -0.00537988543510437, 0.005310701206326485, -0.010515979491174221, -0.005076793488115072, -0.016077060252428055, -0.043592557311058044, -0.02336445078253746, 0.011240764521062374, -0.021466830745339394, 0.026962021365761757, -0.004760523326694965, -0.007241266313940287, -0.01985912397503853, -0.014284864068031311, -0.0032434153836220503, 0.003854541340842843, 0.01739485375583172, 0.037609782069921494, -0.006470357999205589, -0.008255966007709503, -0.0008820969378575683, 0.009672592394053936, 0.002138117328286171, 0.003976437263190746, 0.027620919048786163, 0.02337762899696827, 0.00090268743224442, -0.022020302712917328, -0.007116076070815325, 0.0007029596017673612, -0.008058297447860241, 0.014271685853600502, 0.002355552976951003, -0.01159656886011362, 0.00012848470942117274, 0.003141286550089717, 0.018818069249391556, 0.01801421493291855, -0.00042869418393820524, -0.0012864943128079176, -0.001511342590674758, -0.005508370231837034, 0.007267622277140617, 0.011240764521062374, 0.01215662993490696, 0.004292706958949566, -0.009646236896514893, 0.011326421052217484, -0.038795795291662216, 0.015760790556669235, 0.009290432557463646, -0.0013408532831817865, -0.022059835493564606, 0.02538067288696766, -0.021585430949926376, -0.002050813753157854, -0.012802348472177982, 0.008723782375454903, -0.003861130215227604, 0.018185527995228767, 0.00615408830344677, 0.009145475924015045, 0.0003675404004752636, -0.011715169996023178, 0.0027228875551372766, -0.03125802427530289, 0.0035514491610229015, -0.006127732340246439, -0.02290322259068489, 0.000787792494520545, -0.018027393147349358, 0.00037392345257103443, 0.007089720573276281, 0.01267715822905302, -0.029254980385303497, 0.020676154643297195, -0.003310952102765441, -0.00127743452321738, 0.007280800025910139, -0.003436142345890403, -0.028227102011442184, 0.008967573754489422, 0.002077169483527541, -0.005406241398304701, -0.016670066863298416, -0.014403465203940868, 0.002951853908598423, -0.0069513521157205105, -0.025183003395795822, -0.03956011310219765, 0.004243290051817894, -0.014693379402160645, -0.014719735831022263, 0.02335127256810665, -0.017711123451590538, 0.0022451879922300577, 0.02473495341837406, 0.03323471173644066, 0.0007840861799195409, 0.005804873537272215, -0.03331378102302551, 0.002144706202670932, -0.01813281700015068, 0.009349733591079712, -0.015958460047841072, -0.015233674086630344, -0.007537769619375467, 0.05624336004257202, 0.013981771655380726, -0.02167767658829689, -0.011945783160626888, 0.003159406129270792, -0.058668095618486404, 0.005587437655776739, -0.0031709368340671062, 7.180936518125236e-05, -0.008559058420360088, 0.007504825014621019, 0.00014053014456294477, 0.011379132978618145, -0.01738167554140091, 0.012980250641703606, -0.008440457284450531, -0.03302386403083801, -0.01673595793545246, -0.014324397780001163, -0.0034064920619130135, -0.00021537661086767912, -0.048046693205833435, 0.0012486078776419163, 0.021137382835149765, 0.011893072165548801, 0.0019371540984138846, 0.02519618161022663, -0.02967667393386364, 0.01597163826227188, -0.004483786877244711, -0.0019371540984138846, 0.015233674086630344, -0.0020541080739349127, 0.0005880646640434861, -0.025341138243675232, 0.007214910816401243, -0.0007766736089251935, -9.039228461915627e-05, -0.03594936430454254, -0.013395355083048344, 0.03439436852931976, -0.016406510025262833, -0.000929043279029429, 0.00816372036933899, -0.008723782375454903, 0.009863672778010368, -0.0172235406935215, 0.01098379585891962, -0.01598481647670269, -0.0052415174432098866, 0.015299564227461815, 0.013454655185341835, -0.010852016508579254, 0.019292473793029785, 0.010252420790493488, -0.010054752230644226, 0.008829205296933651, -0.017869258299469948, -0.03971824795007706, -0.01937154121696949, -0.014864692464470863, 0.009013696573674679, -0.004065387882292271, 0.006114554591476917, -0.013243808411061764, 0.008038530126214027, -0.007280800025910139, 0.017658410593867302, 0.00018891782383434474, -0.015905749052762985, -0.0185940433293581, -0.04214298352599144, -0.019595565274357796, 0.01877853460609913, -0.014100373722612858, -0.01566854491829872, 0.011800826527178288, 0.022797800600528717, 0.008176898583769798, -0.012729870155453682, -0.0036305165849626064, 0.01121440902352333, -0.0042070504277944565, -0.009929561987519264, 0.010884961113333702, 0.2127443253993988, 0.008888506330549717, 0.002301194006577134, 0.026369016617536545, -0.002951853908598423, -0.017302608117461205, 0.02644808404147625, 0.031020822003483772, -0.02783176489174366, -0.00791334081441164, 0.001996454782783985, 0.008203255012631416, -0.030361926183104515, -0.002063991501927376, 0.022481529042124748, -0.011945783160626888, -0.02416830323636532, -0.01299342792481184, -0.010522568598389626, 0.03610749915242195, 0.014100373722612858, 0.002958442782983184, 0.009791193529963493, -0.020531198009848595, 0.006124437786638737, 0.012387244030833244, -0.0024774486664682627, 0.0030078599229454994, 0.02951853908598423, -0.0014742796774953604, -0.014192618429660797, -0.0053205848671495914, 0.017447564750909805, 0.0017823135713115335, 0.009283844381570816, -0.005742278415709734, 0.02536749467253685, 0.000862330081872642, 0.009975684806704521, -0.003436142345890403, 0.03270759433507919, 0.002966678934171796, 0.005534726195037365, -0.017948325723409653, 0.022389285266399384, 0.012901182286441326, 0.005287639796733856, -0.0004266351170372218, -0.002597697312012315, 0.0005291757988743484, -0.035606738179922104, -0.037135377526283264, 0.014719735831022263, 0.013784103095531464, 0.0003127696691080928, -0.0010509390849620104, 0.017157649621367455, 0.0035679214634001255, 0.003735939972102642, 0.0338672511279583, 0.0003434495010878891, 0.027410071343183517, -0.03249674662947655, 0.0175134539604187, -0.006206799764186144, 0.02843794971704483, 0.013560079038143158, -0.00906640850007534, 0.005561081692576408, 0.0014355696039274335, 0.005610499065369368, 0.006984297186136246, -0.02261330932378769, -0.0040587992407381535, -0.005119621753692627, -0.013454655185341835, 0.004256467800587416, 0.005900413263589144, 0.021611787378787994, 0.028490660712122917, -0.00634846230968833, 0.02228386141359806, 0.015787146985530853, 0.008512935601174831, 0.007004064042121172, -0.023324916139245033, 0.00038874862366355956, -0.016709601506590843, 0.0193847194314003, -0.010285365395247936, 0.03634469956159592, 0.01522049680352211, -0.006203505210578442, -0.03771520406007767, 0.020241282880306244, 0.008137364871799946, -0.0031528170220553875, 0.0013367352075874805, -0.015892570838332176, 0.019727345556020737, -0.02549927309155464, 0.03276030719280243, -0.014284864068031311, 0.005001020152121782, 0.00022546596301253885, -0.014930582605302334, 0.008829205296933651, 0.01796150393784046, -0.006098081823438406, 0.019187049940228462, 0.0183041300624609, -0.0037655902560800314, 0.0021348227746784687, -0.0009298669174313545, 0.03183785080909729, -0.0057192170061171055, 0.002698178868740797, -0.0035712160170078278, 0.0025186296552419662, 0.01160974707454443, -0.00017728419334162027, -0.020214928314089775, 0.00426305690780282, 0.03394632041454315, 0.0029403232038021088, -0.021743565797805786, -0.056665051728487015, 0.0058806464076042175, 0.008539291098713875, -0.03423623368144035, 0.010410555638372898, -0.007261033169925213, 0.008400923572480679, -0.01553676649928093, 0.03136344626545906, 0.01483833696693182, 0.028543371707201004, 0.0022962524089962244, -0.02258695289492607, 0.015510411001741886, 0.015457699075341225, -0.0034954429138451815, -0.004480492323637009, -0.0032598876859992743, 0.004513437394052744, -0.017566166818141937, 0.003643694566562772, -0.016986338421702385, -0.017737479880452156, 0.004921952728182077, -0.010127230547368526, -0.007116076070815325, -0.021176915615797043, -0.01644604280591011, 0.020214928314089775, -0.005201983731240034, -0.04103603959083557, -0.032233189791440964, 0.008038530126214027, -0.024497751146554947, -0.04654441028833389, 0.002246835269033909, -0.0020623442251235247, 0.0008256789878942072, -0.006697677541524172, 0.0005806520348414779, -0.16572551429271698, 0.02339080534875393, 0.01377092581242323, -0.028068967163562775, 0.03763613849878311, 0.001132477424107492, 0.006509892176836729, 0.00267182313837111, -0.018080104142427444, 0.009092763997614384, 0.018699467182159424, -0.050682276487350464, -0.026698464527726173, -0.01862039975821972, 0.0013663854915648699, 3.6239278415450826e-05, -0.002067286055535078, 0.01647239923477173, 0.03492148593068123, 0.0025762831792235374, 0.040693413466215134, 0.0017114821821451187, 0.005544609390199184, -0.020201750099658966, 0.002979856915771961, -0.009415622800588608, -0.005676388740539551, 0.009435390122234821, -0.021111026406288147, 0.0020293996203690767, -0.03307657688856125, 0.023588474839925766, 0.028042610734701157, 0.0037688848096877337, 0.023140424862504005, -0.0044277808628976345, -0.0013243808643892407, 0.014153084717690945, -0.012176397256553173, 0.03771520406007767, -0.005534726195037365, -0.004661689046770334, 0.014245330356061459, 0.01970098912715912, -0.017302608117461205, 0.020386241376399994, 0.011814004741609097, 0.022863689810037613, 0.0015969990054145455, -0.025868255645036697, -0.006407763343304396, -0.014627490192651749, 0.013362409546971321, 0.016643712297081947, 0.007043597754091024, -0.012960483320057392, -0.004296001512557268, 0.014904226176440716, -0.010799304582178593, 0.004684749990701675, -0.021005602553486824, -0.012327942997217178, 0.009145475924015045, -0.001769135589711368, -0.007228088565170765, -0.010884961113333702, -0.0013425004435703158, -0.00318740913644433, -0.03784698247909546, 0.0037688848096877337, 0.002968326210975647, -0.03107353299856186, 0.009890028275549412, -0.012038028798997402, 0.016986338421702385, 0.00378865166567266, -0.012486077845096588, -0.0067734504118561745, 0.0026042861863970757, -0.02875421941280365, -0.00022320100106298923, 0.019279295578598976, -0.02719922550022602, -0.02166449837386608, -0.01689409278333187, 0.02460317499935627, 0.01377092581242323, -0.014284864068031311, 0.010799304582178593, 0.01199849508702755, 0.021111026406288147, -0.03647647798061371, -0.04809940606355667, -0.027647273615002632, -0.007735438644886017, 0.007452113088220358, 0.012182986363768578, 0.005653327330946922, -0.01014699786901474, -0.00045463821152225137, 0.018343662843108177, 0.0038051242008805275, -0.017315786331892014, 0.027910832315683365, 0.027884475886821747, 0.03658190369606018, 0.0010575279593467712, 0.01798785850405693, 0.003105047158896923, -0.013797281309962273, 4.123247344978154e-05, 0.005910296458750963, 0.008572236634790897, -0.0007017242023721337, -0.010384200140833855, 0.02104513719677925, -0.007959462702274323, -0.009560580365359783, 0.040508922189474106, -0.002889258787035942, 0.04841567575931549, 0.03779427334666252, -0.046043649315834045, 0.01531274151057005, 0.019397897645831108, 0.024682242423295975, -0.07890938222408295, 0.013731391169130802, -0.01153067871928215, 0.0005658268928527832, 0.0013869759859517217, 0.023759787902235985, -0.006971118971705437, 0.0032780072651803493, -0.013296520337462425, 0.028200747445225716, -0.0209001787006855, 0.001138242776505649, 0.005623676814138889, -0.031152600422501564, -0.006440707948058844, -0.013823636807501316, 0.027594562619924545, -0.00876990519464016, -0.01062140241265297, 0.01483833696693182, 0.007017241790890694, -0.04630720615386963, -0.013013195246458054, -0.022415639832615852, 5.626353595289402e-05, -0.014482532627880573, -0.028200747445225716, -0.008783083409070969, 0.005495192017406225, -0.0012922596652060747, -0.025314781814813614, -0.008829205296933651, 0.018211884424090385, 0.002826663665473461, 0.00945515651255846, 0.002021163236349821, -0.027304647490382195, -0.011418666690587997, 0.02581554278731346, -0.017421208322048187, -0.0028843169566243887, -0.006931585259735584, 0.024827199056744576, -0.02490626648068428, 0.0037524125073105097, -0.011669047176837921, -0.003195645287632942, 0.018396375700831413, 0.006038781255483627, -0.006397879682481289, -0.01330310944467783, -0.0030375104397535324, -0.004862652160227299, -0.035316821187734604, 0.016393331810832024, 0.01045008935034275, -0.009231132455170155, 0.03210141137242317, -0.015141429379582405, -0.008974162861704826, -0.00035539199598133564, 0.038215965032577515, 0.0019618626683950424, 0.037293512374162674, 0.041431378573179245, 0.007821095176041126, -0.011464789509773254, -0.00749164680019021, -0.001795491436496377, -0.021717211231589317, -0.020359884947538376, 0.01876535639166832, -0.018330484628677368, 0.0034262589178979397, -0.04625449702143669, -0.026803886517882347, -0.028938710689544678, -0.024866733700037003, -0.00017707828374113888, 0.00465509993955493, 0.009560580365359783, -0.0029633846133947372, 0.013612790033221245, -0.022679198533296585, -0.028806930407881737, 0.01269033644348383, -0.004711105953902006, -0.017289429903030396, 0.01966145448386669, -0.028332525864243507, 0.008275733329355717, 0.02891235426068306, -0.0038479522336274385, 0.019450608640909195, 0.0175134539604187, -0.002291310578584671, -0.003716173116117716, 0.01905527152121067, -0.01693362556397915, 0.010153586976230145, -0.01985912397503853, -0.013415121473371983, -0.05945876985788345, 0.0017559577245265245, -0.02936040423810482, -0.02043895237147808, 0.024339616298675537, -0.00351850432343781, 0.022204793989658356, -0.005429302807897329, -0.022415639832615852, 0.0011753056896850467, -0.018633577972650528, 0.03455250337719917, 0.002597697312012315, -0.01689409278333187, -0.01798785850405693, 0.023720255121588707, -0.01599799282848835, -0.013467833399772644, 0.01583985798060894, 0.015444520860910416, -0.017039049416780472, 0.012420188635587692, -0.00913888681679964, 0.022349750623106956, -0.01037761103361845, 0.020083148032426834, -0.015207318589091301, -0.013731391169130802, 0.007840861566364765, -0.03861130401492119, 0.01153067871928215, -0.03418352082371712, 0.00634846230968833, 0.005778517574071884, -0.01176788192242384, -0.029439471662044525, -0.014284864068031311, 0.010324899107217789, 0.010469856671988964, 0.003347191493958235, -0.021493185311555862, -0.01197872869670391, 0.0039962041191756725, 0.00026046979473903775, -0.02308771386742592, -0.030361926183104515, -0.007043597754091024, 0.013494188897311687, 0.006661437917500734, -0.008150543086230755, 0.02260013110935688, 0.001543463789857924, -0.019872302189469337, 0.025723297148942947, -0.021308695897459984, -0.020017258822917938, -0.043882470577955246, -0.013942237943410873, -0.015088717453181744, -0.025472916662693024, 0.03995545208454132, 0.005089971236884594, 0.003024332458153367, -0.0196746326982975, 0.01724989525973797, 0.020069969817996025, 0.002683353843167424, 0.011168286204338074, 0.0230613574385643, -0.016090238466858864, -0.039112064987421036, 0.022547420114278793, -0.013072495348751545, 0.0027146514039486647, -0.011649280786514282, 0.008176898583769798, -0.009507868438959122, 0.005607204511761665, -0.037609782069921494, 0.020821111276745796, -0.009876850061118603, -0.006579075939953327, 0.004325652029365301, 0.01721036247909069, -0.005462247412651777, -0.0019140926888212562, 0.005557787138968706, -0.0024049701169133186, 0.019885480403900146, 0.004520026035606861, -0.02072886750102043, 0.0002223773772129789, 0.00869742687791586, 0.016683245077729225, 0.02137458510696888, -0.00599265843629837, -0.012888005003333092, 0.008987341076135635, 0.01359961275011301, 0.033287424594163895, 0.0031907036900520325, 0.008038530126214027, -0.03320835530757904, 0.017895614728331566, -0.01630108617246151, 0.007083131466060877, 0.02232339419424534, -0.007326922845095396, -0.005040553864091635, 0.03231225907802582, 0.017750656232237816, -0.008519524708390236, 0.022652842104434967, -0.004319062922149897, -0.016867736354470253, 0.01954285427927971, 0.003643694566562772, -0.024682242423295975, -0.00890827365219593, 0.005979480687528849, 0.0076497821137309074, 0.014930582605302334, 0.0033406023867428303, -0.007728849537670612, 0.021651320159435272, 0.0216908548027277, -0.017487099394202232, 0.006664732471108437, -0.01599799282848835, -0.03183785080909729, 0.010397378355264664, -0.009646236896514893, -0.03378818556666374, -0.00633857911452651, -0.0019107982516288757, 0.0038907804992049932, -0.001479221391491592, 0.014798803254961967, -0.022521063685417175, 0.04889008030295372, 0.01522049680352211, -0.03276030719280243, 0.008921450935304165, 0.014693379402160645, 0.016709601506590843, 0.001565701444633305, 0.013665501959621906, -0.0011942489072680473, -0.013408532366156578, 0.02539384923875332, 0.0017378380289301276, 0.01444299891591072, -0.015167784877121449, -0.022112548351287842, -0.01006134133785963, -0.009329966269433498, 0.012281820178031921, -0.001976687926799059, 0.022521063685417175, 0.00442119175568223, -0.005422713700681925, 0.004111510701477528, -0.023759787902235985, -0.04100968316197395, -0.006651554722338915, 0.01980641297996044, -0.04324993118643761, -0.010707058943808079, -0.039586469531059265, -0.018514975905418396, 0.011043095961213112, 0.006556014530360699, 0.01906844787299633, 0.008012174628674984, 0.025907788425683975, -0.030414637178182602, 0.0018300835508853197, 0.00282501638866961, 0.015167784877121449, -0.0019157399656251073, 0.01796150393784046, -0.0007606130093336105, -0.0485738106071949, 0.0005555316456593573, 0.042511966079473495, -0.030941754579544067, 0.00024317378120031208, -0.02427372708916664]} +{"id": "test:10000077", "text": "\"Aman Ullah is with Ro Nay San Lwin and 93 others.\\nThe UN Security Council has issued a presidential statement on Myanmar complimenting and criticising the country over its stance on the Rohingya crisis and it also gave the government a month to \u201cget its act together\u201d. The 1,300- word statement was read out by the sitting President of UNSC Sebastiano Cardi, the Ambassador of Italy, at a formal UNSC session on Monday ( November 6, 2017) .\\nA presidential statement is a statement made by the President of the Security Council on behalf of the Council, adopted at a formal meeting of the Council and issued as an official document of the Council.\\nA Presidential Statement is often created when the United Nations Security Council cannot reach consensus or are prevented from passing a resolution by a permanent member\u2019s veto, or threat thereof. Such statements are similar in content, format, and tone to resolutions, but are not legally binding. The adoption of a Presidential Statement requires consensus, although Security Council members may abstain. The Statement is signed by the sitting Security Council President.\\nIt was the first presidential statement in 10 years on Myanmar. The Security Council has adopted such statements against Myanmar only three times, with the last one coming a decade ago. Diplomats hailed the move as a unified international stand against a humanitarian crisis that has been called \u201cethnic cleansing\u201d by the U.N., U.S., France and the U.K.\\nIt strongly condemned attacks against the Myanmar security forces carried out by the Arakan Rohingya Salvation Army (ARSA) on August 25, and then expressed \u201cgrave concern\u201d over the government\u2019s response, the alleged burning of villages and threats to villagers to flee, among others.\\nIt called for reform in Myanmar\u2019s security and justice sectors and urged the government to work with Bangladesh and the UN to allow the voluntary return of refugees to their homes, on the basis of an October 24 memorandum of understanding between the two country.\\nThe panel welcomed a \u201cunion enterprise mechanism\u201d for humanitarian assistance, resettlement and development in Rakhine.\\nIt recommended the government ensure the mechanism supported such return and allow UN agencies full access, urging governments and all humanitarian partners to pay special attention to the needs of women, particular survivors of sexual violence.\\nThe council said the government\u2019s primary responsibility is the protection of Myanmar\u2019s population, citizens or not. The statement reiterated concerns and demands previously laid out: an end to excessive military force in Rakhine province, a lack of humanitarian access for the U.N., a right of return for refugees, and steps to address the root causes of the conflict. The statement included most of the demands contained in a draft resolution presented last month by Britain and France, but that measure ran into strong opposition from China, a supporter of Myanmar\u2019s former ruling junta.\\nIt is said that, China had indicated it was willing to resort to its veto power to block a resolution, but Beijing finally agreed to a statement during negotiations.\\nThe action is one step below a Security Council resolution in diplomatic significance, but a step above the Council\u2019s previous action, a \u201cpress statement,\u201d in September.\\nThe statement requires the U.N. Secretary General Ant\u00f3nio Guterres to report back to the Council within 30 days on Myanmar\u2019s progress in addressing the Council\u2019s demands. The statement isn\u2019t as binding as a resolution would be, and carries no punitive measures, such as sanctions, if corrective steps aren\u2019t taken.\\nThe Security Council has held a series of meetings on Myanmar since September, including a briefing attended by high-level officials on the sidelines of the U.N. General Assembly gathering. But efforts to respond forcefully have stalled despite widespread reports of deaths, rapes and torture of Rohingya Muslim minorities, including women and children, at the hands of the country\u2019s military.\\nViolence erupted in August after a Rohingya militant group attacked Myanmar security forces. Since late August, more than 600,000 Rohingya have been driven from their homes by an army campaign that the United Nations has described as ethnic cleansing.\\nDuring negotiations with China, language on citizenship rights was watered down in the statement as was a demand that Myanmar allow a UN human rights mission into the country, diplomats said.\\nThe statement calls on Myanmar to cooperate with the United Nations and encourages UN Secretary-General Antonio Guterres to appoint a special advisor on Myanmar.\\nThe council statement was agreed as Guterres prepares to travel to Manila this week to join leaders of the Southeast Asian (ASEAN) bloc for a summit.\\nThe council also welcomed the Myanmar government\u2019s public support for recommendations by the Advisory Commission on Rakhine State chaired by former UN Secretary-General Kofi Annan and called for their full implementation.\\nIt urged UN Secretary-General Antonio Guterres to consider appointing a special advisor on Myanmar.\\nSince the August attacks, over 604,000 Rohingya have fled to Bangladesh. Some fled by land, others in boats over an inlet of the Bay of Bengal.\\nIn the last two weeks, 4,000 refugees entered Bangladesh while four people drowned in a shipwreck while fleeing, officials said.\\nU.K Foreign Secretary Boris Johnson said: I am pleased that today (6 November) the United Nations Security Council (UNSC) has spoken with one voice on the appalling situation in Rakhine State, Burma. The UK has repeatedly called on the Burmese security forces to protect all civilians and act now to stop the violence and allow humanitarian aid to urgently reach all those who need it. The UNSC has today joined us in that call, with this historic Presidential Statement on Burma.\\nI am encouraged to see State Counsellor Aung San Suu Kyi making important steps forward, including establishing a domestic body to deliver humanitarian and development assistance in Rakhine, and making efforts to promote interfaith and intercommunal harmony, including a recent visit to northern Rakhine. The UK will be watching closely to ensure that the Burmese security forces do not attempt to frustrate these efforts.\\nThe U.K. and France initially circulated a proposed Security Council resolution on Myanmar in late October, but China and Russia refused to engage and negotiate, diplomats said. China finally agreed to a similar text with minor changes as a presidential statement\u2014which carries less weight than a resolution\u2014and Russia followed suit.\\nMr. Guterres has been outspoken about the Myanmar crisis and urged the Security Council to prevent continued bloodshed and curb the refugee flow. Mr. Guterres will be among the world leaders attending the Association of Southeast Asian Nations (Asean) summit later this week in Manila.\\nThe Rohingya refugee crisis is expected to be a top issue of discussion at the summit, to be attended by US President Donald Trump, who will dispatch US Secretary of State Rex Tillerson to Myanmar later this month. Daw Aung San Suu Kyi will also attend that summit.\"", "vector_field": [-0.0212433859705925, 0.00930815003812313, -0.0007751235389150679, -0.018536288291215897, 0.011301801539957523, -0.011395149864256382, -0.022923653945326805, -0.005340851377695799, -0.03093826398253441, -0.01798953302204609, 0.03293858468532562, -0.003167171496897936, -0.013462145812809467, -0.0016510965069755912, -0.021683456376194954, -0.012575337663292885, 0.026590904220938683, 0.0033338647335767746, 0.017616141587495804, -0.01457565650343895, 0.01597587950527668, -0.01798953302204609, -0.01577584818005562, 0.02521735243499279, 0.0056075602769851685, -0.014068908989429474, 0.018042875453829765, -0.02225688099861145, 0.011775209568440914, -0.01187522616237402, -0.013482148759067059, -0.01606922782957554, 0.019496440887451172, -0.023643767461180687, -0.012255286797881126, 0.011808549053966999, -0.017362767830491066, -0.030244819819927216, 0.011888561770319939, -0.01042832899838686, 0.018389597535133362, -0.0042840163223445415, 0.015589151531457901, 0.006434358656406403, 0.0045273881405591965, 0.00021607610688079149, 0.028377855196595192, -0.03229847922921181, -0.026764266192913055, 0.03563234582543373, 0.03997970372438431, 0.010148284025490284, -0.03357868641614914, -0.028511211276054382, -0.027191000059247017, 0.00017044383275788277, -0.006731072906404734, 0.02397715486586094, -0.0011926901061087847, -0.003883952274918556, 0.02707098051905632, 0.01642928458750248, -0.012048587203025818, 0.028804590925574303, -0.0002873374614864588, -0.017682818695902824, -0.014308947138488293, 0.011241791769862175, -0.008661380037665367, 0.0028037801384925842, 0.03651248663663864, 0.01981649175286293, 0.022403569892048836, -0.019936511293053627, 0.045900650322437286, -0.004807432647794485, 1.3804283298668452e-05, -0.005810926202684641, -0.005617561750113964, -0.028057804331183434, 0.009734884835779667, -0.02437721937894821, 0.0014602327719330788, -0.010021597146987915, 0.008007942698895931, 0.00928814709186554, -0.02355041913688183, 0.016869354993104935, -0.02072330191731453, -0.03725927323102951, -0.011088433675467968, 0.037739347666502, -0.01052834466099739, 0.00015221175272017717, -0.015562480315566063, 0.0162692591547966, -0.026630910113453865, 0.0338987372815609, 0.027737753465771675, -0.0384061224758625, 0.006167649757117033, 0.022456912323832512, -0.033338647335767746, -0.007194479927420616, -0.03120497241616249, -0.00792126264423132, -0.03171171993017197, 0.003990636207163334, 0.021350068971514702, -0.036005739122629166, -0.0027754423208534718, 0.03845946118235588, -0.013922219164669514, -0.03419211506843567, 0.012775368988513947, -0.007354505360126495, 0.027764424681663513, -0.009021437726914883, -0.01369551569223404, -0.014482308179140091, 0.01776283048093319, 0.004847439005970955, -0.005714244209229946, 0.0075812083669006824, 0.014842365868389606, -0.02271028608083725, -0.00030900759156793356, -0.009494846686720848, 0.007214483339339495, -0.005854266230016947, -0.006684398744255304, -0.013375464826822281, 0.01350215170532465, 0.007667888887226582, 0.013762193731963634, 0.033231962472200394, -0.042353417724370956, 0.0020553276408463717, -0.028057804331183434, -0.05547550693154335, 0.008661380037665367, 0.012975401245057583, -0.013668845407664776, 0.0012176940217614174, 0.00824798084795475, 0.005924277473241091, 0.014588992111384869, 0.032618530094623566, -0.0018686311086639762, 0.005824261344969273, 0.01942976377904415, 0.01369551569223404, 0.004260679241269827, -0.004427372477948666, -0.01681601256132126, 0.0043306900188326836, 0.010955079458653927, 0.02713765762746334, -0.027897778898477554, 0.003202177118510008, 0.010108278132975101, -0.01176187489181757, -0.006817753426730633, -0.004160663112998009, -0.0160425566136837, 0.02811114676296711, 0.002430387306958437, 0.025977473706007004, -0.02411050908267498, -0.007601211313158274, 0.008648045361042023, 0.026444215327501297, -0.0009668207494542003, 0.018202900886535645, 0.02058994770050049, 0.01780283823609352, 0.01945643499493599, -0.006947773974388838, -0.02612416446208954, -0.011341807432472706, -0.016602646559476852, 0.008374667726457119, -0.002827117219567299, 0.02781776711344719, -0.01920306123793125, 0.015069068409502506, 0.02931133843958378, -0.01776283048093319, 0.012775368988513947, 0.0035138933453708887, 0.017229411751031876, 0.02716432884335518, 0.0058175940066576, -0.015309106558561325, -0.633274257183075, -0.023470407351851463, 0.017496122047305107, 0.02931133843958378, 0.0056208958849310875, 0.011628519743680954, 0.011615184135735035, -0.008494687266647816, -0.018869673833251, 0.031418342143297195, -0.018136223778128624, 0.00039568805368617177, 0.006120975594967604, -0.020736638456583023, -0.0184162687510252, -8.912878547562286e-05, -0.030511530116200447, -0.016029221937060356, 0.017496122047305107, -0.012041918933391571, -0.0005388358840718865, 0.011455158703029156, -0.002098667901009321, -0.025257358327507973, 0.0035138933453708887, 0.013235442340373993, 0.0013218773528933525, -0.0030104797333478928, -0.02017654851078987, 0.03285856917500496, -0.0190963763743639, 0.008354664780199528, -0.012468653731048107, 0.02004319429397583, 0.04141993448138237, 0.0013568829745054245, -0.028964616358280182, 0.008861412294209003, 0.024817287921905518, 0.025097332894802094, 0.01574917696416378, -0.008141297847032547, 0.00936815980821848, -0.022950325161218643, -0.010354983620345592, 0.01708272285759449, 0.02020321972668171, -0.028137817978858948, 0.01665598712861538, -0.0022070184350013733, -0.0058342632837593555, 0.01304874662309885, 0.009634869173169136, -0.0017902853433042765, -0.0024770614691078663, -0.02313701994717121, 0.02196350134909153, -0.0420333668589592, 0.0033755379263311625, -0.011895229108631611, -0.003703923663124442, 0.018069546669721603, -0.0042706807143986225, 0.007974604144692421, -0.007421182934194803, 0.04443374648690224, -0.019963182508945465, 0.01369551569223404, 0.007514530792832375, 0.007801243104040623, 0.0007422016351483762, 0.043713632971048355, -0.023310381919145584, 0.0005600892473012209, -0.005264172330498695, 0.020683296024799347, 0.015055732801556587, -0.01350215170532465, 0.007447853684425354, 0.008174636401236057, -0.016255924478173256, -0.027150994166731834, 0.006854425650089979, 0.0037505978252738714, 0.010695037432014942, -0.00013158346700947732, -0.01792285591363907, 0.0020053195767104626, -0.01769615337252617, 0.0030154807027429342, 0.029097970575094223, -0.02121671475470066, 0.0030804909765720367, 0.012868717312812805, 0.014442301355302334, 0.024350548163056374, 0.003027149010449648, -0.000476742658065632, 0.04053979367017746, -0.020896663889288902, -0.003777268575504422, 0.010588354431092739, -0.0034205452539026737, -0.00040381436701864004, 0.016909360885620117, 0.03405876085162163, -0.03829943761229515, -0.012208612635731697, 0.029951440170407295, -0.02433721162378788, 0.0050241341814398766, 0.006607719697058201, 0.017882850021123886, -0.01750945672392845, 0.03416544571518898, -0.021723462268710136, 0.0007622048142366111, -0.010288306511938572, 0.01974981464445591, -0.020163213834166527, 0.010795054025948048, -0.0033555347472429276, 0.01773615926504135, -0.03413877263665199, -0.0015585817163810134, 0.009828233160078526, 0.011148443445563316, -0.004387366119772196, 0.008681383915245533, 0.014388959854841232, 0.0025470727123320103, 0.0011260127648711205, 0.025257358327507973, -0.004844105336815119, 0.0129353953525424, -0.006040962878614664, 0.02212352678179741, -0.010108278132975101, 0.0059476145543158054, -0.006094304844737053, -0.007327834609895945, -0.0011051761684939265, 0.002807114040479064, -0.007287828251719475, 0.014855700545012951, -0.025777442380785942, -0.009714881889522076, -0.013868876732885838, -0.03315195068717003, 0.015202422626316547, 4.004544280178379e-06, 0.0006996948504820466, -8.563864685129374e-05, 0.012855381704866886, 0.023670438677072525, 0.005987620912492275, -0.0023870470467954874, -0.01228195708245039, 0.004053979646414518, -0.017616141587495804, 0.01694936864078045, 0.021483423188328743, -0.03949962928891182, 0.0057509164325892925, 0.001941976137459278, -0.015269099734723568, 0.013735522516071796, 0.015095739625394344, -0.0008918087696656585, -0.008601371198892593, 0.01465566921979189, 0.008081288076937199, -0.0023287045769393444, 0.0038972878828644753, 0.021976836025714874, -0.001923639909364283, -0.0440603569149971, -0.016282595694065094, 0.01168186217546463, 0.010935076512396336, 0.012195277027785778, 0.012455318123102188, -0.00410732114687562, -0.022003507241606712, -0.01320877205580473, 0.023417064920067787, -0.020936669781804085, 0.032991923391819, -0.007287828251719475, 0.009714881889522076, 0.014148921705782413, 0.012095261365175247, 0.012975401245057583, 0.021430082619190216, 0.003963964991271496, 0.028777919709682465, -0.01486903615295887, 0.015509138815104961, 0.010121612809598446, 0.03317862004041672, 0.0032121785916388035, -0.02128339186310768, 0.03211178630590439, 0.005770919844508171, -0.013028742745518684, -0.021563436836004257, -0.0013443809002637863, -0.014482308179140091, 0.029364680871367455, 0.0033038598485291004, 0.018562957644462585, -0.05867601931095123, -0.010074938647449017, -0.007321166805922985, -0.02040325105190277, 0.03347200155258179, 0.009101450443267822, 0.0020036527421325445, -0.03363202512264252, -0.003143834415823221, 0.0026187507901340723, -0.007174476981163025, -0.0022353562526404858, 0.007061125244945288, -0.01676267199218273, -0.00399396987631917, 0.006377683021128178, 0.028297843411564827, 0.01828291453421116, -0.005430865567177534, 0.001296039903536439, -0.0012385307345539331, 0.0008476350922137499, 0.030538199469447136, -0.02187015302479267, -0.006511037703603506, 0.020016523078083992, -0.01782950758934021, 0.013735522516071796, -0.002272028708830476, 0.008961427956819534, -0.011148443445563316, -0.004877443891018629, -0.0018019538838416338, 0.01075504720211029, -0.02752438746392727, 0.034405484795570374, 0.027844438329339027, 0.00237871240824461, 0.05192827433347702, -0.001711939461529255, 0.011408484540879726, -0.01052834466099739, 0.002628752263262868, -0.0024370551109313965, -0.009508182294666767, -0.015042397193610668, -0.005354186519980431, 0.02287031151354313, 0.008541361428797245, -0.004454043228179216, 0.0067010680213570595, 0.013295452110469341, 0.023897141218185425, 0.01088173408061266, 0.01714939996600151, 0.007207815535366535, -0.020709967240691185, -0.01174853928387165, -0.03131165727972984, -0.016535969451069832, -0.011135107837617397, 0.02872457727789879, -0.022976994514465332, 0.007687891833484173, -0.012535330839455128, 0.010014929808676243, -0.0009126454242505133, -0.001547746709547937, -0.014842365868389606, -0.029551375657320023, -0.0386461578309536, 0.0023303714115172625, 0.014895707368850708, -0.01920306123793125, -0.034832216799259186, -0.02228355221450329, 0.0019303075969219208, -0.020803315564990044, 0.009201466105878353, -0.013722186908125877, 0.01091507263481617, -0.004857440944761038, 0.00917479582130909, -0.014962384477257729, -0.014749017544090748, 0.009888242930173874, 0.011135107837617397, 0.017642810940742493, 0.01929640956223011, 0.015602486208081245, -0.01877632550895214, 0.011481829918920994, 0.004167330916970968, 0.01568249985575676, -0.0181895662099123, -0.001294372952543199, -0.028217829763889313, 0.007327834609895945, -0.0012893722159788013, 0.004390699788928032, -0.034218788146972656, -0.018562957644462585, 0.0008784733363427222, -0.004394033458083868, 0.007901259697973728, -0.0036439141258597374, -0.0004625737201422453, 0.02885793335735798, 0.001584419165737927, 0.0096615394577384, -0.03485888987779617, -0.00917479582130909, -0.0023770455736666918, 0.12417978793382645, 0.013842206448316574, -0.007527866400778294, -0.0034572177100926638, 0.00930815003812313, 0.011508501134812832, -0.006207656115293503, -0.05958282947540283, -0.0133888004347682, -0.005040803458541632, -0.0397663377225399, 0.011008420959115028, -0.018162894994020462, -0.004204003140330315, 0.033045265823602676, -0.0007034454611130059, 0.01730942539870739, -0.015935873612761497, -0.022630274295806885, 0.016535969451069832, 0.0016827682266011834, -0.007421182934194803, 0.017842844128608704, 0.038966208696365356, -0.005297510884702206, -0.002572076627984643, 0.03328530490398407, 0.004173998720943928, 0.0021470088977366686, -0.03789937496185303, -0.014268940314650536, 0.025724099949002266, 0.022363563999533653, 0.0013060414930805564, -0.012922059744596481, 0.007481192238628864, 0.0021353403571993113, -0.011981909163296223, -0.027311019599437714, -0.009281478822231293, 0.011561842635273933, 0.03136499971151352, 0.005590890999883413, -0.009488178417086601, 0.008814738132059574, -0.01075504720211029, 0.006907767616212368, 0.0019736478570848703, -0.008714722469449043, 0.022656943649053574, 0.016349272802472115, -0.03696589171886444, -0.018469611182808876, -0.017682818695902824, -0.020816650241613388, -0.025003984570503235, 0.012695356272161007, 0.0073811765760183334, -0.004887445829808712, 0.003093826351687312, -0.023043673485517502, -0.023083679378032684, -6.290064902714221e-06, -0.003983968403190374, -0.025710763409733772, -0.00471075065433979, -0.01568249985575676, 0.012535330839455128, -0.023177027702331543, -0.005990954581648111, 0.021630113944411278, -0.028617894276976585, -0.016082562506198883, 0.036752525717020035, 0.022016841918230057, -0.014895707368850708, 0.011468494310975075, 0.010081606917083263, 0.001618591253645718, 0.007714563049376011, -0.026204176247119904, -0.005120816174894571, 0.009808230213820934, -0.04037977010011673, -0.008041281253099442, 0.020083200186491013, -0.0013335458934307098, -0.0048807780258357525, -0.028911273926496506, 0.01433561835438013, 0.014909042976796627, 0.01075504720211029, 0.006487700622528791, -0.02625751867890358, -0.022523589432239532, 0.023123685270547867, -0.021950164809823036, -0.017296090722084045, -0.008568032644689083, -0.008141297847032547, 0.011148443445563316, -0.025750771164894104, -0.040486451238393784, -0.006584382615983486, -0.0010610023746266961, 0.0007934597670100629, 0.0035739028826355934, 0.028937945142388344, 0.014815694652497768, 0.0010943410452455282, 0.021096695214509964, -0.03256519138813019, 0.008948092348873615, -0.001096841529943049, -0.029764743521809578, -0.003563901409506798, 0.013762193731963634, 0.03272521495819092, 0.011568509973585606, -0.001436895690858364, -0.009248140268027782, -0.040593136101961136, 0.03459218144416809, 0.004020640626549721, -0.017909521237015724, 0.015509138815104961, -0.008701386861503124, 0.006947773974388838, 0.00046715777716599405, 0.007801243104040623, 0.0009701545932330191, 0.023150356486439705, 0.013748858124017715, -0.005770919844508171, -0.01597587950527668, -0.0066877324134111404, -0.006704401690512896, 0.005847598426043987, -0.00585760036483407, -0.01880299672484398, -0.0028171157464385033, -0.010861731134355068, -0.014375624246895313, -0.0032888574060052633, 0.007747901603579521, -0.025137338787317276, -0.02307034283876419, 0.00023149522894527763, 0.007341170217841864, 0.012788704596459866, -0.018896345049142838, 0.005574221722781658, -0.02739103138446808, 0.011288465932011604, 0.015629157423973083, -0.036752525717020035, 0.02076330967247486, -0.018682977184653282, 0.026337530463933945, 0.0027087649796158075, 0.016802677884697914, 0.011895229108631611, -0.019736478105187416, 0.0041273245587944984, -0.004360694903880358, -0.0025654088240116835, -0.010968415066599846, -0.006671063136309385, -0.02781776711344719, 0.007341170217841864, 0.018909679725766182, 0.01200858037918806, -0.026284189894795418, -0.009888242930173874, -0.017229411751031876, 0.022563595324754715, 0.0003881868615280837, -0.011088433675467968, -0.01182855200022459, -0.028484540060162544, -0.029978111386299133, 0.0033205291256308556, -0.016322601586580276, -0.013882212340831757, -0.023510413244366646, -0.014002231881022453, 0.038832854479551315, -0.01873631961643696, 0.005507544614374638, -0.0031021609902381897, 0.009441505186259747, -0.01780283823609352, 0.010775050148367882, -0.008548028767108917, 0.00027525221230462193, -0.009361491538584232, -0.004704082850366831, 0.0047540911473333836, 0.018216237425804138, -0.005370855797082186, 0.02300366573035717, -0.005354186519980431, 0.007821246050298214, 0.0006105139618739486, 0.011208453215658665, 0.01232863124459982, -0.013762193731963634, 0.0010351649252697825, 0.018976356834173203, -0.02287031151354313, 6.563546048710123e-05, -0.01714939996600151, -0.014855700545012951, -0.014175592921674252, 0.01746945083141327, -0.0032421834766864777, 0.004053979646414518, 0.013802199624478817, 0.003743930021300912, -0.022270215675234795, -0.006204321980476379, -0.0028504543006420135, -0.004904115106910467, 0.006267665419727564, -0.00342387892305851, 0.00391395715996623, -0.01369551569223404, -0.039099566638469696, 0.0015919203869998455, -0.00612430926412344, -0.034992244094610214, 0.018696313723921776, 0.002395381685346365, -0.015349113382399082, -0.011075098067522049, 0.0052808416076004505, 0.020843321457505226, -0.008221310563385487, -0.023670438677072525, 0.008027946576476097, 0.0215767715126276, -0.028324514627456665, -0.00658771675080061, -0.009541520848870277, 0.00028233666671440005, 0.010628360323607922, -0.03341865912079811, 0.009768223389983177, -0.013708851300179958, -0.010028264485299587, -0.01936308667063713, 0.005907608196139336, -0.04350026696920395, 0.010781718418002129, -0.009334821254014969, -0.03357868641614914, -0.0131620978936553, -0.021070023998618126, -0.005667570047080517, -0.0014502311823889613, -0.01456232089549303, 0.012175274081528187, 0.0048241023905575275, 0.0005271673435345292, -0.025710763409733772, 0.0009801562409847975, -0.020016523078083992, -0.013922219164669514, -0.008681383915245533, 0.027471045032143593, -0.04064647853374481, -0.008401338942348957, 0.02004319429397583, 0.011675193905830383, -0.003983968403190374, -0.01433561835438013, -0.005984287243336439, -0.02049659937620163, -0.027177665382623672, -0.0033405323047190905, 0.0033005259465426207, -0.0033755379263311625, -0.03992636501789093, -0.0057609183713793755, -0.011701865121722221, -0.030431516468524933, 0.015242429450154305, 0.0037139251362532377, -0.007541202008724213, 0.01701604574918747, -0.027044309303164482, -0.007607879117131233, -0.0006601052009500563, 0.01854962296783924, -0.010241632349789143, 0.010234964080154896, -0.008548028767108917, 0.025230687111616135, -0.001133513986133039, 0.010354983620345592, -0.01662931777536869, 0.022523589432239532, -6.15723110968247e-05, 0.004103987477719784, 0.0009634869056753814, -0.02485729567706585, 0.015189087018370628, -0.014042237773537636, -0.01850961707532406, 0.005814259871840477, -0.007214483339339495, 0.016322601586580276, -0.015269099734723568, -0.0066443923860788345, 0.03667251020669937, 0.005164156202226877, 0.008908086456358433, 0.012421979568898678, -0.012301960960030556, 0.038992881774902344, -0.009494846686720848, 0.0022920318879187107, -0.016535969451069832, -0.012295292690396309, 0.004730754066258669, 0.0030288160778582096, 0.02391047775745392, 0.011701865121722221, -0.0488077774643898, 0.019829826429486275, 0.012475322000682354, 0.02567075751721859, -0.00890141911804676, -0.03003145381808281, -0.04139326512813568, -0.011675193905830383, -0.011995244771242142, 0.03696589171886444, 0.017949527129530907, -0.03741929680109024, -6.620847125304863e-05, 0.02619084157049656, 0.011548507027328014, 0.0035172272473573685, -0.0021736796479672194, -0.012848714366555214, 0.004247343633323908, -0.023350387811660767, -0.0020303234923630953, -0.005027467850595713, -0.02082998678088188, 0.046514078974723816, 0.016282595694065094, -0.0019536446779966354, -0.015642492100596428, 0.022670280188322067, -0.03907289355993271, 0.007461189292371273, -0.009408165700733662, 0.014762353152036667, 0.02987142652273178, -0.005587557330727577, 0.000157316739205271, 0.027124322950839996, -0.0005067474558018148, 0.00027045977185480297, -0.026737594977021217, 0.00478409556671977, 0.029844757169485092, -0.05059472844004631, 0.026577569544315338, 0.02827117219567299, -0.023310381919145584, 0.01502906158566475, 0.01776283048093319, -0.010981750674545765, -0.019536446779966354, 0.03003145381808281, 0.0015785848954692483, 0.026004144921898842, -0.019936511293053627, 0.01046166755259037, 0.009001434780657291, 0.0013693849323317409, 0.04352693632245064, -0.024523908272385597, -0.016349272802472115, 0.013035411015152931, 0.038379449397325516, -0.024523908272385597, 0.019869834184646606, 0.009261475875973701, -0.007267824839800596, 0.0207899808883667, -0.019016364589333534, -0.006424357183277607, 0.0020569944754242897, -0.01902969926595688, 0.031151631847023964, 0.019416427239775658, 0.0005734247388318181, 0.027444373816251755, -0.01692269742488861, -0.011375145986676216, -0.0033705371897667646, -0.007281160447746515, -0.007194479927420616, 0.005397527012974024, 0.00946817547082901, 0.007621214725077152, 0.016002550721168518, -0.015135745517909527, 0.01880299672484398, -0.009541520848870277, -0.014682340435683727, 0.007214483339339495, -0.007001115940511227, -0.03664584085345268, 0.026430878788232803, -0.017482785508036613, -0.0234970785677433, 0.00536752212792635, -0.019709808751940727, -0.025884125381708145, 0.0031021609902381897, -0.00399396987631917, 0.005710910074412823, -0.015562480315566063, -0.014282275922596455, -0.006821087095886469, -0.0036139092408120632, -0.013508819974958897, 0.01197524182498455, -0.013328790664672852, -0.023777123540639877, 0.029471363872289658, 0.20888662338256836, -0.009294814430177212, 0.023217033594846725, 0.011321804486215115, -0.0118618905544281, 0.013102088123559952, 0.01090173702687025, -0.02189682237803936, -0.0070411222986876965, -0.002792111597955227, -0.03416544571518898, 0.015215758234262466, -0.0005196661804802716, 0.0033205291256308556, 0.002480395371094346, 0.017709489911794662, -0.03387206420302391, -0.019856497645378113, -0.03301859647035599, -0.0008959761471487582, -0.007814578711986542, -0.01494904886931181, -0.00684775784611702, -0.0036639173049479723, 0.039126235991716385, 0.004157329443842173, -0.007274492643773556, -0.016162576153874397, 0.0039306264370679855, 0.013775528408586979, -0.003348866943269968, 0.0010143283288925886, -0.005044137127697468, -0.0010568350553512573, 0.007721230387687683, -0.009188131429255009, -0.003867282997816801, 0.008394671604037285, -0.017136065289378166, 0.002628752263262868, -0.021350068971514702, -0.0036805865820497274, -0.003813941264525056, -0.006724405102431774, 0.015415790490806103, 0.03237849473953247, -0.011301801539957523, 0.007981272414326668, -0.0008080454426817596, 0.017842844128608704, -0.04374030604958534, -0.020349910482764244, 0.009634869173169136, 0.02072330191731453, -0.004097319673746824, -0.024590585380792618, 0.0018302917014807463, 0.027897778898477554, -0.009828233160078526, 0.003007145831361413, -0.0207899808883667, 0.018016204237937927, -0.01464233361184597, 0.028191160410642624, -0.017122728750109673, 0.061183083802461624, 0.01456232089549303, 0.00917479582130909, 0.005940946750342846, -0.01877632550895214, -0.02316369116306305, 0.008788066916167736, -0.022176867350935936, -0.015709171071648598, -0.015562480315566063, -0.011588513851165771, 0.009068111889064312, 0.004947455134242773, 0.010401657782495022, -0.0028354518581181765, -0.007821246050298214, 0.014482308179140091, -0.009281478822231293, -0.0006059298757463694, 0.0033555347472429276, -0.015722505748271942, -0.008501354604959488, 0.0035372304264456034, -0.006721070967614651, -0.012828711420297623, 0.018216237425804138, -0.0015394119545817375, -0.00402730843052268, -0.018816331401467323, 0.008187972009181976, -0.009461508132517338, 0.017949527129530907, 0.007701227441430092, -0.014508979395031929, 0.000693027104716748, -0.008681383915245533, 0.04848772659897804, 0.02727101370692253, 0.0021736796479672194, -0.0015402454882860184, -0.017496122047305107, -0.024350548163056374, -0.011675193905830383, -0.007347837556153536, -0.008774732239544392, -0.0022736957762390375, -0.000953485316131264, -0.012015248648822308, -0.005770919844508171, -0.012582005001604557, 0.0019136383198201656, 0.0002900462131947279, -0.018082881346344948, 0.02176346816122532, -0.005104146897792816, 0.011641855351626873, -0.03595239669084549, -0.011775209568440914, -0.011115104891359806, -0.026337530463933945, -0.0002619167498778552, 0.00034734702785499394, 0.018309583887457848, 0.006764411460608244, -0.030378174036741257, 0.024297205731272697, -0.02453724481165409, 0.003183840773999691, -0.007501195650547743, -0.012988736853003502, -0.008921422064304352, -0.0018019538838416338, -0.011448491364717484, -0.004884111694991589, -0.008674715645611286, 0.0012001913273707032, 0.008808070793747902, 0.013382133096456528, -0.003035483881831169, 0.004950788803398609, -0.016749335452914238, 0.01857629418373108, 0.008688051253557205, -0.0038039395585656166, 0.002125338651239872, 0.0069144354201853275, 0.012268622405827045, 0.011461826972663403, -0.016509298235177994, 0.01449564378708601, -0.00802127830684185, -0.026044150814414024, -0.015429126098752022, 0.01065503153949976, 0.03179173171520233, -0.017042716965079308, 0.004694081377238035, 0.038992881774902344, -0.03744596615433693, -0.012228615581989288, -0.012568669393658638, -0.16888025403022766, 0.017136065289378166, 0.021270057186484337, -0.013855541124939919, 0.01041499339044094, 0.012975401245057583, -0.004204003140330315, 0.005570888053625822, -0.01046166755259037, -0.003940627910196781, 0.03315195068717003, -0.00906144455075264, -0.03960631042718887, -0.005374189931899309, -0.0013918884797021747, -0.023217033594846725, 0.0017702821642160416, 0.026990968734025955, 0.02928466722369194, 0.007294496055692434, 0.05267506092786789, -0.016936032101511955, 0.0215767715126276, 0.010181622579693794, 0.027311019599437714, -0.026070822030305862, -0.01472234632819891, 0.006921103224158287, -0.02560408040881157, -0.022750291973352432, -0.007387843914330006, -0.0018602964701130986, 0.02885793335735798, 0.021936830133199692, -0.009894910268485546, 0.03403209149837494, -0.021643448621034622, -0.021590108051896095, -0.023577090352773666, 0.015935873612761497, 0.015255765058100224, 0.013402136042714119, 0.002217019908130169, 0.009221469983458519, 0.008754728361964226, 0.03368536755442619, -0.0005380024085752666, 0.003153835888952017, 0.0402197428047657, 0.009054776281118393, -0.010408325120806694, -0.01938975602388382, 0.003950629848986864, 0.008461348712444305, 0.02058994770050049, 0.011375145986676216, 0.018696313723921776, 0.006114307790994644, -0.0007334502297453582, 0.0011593514354899526, -0.007274492643773556, 0.0026054151821881533, 0.007214483339339495, 0.005727579351514578, -0.0020636622793972492, -0.00039672988350503147, -0.009194798767566681, 0.01636260747909546, -0.010675034485757351, -0.002577077364549041, -0.030644884333014488, -0.0052541708573699, -0.011341807432472706, -0.03552566096186638, 0.013775528408586979, 0.005277507938444614, -0.0025987476110458374, 0.034245457500219345, -0.013908883556723595, -0.002247024793177843, -0.021350068971514702, 0.02603081613779068, -0.008688051253557205, 0.006407687906175852, 0.02108336053788662, -0.0026154168881475925, 0.004844105336815119, 0.013108755461871624, -0.019923174753785133, -0.007214483339339495, 0.022536925971508026, -0.0039006215520203114, 0.0036572495009750128, -0.008394671604037285, 0.005600892473012209, -0.004854106809943914, -0.00951484963297844, 0.02264360897243023, -0.031178303062915802, 0.01223528292030096, 0.008361333049833775, 0.001941976137459278, -0.032911911606788635, 0.002320369705557823, 0.011255127377808094, -0.002823783317580819, 0.005854266230016947, -0.0010001594200730324, 0.029257996007800102, -0.03139166906476021, -0.005697574932128191, 0.021550100296735764, 0.022336892783641815, 0.022376900538802147, 0.012061922810971737, 0.00456406082957983, -0.007601211313158274, 0.009754887782037258, 0.010221629403531551, -0.01938975602388382, 0.05462203919887543, 0.0020086534786969423, 0.010821724310517311, -0.005630897358059883, 0.009154791943728924, -0.022403569892048836, -0.09916247427463531, 0.016535969451069832, -0.006094304844737053, 0.015375783666968346, -0.0009384829318150878, -0.007714563049376011, 0.022536925971508026, 0.031738393008708954, -0.010681702755391598, -0.0045140525326132774, -0.009708213619887829, -0.02941802144050598, 0.006421023514121771, 0.03045818768441677, 0.030991606414318085, -0.006834422703832388, 0.004324022680521011, -0.014255605638027191, -0.004424038343131542, 0.019696472212672234, 0.0071144672110676765, -0.012908724136650562, 0.019669800996780396, -0.014255605638027191, 0.006097638513892889, 0.013575497083365917, -0.023763787001371384, 0.017269419506192207, 0.01182855200022459, 0.02567075751721859, -0.015215758234262466, -0.008074619807302952, 0.004660742823034525, 0.0027037642430514097, 0.005167490337044001, 0.012028583325445652, -0.049927957355976105, 0.0013935554306954145, 0.013482148759067059, -0.007361173164099455, 0.0055375490337610245, 0.001133513986133039, 0.012922059744596481, -0.01740277372300625, -0.033498670905828476, -0.00040923189953900874, -0.010588354431092739, -0.003113829530775547, 0.022510254755616188, -0.009761556051671505, -0.02716432884335518, -0.01753612793982029, -0.009734884835779667, 0.007507863454520702, 0.007514530792832375, 0.01961646042764187, -0.004250677302479744, 0.004534055944532156, -0.039873022586107254, -0.02277696318924427, 0.0019886502996087074, -0.010421660728752613, -0.008561364375054836, 0.02144341729581356, 0.0016777673736214638, 0.025484060868620872, -0.004534055944532156, -0.0009634869056753814, 0.012948730029165745, -0.017042716965079308, -0.009674875065684319, 0.03419211506843567, 0.0034605516120791435, -0.0031038280576467514, -0.0008513857028447092, -0.0009693211759440601, -0.03416544571518898, -0.02088332735002041, 0.02013654261827469, -0.011168446391820908, -0.01900302805006504, -0.0057509164325892925, 0.01681601256132126, -0.02293698862195015, 0.0190963763743639, 0.02619084157049656, 0.03045818768441677, -0.006391018629074097, 0.022083519026637077, 0.0007809578091837466, 0.0246305912733078, 0.028777919709682465, 0.013948889449238777, -0.02153676562011242, 0.012408643960952759, 0.023403730243444443, 0.007867920212447643, -0.006217657588422298, 0.008261316455900669, -0.006794416345655918, -0.022656943649053574, -0.0032788559328764677, -0.06859759986400604, 0.015189087018370628, -0.017296090722084045, -0.004747423343360424, -0.02781776711344719, -0.0029971443582326174, -0.005130817648023367, -0.008267984725534916, 0.006921103224158287, 0.0027621069457381964, -0.04181999713182449, 0.012475322000682354, -0.004230674356222153, 0.0018152892589569092, -0.020149877294898033, 0.005110814701765776, 0.031898416578769684, -0.019123047590255737, 0.01076838281005621, 0.0057642520405352116, -0.014882371760904789, 0.016842683777213097, -0.014855700545012951, 0.006474365014582872, -0.02619084157049656, 0.003413877449929714, -0.014308947138488293, 0.018909679725766182, -0.02042992226779461, -0.012668685987591743, -0.002837118925526738, -0.028191160410642624, -0.0008851410821080208, 0.027871109545230865, -0.03536563739180565, -0.008441345766186714, 0.02013654261827469, 0.03149835392832756, 0.010675034485757351, 0.05211497098207474, -0.02553740330040455, -0.01990984007716179, 0.0047974311746656895, -0.02287031151354313, 0.004417370539158583, 0.012415312230587006, -0.014468972571194172, 0.004294017795473337, 0.021030018106102943, -0.012141935527324677, 0.013375464826822281, 0.0260974932461977, 0.0014202262973412871, -0.008788066916167736, 0.029338009655475616, -0.0003171338757965714, 0.024523908272385597, 0.008921422064304352, -0.004063981119543314, -0.02736436203122139, 0.028457868844270706, 4.688247281592339e-05, 0.026270853355526924, 0.002685427898541093, 0.01034831628203392, -0.0063043381087481976, -0.01588253118097782, 0.006787748541682959, -0.0007230318733491004, -0.03872617334127426, -0.01658931002020836, -0.03147168084979057, 0.018389597535133362, 0.00914145726710558, 0.006811085622757673, -0.010074938647449017, 0.006067633628845215, 0.01205525454133749, 0.004660742823034525, 0.01090173702687025, 0.00916146021336317, -0.02056327648460865, -0.025190681219100952, 0.01172853633761406, 0.019309744238853455, 0.008761396631598473, -0.002031990559771657, 0.010228296741843224, -0.00924147292971611, 0.022310221567749977, -0.012355302460491657, 0.026337530463933945, -0.002001985674723983, 0.0052541708573699, -0.0032288478687405586, 0.04374030604958534, 0.01090173702687025, 0.0050241341814398766, 0.00828131940215826, 0.02697763219475746, -0.007487860042601824, 0.008854744955897331, 0.0047274199314415455, 0.003077157074585557, -0.02619084157049656, 0.021110031753778458, -0.03901955112814903, -0.0034772208891808987, 0.00802127830684185, -0.015402454882860184, -0.016896026208996773, -0.003693921957165003, 0.038699500262737274, 0.018896345049142838, -0.03712591528892517, 0.0029454694595187902, 0.009194798767566681, -0.01224861852824688, -0.03251184895634651, 0.022403569892048836, -0.00016836015856824815, -0.0035372304264456034, 0.028351185843348503, 0.007947932928800583, -0.008728058077394962, 0.004360694903880358, 0.020229890942573547, -0.005630897358059883, 0.0030321499798446894, -0.006267665419727564, 0.02239023521542549, 0.012962065637111664, -0.048461057245731354, -0.019336415454745293, -0.020323239266872406, 0.0015052398666739464, -0.008688051253557205, 0.02729768306016922, 0.01920306123793125, 0.03493890166282654, 0.014829030260443687, 0.011101769283413887, 0.014068908989429474, 0.013842206448316574, 0.017109394073486328, 0.030511530116200447, -0.0056208958849310875, 0.009001434780657291, -0.02765774168074131, 0.029711401090025902, -0.019763149321079254, 0.010815056972205639, -0.029338009655475616, -0.01590920239686966, 0.01877632550895214, 0.005720911547541618, 0.01780283823609352, -0.02235022932291031, 0.013682181015610695, 0.04027308523654938, -0.01812288910150528, 0.013855541124939919, 0.016015885397791862, -0.028937945142388344, -0.011755206622183323, 0.006517705507576466, -0.006554377730935812, -0.014695675112307072, -0.018722983077168465, -0.015829188749194145, 0.018269577994942665, -0.02187015302479267, -0.009648204781115055, 0.011968574486672878, -0.022536925971508026, -0.0018302917014807463, -0.0028921274933964014, 0.011995244771242142, 0.015629157423973083, 0.013282116502523422, 0.011848554946482182, -0.008314658887684345, -0.02368377521634102, 0.016602646559476852, -0.013522154651582241, -0.01636260747909546, 0.014215598814189434, -0.017589470371603966]} +{"id": "test:10000078", "text": "\"8i announced today it has launched a web player, the 8i Portal, for its volumetric 3D video of people in virtual reality.\\nUsing 8i\u2019s technology and VR goggles, you\u2019ll be able to walk around a person inside a virtual environment to see their entire being. The Wellington, New Zealand-based startup said its VR platform will allow you to view fully volumetric 3D video.\\nPreviously, VR cameras could record a partial torso of a person, but the 8i Portal goes beyond that. It enables \u201ccomplete freedom of movement,\u201d and it gives you a sense of \u201cpresence,\u201d or the feeling that you are there in the virtual space with the person. It creates a more \u201cemotional connection with the person you are watching,\u201d the company said in the video below.\\n8i\u2019s platform will let you create, experience, and share immersive 3D video of real people \u2014 for virtual reality, augmented reality, and the web. I\u2019m not sure how it can be used for games, but you can see how it could give you a unique perspective at a music concert or theatrical performance.\\nLinc Gasking, who previously ran countingdown.com (which DreamWorks acquired), and Eugene d\u2019Eon, formerly of Weta Digital and Nvidia, founded the startup in 2014.\\nTo date, 8i has raised $14.8 million in funding from investors including RRE Ventures, Founders Fund Science, Horizons Ventures, Samsung Ventures, Dolby Family Ventures, Signia Venture Partners, Bertelsmann Digital Media Investments, Sound Ventures, Inevitable Ventures, Freelands, and Advancit Capital.\\nThe new 8i Portal is a volumetric VR player for Oculus Rift and the HTC Vive VR platforms. If you use the Oculus Rift or Vive headsets, you can view cool VR videos of people who look real, not computer-generated, and move around them, make eye contact, and feel \u201ctrue presence,\u201d the company said.\"", "vector_field": [0.00411954103037715, -0.012540963478386402, 0.01354721188545227, -0.034928303211927414, 0.007226080168038607, 0.026783769950270653, -0.04829993098974228, -0.036684174090623856, 0.004062137566506863, -0.026905329897999763, 0.015357108786702156, 0.016653750091791153, -0.0012004072777926922, -0.007280106656253338, 0.022259028628468513, 0.0014815152389928699, 0.0026574416551738977, 0.0005786773399449885, 0.015748802572488785, -0.02124602533876896, -0.02419048547744751, 0.027418583631515503, -0.007529980503022671, -0.007860894314944744, -0.038034845143556595, 0.01624855026602745, 0.014546707272529602, -0.01576230861246586, 0.025203486904501915, -0.007421927060931921, -0.0029967972077429295, 0.00927234347909689, -0.002103667240589857, -0.011602248065173626, -0.015600228682160378, 0.03673820197582245, 0.004862409085035324, 0.010987693443894386, 0.01576230861246586, 0.0039203171618282795, 0.018180007115006447, 0.015505681745707989, -0.005672810599207878, -0.012095242738723755, -0.04692224785685539, 0.024487631395459175, -0.017329085618257523, -0.005318259820342064, 0.01858520694077015, 0.004767862148582935, 0.0033479712437838316, 0.008745582774281502, -0.008029728196561337, 0.004524741787463427, 0.01040690578520298, -0.021232519298791885, -0.011609001085162163, 0.007462447043508291, 0.024636205285787582, -0.006061127875000238, -0.0004106878477614373, -0.02177278697490692, -0.037953805178403854, 0.007786607835441828, -0.0168833639472723, 0.013142011128365993, -0.01696440391242504, 0.0007158546359278262, -0.003528623143211007, -0.004389674868434668, 0.04062812775373459, 0.024730753153562546, 0.007063999772071838, -0.006895166356116533, 0.021786293014883995, -0.006398795172572136, 7.935181201901287e-05, -0.03217294067144394, 0.02192136086523533, 0.008266095072031021, 0.006324508227407932, -0.028147945180535316, -0.005372286774218082, 0.02558167465031147, 0.019003914669156075, 0.010562232695519924, -0.0028245868161320686, 0.015586722642183304, 0.007239586673676968, -0.01249368954449892, -0.00996793806552887, 0.001018911018036306, 0.002424451056867838, 0.008252589032053947, -0.028607172891497612, 0.035684678703546524, -0.015573215670883656, 0.007840634323656559, -0.02439308539032936, -0.018571700900793076, -0.00848220195621252, -0.012318102642893791, -0.02120550535619259, -0.02065173164010048, -0.007253093644976616, -0.007867648266255856, 0.00798245519399643, 0.0007985831471160054, 0.039304472506046295, 0.014249559491872787, -0.0017001548549160361, 0.0018605467630550265, 0.008455188944935799, -0.011966928839683533, -0.008421421982347965, -0.00967079121619463, 0.024298537522554398, 0.004372791387140751, -0.015870362520217896, -0.006469705142080784, 0.019517170265316963, 0.007428680546581745, -0.005693070590496063, 0.012486936524510384, 0.015991922467947006, 0.013155518099665642, -0.010265085846185684, -0.018261047080159187, 0.037548601627349854, -0.031362537294626236, 0.026864809915423393, -0.02259669452905655, 0.02347462996840477, 0.017842339351773262, -0.020962385460734367, 0.008380901999771595, -0.0066655525006353855, 0.001079691224731505, -0.025473620742559433, -0.03860212489962578, 0.018652740865945816, 0.0006766008445993066, -0.007043739780783653, -0.024528151378035545, -0.007989208213984966, 0.028607172891497612, 0.006743215955793858, 0.025595180690288544, 0.03138955309987068, 0.005807877518236637, 0.014087479561567307, -0.008522722870111465, -0.02093537151813507, 0.010292098857462406, 0.003022122196853161, 0.0027283516246825457, 0.014155012555420399, 0.028093919157981873, -0.0019230152247473598, -0.001983795315027237, -0.03430699557065964, 0.015870362520217896, -0.013695785775780678, 0.009954432025551796, 0.014033452607691288, 0.015505681745707989, 0.005267609842121601, -0.01739661954343319, -0.0011885889107361436, 0.016694270074367523, -0.023055922240018845, 0.035765718668699265, -0.03960162028670311, 0.008664542809128761, 0.010460932739078999, 0.018976902589201927, -0.0033277112524956465, 0.01537061482667923, -0.008907663635909557, -0.016896871849894524, 0.01175757497549057, -0.014843854121863842, 0.0019382102182134986, 0.022069934755563736, -0.012979931198060513, -0.01273005735129118, 0.00312342238612473, -0.0013017074670642614, -0.010460932739078999, -0.03022797591984272, 0.012277582660317421, 0.010893146507441998, 0.02795885130763054, -0.016505178064107895, -0.6396769285202026, -0.019287556409835815, 0.012601743452250957, -0.01066353265196085, 0.004315387923270464, 0.014533200301229954, 0.0010501452488824725, -0.006827632896602154, 0.00028828345239162445, 0.019368596374988556, -0.00045036376104690135, 0.009258837439119816, -0.016788817942142487, -0.0342259556055069, -0.01655920408666134, -0.006256974767893553, 0.008988703601062298, -0.015748802572488785, 0.0011657963041216135, 0.003839277196675539, -0.042167890816926956, -0.0025915964506566525, -0.009441177360713482, 0.003940577153116465, 0.0013970984145998955, -0.0007137442589737475, 0.02394736371934414, 0.004315387923270464, -0.005706577096134424, 0.013749811798334122, 0.0021678239572793245, 0.01759921945631504, -0.006581135559827089, 0.011136267334222794, 0.035252466797828674, -0.015951402485370636, -0.021975386887788773, 0.02177278697490692, 0.001559178694151342, 0.0464630201458931, -0.005284493323415518, -0.005834890995174646, 0.017612725496292114, -0.009994952008128166, -0.02021951787173748, 0.012439663521945477, 0.023096442222595215, 0.0007247184403240681, -0.016059456393122673, -0.026027394458651543, -0.00958975125104189, -0.019125474616885185, -0.007239586673676968, 0.010744573548436165, 0.007145039737224579, -0.02157018706202507, 0.017221031710505486, 0.0022826308850198984, 0.010420412756502628, 0.006229961756616831, -0.01580282859504223, -0.014722294174134731, -0.020543677732348442, -0.019165996462106705, -0.013702538795769215, 0.021435119211673737, 0.0024362695403397083, 0.008806362748146057, 0.0051224129274487495, 0.006084764841943979, 0.015127494931221008, 0.053648579865694046, -0.013310845009982586, 0.012649016454815865, 0.009799105115234852, 0.022096946835517883, 0.031794752925634384, 0.004805005621165037, -0.004389674868434668, 0.011602248065173626, 0.0179503932595253, -0.026594676077365875, -0.008124275133013725, 0.0094952043145895, 0.0006905295886099339, 0.006469705142080784, -0.03587377443909645, -0.019908864051103592, 0.026432596147060394, -0.013054217211902142, 0.012790837325155735, 0.010217811912298203, -0.020368091762065887, -0.007421927060931921, -0.007955441251397133, 0.019571196287870407, -0.01763973943889141, 0.04079020768404007, 0.006283988244831562, -0.006128661334514618, -0.00446733832359314, -0.043275441974401474, 0.012885384261608124, 0.0038190169725567102, 0.021191999316215515, 0.048840198665857315, -0.03538753092288971, -0.002716533374041319, 0.02852613292634487, -0.0481378510594368, -0.0008821557858027518, -0.02327203005552292, 0.002177953952923417, -0.0024599062744528055, 0.011305101215839386, -0.020408611744642258, 0.02224552072584629, -0.0020901605021208525, 0.01716700568795204, -0.026351556181907654, 0.022988390177488327, -0.023447616025805473, 0.009150783531367779, 0.00836064200848341, -0.009765338152647018, 0.01712648570537567, -0.0024193860590457916, -0.009076496586203575, -0.04759758338332176, -0.007813621312379837, 0.005588393658399582, -0.022150974720716476, 0.006084764841943979, -0.0004647145979106426, 0.015397628769278526, 0.023420603945851326, 0.008218822069466114, -0.02069225162267685, -0.010704053565859795, -0.01842312701046467, -0.03311840817332268, -0.02912042662501335, 0.010312358848750591, -0.010143525898456573, 0.00192639185115695, -0.030498109757900238, -0.02144862711429596, 0.02208344079554081, 0.01019079890102148, 0.0013802150497213006, -0.009123770520091057, -0.009549231268465519, 0.027715731412172318, 0.01523554790765047, -0.0018166500376537442, -0.002220162423327565, -0.014681774191558361, -0.02367722988128662, -0.007050493266433477, -0.007185560185462236, 0.014951908029615879, -0.004879292566329241, -0.01811247318983078, -0.03206488490104675, 0.0037379770074039698, -0.01763973943889141, 0.006013854406774044, 0.013452664948999882, -0.026324542239308357, -0.02240760065615177, -0.006196194794028997, 0.01576230861246586, -0.0021171739790588617, 0.032632168382406235, -0.007820374332368374, 0.022191494703292847, 0.027850797399878502, -0.013992932625114918, -0.0013017074670642614, -0.02613544836640358, -0.004328894894570112, -0.016343096271157265, -0.04205983877182007, -0.007813621312379837, 0.016748297959566116, 0.0036704435478895903, 0.01019079890102148, 0.03260515257716179, 0.004956956021487713, 0.016977911815047264, 0.0137227987870574, 0.025473620742559433, -0.013013697229325771, 0.03228099271655083, -0.012817850336432457, 0.02220500074326992, 0.012041215784847736, 0.006726332474499941, 0.007394913583993912, 0.005227089859545231, 0.037710681557655334, 0.012527456507086754, 0.007644787430763245, 0.0032483593095093966, 0.01735609956085682, -0.017626233398914337, -0.0004621820989996195, -0.016397124156355858, 0.03184878081083298, 0.010562232695519924, 0.03122747130692005, 0.0043694148771464825, 0.0010265085147693753, -0.006844515912234783, 0.019854836165905, 0.02803989127278328, 0.02390684373676777, -0.01866624876856804, -0.02267773449420929, 0.010697299614548683, -0.014506187289953232, -0.03274022042751312, 0.030606163665652275, 0.007124779745936394, -0.028607172891497612, 0.024730753153562546, -0.005953074432909489, 0.03511739894747734, 0.021191999316215515, -0.0371704138815403, -0.024325551465153694, 0.012838110327720642, 0.006338015198707581, 0.010298852808773518, 0.0007217638194561005, -0.016950897872447968, 0.022866828367114067, 0.010048978962004185, 0.03663014620542526, 0.005281116347759962, 0.00996793806552887, 0.03738652169704437, 0.032118912786245346, -0.014627747237682343, 0.014209039509296417, 0.005807877518236637, 0.03074122965335846, -0.020449131727218628, -0.01672128401696682, 0.020759785547852516, -0.0038122637197375298, 0.006128661334514618, -0.009508711285889149, -0.017572205513715744, 0.005770734045654535, -0.019962890073657036, -0.01723453775048256, -0.007415173575282097, -0.0012426156317815185, 0.032308004796504974, -0.006797242444008589, -0.008792856708168983, 0.02550063468515873, -0.010244825854897499, -0.016491670161485672, 0.008786102756857872, -0.0132905850186944, -0.0039203171618282795, -0.0151950279250741, 0.002220162423327565, -0.009177797473967075, -0.010278592817485332, 0.021462133154273033, -0.011791341938078403, 0.01755869947373867, -0.006540615577250719, 0.008657789789140224, 0.007401667069643736, 0.019152488559484482, 0.018774300813674927, -0.005223712883889675, -0.032713208347558975, -0.009805858135223389, 0.006435938645154238, 0.01006923895329237, -0.02506841905415058, -0.020503157749772072, -0.007678554393351078, -0.02124602533876896, -0.005514106713235378, -0.006982959806919098, -0.0043120114132761955, 0.00958975125104189, 0.013006944209337234, -0.014344106428325176, -0.007192313205450773, 0.025878820568323135, -0.013479677960276604, -0.020368091762065887, -0.00863752979785204, -0.01167653501033783, 0.003223034320399165, 0.016626738011837006, -0.025162966921925545, 0.024203991517424583, -0.02085433155298233, -0.02077329158782959, -0.003201085841283202, -0.012486936524510384, 0.0038527837023139, -0.004396428354084492, -0.009144030511379242, -0.011892641894519329, -0.008819869719445705, -0.005632290616631508, -0.0003216280892957002, -0.0077595943585038185, -0.00827960204333067, 0.012432909570634365, 0.006331261713057756, -0.004609158728271723, -0.03066018968820572, -0.015303081832826138, -0.01576230861246586, 0.07563747465610504, -0.010379892773926258, 0.011365881189703941, 0.02740507759153843, -0.03690028190612793, -0.02173226699233055, -0.012919150292873383, -0.022542668506503105, 0.0094952043145895, 0.017139991745352745, -0.008826622739434242, 0.019017422571778297, 0.0068613993935287, -0.0164781641215086, -0.005757227540016174, 0.00481513561680913, 0.017531685531139374, -0.03160565719008446, -0.005787617526948452, -0.01862572878599167, -0.0033260227646678686, 0.004754355642944574, -0.013081231154501438, 0.0255411546677351, 0.01954418234527111, -0.020989399403333664, 0.03257814049720764, 0.017302071675658226, -0.015303081832826138, 0.0011100812116637826, 0.019436130300164223, -0.004734095651656389, 0.014492680318653584, -0.0069491928443312645, -0.020408611744642258, -0.016856351867318153, 0.002605103189125657, 0.011784588918089867, 0.014627747237682343, 0.011980435810983181, -0.020881345495581627, 0.0255411546677351, 0.001887560123577714, -0.010555479675531387, 0.002392372814938426, 0.00459902873262763, -0.006199571304023266, 0.016977911815047264, 0.011406401172280312, -0.002443022793158889, 0.021624213084578514, 0.0033648544922471046, -0.025135952979326248, -0.028742240741848946, -0.032226964831352234, 0.013621498830616474, 0.008813116699457169, -0.028066905215382576, -0.0026726366486400366, -0.027107929810881615, -0.000771569786593318, -0.03722444176673889, -0.008671295829117298, 0.010346125811338425, -0.014046959578990936, -0.021826812997460365, 0.0015532695688307285, 0.020165489986538887, -0.006888412870466709, -0.001931456965394318, 0.011750821955502033, -0.028228985145688057, -0.04862409085035324, -0.0259733684360981, 0.003582649864256382, 0.011372634209692478, 0.0028262753039598465, 0.0006470549269579351, 0.01052846573293209, 0.0024362695403397083, 0.003643430070951581, -0.000500591762829572, -0.006135414820164442, -0.012176282703876495, 0.008353888988494873, 0.007989208213984966, -0.001813273411244154, 0.008590255863964558, -0.0051933228969573975, 0.010481192730367184, 0.028553146868944168, -0.000636502867564559, 0.0160459503531456, 0.0082390820607543, 0.03047109581530094, 0.016937391832470894, 0.005493846721947193, 0.0016832714900374413, 0.008752335794270039, -0.0219618808478117, -0.01048794575035572, -0.030336029827594757, -0.009306110441684723, -0.020665237680077553, 0.00988689810037613, -0.0015102169709280133, -0.0018689885037019849, -0.001811585039831698, 0.00466656219214201, -0.0004575391940306872, -0.02128654532134533, 0.002643934916704893, 0.0034779731649905443, -0.014654760248959064, 0.0023484760895371437, -0.0026287399232387543, 0.003896680660545826, 0.0034779731649905443, 0.01131860725581646, -0.007692060898989439, -0.0033445945009589195, -0.02828301303088665, 0.018017927184700966, -0.005433066748082638, 0.004014864098280668, -0.002235357416793704, 0.009711311198771, -0.0034948564134538174, 0.0012822916032746434, 0.008664542809128761, 0.0215296670794487, 0.013992932625114918, 0.01189939584583044, -0.003933824133127928, -0.02939056046307087, -0.014789827167987823, -0.00021178070164751261, -0.003589403349906206, 0.0077055674046278, -0.01332435104995966, -0.014789827167987823, 0.002448087790980935, 0.0154111348092556, -0.0010070926509797573, -0.009623518213629723, -0.02537907287478447, -0.014708787202835083, 0.022853322327136993, -0.0077055674046278, 0.008894156664609909, -0.020327571779489517, 0.013607991859316826, -0.007172053214162588, -0.024879327043890953, -0.005828137509524822, -0.06342742592096329, -0.03565766662359238, -0.010926913470029831, 0.023069430142641068, 0.0025730247143656015, 0.026864809915423393, -0.005821384023874998, 0.02292085625231266, 0.01870676875114441, -0.004014864098280668, -0.003643430070951581, -0.0005263388738967478, -0.005524236708879471, -0.03760262951254845, 0.020867839455604553, 0.008657789789140224, 0.020638225600123405, 0.01102146040648222, -0.007827128283679485, -0.017423631623387337, 0.009873392060399055, -0.02165122702717781, 0.01127808727324009, -0.03438803553581238, -0.04187074303627014, -0.01819351315498352, 0.007016726303845644, 0.0022522408980876207, -0.0160459503531456, -0.021002905443310738, 0.014533200301229954, 0.029957842081785202, -0.0012957982253283262, 0.024852313101291656, -0.009974692016839981, 0.01612699031829834, -0.03260515257716179, -0.012973177246749401, 0.0004419220786076039, 0.018814820796251297, 0.004875915590673685, 0.0038831739220768213, -0.025986874476075172, 0.010953927412629128, 0.01696440391242504, 0.02259669452905655, -0.005368909798562527, 0.0154111348092556, -0.0009226758847944438, -0.011034967377781868, 0.036603134125471115, -0.0018082082970067859, -0.026864809915423393, 0.014452160336077213, -0.0308222696185112, 0.00015501039160881191, -0.030876297503709793, -0.024001391604542732, -0.009312864392995834, -0.024879327043890953, -0.00012873565719928592, -0.003373296232894063, 0.013635004870593548, -0.0022235391661524773, -0.030363043770194054, 0.011946668848395348, 0.01167653501033783, 0.0324971005320549, 0.010359632782638073, 0.03506337106227875, 0.011197047308087349, 0.007293613627552986, -0.02836405299603939, -0.03319944813847542, -0.0020141853019595146, -0.0002587375638540834, 7.032976282062009e-05, 0.0032652427908033133, -0.01388487871736288, -0.021273039281368256, 0.0320919007062912, 0.0007069908897392452, -0.036603134125471115, -0.03852108493447304, -0.002026003785431385, 0.019355088472366333, 0.005649173632264137, -0.02458217926323414, -0.0017212589737027884, -0.018004419282078743, 0.029201466590166092, -0.011575235053896904, 0.006729708984494209, -0.013169024139642715, -0.011048473417758942, -0.007908168248832226, 0.001177614671178162, -0.013391884975135326, -0.0004465649835765362, 0.024528151378035545, 0.0073408870957791805, -0.0034070629626512527, -0.008826622739434242, -0.01281109731644392, 0.009042730554938316, 0.0003826192405540496, -0.00031951768323779106, -0.012925904244184494, 0.016302576288580894, 0.012919150292873383, 0.011825108900666237, -0.00023277939180843532, -0.013790332712233067, 0.002605103189125657, 0.03276723250746727, -0.016099976375699043, -0.002355229342356324, 0.021597199141979218, -0.011507701128721237, -0.002853288548067212, -0.015222041867673397, -0.008407915942370892, -0.017747793346643448, 0.02390684373676777, 0.004396428354084492, 0.0060037244111299515, -0.013770071789622307, 0.00015902018640190363, -0.004244477953761816, 0.0010315736290067434, -0.004072267562150955, -0.015438148751854897, 0.01493840105831623, 0.0026692599058151245, -0.030525123700499535, -0.012230309657752514, 0.00431876489892602, 0.0006483211764134467, 0.011980435810983181, -0.005723460577428341, 0.022069934755563736, 0.0006892633391544223, 0.012250569649040699, -0.04586872458457947, -0.006885035894811153, 0.016113482415676117, 0.01651868410408497, -0.024001391604542732, -0.00875908974558115, 0.003260177792981267, -0.014290079474449158, 0.026554156094789505, -0.01581633649766445, -0.03527947887778282, -0.008043235167860985, 0.0039000571705400944, 0.010163785889744759, 0.017896367236971855, 0.00011828907008748502, 0.007469200529158115, -0.003741353517398238, -0.015721788629889488, -0.005578263662755489, -0.00589904747903347, -0.006928932853043079, 0.006226584780961275, -0.0011227437062188983, 0.0008336161263287067, -0.02566271461546421, 0.026473116129636765, -0.03579273447394371, 0.01723453775048256, -0.00927234347909689, -0.04016890004277229, -0.011595495045185089, -0.001462943502701819, 0.004875915590673685, -0.008907663635909557, -0.006577759049832821, -0.010460932739078999, -0.0029934204649180174, -0.0225696824491024, -0.011365881189703941, 0.0010754703544080257, -0.015951402485370636, -0.001256122370250523, 0.04154658317565918, 0.01641063019633293, 0.0030575774144381285, -0.008313369005918503, -0.010143525898456573, -0.007840634323656559, -0.011690041981637478, -0.01874728873372078, 0.017464151605963707, -0.006311001721769571, 0.0439237616956234, 0.01477632112801075, -0.018261047080159187, -0.031362537294626236, -0.005338519811630249, -0.04162762314081192, 0.0011294970754534006, 0.0031082273926585913, 0.017031937837600708, 0.019328076392412186, -0.0023180858697742224, 0.02589232847094536, 0.020043930038809776, 0.010717559605836868, -0.009380397386848927, -0.010055731981992722, -0.02736455760896206, 0.0175181794911623, -0.01285161729902029, -0.0021053554955869913, 0.012979931198060513, -0.029147440567612648, -0.005436443258076906, 0.00547021022066474, 0.012122255750000477, 0.02093537151813507, 0.018247541040182114, -0.006577759049832821, -0.014411640353500843, 0.0016148937866091728, 0.002311332616955042, 0.0002735104935709387, 0.01862572878599167, 0.027823785319924355, -0.024163471534848213, 0.0010687169851735234, -0.002755365101620555, 0.0003452647943049669, 0.003413816448301077, 0.0210974533110857, 0.00941416434943676, 0.0037109635304659605, 0.016154002398252487, 0.008664542809128761, 0.007779854349792004, 0.0069356863386929035, -0.009326370432972908, 0.03252411261200905, 0.004943449050188065, 0.005250726360827684, 0.015086974948644638, 0.03409089148044586, 0.005041372496634722, -0.014182026498019695, 0.031281497329473495, 0.012925904244184494, 0.007455694023519754, 0.02331255003809929, -0.002542634727433324, 0.01858520694077015, -0.007455694023519754, -0.012176282703876495, -0.01189939584583044, -0.004723965656012297, -0.020867839455604553, -0.0015076844720169902, -0.007752840872853994, 0.014425146393477917, 0.009117016568779945, -0.0034070629626512527, -0.01727505959570408, -0.028985360637307167, -0.008914416655898094, -0.043707653880119324, -0.017855847254395485, 0.009035976603627205, -0.031038377434015274, -0.005379039794206619, 0.0005157868145033717, -0.01044742576777935, -0.027391571551561356, 0.023731257766485214, 0.011845368891954422, 0.012216802686452866, 0.011392894200980663, 0.20875942707061768, 0.018207021057605743, -0.002004055306315422, 0.03444206342101097, -0.006729708984494209, 0.0015321653336286545, 0.023245016112923622, -0.0003771321498788893, 0.017626233398914337, -0.003889927174896002, -0.011007953435182571, 0.025271020829677582, -0.0011041720863431692, -0.0035016098991036415, 0.0029545887373387814, -0.038953300565481186, -0.02525751292705536, -0.022191494703292847, -0.014479173347353935, 0.008509215898811817, -0.00037206715205684304, -0.04314037412405014, -0.01842312701046467, -0.015316588804125786, 0.013263571076095104, -0.010224565863609314, -0.00975183118134737, 0.0020361337810754776, 0.03638702630996704, 0.02097589150071144, 0.0034357646945863962, -0.01380383875221014, 0.003064330667257309, 0.0023434110917150974, -0.0005014359485358, -0.021597199141979218, -0.006084764841943979, -0.003292256034910679, 0.0063177552074193954, 0.03892628476023674, -0.00031635205959901214, 0.02836405299603939, 0.014087479561567307, 0.00044951957534067333, 0.0259733684360981, 0.04246503859758377, -0.024082431569695473, -0.01123756729066372, 0.011068733409047127, -0.01946314238011837, -0.016910377889871597, 0.024285031482577324, 0.020354583859443665, 0.02812093123793602, -0.024339059367775917, 0.014668267220258713, -0.009691051207482815, 0.008333628997206688, 0.006105024833232164, 0.007401667069643736, -0.02672974206507206, 0.020476143807172775, 0.003562389872968197, 0.01978730410337448, -0.028688212856650352, 0.033469583839178085, 0.006128661334514618, 0.01259499043226242, -0.012723303399980068, -0.013783578760921955, 0.009441177360713482, -0.0026844548992812634, -0.02928250841796398, -0.0020124970469623804, -0.015208534896373749, -0.032983340322971344, 0.017221031710505486, 0.00989365205168724, 0.02192136086523533, 0.015870362520217896, 0.016140496358275414, -0.0215296670794487, 0.002176265697926283, 0.005666057113558054, -0.03814289718866348, -0.017585713416337967, 0.02617596834897995, -0.01273005735129118, -0.006631785538047552, 0.017072457820177078, -0.005848397500813007, -0.0014156701508909464, -0.017099471762776375, -0.012486936524510384, -0.001078846980817616, -0.006726332474499941, 0.009008963592350483, 0.006695942487567663, -0.002313020871952176, 0.01934158243238926, -0.0014131376519799232, 0.024163471534848213, 0.025811288505792618, -0.007300366647541523, 0.007935181260108948, 0.02177278697490692, 0.004994099494069815, 0.022974882274866104, 0.012459923513233662, -0.013013697229325771, -0.006618279032409191, -0.010332618840038776, 0.014371120370924473, 0.0020496405195444822, 0.007509720511734486, -0.006256974767893553, 0.013513444922864437, -0.019328076392412186, 0.011311854235827923, -0.0010062485234811902, -0.01565425656735897, 0.002329904353246093, 0.012135762721300125, 0.0031909558456391096, -0.008954936638474464, -0.0308222696185112, -0.01072431355714798, 0.025689728558063507, -0.029012374579906464, 0.009920665062963963, 0.010116511955857277, 0.011588741093873978, 0.010602752678096294, -0.011595495045185089, 0.00023531189071945846, -0.013182531110942364, 0.013378378003835678, 0.0002739325864240527, 0.004187074489891529, 0.02128654532134533, -0.011548221111297607, -0.004035124089568853, 0.010839120484888554, 0.007550240494310856, 0.0064528221264481544, -0.01158198807388544, 0.012216802686452866, 0.011116007342934608, -0.0074759540148079395, -0.014290079474449158, -0.021475639194250107, -0.008353888988494873, -0.0045179883018136024, -0.012392389588057995, 0.028310025110840797, 0.008090508170425892, -0.038304977118968964, -0.009441177360713482, 0.001236706506460905, 0.004372791387140751, -0.0270133838057518, 0.014168519526720047, 0.024203991517424583, 0.005969957914203405, -0.04708432778716087, -0.015100480988621712, -0.1723453849554062, 0.01934158243238926, 0.023690737783908844, -0.029741734266281128, -0.0002386885607847944, -0.007172053214162588, 0.012311349622905254, -0.02375826984643936, 0.005831514019519091, -0.008651035837829113, 0.017383111640810966, -0.009265590459108353, -0.027823785319924355, -0.009738325141370296, 0.006746592465788126, 0.0010670286137610674, 0.0042343479581177235, 0.010710806585848331, 0.047165367752313614, -0.006094894837588072, 0.038845244795084, -0.005176439415663481, 0.015262561850249767, -0.004139801021665335, 0.0037852502427995205, 0.0031149808783084154, 0.0009480009321123362, 0.024014897644519806, 0.017248045653104782, -0.02812093123793602, 0.004288374446332455, -0.020327571779489517, 0.05202777683734894, 0.01763973943889141, 0.011345621198415756, -0.02736455760896206, 0.02312345616519451, -0.01776129938662052, -0.00694243935868144, 0.029903816059231758, 0.02097589150071144, 0.01241264957934618, 0.005264232866466045, 0.0263785682618618, -0.001985483570024371, 0.02656766213476658, 0.021070439368486404, -0.014452160336077213, -0.004683445207774639, -0.013810592703521252, 0.009218317456543446, -0.042681146413087845, 0.009488451294600964, -0.0010568986181169748, 0.022177988663315773, 0.019476650282740593, 0.013810592703521252, -0.003508363151922822, -0.01241264957934618, -0.021475639194250107, -0.031173445284366608, -0.005517483688890934, -0.016343096271157265, -0.03368568792939186, -0.010062485001981258, 0.0069356863386929035, -0.007779854349792004, 0.019760290160775185, -0.01668076403439045, -0.01477632112801075, -0.003084590658545494, -0.025406086817383766, 0.004528118297457695, -0.006050997879356146, -0.0026540649123489857, 0.027297023683786392, -0.0020209387876093388, -0.023690737783908844, 0.017855847254395485, 0.01581633649766445, 0.002507179742679, 0.0032500477973371744, -0.020516665652394295, 0.017329085618257523, -0.0320919007062912, 0.0055850171484053135, -0.02323151007294655, 0.031119417399168015, -0.0003117091255262494, -0.008597008883953094, 0.005503976717591286, -0.029417574405670166, -0.012162775732576847, -0.0016815831186249852, 0.0154111348092556, 0.008110768161714077, 0.00014234786794986576, -0.0038190169725567102, -0.001213913899846375, -0.03427998349070549, 0.011392894200980663, -0.0021526289638131857, -0.0009100133320316672, 0.02498737908899784, 0.031443577259778976, -0.015640748664736748, -0.0079216742888093, 0.02546011470258236, 0.01759921945631504, -0.0035387531388550997, -0.01298668421804905, -0.01715349778532982, 0.016829337924718857, 0.008651035837829113, -0.019449636340141296, 0.005426313262432814, -0.03122747130692005, -0.004075644072145224, 0.013830852694809437, -0.01716700568795204, 0.049272410571575165, 0.0003777652746066451, -0.047057315707206726, -0.026365062221884727, -0.021637720987200737, -0.015829842537641525, -0.09589751064777374, -0.03341555595397949, 0.023933857679367065, 0.04489624500274658, -0.007158546708524227, 0.026202982291579247, 0.0016807389911264181, 0.005895670969039202, -0.000332391238771379, 0.024366071447730064, -0.0056424206122756, -0.036251962184906006, -0.012284335680305958, 0.02768871746957302, 0.011777834966778755, -0.008394408971071243, -0.008097262121737003, -0.005851774010807276, -0.01696440391242504, 0.03349659591913223, -0.010265085846185684, -0.011406401172280312, 0.027499623596668243, -0.009434424340724945, -0.021232519298791885, -0.03595481440424919, -0.02582479454576969, 0.008846882730722427, -0.000858096987940371, -0.003491479903459549, 0.005081892944872379, -0.006061127875000238, 0.018085461109876633, -0.021597199141979218, -0.016194524243474007, -0.019922370091080666, 0.0010357943829149008, -0.003104850649833679, 0.005088645964860916, -0.014924894087016582, 0.032713208347558975, 0.02288033626973629, 0.002434581285342574, -0.0017280123429372907, -0.017855847254395485, -0.006574382074177265, -0.036360014230012894, 0.018679754808545113, -0.017585713416337967, -0.0015380745753645897, -0.04567963257431984, -0.025986874476075172, -0.01163601502776146, -0.0032112160697579384, 0.013486431911587715, -0.01711297780275345, 0.015357108786702156, 0.012622003443539143, 0.003457713173702359, 0.0002638025616761297, -0.012736810371279716, 0.012574730440974236, -0.008529475890100002, 0.02398788370192051, -0.006625032518059015, -0.009468191303312778, 0.004872539080679417, -0.007111273240298033, -0.007685307413339615, 0.01851767487823963, -0.023852817714214325, 0.007543487474322319, 0.023461123928427696, 0.01032586582005024, -0.005770734045654535, -0.024379579350352287, -0.012142515741288662, -0.011305101215839386, 0.024095937609672546, 0.00014340308553073555, -0.014924894087016582, -0.021664733067154884, 0.013162271119654179, -0.014492680318653584, 0.02212396077811718, 0.01511398795992136, 0.0011657963041216135, -0.006743215955793858, 0.026108434423804283, -0.032118912786245346, 0.00818505510687828, 0.01058249268680811, 0.005014359485358, -0.013871372677385807, -0.005267609842121601, -0.014236053451895714, 0.001022287760861218, -0.020206009969115257, -0.008603762835264206, 0.0059125544503331184, -0.036441054195165634, 0.006982959806919098, -0.06423782557249069, 0.024447111412882805, 0.013162271119654179, 0.005264232866466045, -0.022015906870365143, -0.006294118240475655, -0.001386968418955803, -0.005534366704523563, -0.006105024833232164, 0.01489788107573986, -0.016856351867318153, 0.017572205513715744, -0.01704544574022293, 0.021678240969777107, -0.03006589598953724, -0.03425297141075134, 0.04419389367103577, 0.0009361825650557876, 0.010697299614548683, -0.007935181260108948, 0.013601238839328289, 0.002280942630022764, 0.017734285444021225, 0.014924894087016582, -0.016572710126638412, 0.01285161729902029, -0.018044939264655113, 0.018382607027888298, -0.009481697343289852, -0.032902300357818604, -0.00012820804840885103, -0.015897376462817192, -0.009664038196206093, 0.026432596147060394, 0.014681774191558361, -0.02093537151813507, 0.0047881221398711205, 0.03225398063659668, 0.011372634209692478, 0.0024582177866250277, -0.01776129938662052, -0.035009343177080154, -0.00547021022066474, 0.02001691795885563, -0.00988689810037613, 0.020908359438180923, 0.009812611155211926, -0.008205315098166466, 0.002542634727433324, 0.012432909570634365, 0.026229996234178543, 0.0021526289638131857, -0.049110330641269684, -0.015748802572488785, -0.031281497329473495, -0.002412632806226611, 0.006013854406774044, -0.003022122196853161, 0.0028499120380729437, 0.007151793222874403, 0.06407574564218521, -0.010346125811338425, 0.005355403292924166, 0.0001672508369665593, 0.03201086074113846, -0.016397124156355858, -0.00788115430623293, -0.008326875045895576, 0.00788115430623293, 0.013398637995123863, -0.019530676305294037, -0.01576230861246586, 0.02439308539032936, 0.018814820796251297, -0.0009311175672337413, 0.0035725198686122894, 0.010008458979427814, -0.006824255920946598, -0.016626738011837006, 0.017801819369196892, 0.02251565456390381, 0.013128504157066345, -0.039817728102207184, 0.02383931167423725, 0.012500443495810032, 0.023812297731637955, -0.009042730554938316, 0.0447341613471508, 0.004521365277469158, 0.003121734131127596, -0.014789827167987823, 0.028472106903791428, 0.0119331618770957, -0.011338867247104645, 0.004585521761327982, 0.0015119053423404694, 0.006354898680001497, 0.0032314760610461235, 0.03333451598882675, 0.007226080168038607, 0.026202982291579247, 0.009312864392995834, -0.01397942565381527, -0.01759921945631504, -0.03700833395123482, -0.006121907848864794, -0.0316866971552372, -0.017031937837600708, -0.00810401514172554, 0.006307625211775303, 0.01266927644610405, 0.00923182349652052, -0.0137227987870574, -0.0011286529479548335, -0.030444083735346794, 0.013020451180636883, 0.017774805426597595, -0.016532190144062042, -0.03657612204551697, 0.031281497329473495, 0.02069225162267685, 0.029066400602459908, 0.01358773186802864, -0.02323151007294655, 0.024501139298081398, -0.003889927174896002, 0.027378063648939133, -0.016856351867318153, 0.014195532537996769, -0.022150974720716476, 0.0009783910354599357, -0.004173567518591881, 0.0030204339418560266, -0.008353888988494873, -0.01201420184224844, -0.00624684477224946, -0.04087124764919281, 0.015856856480240822, 0.007536733988672495, 0.08579450845718384, 0.018085461109876633, 0.01485736109316349, -0.0003843075828626752, -0.01903092861175537, -0.005247349850833416, 0.007057246286422014, 0.016626738011837006, -0.0034745964221656322, -0.025608686730265617, -0.0023012026213109493, -0.0008475449285469949, 0.01131860725581646, 0.021799800917506218, -0.0013076165923848748, 0.014046959578990936, -0.00012694179895333946, 0.02760767750442028, 0.007800114341080189, -0.0017440515803173184, 0.018004419282078743, -0.0018723651301115751, 0.024636205285787582, -0.012797590345144272, -0.015586722642183304, 0.020557185634970665, 0.019962890073657036, 0.017801819369196892, -0.032875288277864456, -0.03806185722351074, 0.019854836165905, 0.003005238948389888, -0.02351514995098114, -0.03198384493589401, 0.004207334481179714, -0.0018740535015240312, -0.01537061482667923, 0.00879960972815752, -0.011771081946790218, 0.0017710649408400059, 0.023528657853603363, 0.034415051341056824, -0.01874728873372078, -0.018531180918216705, -0.002220162423327565, -0.02379879169166088, -0.018612220883369446, -0.005757227540016174, -0.010913406498730183]} +{"id": "test:10000079", "text": "\"In the past five years the percentage of weekly mobile banking users has more than tripled \u2014 from 9% in 2010 to 30% in 2015 \u2014 while weekly branch visitors fell from 40% to 24% in the same time frame. 2015 marked the first time that weekly mobile bankers (30%) exceeded branch bankers (24%) in Javelin\u2019s survey. \u201cWhat is driving this incredible growth? Smartphones and tablets are gaining rapidly in consumer adoption but it\u2019s also a function of the fact that mobile banking services have not only proliferated but increased in the quality and features they offer consumers,\u201d said Javelin analyst Daniel Van Dyke during a January 27 webinar reviewing the firm\u2019s Mobile Banking, Smartphone and Tablet Forecast.\\nJavelin\u2019s recent report Digital Account Opening Reaches The Tipping Point also found that more Americans are applying for credit cards, loans and investment products through digital channels than physical ones. Javelin has argued that \u201cFIs that fail to divert resources from branch to digital suffer a large opportunity cost, missing out in lowered servicing costs for basic transactions and in sales of new products.\u201d The service costs argument rings particularly true for mobile deposit, with Chase citing a cost of 65 cents for each teller deposit vs only 3 cents using mobile. While 38 percent of big bank customers utilize mobile deposit, only 18 percent of community bank and 16 percent of credit union customers do, placing these institutions at a significant disadvantage in terms of service costs.\\nThe branch network, which peaked at 99,550 U.S. locations in 2009, has dropped each year since. The latest FDIC figure of 93,283 represents a 6.3 percent decline in the last six years, although Bryan Yurcan of American Banker was quick to point out that \\\"most of that reduction has come with the nation's largest banks shedding inventory ... many smaller banks have held steady, and even increased, branch numbers.\\\"\\nPredicting that account maintenance will increasingly be performed on mobile devices but there will be times that customers seek face-to-face advice, Schwanhausser said FIs need to \\\"get into the business where they're influencing the behavior of their customers, guiding them to a smarter way to manage their finances. Those are very personal kinds of conversations. It's not about eliminating (the branch) but how do we put the pieces together in a more coherent way?\\\"\\nWhile early adopters of smartwatches tend to be affluent or part of Gen Y2 (ages 25-34), mobile bankers no longer carry such unique characteristics. As Van Dyke put it, \\\"The mobile banker of today is the banking customer of today. They're becoming indistinguishable.\\\" Mobile banking has become so commonplace that Javelin projects 80 percent of consumers will be engaging in it \u2014 on a 90 day basis \u2014 by 2020.\\nIn 2015 74 percent of mobile banking users employed an app and 69 percent utilized a mobile browser, dwarfing the 31 percent using SMS text messaging. Schwanhausser said FIs will continue to wrestle with the question of what to make available in the app: \\\"Do you make it fine tuned and focused and streamlined or replicate what can be viewed in online banking?\\\" Van Dyke predicts that banks will continue to take advantage of device-specific capabilities, whether using the camera for mobile deposit or geolocation to identify nearby ATMs. \\\"Things like that will underscore the need to offer superior experiences on the app,\\\" said Van Dyke.\\n2015 marked a continuation of Android's dominance, as the operating system grew its share of the U.S. tablet market from 45 to 48 percent and its share of the U.S. smartphone market from 47 to 49 percent. Apple's tablet share fell from 46 to 43 percent, with its smartphone share dropping from 46 to 45 percent. While Apple's numbers have been moving in the wrong direction Van Dyke reiterated that iOS users are the most valuable that an FI can serve. \\\"iPhone users show extremely unique characteristics. They bank more often, they make payments more often,\\\" said Van Dyke. \\\"When it comes to banks and payment providers iPhone owners are the gold standard.\\\" In addition to serving Android users, Javelin recommended that FIs prioritize an app on Amazon \u2014 but only after all iOS and Android device types have been addressed. Javelin suggested that there was no value in supporting Windows or Blackberry users, who only represent 4 and 2 percent of the smartphone market respectively.\"", "vector_field": [-0.02232522889971733, 0.006659099832177162, 0.027803732082247734, -0.005292790476232767, -0.02545580267906189, 0.00011358275514794514, -0.020600762218236923, -0.022166047245264053, -0.022816039621829987, -0.04579126089811325, 0.015480417758226395, 0.02846698835492134, -0.018199771642684937, 0.016196735203266144, -0.016170205548405647, 0.035815875977277756, 0.024447651579976082, 0.007766738999634981, 0.01045956276357174, -0.012913613580167294, -0.032632242888212204, 0.01643550768494606, -0.006367267109453678, 0.021622177213430405, -0.014313085936009884, 0.028705762699246407, 0.007388682570308447, -0.014923281967639923, -0.009915691800415516, -0.005916252266615629, 0.002289894735440612, 0.0025651464238762856, -0.030217988416552544, 0.006509867031127214, -0.026158854365348816, 0.0001869555562734604, -0.00817795842885971, -0.009484575130045414, 0.03385263681411743, -0.01501613762229681, 0.03364039212465286, 0.005631051491945982, -0.01626306027173996, 0.00681828148663044, -0.02846698835492134, 0.0012013241648674011, -0.01177944429218769, 0.016077348962426186, -0.03443630039691925, 0.01877017319202423, 0.0018007426988333464, 0.005491767544299364, -0.022736448794603348, -0.00043982226634398103, -0.023996638134121895, -0.004792031832039356, -0.01796099916100502, 0.00963712390512228, -0.007395315449684858, -0.0003339084214530885, -0.01618346944451332, 0.04557901993393898, -0.014246759936213493, -0.01825283281505108, -0.020985450595617294, -0.0138488058000803, 0.009099885821342468, 0.0036711273714900017, -0.000620974344201386, 2.1892660697631072e-06, 0.010028445161879063, 0.02883841283619404, 0.021157898008823395, 0.006509867031127214, -0.003344473196193576, -0.002405964769423008, 0.0013497279724106193, 0.006387164816260338, -0.02151605673134327, 0.019844649359583855, -0.0006122690974734724, -0.015321236103773117, -0.015029403381049633, 0.020521171391010284, 0.006251196842640638, -0.000682325626257807, -0.014273290522396564, 0.009146314114332199, 0.002271655248478055, 0.004413974937051535, 0.0145518584176898, 0.00586650799959898, -0.009756510145962238, 0.024129288271069527, 0.0150824636220932, 0.0016515098977833986, -0.006015740800648928, -0.010651906952261925, -0.010021813213825226, 0.00046718164230696857, 0.00688460748642683, -0.010957005433738232, -0.015427357517182827, 0.0011573834344744682, -0.02939554862678051, 0.005796866025775671, -0.002210303908213973, -0.017045704647898674, 0.023267054930329323, -0.018836498260498047, -0.011965155601501465, 0.019115066155791283, 0.0143794110044837, -0.028095565736293793, -0.002473948523402214, -0.03825666382908821, -0.01363656297326088, -0.02727312594652176, -0.003515261923894286, -0.017934467643499374, 0.02684864215552807, 0.03849543631076813, 0.05507685989141464, -0.016780400648713112, 0.005276209209114313, 0.006330787669867277, -0.009464677423238754, 0.0070371562615036964, 0.004377495963126421, -0.0007768397335894406, 0.005617786664515734, 0.0016224923310801387, 0.018398748710751534, 0.006075433921068907, -0.010492725297808647, -0.004593054763972759, -0.018438544124364853, -0.01562633365392685, -0.009524370543658733, -0.028201686218380928, 0.005243046209216118, 0.01630285568535328, 0.015573273412883282, -0.007733576465398073, 0.02072014845907688, 0.020216071978211403, 0.018014058470726013, 0.013888601213693619, 0.021794624626636505, -0.016037553548812866, 0.0012767696753144264, -0.001891940482892096, 0.013424321077764034, -0.004729022271931171, -0.01583857648074627, 0.005491767544299364, -0.010519255883991718, 0.027564959600567818, -0.030615942552685738, -0.007156542968004942, -0.02960779145359993, 0.018836498260498047, -0.004105560481548309, -0.002463999669998884, 0.02087933011353016, 0.034622013568878174, -0.025601718574762344, 0.010300381109118462, 0.006957565434277058, -0.0027309604920446873, -0.0020361989736557007, 0.007607557345181704, -0.023094607517123222, 0.009073355235159397, 0.015573273412883282, -0.0009003713494166732, 0.006964198313653469, -0.00016726511239539832, -0.006145075894892216, -0.018093649297952652, 0.02397010661661625, -0.00340914074331522, 0.013145753182470798, 0.009007029235363007, -0.0038303090259432793, 0.0029034074395895004, 0.03332202881574631, -0.0076606180518865585, 0.001893598702736199, -0.0060953316278755665, 0.018146710470318794, 0.019327308982610703, -0.022630328312516212, -0.021914010867476463, -0.6430939435958862, 0.03133225813508034, 0.01965893618762493, 0.016634484753012657, 0.017815081402659416, 0.003591536544263363, 0.00961722619831562, -0.008078469894826412, -0.009199374355375767, 0.022670123726129532, 0.0011507509043440223, -0.018438544124364853, 0.018955884501338005, -0.019897708669304848, -0.019711997359991074, -0.008575912564992905, 0.01562633365392685, -0.005130292847752571, -0.006678997538983822, -0.0018339055823162198, -0.03608117997646332, 0.026065999642014503, -0.030217988416552544, 0.005130292847752571, -0.004778766538947821, -0.005916252266615629, -0.02740577794611454, -0.006589457858353853, -0.015812046825885773, 0.036532193422317505, -0.00016643604612909257, 0.022351760417222977, 0.01656815968453884, 0.029448609799146652, 0.0369301475584507, -0.0006897872663103044, -0.020600762218236923, 0.03435670956969261, 0.009351923130452633, 0.017271211370825768, -0.02079973928630352, -0.014326350763440132, -0.0019184708362445235, 0.0001680941932136193, 0.0058267125859856606, 0.010446297004818916, 0.02574763633310795, -0.013251874595880508, -0.010744762606918812, 0.009411616250872612, -0.0019416847499087453, -0.026238445192575455, -0.004019337240606546, 0.006957565434277058, 0.007919288240373135, 0.028864944353699684, -0.0030211354605853558, -0.005339218769222498, 0.013285037130117416, 0.008794787339866161, -0.015467152930796146, 0.0045366776175796986, -0.01796099916100502, -0.026835376396775246, -0.01902220956981182, 0.011699853464961052, -0.025721104815602303, 0.0024026483297348022, 0.01880996860563755, -0.014896751381456852, 0.010578949004411697, 0.00445708679035306, -0.005090497434139252, -0.02050790563225746, 0.008370302617549896, -0.00453336164355278, 0.04297905042767525, 0.012217193841934204, -0.015148789621889591, 0.025362946093082428, 0.005899670999497175, -0.016249796375632286, -0.003052640240639448, 0.0029747076332569122, 0.03679749742150307, 0.005989210680127144, -0.007057053968310356, -0.0075147016905248165, 0.016156939789652824, -0.029422080144286156, 0.02554865926504135, -0.007726943586021662, 0.0052994233556091785, -0.009776407852768898, -0.0054453397169709206, 0.025734370574355125, 0.0023048180155456066, 0.0022683388087898493, 0.00819785613566637, -0.024925196543335915, -0.01927424781024456, 0.01501613762229681, 0.02456703782081604, 0.009451411664485931, 0.02091912552714348, 0.03178327530622482, -0.009650388732552528, -0.0022185945417732, 0.04847082123160362, -0.050301410257816315, 0.002813867758959532, -0.031836334615945816, -0.001653997111134231, -0.0006350685725919902, 0.0025270089972764254, -0.01672734133899212, 0.04380149021744728, -0.008138163015246391, -0.003654545871540904, -0.02460683323442936, 0.002934912219643593, -0.023824190720915794, -0.007421845570206642, -0.03271183371543884, -0.01693958230316639, 0.005763703025877476, 0.006891239900141954, -0.00729582691565156, 0.029289428144693375, -0.009564165957272053, -0.012130970135331154, -0.0013621640391647816, 0.03478119522333145, -0.00129998370539397, 0.011474345810711384, 0.003860155586153269, 0.008900908753275871, -0.01910180039703846, 0.008416730910539627, -0.03968929871916771, 0.008118265308439732, -0.01294014323502779, 0.004380812402814627, -0.024421121925115585, -0.011083023622632027, -0.04374843090772629, 0.0012510684318840504, 0.01703243888914585, -0.01448553241789341, -0.026278240606188774, -0.008781522512435913, -0.0038800532929599285, 0.009219272062182426, -0.0077136787585914135, 0.019844649359583855, 0.018584460020065308, -0.018451809883117676, -0.029077185317873955, -0.02334664575755596, -0.007435110863298178, -0.014936546795070171, 0.031703684478998184, -0.014193698763847351, -0.02338644117116928, 0.002233517821878195, -0.011414652690291405, 0.005438707303255796, 0.01715182512998581, 0.008947336114943027, -0.02902412600815296, -0.0052032507956027985, -6.0159480199217796e-05, -0.0029647587798535824, 0.014750835485756397, -0.021874215453863144, 0.017443658784031868, -0.023028282448649406, 0.0028420560993254185, 0.019552815705537796, -0.001890282379463315, 0.0005762045038864017, 0.0010728181805461645, 0.01105649396777153, -0.0315975621342659, 0.02024260349571705, -0.023107873275876045, 0.032977137714624405, 0.021993601694703102, -0.014405941590666771, 0.011812606826424599, 0.021661972627043724, 0.019433429464697838, -0.008642238564789295, -0.006473388057202101, -0.023479297757148743, 0.0055846236646175385, 0.010180994868278503, -0.010200892575085163, 0.00045060020056553185, 0.01740386337041855, 0.04865653067827225, 0.04624227434396744, 0.00690450519323349, -0.009159578941762447, 0.0003873835375998169, -0.018902823328971863, -0.0016614587511867285, -0.003128914861008525, 0.02812209539115429, 0.02647721767425537, 0.009053457528352737, -0.01732427254319191, 2.9691113013541326e-05, -0.019154861569404602, -0.008642238564789295, -0.011262102983891964, 0.01166668999940157, 0.007620822638273239, -0.004042551387101412, 0.024036433547735214, -0.00878815446048975, -0.0301649272441864, 0.02553539350628853, -0.009272332303225994, 0.009477942250669003, 0.01956608146429062, -0.007057053968310356, 0.015135523863136768, -0.010552418418228626, -0.030217988416552544, -0.01656815968453884, 0.0015370980836451054, -0.020998716354370117, 0.004864989779889584, 0.009259067475795746, 0.01441920641809702, 0.015095728449523449, -0.016873257234692574, 0.018266096711158752, -0.014074312523007393, 0.00830397754907608, 0.037964828312397, 0.014246759936213493, -0.01715182512998581, 0.04603003337979317, 0.023068077862262726, 0.02553539350628853, 0.013795744627714157, -0.003339498769491911, 0.006718792952597141, -0.006483336910605431, 0.016289591789245605, -0.013391158543527126, 0.01778855174779892, -0.002745883772149682, -0.01522838044911623, 0.024593569338321686, -0.012004951015114784, 0.003996123094111681, 0.013039631769061089, 0.028918003663420677, 0.014790630899369717, 0.020998716354370117, -0.002929937792941928, 0.012416170910000801, 0.0027773885522037745, -0.009544268250465393, 0.00615170830860734, 0.003843574086204171, -0.01660795509815216, -0.000608123722486198, 0.0013290011556819081, 0.02317419834434986, -0.012807492166757584, 0.0135768698528409, -0.020653821527957916, 0.026782317087054253, -0.003604801604524255, 0.017297741025686264, 0.030615942552685738, -0.010307013057172298, -0.031995516270399094, 0.0031803171150386333, 0.004135407041758299, 0.008509586565196514, -0.026835376396775246, -0.020773207768797874, 0.006678997538983822, 0.01740386337041855, 0.02296195551753044, -0.02113136649131775, 0.014949812553822994, -0.013318199664354324, -0.010612111538648605, 0.0011432892642915249, 0.010797823779284954, 0.011487610638141632, -0.008158060722053051, 0.0026016253978013992, 0.004937948193401098, 0.018995679914951324, -0.004954529460519552, -0.030350638553500175, -0.021701768040657043, 0.03512609004974365, -0.014618183486163616, -0.008290711790323257, -0.015374296344816685, -0.018159976229071617, -0.006977463141083717, -0.0006624279194511473, -0.02698129415512085, -0.005979261826723814, 0.01749671809375286, -0.0013124197721481323, 0.024898666888475418, 0.006529764737933874, 0.005571358371526003, 0.03902604058384895, 0.010797823779284954, -0.015201849862933159, -0.016873257234692574, -0.022219108417630196, 0.012827389873564243, 0.098639577627182, 0.0163426510989666, -0.00951110478490591, 0.016117144376039505, -0.0075412318110466, -0.007441743277013302, -0.003382610622793436, -0.024553773924708366, 0.006685630418360233, 0.0003079999587498605, 0.01685999147593975, -0.021595647558569908, 0.008655503392219543, 0.003720871638506651, 0.009962120093405247, 0.010127933695912361, -0.0007967374403961003, -0.00587645685300231, 0.0038236763793975115, -0.028864944353699684, -0.005319321062415838, -0.013563605025410652, 0.02054770104587078, 0.023068077862262726, 0.027936384081840515, 0.00029245487530715764, 0.013941661454737186, 0.00616165716201067, 0.01664775051176548, -0.008463159203529358, 0.02004362642765045, 0.027671080082654953, -0.00949120707809925, 0.01910180039703846, 0.007766738999634981, 0.015440622344613075, -0.0205874964594841, -0.0012494103284552693, -0.004815245512872934, -0.0010670146439224482, 0.006964198313653469, 0.004904785193502903, -0.012555454857647419, -0.015745719894766808, 0.015865106135606766, -0.014459001831710339, 0.00819785613566637, 0.008841215632855892, -0.016462037339806557, -0.009610593318939209, 0.016833461821079254, 0.004583105910569429, -0.01664775051176548, 0.005621102638542652, -0.012867185287177563, 0.03167715296149254, -0.0029017492197453976, -0.014352881349623203, -0.005853242706507444, -0.029369018971920013, -0.010612111538648605, -0.013795744627714157, 0.005753754172474146, -0.005319321062415838, -0.014777365140616894, -0.024248674511909485, -0.03117307834327221, -0.007620822638273239, 0.00042116816621273756, -0.0034190895967185497, 0.0066060395911335945, -0.02186094969511032, -0.010287115350365639, -0.0057836007326841354, 0.03990153968334198, 0.0030095286201685667, 0.0006947616930119693, 0.0008464817074127495, 0.009955487214028835, 0.06075434014201164, 0.0023197412956506014, -0.002979682059958577, -0.0011316821910440922, -0.03435670956969261, -0.024036433547735214, 0.011122819036245346, -0.020826268941164017, -0.03960970789194107, -0.012383007444441319, 0.018491605296730995, -0.0033461314160376787, -0.020322194322943687, 0.011427917517721653, -0.007435110863298178, 0.014711040072143078, -0.010001915507018566, 0.0022020130418241024, 0.006781802512705326, 0.004884887486696243, -0.0055846236646175385, 0.02211298793554306, -0.009935589507222176, -0.001639902824535966, -0.02553539350628853, 0.005674163345247507, 0.0016017656307667494, 0.005534879397600889, -0.003011186607182026, -0.01084425114095211, 0.008569279685616493, -0.018995679914951324, -0.01045956276357174, -0.00028188421856611967, 0.001267649931833148, 0.006446857936680317, 0.0048052966594696045, 0.038866858929395676, -0.0017775286687538028, 0.03236693888902664, 0.003254933515563607, -0.004894836340099573, -0.007833064533770084, 0.017854876816272736, 0.01589163765311241, 0.005627735517919064, 0.011228940449655056, 0.008549381978809834, -0.03295060619711876, -0.036877088248729706, -0.0075147016905248165, 0.006579509004950523, 0.005047385580837727, -0.016966113820672035, -0.02960779145359993, -0.021887479349970818, -0.009729979559779167, -0.017974263057112694, -0.007780004292726517, -0.00612186174839735, -0.0015744062839075923, -0.027007823809981346, -0.0019930871203541756, 6.725839921273291e-05, -0.0161436740309, 0.021423200145363808, -0.023877251893281937, 0.006174922455102205, -0.004231579601764679, -0.008423363789916039, 0.0027807047590613365, -0.024898666888475418, -0.0032366940286010504, 0.010797823779284954, -0.0022534155286848545, -0.004931315779685974, -0.00725603150203824, -0.015374296344816685, -0.014047782868146896, 0.034542422741651535, 0.03647913411259651, 0.03260571137070656, 0.0076009249314665794, 0.01512225903570652, -0.013901866041123867, -0.0019897709134966135, -0.012316682375967503, -0.017297741025686264, 0.008735094219446182, -0.005687428638339043, 0.03037717007100582, 0.00718970550224185, 0.0019251033663749695, -0.012734534218907356, 0.004092295654118061, -0.01158709917217493, 0.020945655182003975, -0.024434387683868408, -0.01859772577881813, -0.036187298595905304, -0.04125458374619484, -0.009338658303022385, -0.012741166166961193, -0.020388519391417503, -0.0038866859395056963, -0.04905448481440544, -0.010751395486295223, 0.016329387202858925, -0.00203785696066916, 0.03215469792485237, -0.012356477789580822, 0.009179476648569107, 0.009046824648976326, 0.01842527836561203, 0.010054975748062134, -0.010340176522731781, -0.009418249130249023, -0.007985614240169525, -0.035258740186691284, -0.0036711273714900017, 0.01985791325569153, 0.016806932166218758, 0.015148789621889591, 0.018478339537978172, 0.003372661769390106, -0.018372219055891037, 0.028015974909067154, -0.0348077267408371, -0.018226301297545433, 0.005233097355812788, -0.016249796375632286, 0.022391555830836296, -0.020176276564598083, -0.00893407128751278, 0.009332025423645973, -0.015812046825885773, 0.03308325633406639, 0.001948317396454513, 0.008191223256289959, -0.022723183035850525, -0.012621779926121235, -0.0009235853212885559, 0.009219272062182426, 0.032260820269584656, -0.0036346481647342443, 0.00878815446048975, -0.013835540041327477, 0.018690582364797592, -0.0216487068682909, -0.015135523863136768, 0.011162614449858665, -0.007289194036275148, 0.031411848962306976, 0.01728447712957859, -0.005541511811316013, -0.00900039728730917, -0.0015760643873363733, -0.005936149973422289, -0.030589411035180092, -0.0397423580288887, 0.04202396050095558, 0.02109157107770443, -0.01960587687790394, -0.006529764737933874, -0.0027226698584854603, -0.031066955998539925, 0.02020280808210373, -0.007017258554697037, 0.009418249130249023, -0.008562647737562656, -0.020932389423251152, -0.012383007444441319, 0.03918522223830223, 0.007872859947383404, 0.010340176522731781, -0.02122422307729721, -0.018491605296730995, -0.005362432450056076, -0.0055050328373909, -0.0004974427283741534, 0.003511945717036724, -0.012880450114607811, 0.008270814083516598, -0.011454448103904724, 0.0044371890835464, -0.005793549586087465, 0.012336580082774162, -0.027458839118480682, -0.0028984330128878355, 0.016170205548405647, 0.02854658104479313, -0.02079973928630352, 0.004440505523234606, 0.03430365025997162, -0.012635045684874058, -0.0044239237904548645, 0.0003515261923894286, -0.011746280826628208, -0.018266096711158752, 0.0025933347642421722, -0.011288633570075035, 0.026251710951328278, -0.016528364270925522, -0.04104234278202057, 0.0037639832589775324, -0.003316284855827689, -0.014114107936620712, -0.008025409653782845, -0.00033784651895985007, 0.023784395307302475, -0.022603796795010567, -0.019884444773197174, 0.024089492857456207, -0.00681828148663044, 0.011169247329235077, -0.007647352758795023, -0.006304257549345493, -0.0012784277787432075, 0.014206964522600174, 0.0009078329894691706, 0.007594292517751455, 0.02334664575755596, 0.001769237918779254, -0.03231387957930565, -0.005534879397600889, -0.0011565543245524168, -0.022855835035443306, 0.01024731993675232, -0.022006865590810776, -0.02469968982040882, 0.012562086805701256, -0.006695579271763563, -0.011985053308308125, -0.00882794987410307, 0.013397790491580963, -0.004669329151511192, 0.018438544124364853, -0.010194259695708752, 0.014910017140209675, -0.029236366972327232, 0.01850486919283867, -0.0165150985121727, 0.004709124565124512, 0.011288633570075035, 0.00522978138178587, 0.0075147016905248165, -0.02562825009226799, 0.0021323710680007935, -0.014803895726799965, -0.012601882219314575, -0.03820360079407692, -0.0291037168353796, 0.018478339537978172, -0.02694149874150753, -0.0070371562615036964, -0.008648870512843132, 0.02211298793554306, -0.007733576465398073, 0.0029664167668670416, 0.0011938625248149037, -0.003959644120186567, -0.004702492151409388, -0.0037042901385575533, 0.013649828732013702, 0.01609061472117901, 0.02854658104479313, 0.00827744696289301, -0.026503749191761017, -0.031120017170906067, -0.0078065344132483006, 0.00654303003102541, -0.005398911889642477, 0.02668946050107479, 0.03080165386199951, -0.009504472836852074, -0.020322194322943687, -0.012137603014707565, -0.05497073754668236, -0.007839697413146496, 0.006470071617513895, 0.027511898428201675, 0.024898666888475418, 0.005156822968274355, -0.0022384922485798597, 0.03401181846857071, -0.018067119643092155, 0.027299657464027405, -0.015918167307972908, -0.0006624279194511473, 0.02083953469991684, -0.0235986839979887, 0.005352483596652746, -0.010194259695708752, -0.04300558194518089, 0.00127096613869071, 0.012024848721921444, -0.0027127210050821304, 0.010015180334448814, 0.0207599438726902, -0.004801980685442686, 0.007839697413146496, 0.016806932166218758, 0.016156939789652824, 0.013145753182470798, 0.004188467748463154, 0.03151797130703926, -0.03271183371543884, -0.0037639832589775324, 0.006065485067665577, 0.00453336164355278, -0.0034389873035252094, 0.006964198313653469, 0.0038070951122790575, 0.0030327425338327885, -0.004122142214328051, -0.012780961580574512, -0.020415049046278, 0.003334524342790246, -0.0057670194655656815, 0.021529320627450943, -0.021038511767983437, -0.00044686938053928316, 0.029501670971512794, -0.014472267590463161, -0.021197693422436714, 0.021356875076889992, 0.021317079663276672, -0.013430953957140446, -0.023373175412416458, 0.007594292517751455, -0.04380149021744728, 0.036585256457328796, -0.02400990203022957, 0.00902029499411583, 0.006330787669867277, 0.021847683936357498, -0.015480417758226395, -0.04637492820620537, 0.011215675622224808, 0.008191223256289959, 0.003271515015512705, -0.005677479784935713, 0.011931993067264557, 0.010937107726931572, 0.0020677035208791494, 0.011315164156258106, -0.006032322067767382, 0.00259001855738461, -0.006665732711553574, -0.0016083981608971953, 0.009139681234955788, 1.4651242963736877e-05, -0.011288633570075035, 0.023359911516308784, -0.00908662099391222, 0.003697657724842429, 0.0040790303610265255, 0.21563810110092163, -6.249124271562323e-05, 0.00830397754907608, 0.03204857558012009, 0.0045366776175796986, 0.023479297757148743, 0.014286555349826813, -0.008987131528556347, -0.01914159581065178, -0.0034721503034234047, -0.017483454197645187, 0.008774889633059502, -0.01740386337041855, 0.0025170601438730955, 0.022272169589996338, -0.015573273412883282, -0.008164693601429462, -0.010645274072885513, 0.013729419559240341, 0.039795417338609695, -0.015865106135606766, -0.016064083203673363, -0.010101404041051865, 0.020109951496124268, 0.021237488836050034, 0.007183073088526726, 0.005478502716869116, 0.009332025423645973, 0.043695367872714996, 0.0008017118670977652, -0.014193698763847351, 0.0008245113422162831, 0.008502954617142677, -0.0059129358269274235, -0.002540274290367961, -0.016501832753419876, -0.002739251358434558, 0.01179934199899435, 0.023824190720915794, 0.006324155256152153, -0.0195262860506773, 0.015414091758430004, 0.010406501591205597, -0.018106915056705475, 0.01906200498342514, 0.005183353088796139, 0.014525327831506729, -0.011328428983688354, 0.004072397947311401, -0.00976314302533865, -0.017058968544006348, 0.00274256756529212, 0.012847287580370903, -0.01253555715084076, -0.009577430784702301, 0.0071499100886285305, -0.006718792952597141, 0.007494803983718157, -0.009484575130045414, 0.023373175412416458, -0.004234895575791597, 0.002546906704083085, 0.006682313978672028, 0.024513978511095047, -0.0489748939871788, 0.01635591685771942, -0.015281440690159798, 0.022723183035850525, 2.8602957172552124e-05, -0.020070156082510948, -0.002750858198851347, 0.009312127716839314, -0.028599640354514122, -0.0065695601515471935, -0.016329387202858925, -0.03650566563010216, 0.018199771642684937, 0.02804250456392765, 0.025217030197381973, 0.007799901999533176, 0.017019173130393028, 0.007753474172204733, 0.0041486723348498344, -0.020216071978211403, 0.00872846134006977, -0.026623135432600975, 0.016024287790060043, -0.04077703878283501, -0.023771129548549652, -0.011553936637938023, 0.010598846711218357, 0.002613232471048832, -0.023943576961755753, -0.03180980309844017, -0.00015068369975779206, -0.0006607697578147054, -0.002087601227685809, 0.01965893618762493, -0.025654779747128487, -0.01448553241789341, -0.007912655360996723, 0.0489748939871788, 0.007634087931364775, 0.0016208342276513577, -0.0022600481752306223, -0.002354562282562256, -0.00972334761172533, 0.015745719894766808, -0.006669048685580492, -0.00889427587389946, 0.0028851677197963, -0.025084378197789192, 0.013371260836720467, -0.02083953469991684, -0.0005413835169747472, 0.014803895726799965, -0.0010487750405445695, -0.003909899853169918, 0.02533641643822193, -0.015201849862933159, 0.00020405514806043357, -0.01914159581065178, 0.005289474502205849, 0.02012321725487709, -0.016753870993852615, -0.031836334615945816, -0.01579878106713295, -0.008045307360589504, -0.002934912219643593, -0.010592213831841946, -0.0034157733898609877, -0.0163426510989666, 0.025018053129315376, -0.02638436295092106, -0.005803498439490795, 0.0117131182923913, 0.016714075580239296, -0.018199771642684937, -0.009060090407729149, 0.0016515098977833986, 0.013835540041327477, 0.005611153785139322, 0.03393222764134407, -0.00034696629154495895, 0.006393797229975462, -0.00872846134006977, 0.023704804480075836, 0.013537074439227581, -0.04133417457342148, -0.009995282627642155, 0.001327343052253127, 0.007627455051988363, -0.023691538721323013, -0.025402741506695747, 0.0390525721013546, 0.005070599727332592, -0.016170205548405647, 0.0012054695980623364, 0.015414091758430004, -0.004775450099259615, -0.030775124207139015, 0.007070319261401892, 0.023651743307709694, -0.004685910418629646, 0.0017957682721316814, 0.0319424569606781, -0.16777747869491577, 0.0376199372112751, 0.008456526324152946, -0.021370138972997665, 0.02817515656352043, 0.0023711437825113535, 0.012256989255547523, -0.0004920537467114627, -0.009935589507222176, -0.013755949214100838, 0.0189824141561985, -0.018451809883117676, -0.01497634220868349, -0.022683387622237206, 0.023691538721323013, 0.022311965003609657, -0.004281323868781328, 0.03435670956969261, 0.022670123726129532, 0.017271211370825768, 0.0189824141561985, -0.00021535124687943608, 0.008416730910539627, -0.002729302505031228, -0.01158709917217493, 0.016382446512579918, -0.025309886783361435, 0.031915925443172455, 0.008045307360589504, -0.03783217817544937, -0.0018156659789383411, 0.0039662765339016914, 0.014180433936417103, 0.008496321737766266, 0.01103659626096487, 0.007534599397331476, -0.008861113339662552, -0.016833461821079254, 0.0011233914410695434, 0.03669137507677078, 0.01892935484647751, 0.02854658104479313, -0.022006865590810776, -0.00259996741078794, 0.009484575130045414, 0.009995282627642155, -0.013338097371160984, -0.02155585214495659, -0.018584460020065308, -0.005654265638440847, 0.001044629723764956, -0.02655680850148201, 0.017523249611258507, 0.004768817685544491, 0.014790630899369717, 0.005617786664515734, -0.01589163765311241, 0.002682874444872141, 0.023863986134529114, -0.008078469894826412, -0.005946098826825619, -0.007408580277115107, 0.000441480427980423, -0.019804853945970535, -0.01800079457461834, -0.01505593303591013, -0.015361031517386436, -0.020826268941164017, -0.03342815116047859, 0.01605081930756569, -0.03310978785157204, -0.0028403978794813156, 0.02008342184126377, -0.004241528455168009, 0.010194259695708752, 0.001959924353286624, 0.01850486919283867, 0.015374296344816685, 0.01685999147593975, 0.00027628798852674663, 0.006489969324320555, 0.013397790491580963, -0.03414446860551834, 0.021701768040657043, -0.03968929871916771, -0.003001237753778696, 0.016767136752605438, -0.011494243517518044, 0.0005471869953908026, 0.004612952470779419, 0.029422080144286156, -0.014180433936417103, -0.011905462481081486, 7.984992407727987e-05, -0.002022933680564165, 0.01086414884775877, 0.010784558020532131, -0.0145518584176898, 0.014565123245120049, 0.008715196512639523, 0.015201849862933159, -0.01014783140271902, -0.004911418072879314, 0.019897708669304848, 0.01736406795680523, -0.0009310469613410532, 0.0044371890835464, -0.005097129847854376, 0.03236693888902664, -0.01016772910952568, -0.02541600726544857, 0.0011971788480877876, 0.014193698763847351, 0.007468273397535086, -0.023240523412823677, 0.015732455998659134, 0.005279525648802519, -0.014405941590666771, -0.0030476658139377832, -0.015772251412272453, 0.06112576276063919, -0.008556014858186245, -0.007799901999533176, 0.03218122944235802, -0.01238964032381773, -0.010817721486091614, -0.11556589603424072, -0.008124898187816143, 0.00339919188991189, 0.030828183516860008, 0.02033545821905136, 0.01693958230316639, -0.020109951496124268, 0.028413929045200348, 0.0011822555679827929, 0.030138397589325905, -0.041785188019275665, -0.015705924481153488, -0.007687148172408342, -0.0022534155286848545, 0.03133225813508034, 0.016156939789652824, -0.017987528815865517, -0.011089656502008438, 0.004311170428991318, 0.033613864332437515, 0.0029713911935687065, -0.015095728449523449, -0.015599803999066353, -0.011812606826424599, -0.017058968544006348, -0.02740577794611454, -0.030828183516860008, 0.0009252434829249978, 0.002205329481512308, 0.002729302505031228, 0.024301735684275627, 0.00024271059373859316, 0.01579878106713295, -0.005173404235392809, 0.002536957850679755, -0.003445619950070977, -0.02566804550588131, 0.010393236763775349, 0.01851813495159149, -0.0411749929189682, 0.0040259696543216705, 0.011195777915418148, 0.0006168289692141116, -0.04276680946350098, -0.005853242706507444, 0.00010306391777703539, -0.007143277674913406, -0.01919465698301792, -0.010015180334448814, -0.028069034218788147, 0.004175202455371618, -0.0078065344132483006, -0.02524355985224247, -0.02316093258559704, 0.02228543348610401, 0.008695298805832863, 0.000734142551664263, -0.0036810762248933315, -0.01451206300407648, -0.030775124207139015, -0.01433961559087038, -0.01583857648074627, -0.01518858503550291, -0.0083437729626894, 0.020003831014037132, -0.004636166151612997, -0.024885401129722595, 0.0011515798978507519, -0.013178915716707706, -0.008390200324356556, 0.006742007099092007, 0.00444713793694973, -0.018279362469911575, 0.0015478759305551648, -0.03570975363254547, 0.030907774344086647, -0.02507111430168152, 0.012051379308104515, -0.009895794093608856, 0.010791190899908543, -0.017045704647898674, -0.023611947894096375, 0.007302459329366684, 0.010287115350365639, 0.01931404322385788, -0.00456320820376277, 0.001202982384711504, -0.0022185945417732, 0.01697937771677971, -0.04857693985104561, 0.013139120303094387, 0.021622177213430405, -0.016103878617286682, -0.020322194322943687, 0.0036943412851542234, 0.02711394429206848, 0.02072014845907688, -0.000310487172100693, 0.029130246490240097, -0.011985053308308125, -0.00614175945520401, 0.0034190895967185497, -0.05048711970448494, 0.021622177213430405, -0.006019057240337133, -0.02033545821905136, -0.004248160868883133, -0.00549508398398757, 5.4045078286435455e-05, -0.026609869673848152, -0.003307994222268462, 0.02804250456392765, -0.0009542609914205968, 0.028413929045200348, -0.02456703782081604, -0.024513978511095047, -0.033189378678798676, -0.010685069486498833, 0.027591489255428314, 0.012150867842137814, 0.0064070625230669975, 0.013245241716504097, -0.008522852323949337, 0.0012154184514656663, 0.013245241716504097, -0.01871711201965809, -0.022378290072083473, 0.03180980309844017, -0.012635045684874058, 0.021250752732157707, -0.006290992256253958, -0.02595987729728222, 0.006068801507353783, -0.018478339537978172, -0.004649431444704533, 0.0006417011027224362, -0.017311006784439087, -0.036664847284555435, 0.004599687177687883, 0.018823232501745224, -0.0008754992159083486, 0.01448553241789341, -0.0012154184514656663, -0.0250313188880682, 0.017735490575432777, -0.011381489224731922, -0.02566804550588131, -0.007076951675117016, 0.011202409863471985, -0.0026397628244012594, 0.002410939196124673, -0.006791751366108656, 0.035391390323638916, -0.0033577384892851114, -0.01579878106713295, -0.03451589122414589, -0.013178915716707706, -0.015560008585453033, -0.013344730250537395, 0.011374857276678085, 0.02883841283619404, -0.016966113820672035, 0.024381326511502266, 0.01919465698301792, 0.017470188438892365, -0.033534273505210876, 0.009212639182806015, -0.016661014407873154, -0.026331301778554916, 0.02706088498234749, 0.02422214485704899, -0.028228215873241425, 0.025946613401174545, -0.02155585214495659, 0.015506948344409466, 0.0011507509043440223, 0.016833461821079254, -0.014777365140616894, -0.022179313004016876, 0.013258506543934345, -0.009849365800619125, 0.016952848061919212, -0.003860155586153269, 0.005992526654154062, -0.011593732051551342, 0.028148625046014786, 0.01579878106713295, 0.029554730281233788, 0.0021174477878957987, 0.010532520711421967, -0.027352716773748398, 0.009690184146165848, -0.005279525648802519, 0.008191223256289959, -0.00576038658618927, 0.021582381799817085, 0.00027338622021488845, 0.025853756815195084, -0.03260571137070656, 0.001639902824535966, -0.011308531276881695, 0.030722063034772873, 0.005720591172575951, -0.00404586736112833, -0.0027574908453971148, -0.006058852653950453, -0.0011225624475628138, -0.004523412324488163, 0.013484014198184013, -0.043350476771593094, -0.009842733852565289, 0.001498960773460567, 0.003122282214462757, 0.02668946050107479, 0.0034224060364067554, 0.007415213156491518, -0.019765058532357216, 0.001262675505131483, 0.00030489094206131995, -0.010346808470785618, -0.028732292354106903, 0.024116024374961853, 0.025827227160334587, 0.03541792184114456, 0.010108035989105701, -0.0179212037473917, 0.035152617841959, 0.0037540344055742025, 0.030350638553500175, -0.012847287580370903, -0.012044746428728104, -0.006562927737832069, -0.00046510895481333137, 0.010048342868685722, -0.0021804573480039835, -0.002747541991993785, -0.010651906952261925, -0.007707045879215002, 0.004108876921236515, 0.031279198825359344, 0.02381092496216297, 0.0639379695057869, -0.01749671809375286, -0.014750835485756397, 0.004059132654219866, -0.007998879067599773, 0.03358733281493187, 0.009968752041459084, 0.017019173130393028, -0.0031604194082319736, -0.023227259516716003, 0.010054975748062134, 0.002455709036439657, -0.00821775384247303, -0.015148789621889591, 0.00716980779543519, 0.006957565434277058, -0.016289591789245605, -0.0034389873035252094, -0.009902426972985268, 0.0008531142957508564, 0.010307013057172298, -0.014989607967436314, 0.006271094549447298, 0.016130410134792328, -0.025721104815602303, 0.01299983635544777, 0.013105957768857479, -0.00576038658618927, -0.027220066636800766, -0.029156776145100594, 0.017271211370825768, -0.00836367066949606, -0.027167005464434624, -0.004682594444602728, 0.018531400710344315, 0.004483616910874844, -0.013543707318603992, -0.033613864332437515, -0.0029631005600094795, 0.0016763820312917233, -0.03472813591361046, 0.02003036066889763, -0.014830426312983036, -0.02477928064763546, 0.024129288271069527, 0.0008647212525829673, -0.009325393475592136, 0.0002980511053465307, -0.015148789621889591]} +{"id": "test:10000080", "text": "\"Roser, M. \uff082017\uff09\u3002 The short history of global living conditions and why it matters that we know it. Published online at OurWorldInData. org.\\nLevari, D. E., Gilbert, D. T., Wilson, T. D., Sievers, B., Amodio, D. M., & Wheatley, T. \uff082018\uff09\u3002 Prevalence-induced concept change in human judgment. Science, 360\uff086396\uff09\uff0c 1465-1467.\"", "vector_field": [0.017346324399113655, -0.0067606717348098755, 0.03623088076710701, -0.030325166881084442, -0.0010661191772669554, 0.0017751484410837293, -0.011571083217859268, 0.01720898225903511, -0.024185968562960625, -0.022345582023262978, -0.012807163409888744, 0.013960838317871094, -0.00846028234809637, -0.013129917904734612, -0.00556922797113657, 0.017840756103396416, 0.03672531619668007, 0.0021700074430555105, 0.018568670377135277, -0.021727541461586952, -0.006310875993221998, 0.009538418613374233, -0.006111729424446821, -0.00027640126063488424, -0.00013541086809709668, -0.001406899536959827, 0.0354342982172966, -0.03947215899825096, 0.008233667351305485, -0.010767631232738495, 0.0006090269889682531, 0.004724572878330946, -0.02101336233317852, -0.033786192536354065, 0.00010794241825351492, -0.014050111174583435, 0.004748607985675335, -0.027015218511223793, -0.008425946347415447, -0.02403489127755165, 0.020450258627533913, 0.009387342259287834, 0.00962082389742136, -0.0025339643470942974, -0.013596881181001663, 0.006613028701394796, 0.02509242668747902, -0.007265404332429171, -0.01844506338238716, 0.012971974909305573, 0.028484780341386795, 0.04485597461462021, -0.014723087660968304, -0.0232657752931118, -0.0073203411884605885, -0.009909242391586304, -0.012965107336640358, 0.030270230025053024, 0.01847253181040287, -0.01746993325650692, 0.014654416590929031, 0.012649220414459705, -0.034005939960479736, 0.010953043587505817, -0.02198849245905876, -0.011619153432548046, -0.02101336233317852, 0.03241276741027832, 0.02840237505733967, 0.011426874436438084, 0.023705270141363144, 0.00821993313729763, 0.01619265042245388, -0.02936377003788948, 0.03537936136126518, 0.01418745331466198, -0.020834818482398987, 0.018857089802622795, 0.007014754693955183, 0.016906829550862312, 0.013322196900844574, 0.001555400900542736, -0.008110059425234795, 0.022345582023262978, 0.012264661490917206, 0.013995174318552017, 0.007663697004318237, 0.020148105919361115, -0.011790831573307514, -0.008055122569203377, -0.012003711424767971, 0.008082590997219086, 0.022386785596609116, 0.01953006722033024, 0.008659427985548973, 0.007684298325330019, -0.013473273254930973, 0.02205716446042061, -0.0004388513625599444, -0.04845434054732323, -0.01645359955728054, 0.007354676723480225, -0.0028069319669157267, -0.016810689121484756, -0.00020408198179211468, -0.003150287549942732, -0.004638734273612499, -0.0021597067825496197, 0.00028541433857753873, 0.0032069412991404533, -0.014530808664858341, 0.02231811359524727, -0.005012991838157177, -0.028292501345276833, 0.01797809824347496, -0.008268002420663834, 0.006190701387822628, -0.0034438567236065865, -0.02458425983786583, 0.00046481762547045946, 0.02536711096763611, 0.017538603395223618, 0.019049368798732758, -0.013892167247831821, 0.004889383912086487, 0.013507609255611897, -0.04095545411109924, -0.006829342804849148, -0.007958983071148396, -0.008233667351305485, 0.001654973952099681, 0.02656198851764202, 0.010595954023301601, -0.004580363631248474, -0.010589086450636387, 0.01950259879231453, -0.02554565668106079, 0.0194888636469841, -0.008089457638561726, -0.016783220693469048, -0.0062937079928815365, -0.0014850129373371601, -0.011371937580406666, -0.004394951742142439, -0.005404416937381029, 0.013438938185572624, -0.015080178156495094, 0.03524201735854149, 0.015753155574202538, 0.006396714597940445, 0.021439123898744583, 0.01382349617779255, 0.014517074450850487, 0.02582034096121788, 0.022482924163341522, 0.0002042965788859874, 0.01443466916680336, 0.014791758731007576, -0.003625835059210658, -0.0137891611084342, 0.0001883090881165117, 0.004861915484070778, 0.013638084754347801, 0.007279138546437025, 0.0022489791736006737, 0.022221975028514862, 0.014297327026724815, -0.015492204576730728, 0.008110059425234795, -0.007004454266279936, -0.03345656767487526, 0.029281364753842354, -0.02282628044486046, 0.021892352029681206, 0.0062284707091748714, 0.027949145063757896, -0.0025597158819437027, 0.029226427897810936, -0.020093169063329697, -0.012305865064263344, -0.030819598585367203, 0.02807275392115116, 0.007368410937488079, 0.006997587159276009, -0.0020773012656718493, -0.023897549137473106, 0.003979491535574198, -0.0134046021848917, 0.01593169942498207, -0.0091263921931386, 0.017057906836271286, 0.011049183085560799, 0.018884558230638504, 0.0006154649308882654, -0.6627587080001831, -0.030050482600927353, 0.0314239040017128, -0.006183834280818701, -0.014118782244622707, 0.014970304444432259, 0.018280252814292908, 0.015835560858249664, -0.00973756518214941, 0.03727468475699425, -0.005977821070700884, 0.014050111174583435, -0.017923161387443542, -0.028031550347805023, -0.022112101316452026, -0.016014104709029198, 0.010589086450636387, -0.00810319185256958, 0.010252597741782665, 0.003502227133139968, -0.009414810687303543, 0.03793392702937126, -0.013315330259501934, -0.000530484423507005, 0.011584817431867123, 0.00745768379420042, 0.00020397469052113593, -0.03719227761030197, 0.017648477107286453, 0.026397177949547768, -0.010863770730793476, 0.018651075661182404, 0.021425388753414154, 0.020093169063329697, 0.04507572203874588, -0.0343630276620388, -0.013624350540339947, 0.018527468666434288, 0.01950259879231453, 0.041889384388923645, -0.03867557272315025, -0.01202431321144104, -0.002549415221437812, 0.001795749762095511, 0.007443949114531279, 0.02025797963142395, 0.004158036317676306, -0.002262713387608528, 0.015038975514471531, 0.002963158767670393, 0.011275798082351685, -0.015299925580620766, -0.0022661469411104918, 0.005720304325222969, 0.016659613698720932, 0.002195758977904916, 0.008013919927179813, -0.022675203159451485, 0.007306606974452734, -0.004274777136743069, -0.009057720191776752, -0.00016566907288506627, -0.009668893180787563, 0.022125834599137306, -0.019900890067219734, 0.03881291672587395, -0.022153303027153015, 0.025010021403431892, 0.007148663513362408, -0.0011897271033376455, 0.011330734938383102, 0.030215293169021606, -0.017772085964679718, 0.012635486200451851, 0.0049752225168049335, 0.022235708311200142, 0.007711766753345728, 0.004340014886111021, -0.012381402775645256, 0.0033408498857170343, -0.0005146042094565928, -0.014008908532559872, 0.022455455735325813, 0.0018695712788030505, 0.0013244942529127002, -0.01443466916680336, -0.013047512620687485, 0.00018895287939812988, 0.004384650848805904, -0.008158128708600998, 0.010836302302777767, 0.0019245081348344684, -0.028237564489245415, 0.012058648280799389, -0.019076837226748466, 0.024886414408683777, -0.010637156665325165, -0.01543726772069931, 0.03287973254919052, -0.010801967233419418, -0.013995174318552017, 0.009943578392267227, 0.024996288120746613, 0.02274387516081333, -0.0035228284541517496, 0.0010154742049053311, -0.010582219809293747, 0.030435040593147278, 0.02079361490905285, -0.016151446849107742, 0.004003526177257299, -0.0006523756310343742, -0.031231625005602837, 0.01468188501894474, 0.005843912251293659, -0.011399406008422375, 0.016563473269343376, 0.008075723424553871, -0.02432331070303917, -0.012635486200451851, 0.012875834479928017, 0.01947513036429882, 0.02275760844349861, -0.025971418246626854, -0.005576095078140497, 0.011804565787315369, -0.003083333373069763, 0.01078136544674635, 0.002980326535180211, -0.014228655956685543, -0.0032859130296856165, -0.004429287277162075, 0.021054565906524658, -0.011838900856673717, 0.018184112384915352, -0.003176039317622781, 0.02506495825946331, -0.01367241982370615, 0.018898291513323784, -0.01014272402971983, 0.003086766926571727, -0.03161618486046791, -0.0006841360009275377, -0.029528580605983734, -0.02862212248146534, -0.016577208414673805, 0.0006446501356549561, 0.003986358642578125, 0.005723737645894289, -0.016810689121484756, -0.010067186318337917, -0.03222049027681351, -0.03301707282662392, 0.032302893698215485, -0.022647734731435776, -0.0003261878155171871, -0.013123051263391972, -0.016879361122846603, -0.018115440383553505, -0.010850036516785622, 0.010451744310557842, 0.005562360864132643, -0.02152152918279171, -0.011097253300249577, 0.008137527853250504, -0.0050644949078559875, -0.008268002420663834, 0.013507609255611897, 0.000252151774475351, -0.034555308520793915, 0.011152190156280994, -0.029034148901700974, -0.005740905646234751, 0.012923904694616795, -0.01649480313062668, 0.021494060754776, -0.026438381522893906, 0.005040460266172886, -0.00011749199620680884, -0.024254638701677322, 0.010094654746353626, -0.007663697004318237, -0.03233036398887634, -0.025724200531840324, 0.01543726772069931, 0.007649962790310383, -0.002784613985568285, 0.02654825523495674, -0.007100593764334917, 0.018417594954371452, 0.028429843485355377, 0.03309947997331619, 0.0019348087953403592, 0.004638734273612499, -0.0077873049303889275, 0.002484177704900503, 0.008645693771541119, -0.0005828461144119501, -0.0050885300152003765, 0.01597290299832821, 0.03881291672587395, 0.007313474081456661, -0.017305122688412666, -0.01382349617779255, 0.011873236857354641, -0.00825426820665598, 0.0033116647973656654, -0.02457052655518055, -0.0012240627547726035, 0.030077951028943062, 0.01870601251721382, -0.030325166881084442, -0.011076651513576508, -0.016123978421092033, -0.010286933742463589, 0.03298960626125336, -0.0002980755816679448, 0.01649480313062668, -0.018101707100868225, -0.009284335188567638, 0.005857646465301514, -0.01847253181040287, 0.01950259879231453, -0.017552338540554047, -0.025998886674642563, 0.011646621860563755, -0.018788417801260948, 0.0013674136716872454, 0.015258722938597202, -0.02230438031256199, -0.017868224531412125, -0.007588158827275038, 0.03087453544139862, 0.015409799292683601, -0.0012094700941815972, -0.018815886229276657, 0.00436748331412673, -0.016137713566422462, -0.008007052354514599, 0.003979491535574198, -0.014585745520889759, -0.00885857455432415, 0.017868224531412125, 0.0019348087953403592, 0.02586154267191887, -0.011495545506477356, 0.02933630160987377, 0.018554937094449997, -0.023870080709457397, 0.009112657979130745, -0.012738492339849472, 0.00532544543966651, 0.0047074053436517715, 0.00404816260561347, -0.005881681572645903, -0.019818484783172607, 0.009421677328646183, -0.01642613112926483, 0.019900890067219734, 0.015231254510581493, 0.0022077765315771103, 0.025682998821139336, -0.014572011306881905, -0.01719524897634983, -0.016879361122846603, -0.017126576974987984, -0.01926911622285843, -0.02708388864994049, 0.004195805173367262, -0.0037803451996296644, -0.007993318140506744, -0.027399776503443718, 0.003564031096175313, -0.021494060754776, 0.0035434297751635313, 0.005740905646234751, 0.006925482302904129, -0.014338529668748379, 0.010307534597814083, -0.015753155574202538, -0.022936154156923294, -0.008432813920080662, 0.003263595048338175, 0.01646733470261097, -0.003464458044618368, -0.027001483365893364, -0.011674090288579464, 0.007279138546437025, -0.045734964311122894, -0.002776029985398054, 0.02128804661333561, 0.004085931461304426, -0.006094561889767647, -0.0116328876465559, 0.007773570716381073, 0.0038593169301748276, -0.00272624334320426, -0.0008528094622306526, -0.0019296584650874138, 0.004539160989224911, 0.008556421846151352, -0.0026404045056551695, 0.00532887876033783, -0.007073125336319208, 0.04123013839125633, 0.0015133398119360209, -0.017909428104758263, -0.000857530627399683, -0.0431254617869854, -0.01671455055475235, 2.1962686957976985e-07, -0.011481811292469501, 0.0019073403673246503, 0.0018712880555540323, -0.0019588437862694263, 0.010815701447427273, 0.007169264834374189, -0.006324610207229853, 0.008934112265706062, -0.03771417960524559, 0.0009270600858144462, -0.024227170273661613, -0.012965107336640358, -0.01698923483490944, 0.07932887971401215, 0.021645136177539825, -0.021480325609445572, 0.0435924269258976, 0.015107646584510803, -0.017634743824601173, -0.02983073517680168, -0.00938047468662262, 0.0030884835869073868, -0.005744338966906071, -0.014228655956685543, 0.004782943520694971, 0.02102709747850895, -0.010074052959680557, 0.02403489127755165, -0.011468077078461647, 0.012092984281480312, -0.01726391911506653, 0.00669886777177453, 0.012326465919613838, -0.017291387543082237, 0.00625593913719058, 0.02458425983786583, 0.05298663675785065, 0.00550055643543601, -0.011351335793733597, 0.019310317933559418, 0.02808648720383644, -0.003464458044618368, 0.006128897424787283, -0.0023846046533435583, 0.016123978421092033, 0.004827579483389854, 0.00018358795205131173, 0.001467845169827342, 0.028429843485355377, 0.01431106124073267, 0.0035262620076537132, 0.02433704398572445, 0.0032498608343303204, 0.015505938790738583, 0.013638084754347801, 0.013363399542868137, -0.021178172901272774, -0.009208797477185726, 0.005459353793412447, 0.0045563289895653725, 0.041669636964797974, -0.010280066169798374, -0.0008553846273571253, 0.0057340385392308235, -0.023128433153033257, -0.02049146220088005, -0.03312694653868675, 0.0125187449157238, -0.003818114288151264, 0.02054639905691147, -0.005270508583635092, -0.024900147691369057, -0.0139402374625206, 0.0017219283618032932, -0.009092056192457676, -0.010932441800832748, -0.0011923023266717792, -0.00625593913719058, -0.016137713566422462, -0.014599479734897614, 0.009565887041389942, -0.026012619957327843, -0.00037533059366978705, 0.030572382733225822, -0.024900147691369057, -0.0028361172880977392, -0.008480883203446865, 0.023664068430662155, 0.018541201949119568, 0.013356532901525497, -0.0011219143634662032, -0.008769301697611809, 0.018925759941339493, -0.017414996400475502, -0.01531365979462862, 0.011481811292469501, -0.022427987307310104, 0.027056420221924782, 0.0004982948303222656, -0.025518188253045082, -0.02307349629700184, -0.005974387284368277, 0.023595396429300308, -0.0012317881919443607, 0.001291875378228724, 0.0016472485149279237, 0.0052911099046468735, 0.0013734223321080208, 0.00989550817757845, 0.014324795454740524, 0.024433184415102005, -0.009662026539444923, -0.019310317933559418, 0.020065700635313988, -0.015148849226534367, -0.03246770426630974, 0.008577022701501846, 0.0004974364419467747, 0.010080920532345772, -0.012965107336640358, -0.015821825712919235, 0.008137527853250504, -0.010966777801513672, 0.029061617329716682, -0.027454713359475136, 0.009538418613374233, -0.01241573877632618, 0.00372884189710021, 0.011468077078461647, 0.017881959676742554, 0.010733296163380146, 0.024694135412573814, -0.017030438408255577, -0.003194923745468259, -0.03059985116124153, 0.00424387538805604, 0.009092056192457676, 0.008872308768332005, 0.015093912370502949, -0.00973756518214941, -0.011811432428658009, -0.03672531619668007, -0.012312731705605984, -0.0031331197824329138, 0.033291757106781006, 0.005524591542780399, -0.015519673004746437, 0.0014000324299558997, -0.012127319350838661, -0.021631402894854546, -0.0036327021662145853, -0.034280624240636826, -0.00028155159088782966, -0.016796955838799477, -0.023430585861206055, -0.011049183085560799, 0.006616462487727404, 0.04491091147065163, -0.033346693962812424, -0.010692093521356583, 0.012834631837904453, 0.01415998488664627, 0.02102709747850895, -0.017291387543082237, -0.004937453661113977, -0.014407200738787651, 0.02178247831761837, -0.01483296137303114, -0.03134150058031082, -0.005610430613160133, -0.013988306745886803, 0.01898069679737091, 0.011028582230210304, 0.03672531619668007, -0.008508351631462574, 0.018184112384915352, 0.011351335793733597, 0.0017030438175424933, 0.00029936316423118114, 0.01431106124073267, 0.008322939276695251, -0.006795007269829512, 0.0073821451514959335, 0.0030095118563622236, -0.005981254391372204, -0.010252597741782665, 0.012930771335959435, 0.02483147755265236, 0.023458054289221764, -0.022235708311200142, -0.01671455055475235, -0.021452857181429863, -0.06032071262598038, 0.0009596789022907615, 0.0030009278561919928, -0.0051778024062514305, 0.0029820434283465147, -0.02561432681977749, -0.022235708311200142, 0.031066814437508583, -0.0028773199301213026, 0.003316815011203289, -0.023142166435718536, -0.005881681572645903, -0.021713808178901672, 0.035351891070604324, -0.004164903424680233, -0.016110245138406754, -0.02582034096121788, -0.007629361469298601, -0.0159454345703125, 0.013988306745886803, -0.0012566815130412579, -0.02859465405344963, 0.017607275396585464, 0.010719561949372292, 0.0036018001846969128, -0.008961580693721771, 0.017840756103396416, -0.004501391667872667, 0.000240992711042054, 0.018142910674214363, -0.013267260044813156, 0.020326651632785797, 0.01315738633275032, 0.00505762780085206, 0.014929101802408695, 0.020821083337068558, 0.006819042377173901, -0.010190794244408607, 0.013953971676528454, -0.04109279811382294, -0.016343725845217705, -0.005857646465301514, 0.02051893062889576, 0.007636228576302528, -0.019667409360408783, 0.05136599764227867, -0.004803544841706753, -0.015492204576730728, -0.019365254789590836, 0.021851150318980217, 0.0018867390463128686, -0.0007940098294056952, 0.022386785596609116, 0.01150927972048521, -0.017621008679270744, -0.02459799498319626, 0.010918707586824894, 0.022991091012954712, -0.012635486200451851, -0.04353749006986618, 0.02329324372112751, 0.016536004841327667, -0.002458426170051098, -0.014585745520889759, 0.004412119276821613, -0.014379732310771942, 0.018912026658654213, -0.02329324372112751, -0.006389847490936518, -0.0068465108051896095, -0.011488677933812141, 0.02583407424390316, 0.02840237505733967, -0.020862286910414696, 0.022194506600499153, -0.006588994059711695, 0.0003341279225423932, 0.0009914393303915858, -0.020834818482398987, -0.006547791417688131, 0.013246659189462662, 0.008055122569203377, -0.014077579602599144, -0.004374350421130657, -0.0020549832843244076, 0.006619895808398724, -0.0009279184741899371, -0.015794357284903526, 0.020106904208660126, -0.02207089774310589, 0.03112175129354, -0.0194888636469841, -0.028182627633213997, 0.013967705890536308, -0.006376113276928663, 0.00782850757241249, 0.00029592961072921753, -0.01327412761747837, -0.02939123846590519, -0.0027794635389000177, 0.0004437871102709323, 0.01851373352110386, 0.01078136544674635, -0.00898218248039484, -0.014874164015054703, -0.0025579992216080427, -0.027784334495663643, -0.00989550817757845, -0.010094654746353626, -0.006358945742249489, -0.018815886229276657, -0.00861135870218277, 0.005517724435776472, 0.00228159804828465, 0.024419449269771576, 0.014572011306881905, -0.012195990420877934, -0.0055554937571287155, 0.018637342378497124, -0.033291757106781006, 0.019104305654764175, 0.012738492339849472, 0.0022060596384108067, -0.013679287396371365, -0.00392455467954278, -0.01126206386834383, -0.021480325609445572, 0.010348737239837646, -0.013294728472828865, -0.04581737145781517, -0.014173719100654125, 0.004195805173367262, 0.03565404564142227, -0.033319227397441864, -0.00950408261269331, -0.0070250555872917175, 0.014352263882756233, 0.012244060635566711, -0.0389777272939682, -0.005837045144289732, 0.019873421639204025, -0.019928358495235443, -0.005184669513255358, 0.03906013444066048, -0.023252040147781372, 0.03186339884996414, 0.0009407943580299616, 0.004336581099778414, 0.010128989815711975, -0.02358166314661503, -0.004470489919185638, 0.0003802663122769445, 0.028979212045669556, -0.0017802987713366747, -0.00461469916626811, -0.03134150058031082, 3.3262575016124174e-05, -0.006084260996431112, 0.0017047605942934752, 0.012134186923503876, 0.0023846046533435583, 5.069859980721958e-05, 0.0014978888211771846, 0.0030438473913818598, -0.00633147731423378, -0.009456013329327106, -0.006942650303244591, -0.014352263882756233, 0.01565701514482498, -0.022853748872876167, -0.014613213948905468, 0.0009056003764271736, 0.0334291011095047, -0.0011253479169681668, 0.001035217079333961, -0.048262063413858414, -0.0007034497684799135, -0.033566441386938095, 0.004096232354640961, 0.03134150058031082, 0.0010446594096720219, 0.017016703262925148, 0.005655066575855017, 0.002264430047944188, 0.010122123174369335, -0.013514475896954536, 0.019612472504377365, -0.009174461476504803, -0.028512248769402504, -0.015863029286265373, -0.012367668561637402, 0.025463251397013664, 0.017566071823239326, -0.011488677933812141, -0.0020910354796797037, 0.010760764591395855, 0.023375649005174637, 0.0004626716545317322, 0.014105048030614853, -0.010231996886432171, 0.016302524134516716, -0.013129917904734612, 0.023622864857316017, 0.023210838437080383, 0.0006978702731430531, 0.00224554562009871, -0.021397920325398445, -0.03285226225852966, 0.011502412147819996, -0.0029477078933268785, -0.019585004076361656, 0.006194135174155235, -0.0015107645886018872, 0.010472346097230911, 0.012003711424767971, 0.01823904924094677, -0.016384929418563843, 0.020051967352628708, -0.008480883203446865, 0.025188567116856575, -0.010348737239837646, 0.009586487896740437, 0.0026764569338411093, 0.01038307324051857, 0.001051526516675949, -0.014214921742677689, 0.015135115012526512, 0.0039726244285702705, 0.014503340236842632, 0.012402004562318325, -0.0007832799456082284, 0.008329806849360466, -0.026479583233594894, -0.0006000138819217682, -0.00913325883448124, -0.00689801387488842, -0.026932813227176666, -0.03263251483440399, -0.035049740225076675, 0.027276169508695602, 0.02759205549955368, 0.008686896413564682, 0.014970304444432259, -0.001698751817457378, -0.007739235181361437, -0.005631031934171915, -0.015780624002218246, 0.008233667351305485, 0.009201929904520512, -0.026287304237484932, 0.014929101802408695, -0.0014901632675901055, 0.013857832178473473, -0.0029768929816782475, -0.006146064959466457, -0.017593540251255035, 0.012436339631676674, 0.20359613001346588, 0.011591685004532337, -0.01049981452524662, 0.031726058572530746, -0.006389847490936518, -0.0007171839824877679, 0.03771417960524559, -0.004916852340102196, 0.0028807534836232662, 0.010836302302777767, 0.0011682674521580338, -0.0003283337864559144, 0.006613028701394796, 0.0034627411514520645, 0.01014272402971983, -0.01947513036429882, -0.04578990116715431, -0.023883815854787827, -0.00429194513708353, -0.0029700258746743202, 0.004116833675652742, -0.00023755915754009038, -0.015863029286265373, -0.027399776503443718, 0.030572382733225822, 0.012044914066791534, -0.012704157270491123, -0.004834446590393782, 0.019351521506905556, -0.009023385122418404, 0.016590941697359085, -0.003749443218111992, -0.0037528767716139555, 0.016151446849107742, -0.025889011099934578, -0.013761692680418491, 0.05265701562166214, -0.01238827034831047, 0.014530808664858341, 0.005984688177704811, 0.0024361079558730125, 0.007615627255290747, 0.005225872155278921, -0.013844097964465618, 0.006108296103775501, 0.019667409360408783, -0.00702162180095911, -0.002891054144129157, -0.014407200738787651, 0.00873496662825346, -0.017099108546972275, 0.011729027144610882, 0.010685225948691368, 0.03433556109666824, 0.0027193762362003326, -0.0006772688939236104, 0.028210096061229706, 0.007478285115212202, -0.03411581367254257, -0.014283592812716961, -0.00348677602596581, 0.014626948162913322, -0.002013780642300844, 0.01327412761747837, -0.023622864857316017, 0.03345656767487526, -0.023609131574630737, 0.005208704620599747, -0.012580549344420433, -0.012038047425448895, 0.021590199321508408, -0.01998329535126686, -0.017909428104758263, -0.0005231880932115018, -0.03271492198109627, -0.005174368619918823, 0.02730363793671131, 0.04186191409826279, 0.024103563278913498, 0.03362137824296951, -0.014887898229062557, 0.012978841550648212, -0.03884038329124451, -0.011337601579725742, -0.018843354657292366, -0.052025239914655685, 0.030819598585367203, -0.007148663513362408, -0.02477654069662094, 0.01946139521896839, -0.0004845605872105807, -0.005043893586844206, -0.02484521083533764, -0.014654416590929031, 0.009277468547224998, -0.008810504339635372, 0.013741090893745422, 0.00020236520504113287, 0.01087063830345869, 0.0003809101181104779, -0.034308090806007385, 0.05955159291625023, -0.0023760206531733274, 0.005624164827167988, 0.002671306487172842, -0.001979445107281208, -0.00532887876033783, 0.01619265042245388, 0.02712509222328663, 0.01470935344696045, -0.012807163409888744, -0.02606755681335926, -0.004559762310236692, -0.0051159984432160854, -0.0032395601738244295, 0.00581987714394927, 0.018843354657292366, -0.009929844178259373, 0.02455679140985012, 0.0085907569155097, 0.004497958347201347, -0.03463771194219589, 0.01494283601641655, 0.02226317673921585, -0.013576280325651169, -0.013686154037714005, 0.0030644487123936415, 0.0018815887160599232, -0.022853748872876167, -0.01455827709287405, 0.012724758125841618, -0.0002328380214748904, -8.476591028738767e-05, 0.007258537225425243, -0.007725500967353582, -0.0011116137029603124, 0.014462137594819069, -0.019310317933559418, 0.007079992443323135, -0.01847253181040287, -0.015286191366612911, -0.031204156577587128, 0.015231254510581493, 0.003359734546393156, -0.010740162804722786, -0.011996844783425331, 0.0053529138676822186, 0.003255011048167944, -0.0025356810074299574, 0.008556421846151352, -0.02731737121939659, 0.0036018001846969128, -0.01675575226545334, -0.012820897623896599, 0.01175649557262659, 0.010300667956471443, -0.008364141918718815, -0.005991555284708738, -0.0005858505028299987, 0.008515219204127789, -0.018060503527522087, 0.02198849245905876, 0.017401261255145073, -0.02656198851764202, -0.008116926066577435, 0.0026215200778096914, -0.17788566648960114, 0.005390682723373175, 0.03515961393713951, -0.04026874527335167, 0.019090570509433746, 0.006372679956257343, 0.022675203159451485, -0.013610616326332092, -0.022991091012954712, -0.0004360616148915142, 0.01770341396331787, 0.01873348094522953, -0.026905344799160957, -0.021645136177539825, -0.00848775077611208, -0.00825426820665598, -0.020587600767612457, 0.016865627840161324, 0.008048254996538162, 0.018348922953009605, 0.02279881201684475, 0.012724758125841618, 0.004954621195793152, -0.015451001934707165, -0.003811247181147337, 0.013040645979344845, 0.003370035206899047, 0.025463251397013664, -0.0036498699337244034, -0.02452932298183441, -0.03169858828186989, 0.0005120290443301201, 0.012017445638775826, 0.019337786361575127, -0.009298069402575493, -0.0038936524651944637, -0.00797958392649889, -0.014668150804936886, -0.013727356679737568, 0.008844840340316296, 0.01999703049659729, 0.016838159412145615, 0.009298069402575493, -0.002556282328441739, -0.009002783335745335, 0.021480325609445572, 0.006183834280818701, -0.009112657979130745, 0.004539160989224911, -0.006190701387822628, 0.002485894598066807, 0.006396714597940445, 0.012875834479928017, -0.007148663513362408, 0.0039760577492415905, 0.015533407218754292, 0.0027708797715604305, 0.015368596650660038, -0.0012721325038000941, -0.014077579602599144, -0.02402115799486637, -0.021713808178901672, 0.009030251763761044, -0.0006901447777636349, -0.0010154742049053311, 0.010156458243727684, -0.013734224252402782, 0.010389940813183784, -0.023169634863734245, 0.005675667896866798, -0.020326651632785797, -0.022208239883184433, -0.013864698819816113, -0.015299925580620766, 0.002351985778659582, 0.0042644767090678215, 0.007286005653440952, 0.0029837600886821747, -0.007389012258499861, -0.008789903484284878, -0.011296398937702179, 0.008151262067258358, -0.022441722452640533, 0.0029665923211723566, 0.00013723493611905724, -0.0030713158193975687, 0.003131403122097254, 0.03568151220679283, -0.006183834280818701, -0.027440980076789856, 0.03087453544139862, -0.029171491041779518, 0.0035708982031792402, -0.02079361490905285, 0.01455827709287405, 0.009559019468724728, -0.003986358642578125, 0.018101707100868225, -0.010444877669215202, -0.011440608650445938, 0.0017579806735739112, -0.0071417964063584805, -0.0135556785389781, 0.014544542878866196, 0.017016703262925148, 0.006008722819387913, -0.0041786376386880875, -0.02433704398572445, 0.03892279043793678, -0.001353679457679391, -0.01174962893128395, 0.00134423712734133, 0.0055589270778000355, -0.0024120730813592672, -0.015121380798518658, 0.00809632521122694, -0.001574285444803536, -0.0022609964944422245, -0.010101521387696266, -0.0038936524651944637, 0.04565256088972092, -0.006166666746139526, -0.01506644394248724, -0.004515125881880522, 0.018939495086669922, -0.01443466916680336, -0.10503934323787689, -0.017332591116428375, 0.005953785963356495, 0.016893096268177032, 0.005486822221428156, 0.022469190880656242, -0.008082590997219086, 0.0012772828340530396, -0.01797809824347496, 0.022112101316452026, -0.027276169508695602, -0.011941907927393913, -0.008652561344206333, -0.01026633195579052, 0.03147884085774422, -0.03884038329124451, 0.00825426820665598, -0.03038010373711586, -0.0276469923555851, 0.05444246530532837, -0.0036567370407283306, -0.032275427132844925, -0.0012678405037149787, -0.02908908575773239, -0.004312546458095312, -0.0014257840812206268, -0.03191833570599556, -0.01330159604549408, 0.0016240719705820084, -0.005806142929941416, -0.0017820155480876565, -0.012923904694616795, -0.0008292037528008223, -0.03367631882429123, -0.0036292686127126217, -0.020340384915471077, -0.014846695587038994, -0.005837045144289732, 0.011303266510367393, -0.021411655470728874, -0.000300865329336375, 0.02330697700381279, 0.006695433985441923, -0.012944505549967289, -0.0033030807971954346, -0.011248329654335976, -0.007402746472507715, 0.0294187068939209, -0.004456755705177784, -0.01646733470261097, 0.0005115998210385442, -0.011083519086241722, -0.005778674501925707, -0.01671455055475235, 0.0026833240408450365, -0.004841313697397709, 0.011124721728265285, 0.007601893041282892, -0.015299925580620766, 0.010122123174369335, -0.009655158966779709, 0.006877412553876638, -0.03285226225852966, 0.03224795684218407, 0.02908908575773239, -0.012765960767865181, -0.02963845431804657, -0.011207127012312412, 0.008886042982339859, -0.032522641122341156, -0.012312731705605984, 0.0291165541857481, -0.012676688842475414, 0.00550055643543601, -0.02050519548356533, 0.011667223647236824, -0.025641795247793198, -7.720135909039527e-05, -0.0012978841550648212, -0.010863770730793476, -0.010211395099759102, -0.017099108546972275, 0.00018069088400807232, -0.011365070007741451, 0.0064688194543123245, 0.010520415380597115, -0.019955826923251152, 0.008563288487493992, -0.02432331070303917, -0.026163697242736816, -0.000955386960413307, 0.010218262672424316, 0.0012438056292012334, -0.019131774082779884, 0.002406922634691, 0.013198588974773884, -0.009442279115319252, -0.02251039259135723, 0.014791758731007576, 0.01013585738837719, -0.03092947229743004, -0.0013862982159480453, -0.04735560342669487, 0.02933630160987377, -0.012092984281480312, -0.023650333285331726, -0.006928916089236736, -0.0029992111958563328, -0.009449145756661892, 0.004336581099778414, -0.0019416759023442864, 0.01190070528537035, -0.0329621359705925, 0.02072494477033615, -0.007176131941378117, -0.040927987545728683, -0.02908908575773239, 0.002053266391158104, 0.039362285286188126, 0.031506311148405075, 0.009806236252188683, 0.009696361608803272, -0.006043058354407549, 0.004020694177597761, 0.006482553668320179, 0.03469264879822731, 0.005994988605380058, -0.0005283384234644473, -0.021700073033571243, 0.023622864857316017, -0.0036464363802224398, -0.002715942682698369, 0.01330846268683672, -0.026699330657720566, 0.003670471254736185, 0.016110245138406754, -0.016590941697359085, -0.007588158827275038, 0.015835560858249664, 0.0354342982172966, 0.019118038937449455, 0.05537639185786247, -0.018321454524993896, -0.0019348087953403592, -0.004580363631248474, -0.013569412752985954, -0.007622494362294674, 0.010843169875442982, -0.020354120060801506, 0.020834818482398987, 0.005342612974345684, 0.003670471254736185, 0.01648106798529625, 0.02789420820772648, -0.032302893698215485, -0.009112657979130745, 0.005174368619918823, 0.00449452456086874, 0.003670471254736185, 0.02686414122581482, 0.005517724435776472, -0.009819970466196537, 0.028704527765512466, 0.01900816522538662, 0.018541201949119568, 0.011818300001323223, 0.007244803011417389, -0.009263734333217144, -0.0033803358674049377, 0.020958425477147102, -0.006060226354748011, -0.01010838896036148, -0.03038010373711586, -0.000981138669885695, -0.002111637033522129, 0.027454713359475136, 0.021452857181429863, -0.003510810900479555, 0.0059366184286773205, 0.012944505549967289, -0.02227691188454628, 0.029803266748785973, 0.020642537623643875, -0.0012352217454463243, -0.01671455055475235, 0.03238530084490776, 0.009668893180787563, 0.011989977210760117, -0.02327950857579708, 0.004652468487620354, -0.02429584227502346, 0.018074238672852516, -0.0049099852330982685, 0.016618410125374794, 0.005067928694188595, 0.019639940932393074, 0.0008738400065340102, 0.01480549294501543, -0.0009493782417848706, 0.009696361608803272, 0.02756458707153797, 0.01330846268683672, 0.011941907927393913, -0.006056792568415403, -0.00316745531745255, -0.026259835809469223, -0.016261320561170578, 0.007134929299354553, -0.03293466940522194, -0.021919820457696915, 0.023760206997394562, 0.027180029079318047, 0.011090385727584362, -0.008817371912300587, -0.007395879365503788, -0.010465478524565697, -0.021425388753414154, 0.015574609860777855, -0.0019399591255933046, -0.005610430613160133, -0.02175500988960266, 0.019406458362936974, 0.004017260391265154, -0.019804751500487328, 0.017675945535302162, 5.719016553484835e-05, 0.018788417801260948, -5.515149314305745e-05, 0.028539717197418213, -0.010561618022620678, 0.014201187528669834, 0.004030994605273008, 0.016055308282375336, -0.02021677792072296, -0.005984688177704811, -0.00312796956859529, 0.0032481439411640167, -0.007986451499164104, 0.009565887041389942, 0.01821158081293106, -0.018912026658654213, 0.059386782348155975, 0.015038975514471531, 0.005754639860242605, 0.01694803312420845, -0.01716778054833412, 0.026974014937877655, 0.008542687632143497, 0.00709372665733099, 0.009531551040709019, -0.03922494500875473, 0.005174368619918823, 0.004487657453864813, 0.004336581099778414, -0.030517445877194405, -0.018815886229276657, 0.0019245081348344684, -0.006918615195900202, 0.01646733470261097, -0.003697939682751894, 0.005933184642344713, 0.01999703049659729, -0.0027468446642160416, 0.03208314627408981, 0.007045656908303499, -0.011481811292469501, -0.023155901581048965, -0.009435411542654037, -0.007601893041282892, -0.005874814465641975, -0.008597624488174915, -0.017099108546972275, 0.007128062192350626, -0.030215293169021606, -0.003200074192136526, 0.004230141174048185, -0.0012970258248969913, -0.008920378051698208, -0.02459799498319626, -0.00040902235195972025, -0.012168521992862225, 0.0026112194173038006, 0.012985709123313427, -0.01723645068705082, -0.025518188253045082, -0.013699888251721859, -0.00019142073870170861, -0.01382349617779255, -0.013164253905415535, -0.03191833570599556]} +{"id": "test:10000081", "text": "\"Well it's been a funny few days. On Saturday I made the best beefburgers yet - the recipe's from a fellow Sausagemaking.org member, Oddley.\\nI've always been a \\\"don't use anything other than chuck steak, ground with salt and pepper\\\" man, but he's converted me. The spices aren't discernable in the taste, but it wouldn't be as good without them.\\nThings were going so well, that is until the element went in the fan oven on Sunday. \u00a350 later and it's up and running again.\\nHowever, things are looking up again. Maurice rang me yesterday lunch-time to say that the fishmonger had replaced the roe that wasn't very good, free of charge. So yesterday evening we dry salted them for 8 hours, and they're in the cold smoker as I speak.\\nThey certainly look more promising than the last lot.\\nVery interested in how the online course went Phil, I\u2019ve been putting this off for too long and it\u2019s high time I got one of those certificates.\"", "vector_field": [0.007798553444445133, 0.011819357052445412, -0.004368023481220007, -0.027430353686213493, -0.01160408090800047, 0.0261942520737648, -0.00930548645555973, -0.030138665810227394, -0.0460829921066761, -0.03633306547999382, -0.004284690599888563, 0.0250831488519907, -0.004267329815775156, 0.003107615979388356, -0.0035138630773872137, 0.025902586057782173, 0.05230516940355301, 0.024680374190211296, 0.006802033167332411, -0.025319257751107216, -0.007270779460668564, 0.004687465261667967, 0.0018246392719447613, -0.006458285730332136, 0.006347175221890211, 0.006413147319108248, 0.011645747348666191, -0.012062410824000835, -0.02451370842754841, 0.004368023481220007, -0.0017482510302215815, 0.009215209633111954, -0.013715176843106747, -0.007055503316223621, -0.040333036333322525, -0.005388848949223757, -0.029444226995110512, 0.01059714425355196, 0.021013732999563217, -0.010708254761993885, 0.011388804763555527, -0.016916541382670403, 0.012729072943329811, -0.014527671039104462, -0.007715221028774977, -0.004090247675776482, 0.0013107542181387544, -0.0020104018040001392, -0.021999837830662727, 0.01619432494044304, 0.03597195819020271, -0.006190926767885685, 0.005510375835001469, 0.0051423232071101665, -0.003881915705278516, 0.009812427684664726, -0.0018940832233056426, 0.01093047484755516, 0.021360954269766808, -0.004711770918220282, 0.009159654378890991, 0.0011102348798885942, -0.02409704402089119, 0.00561106950044632, -0.007520777639001608, -0.01909708045423031, -0.011319360695779324, -0.01597210392355919, -0.030360886827111244, 0.027291465550661087, 0.026027586311101913, 0.004499966744333506, -0.001207456341944635, -0.004326357040554285, 0.03288864716887474, 0.000843309739138931, -0.03197198733687401, 0.0006163148791529238, 0.026555359363555908, -0.029055342078208923, 0.013006848283112049, -0.02515259198844433, -0.015763772651553154, 0.05638847127556801, 0.030527552589774132, -0.002751715714111924, -0.007513833232223988, 0.005041629541665316, -0.006010372191667557, -0.00913187675178051, 0.01340267900377512, 0.06488841027021408, 0.006024261005222797, -0.009041599929332733, -0.04177746921777725, 0.008187439292669296, -0.017319316044449806, 0.0282359030097723, -0.03008311055600643, -0.022360946983098984, -0.012326298281550407, -0.017513759434223175, -0.008027718402445316, -0.009965204633772373, -0.02311094105243683, -0.0023246356286108494, 0.019972074776887894, -0.03002755530178547, 0.025791475549340248, -0.016305435448884964, -0.009076321497559547, 0.025388700887560844, -0.0027412991039454937, -0.01954152248799801, 0.01160408090800047, -0.0032586564775556326, 0.0030138667207211256, -0.005444404203444719, -0.012902682647109032, -0.01956930011510849, 0.014861001633107662, 0.03280531242489815, 0.014020729809999466, -1.1928894309676252e-05, 0.022458167746663094, -0.01177769061177969, -0.03283309191465378, -0.00981937162578106, 0.028583122417330742, -0.002194428350776434, 0.009756872430443764, 0.026597026735544205, 0.014847112819552422, 0.01823597587645054, -0.013347123749554157, 0.025138704106211662, -0.024485930800437927, -0.015416553243994713, -0.011430471204221249, -0.030805328860878944, 0.0201109629124403, 0.02406926639378071, -0.020027630031108856, -0.015583218075335026, -0.013423511758446693, 0.0023281078319996595, 0.03186087682843208, 0.013444345444440842, 0.0008320251363329589, -0.02173595130443573, 0.012277686968445778, -0.012131854891777039, 0.008979100733995438, -0.020597070455551147, 0.02693035639822483, 0.003635389730334282, 0.001842000288888812, 0.012965181842446327, 0.00018001168791670352, -0.012784628197550774, 0.006038149818778038, 0.019458189606666565, 0.01340267900377512, 0.0010034648003056645, 0.002506925957277417, 0.01688876375555992, 0.03930526599287987, -0.01254157442599535, 0.010701309889554977, -0.02023596130311489, 0.01666654460132122, 0.018541529774665833, -0.03763861209154129, 0.02983311377465725, 0.01883319392800331, 0.017222095280885696, -0.012340187095105648, -0.0024982455652207136, 0.011388804763555527, -0.00241838488727808, -0.020374849438667297, 0.019652632996439934, 0.034499745815992355, 0.037305280566215515, -0.006062455475330353, -0.022513723000884056, 0.023069273680448532, 0.005687457975000143, 0.0036944171879440546, -0.018763750791549683, 0.005534681491553783, 0.019263746216893196, -0.008944378234446049, -0.009479097090661526, -0.6373286247253418, -0.02313871867954731, 0.003289906308054924, -0.017444316297769547, 0.035999733954668045, 0.025985918939113617, -0.0015555440913885832, -0.0064652301371097565, -0.006638839840888977, 0.022235946729779243, 0.012097133323550224, -2.290835982421413e-05, 0.017027651891112328, 0.008534659631550312, -0.009104099124670029, -0.01599988155066967, 0.029527559876441956, -0.027513686567544937, 0.007604110520333052, 0.010208258405327797, 0.009916593320667744, 0.016402656212449074, 0.0009574582218192518, -0.0171804279088974, 0.005479126237332821, 0.0003016470873262733, 0.008819378912448883, -0.006069399882107973, 0.0144721157848835, 0.0073957787826657295, -0.022833164781332016, 0.009090210311114788, -0.007333279121667147, 0.002197900554165244, 0.0618884339928627, 0.001282976591028273, -0.026916468515992165, 0.00952770747244358, 0.010687421075999737, 0.014624891802668571, -0.0004726527549792081, -0.037277501076459885, 0.03574973717331886, -0.007854108698666096, -0.01161796972155571, 0.0039409431628882885, 0.021055400371551514, 0.026624804362654686, -0.00879854615777731, -0.000843309739138931, 0.004628438036888838, 0.009569373913109303, 0.011916578747332096, 0.015624884516000748, 0.01106241811066866, -0.015180443413555622, 0.030221998691558838, -0.025263702496886253, 0.0014331490965560079, 0.03722194582223892, -0.02068040333688259, 0.018652640283107758, -0.02161095105111599, -0.020485959947109222, -0.013284624554216862, -0.00018760711827781051, -0.0013012057170271873, -0.006118010263890028, 0.023124828934669495, -0.008992989547550678, 0.02119428850710392, 0.05297183245420456, -0.04019414633512497, 0.011812413111329079, -0.007854108698666096, 0.011430471204221249, 0.00430205138400197, 0.016805430874228477, 0.019652632996439934, 0.012583240866661072, 0.0037951108533889055, -0.002562480978667736, 0.010506866499781609, 0.016208212822675705, 0.01720820553600788, -0.002236094558611512, -0.030416442081332207, -0.001923596952110529, -0.0005451348843052983, -0.024972038343548775, -0.008062440901994705, 0.01727765053510666, -0.012152687646448612, -0.004690937697887421, -0.0006488667568191886, 0.01927763596177101, 0.008013829588890076, 0.0060346778482198715, 0.021902617067098618, -0.0456385537981987, 0.0006341098924167454, 0.005114545579999685, 0.032138653099536896, 0.016208212822675705, 0.013645732775330544, 0.012597129680216312, -0.018624862655997276, 0.008541603572666645, 0.02801368199288845, -0.011874912306666374, 0.024013711139559746, -0.021333176642656326, -0.006263842806220055, -0.022902607917785645, -0.03438863530755043, -0.032749757170677185, -0.0009678748319856822, 0.0017413066234439611, 0.006979115307331085, -0.01963874325156212, 0.024902593344449997, 0.0024374821223318577, 0.018402641639113426, -0.003079838352277875, -0.012666572816669941, 0.04002748429775238, -0.0039027489256113768, 5.8647568948799744e-05, -0.004315940197557211, -0.012930460274219513, 0.009874926880002022, 0.002402760088443756, 0.016430433839559555, -0.026472026482224464, -0.013520733453333378, 0.03824971616268158, 0.030638663098216057, -0.016819320619106293, 0.04013859108090401, -0.004961769096553326, 0.00914576556533575, 0.033833082765340805, 0.011611025780439377, -0.00787494145333767, -0.020958179607987404, -0.023860935121774673, 0.0025138703640550375, -0.0012821085983887315, -0.024624818935990334, 0.016305435448884964, 0.015944326296448708, -0.005760374013334513, -0.01590266078710556, -0.00553815346211195, 7.134278712328523e-05, -0.006927032023668289, -0.018402641639113426, -0.025569256395101547, -0.01247907429933548, -0.012409630231559277, 0.014527671039104462, 0.024680374190211296, -0.04405523091554642, -0.012992959469556808, -0.00990964937955141, -0.03069421835243702, 0.007361056748777628, 0.006895782425999641, 0.002210053149610758, -0.005774262826889753, 0.005760374013334513, -0.008833267726004124, 0.01215963251888752, -0.006388841662555933, -0.012090188451111317, 0.0024114404805004597, -0.015610995702445507, 0.0010269021149724722, -0.013805453665554523, 0.006732589099556208, -0.007520777639001608, -0.00973603967577219, -0.0018663057126104832, 0.01512488815933466, 0.032499760389328, -0.0029895612969994545, 0.008208272978663445, 0.016485989093780518, -0.007479111663997173, 0.04130525141954422, -0.006881893612444401, -0.021458175033330917, 0.003909693565219641, 0.014791557565331459, 0.010062425397336483, -0.0013272471260279417, 0.011187417432665825, -0.006767311133444309, 0.014159617945551872, 0.007923552766442299, 0.009381875395774841, -0.011388804763555527, 0.013277679681777954, -0.0045034391805529594, 0.012631851248443127, -0.012194354087114334, 0.017708202823996544, -0.009638817980885506, 0.02456926368176937, 0.01631932333111763, -0.0041909413412213326, -0.008645769208669662, -0.011583248153328896, 0.0020381794311106205, 0.006447868887335062, 0.03649973124265671, -0.0029635198879987, 0.013666565530002117, 0.006260370370000601, 0.03616639971733093, 0.016166547313332558, -0.023944268003106117, -0.011736024171113968, -0.0035485848784446716, -0.025472033768892288, 0.020416516810655594, 0.016235990449786186, -0.0018072783714160323, 0.002314219018444419, -0.00396872078999877, 0.012590184807777405, 0.029055342078208923, 0.012187410145998001, -0.0017864451510831714, 0.03955526277422905, -0.01043742336332798, 0.013583232648670673, -0.010492977686226368, 0.028722010552883148, -0.0011857551289722323, 0.005180517211556435, 0.03530529513955116, 0.01602765917778015, -0.015374886803328991, 0.017416538670659065, 0.003159698797389865, 0.022930385544896126, -0.008374937810003757, -0.011923522688448429, 0.029749780893325806, 0.007590221706777811, 0.020833179354667664, -0.023666491732001305, -0.014694335870444775, 0.013444345444440842, -0.024180376902222633, -0.0010703046573325992, 0.009277709759771824, 0.01338879019021988, 0.015430442057549953, -0.016013771295547485, 0.023097051307559013, -0.00988187175244093, -0.03186087682843208, 0.009937427006661892, 0.03677750751376152, -0.005308988504111767, -0.020638735964894295, -0.008298549801111221, -0.030888661742210388, -0.00809716247022152, -0.0012326297583058476, -0.020708180963993073, -0.009263820946216583, 0.022874830290675163, -0.011111029423773289, 0.014076285064220428, 0.006659673061221838, -0.019360968843102455, 0.000763883232139051, -0.028499789535999298, -0.01986096426844597, 0.04002748429775238, 0.020166518166661263, -0.021680396050214767, -0.005795096047222614, 0.0019322774605825543, -0.0005012984038330615, -0.005722180008888245, 0.04063858836889267, -0.010013815015554428, 0.013923508115112782, 0.004861075431108475, 0.024874815717339516, 0.0063853696919977665, 0.0307219959795475, 0.025999808683991432, -0.01711098477244377, 0.0033680307678878307, 0.00396872078999877, 0.0004852394631598145, -0.02062484808266163, -0.0013706495519727468, -0.03113865852355957, 0.037971943616867065, -0.008354105055332184, -0.017666535452008247, -0.02368038147687912, 0.002777757355943322, 0.001302073709666729, 0.012611018493771553, -0.002588522620499134, -0.02269427664577961, 0.01599988155066967, 0.010354090481996536, 0.013159625232219696, -0.017666535452008247, -0.02759701944887638, 0.010111036710441113, 0.005333294160664082, 0.00241838488727808, -0.012388797476887703, -0.026499804109334946, 0.005374960135668516, 0.11033251881599426, 0.0019479022594168782, -0.004124969709664583, 0.015472108498215675, 0.024555373936891556, -0.00015006399189587682, 0.0016666543669998646, -0.02055540308356285, 0.01512488815933466, 0.0012959974119439721, 0.006767311133444309, 0.010090203024446964, -0.012076299637556076, -0.00044921544031240046, 0.017902646213769913, -0.03130532428622246, 0.0025121341459453106, -0.01848597452044487, -0.014194339513778687, -0.026110919192433357, -0.008645769208669662, 0.030666440725326538, -0.017027651891112328, 0.03419419378042221, 0.030610885471105576, 0.025999808683991432, 0.03024977631866932, 0.021458175033330917, 0.01602765917778015, -0.023763714358210564, -0.02151373028755188, -0.004739548545330763, 0.013888786546885967, 0.0063645364716649055, 0.0036110845394432545, -0.010874919593334198, 0.005666624754667282, 0.0010112773161381483, -0.004586771596223116, -0.01093047484755516, -0.0035902513191103935, 0.001118047279305756, 0.012472130358219147, -0.005590236745774746, -0.0008420076919719577, -0.03263864666223526, -0.013902675360441208, 0.03369419649243355, -0.007867997512221336, -0.01629154570400715, 0.014652669429779053, 0.003989554010331631, 0.004163163714110851, -0.02212483622133732, 0.014513782225549221, 0.00521176727488637, -0.007895775139331818, -0.0012083244509994984, -0.01956930011510849, 0.03180532157421112, -0.008124940097332, -0.037527501583099365, 0.0214859526604414, -0.01656932197511196, -0.02899978682398796, -0.001313358312472701, -0.009569373913109303, 0.01534710917621851, -0.026847023516893387, 0.018222087994217873, -0.00796521920710802, -0.02284705452620983, -0.02705535665154457, 0.013812398537993431, 0.009583262726664543, 0.021874839439988136, 0.011423527263104916, 0.0030503247398883104, -0.0018958193250000477, 0.005149267613887787, -0.005972178187221289, -0.0037916386500000954, 0.015208221040666103, -0.03247198089957237, -0.02311094105243683, 0.009965204633772373, 0.02658313699066639, -0.0005551174399442971, -0.01335406769067049, -0.013159625232219696, 0.03113865852355957, 0.017360983416438103, 0.017541537061333656, -0.0038402494974434376, -0.00043771378113888204, -0.0011301999911665916, -0.0011909634340554476, 0.019110970199108124, 0.018763750791549683, 0.0036458063405007124, 0.0005772527074441314, 0.005791624076664448, -0.01679154299199581, -0.02466648444533348, 0.011833245866000652, -0.00023372223949991167, -0.015374886803328991, -0.004607604816555977, 0.0019201247487217188, -0.004902741406112909, 0.03005533292889595, 0.0011354082962498069, -0.02454148605465889, 0.0013185666175559163, -0.003288170089945197, 0.008347160182893276, 0.016583211719989777, 0.03877749294042587, -0.012784628197550774, 0.007722165435552597, -0.002277760999277234, -0.024680374190211296, 0.025222036987543106, -0.003003450110554695, -0.011729080229997635, 0.004663160070776939, -0.012999904341995716, -0.0388886034488678, -0.03788860887289047, -0.002234358573332429, 0.003913165535777807, 0.030610885471105576, -0.015499885194003582, -0.025013703852891922, -0.03766638785600662, -0.03691639378666878, 0.005208294838666916, 0.016402656212449074, -0.019610965624451637, -0.020958179607987404, -0.012284631840884686, -0.018874861299991608, -0.008326327428221703, 0.012201298959553242, 0.010736031457781792, -0.028180347755551338, -0.016805430874228477, -0.00467704888433218, 0.015277665108442307, 0.023374827578663826, -0.017833201214671135, -0.012624907307326794, -0.0014982528518885374, 0.015263776294887066, 0.01182630192488432, -0.051582951098680496, -0.001394954975694418, -0.0094999298453331, 0.0009435694082640111, 0.025985918939113617, 0.03949970752000809, -0.018374864012002945, 0.011465192772448063, -0.011798524297773838, -0.011715191416442394, -0.019194303080439568, -0.010319367982447147, 0.0009982564952224493, -0.03424974903464317, 0.02291649766266346, 0.02355538122355938, 0.01733320578932762, -0.009444374591112137, 0.02852756716310978, -0.011861023493111134, 0.025319257751107216, 0.002427065512165427, -0.008402715437114239, -0.01701376400887966, -0.023416493088006973, -0.004722187295556068, -0.002795118372887373, 0.0013272471260279417, 0.002793382154777646, -0.0026822718791663647, -0.014902668073773384, 0.02540259063243866, 0.010173535905778408, 0.016902653500437737, -0.0008932225755415857, 0.02855534479022026, -0.029305338859558105, 0.01774986833333969, -0.022638721391558647, -0.020513737574219704, 0.0024305377155542374, -0.01819431036710739, -0.016430433839559555, -0.013506844639778137, 0.012465185485780239, 0.007173558231443167, 0.004593716003000736, 0.003378447378054261, -0.007097169756889343, -0.006645784247666597, 0.021055400371551514, -0.014749891124665737, -0.011902689933776855, 0.0006740401731804013, -0.03269420191645622, -0.011729080229997635, -0.007673554588109255, -0.00583329051733017, 0.006857588421553373, 0.017263760790228844, -0.01733320578932762, 0.0036978893913328648, 0.00723605789244175, -0.003409696975722909, -0.010513811372220516, -0.0031527546234428883, 8.66964110173285e-05, 0.038805268704891205, 0.01176380179822445, 0.013104069977998734, 0.03049977496266365, -0.007347167935222387, -0.025999808683991432, 0.014888779260218143, 0.01858319714665413, -0.009631873108446598, 0.030777551233768463, 0.004739548545330763, 0.010402700863778591, -0.011347138322889805, 0.0036735839676111937, 0.0130971260368824, -0.022110948339104652, -0.016485989093780518, 0.029083117842674255, 0.0027881739661097527, 0.003229142865166068, -0.01377073209732771, -0.014861001633107662, -0.004788158927112818, 0.00467010447755456, -0.01679154299199581, 0.0033992803655564785, -0.00904854480177164, -0.0044027455151081085, -0.02202761545777321, 0.010624921880662441, -0.0044305226765573025, 0.020888734608888626, 0.01838875375688076, -0.011888801120221615, -0.006142315920442343, -0.012499907985329628, -0.008645769208669662, 0.025874808430671692, 0.014569336548447609, 0.00973603967577219, -0.0207776241004467, 0.0180693119764328, 3.34741453116294e-05, 0.02636091597378254, -0.025791475549340248, -0.03288864716887474, 0.002564217196777463, 0.025902586057782173, -0.020610958337783813, 0.015527662821114063, -0.01791653409600258, -0.014666558243334293, -0.003491293638944626, 0.006350647658109665, -0.010333256796002388, -0.01570821739733219, 0.007527722045779228, -0.006562451366335154, 0.01147213764488697, 0.020888734608888626, -0.036027513444423676, 0.007791609037667513, -0.00537148816511035, -0.0006432243972085416, 0.004958296660333872, -0.003965248353779316, -0.005854123272001743, -0.03163865581154823, -0.01915263570845127, -0.027291465550661087, -0.014034618623554707, -0.007013837341219187, 0.004392328672111034, -0.01765264756977558, 0.0028923398349434137, 0.008020774461328983, -0.026819245889782906, 0.010222147218883038, -0.0176248699426651, 0.01880541630089283, -0.01986096426844597, -0.0020329710096120834, -0.015638774260878563, -0.015097110532224178, -0.0028853954281657934, -0.0005243016639724374, 0.010006871074438095, -0.016138769686222076, 0.013888786546885967, 0.0034930298570543528, -0.015694329515099525, -0.009256876073777676, 0.03724972531199455, 0.0020399154163897038, 0.011541581712663174, 0.006312453653663397, -0.033221978694200516, 0.015166554600000381, 3.428794298088178e-05, 0.0014548504259437323, -0.001862833509221673, -0.005100656766444445, -0.010958252474665642, 0.00397566519677639, 0.008784657344222069, 0.007159669417887926, -0.010708254761993885, 0.001966999378055334, -0.0031128241680562496, 0.02013874053955078, -0.025194259360432625, -0.005576347932219505, -0.013166569173336029, -0.011270750313997269, 0.00015570632240269333, 0.035805292427539825, -0.004954824689775705, -0.021999837830662727, 0.024722039699554443, 0.02808312699198723, 0.0012630114797502756, 0.005569403525441885, -0.005076351575553417, -0.001497384742833674, -0.009486041031777859, -0.01021520234644413, -0.02938867174088955, -0.0004900137428194284, -0.023638714104890823, 0.031055325642228127, 0.0060346778482198715, -0.013777676038444042, -0.008944378234446049, 0.0093679865822196, -0.021249843761324883, -0.015319331549108028, -0.020930401980876923, 0.013152680359780788, 0.020263738930225372, -0.03899971395730972, -0.004378439858555794, 0.018763750791549683, -0.02769424021244049, 0.005506903864443302, 0.0085902139544487, -0.029027564451098442, -0.01769431307911873, -0.005020796321332455, -0.0010451311245560646, 0.0059548174031078815, -0.0225970558822155, 0.005100656766444445, 0.022777609527111053, 0.0008975628297775984, 0.012340187095105648, 0.0020972066558897495, -0.012117966078221798, -0.007694387808442116, 0.010062425397336483, -0.013090181164443493, 0.019930409267544746, -0.0036978893913328648, 0.014361005276441574, -0.020513737574219704, -0.021777616813778877, 0.0221526138484478, 0.002607619622722268, -0.012187410145998001, -0.006281203590333462, 0.010784642770886421, 0.018666528165340424, 0.008548548445105553, -0.02737479843199253, 0.004618021659553051, 0.025360923260450363, 0.00021907390328124166, 0.03527751564979553, 0.00670828390866518, -0.0221526138484478, -0.0008762956131249666, 0.01765264756977558, -0.018180420622229576, -0.029888669028878212, 0.0037777498364448547, -0.00935409776866436, -0.012291575782001019, 0.0004175316425971687, -0.011131862178444862, -0.0006905330810695887, -0.0037152504082769156, 0.03388863801956177, 0.005093712359666824, -0.02518036961555481, -0.045527443289756775, 0.005791624076664448, 0.0038367772940546274, 0.004628438036888838, 0.0007304633618332446, 0.011256861500442028, 0.0007851504487916827, -0.016180437058210373, -0.015249887481331825, 0.006142315920442343, -0.021944282576441765, -0.0032152540516108274, -0.029916446655988693, -0.02702757902443409, 0.0123193534091115, 0.007992996834218502, -0.039694152772426605, 0.012590184807777405, 0.0005238676676526666, -0.008541603572666645, 0.01666654460132122, 0.21410952508449554, -0.0021232482977211475, -0.016235990449786186, 0.055860698223114014, 0.0008259487804025412, 0.01459711417555809, -0.010152703151106834, 0.013673510402441025, 0.010479089803993702, 0.028444234281778336, 0.02511092647910118, 0.0038437217008322477, -0.03705528378486633, 0.0012274214532226324, 0.018180420622229576, -0.017791535705327988, -0.02855534479022026, -0.012930460274219513, -0.013652676716446877, -0.018499864265322685, 0.035360850393772125, 0.006402730476111174, -0.02036096155643463, -0.012138799764215946, 0.045083001255989075, -0.009465208277106285, -0.013499900698661804, 0.01622210256755352, 0.008499937132000923, -0.010326312854886055, -0.016874875873327255, -0.009888815693557262, 0.012888793833553791, -0.007024253718554974, -0.010291590355336666, -0.0015737731009721756, 0.026874801144003868, -0.01395823061466217, 0.016944319009780884, 0.003808999666944146, 0.03149976581335068, 0.0023506770376116037, 0.02180539444088936, -0.016597099602222443, -0.009930482134222984, 0.007041614502668381, -0.007173558231443167, -0.000534284277819097, 0.0008719553588889539, -0.0030711579602211714, -0.03416641429066658, -0.019416524097323418, 0.01590266078710556, 0.0016458211466670036, 0.01673598773777485, -0.02020818367600441, 0.014513782225549221, -0.006993004120886326, -0.0022638721857219934, -0.004611077252775431, -0.015291553921997547, 0.02163872867822647, -0.019083192571997643, 0.015638774260878563, -0.034277524799108505, 0.04516633227467537, -0.01797208935022354, -0.011756857857108116, 0.009277709759771824, 0.012951293028891087, -0.005510375835001469, -0.03283309191465378, -0.014201284386217594, -0.01217352133244276, -0.02116651087999344, -0.027291465550661087, -0.007270779460668564, 0.0029912972822785378, 0.03313864395022392, 0.020222073420882225, -0.022610943764448166, 0.00827077217400074, -0.016458211466670036, 0.02304149605333805, -0.01781931333243847, -0.06227731704711914, 0.027402576059103012, -0.00027321846573613584, -0.0025781060103327036, -0.03091643936932087, 0.01022909115999937, -0.0006940052844583988, -0.019555410370230675, -0.003168379422277212, 0.016138769686222076, -0.029777558520436287, 0.0009140557376667857, 0.00756938848644495, -0.0030086582992225885, -0.004413161892443895, -0.0007096301997080445, -0.01720820553600788, 0.014222117140889168, -0.011569359339773655, -0.007166613824665546, 0.006065927445888519, -0.002336788224056363, 0.03186087682843208, 0.005767318420112133, -0.0053020440973341465, -0.0005559854907914996, -0.020485959947109222, -0.0037360836286097765, -0.00490968581289053, 0.005746485199779272, 0.004479133524000645, 0.011749912984669209, -0.0325830914080143, 0.014791557565331459, -0.033416420221328735, 0.009458263404667377, -0.00010595624917186797, -0.01177769061177969, 0.017458204180002213, -0.019166525453329086, -0.014652669429779053, -0.014388782903552055, 0.015083221718668938, -0.0119860228151083, -0.035166408866643906, 0.03974970802664757, 0.01727765053510666, -0.010805475525557995, 0.0015954743139445782, -0.012361019849777222, -0.021180398762226105, -0.0171804279088974, -0.010298535227775574, -0.015263776294887066, -0.0031336573883891106, 0.0023419966455549, 0.011541581712663174, 0.02094428986310959, -0.0038645549211651087, 0.0201109629124403, -0.030138665810227394, 0.003972192760556936, 0.010895753279328346, -0.022291501984000206, 0.009652706794440746, -0.0043714954517781734, 0.0005846311105415225, -0.006513840984553099, -0.009291598573327065, -0.0032586564775556326, -0.0123193534091115, -0.04277746379375458, -0.03024977631866932, -0.01644432358443737, -0.013715176843106747, -0.020888734608888626, 0.008583270013332367, 0.03388863801956177, -0.02126373164355755, -0.0031162963714450598, 0.0037777498364448547, -0.17877645790576935, 0.020833179354667664, 0.01308323722332716, -0.04016637057065964, 0.029972001910209656, -0.004920102655887604, 0.051110733300447464, 0.01115269586443901, -0.02466648444533348, -0.012909626588225365, -0.0126526840031147, 0.01270129531621933, -0.012152687646448612, -0.044638559222221375, 0.013166569173336029, 4.709166751126759e-05, -0.024902593344449997, -0.001199643942527473, 0.031027548015117645, 0.006308981217443943, 0.025874808430671692, -0.00919437687844038, 0.0006792484782636166, -0.009756872430443764, 0.02062484808266163, 0.018013756722211838, 0.021180398762226105, -0.018444309011101723, -0.018333198502659798, -0.011930467560887337, -0.014861001633107662, 0.006670089904218912, 0.021916504949331284, 0.003937470726668835, 0.003663167357444763, 0.009652706794440746, 0.015555440448224545, 0.0033367809373885393, -0.008930489420890808, 0.030860884115099907, 0.0026770636904984713, 0.013131847605109215, 0.006010372191667557, -0.015319331549108028, -0.006156204733997583, -0.001881930511444807, 0.014083229005336761, -0.006024261005222797, -0.008569381199777126, -0.00044530921149998903, 0.0020572764333337545, -0.006357592064887285, 0.011104084551334381, 0.016819320619106293, -0.00010031393321696669, -0.0006553771090693772, -0.02495814859867096, -0.013347123749554157, -0.003722194815054536, 0.005124961957335472, 0.0030572691466659307, -0.009548540227115154, -0.010187424719333649, 0.0010659643448889256, -0.0048714918084442616, -0.025291480123996735, -0.021999837830662727, 0.0010920058703050017, -0.01769431307911873, -0.0015494676772505045, -0.015055444091558456, -0.013062403537333012, 0.0016874875873327255, 0.0023663020692765713, -0.006597173400223255, 0.008027718402445316, -0.041194140911102295, -0.009111043997108936, 0.004076358862221241, -0.01534710917621851, 0.0009531179675832391, 0.02208317071199417, -0.026638692244887352, 0.018555419519543648, 1.5421435819007456e-05, 0.008770768530666828, -0.0059548174031078815, 0.02119428850710392, -0.0138957304880023, -0.0016692584613338113, 0.027291465550661087, 0.006645784247666597, -0.008034663274884224, -6.461540760938078e-05, -0.003763861022889614, 0.02158317342400551, -0.0010876655578613281, 0.007777720224112272, -0.004694409668445587, -0.016805430874228477, 0.02049984782934189, 0.0021874839439988136, -0.034721966832876205, 0.04316634684801102, 0.017805423587560654, -0.005253433249890804, -0.01861097291111946, 0.023541493341326714, 0.01755542680621147, -0.014722113497555256, -0.008902711793780327, 0.010180480778217316, 0.015277665108442307, 0.04088858887553215, -0.016069326549768448, 0.008645769208669662, 0.014902668073773384, -0.029527559876441956, 0.015402664430439472, -0.013645732775330544, 0.004194413311779499, -0.009013822302222252, 0.016513766720891, 0.0056319027207791805, -0.026124807074666023, -0.018222087994217873, -0.12033244222402573, -0.028916453942656517, 0.0017213415121659636, 0.03908304497599602, -0.03877749294042587, 0.008180495351552963, 0.013513789512217045, 0.015958216041326523, -0.01791653409600258, 0.025097036734223366, -0.009430485777556896, -0.012937404215335846, -0.0018593613058328629, 0.012444352731108665, 0.018555419519543648, 0.004288163036108017, 0.006607590243220329, 0.0026770636904984713, -0.014874890446662903, 0.01998596452176571, -0.02062484808266163, 0.006329814437776804, -0.016694320365786552, -0.021652618423104286, -0.0011892273323610425, -0.002479148330166936, -0.01991651952266693, 0.01255546323955059, 0.0004548577417153865, 0.022416500374674797, 0.0009505138150416315, -0.011020751670002937, 0.01176380179822445, -0.015485997311770916, -0.0028836592100560665, -0.026305360719561577, -0.00616662111133337, 0.009548540227115154, 0.019944297149777412, 0.004138858523219824, 0.015666551887989044, 0.009937427006661892, 0.01991651952266693, -0.028277568519115448, -0.012020744383335114, 0.010979085229337215, -0.02786090597510338, 0.047444093972444534, 0.00723605789244175, 0.0037742776330560446, -0.003996498417109251, 0.00405552564188838, 0.0069339764304459095, -0.0110763069242239, 0.015958216041326523, -0.014361005276441574, 0.011576303280889988, 0.024847038090229034, -0.017347093671560287, -0.020374849438667297, -0.015305442735552788, 0.0016605780692771077, -0.0026197724509984255, 0.016680432483553886, 0.0025919948238879442, 0.0017751605482771993, -0.016916541382670403, 0.00404858123511076, 0.002256927778944373, -0.01574988290667534, 0.008374937810003757, -0.005576347932219505, -0.0006536410073749721, 0.010104091838002205, -0.0022239419631659985, -0.0026423416566103697, -0.01934707909822464, -0.025291480123996735, 0.013437400572001934, -0.02661091461777687, -0.005760374013334513, -0.021444285288453102, 0.006680506281554699, -0.026597026735544205, 0.01654154434800148, 0.005055518355220556, 0.018499864265322685, 0.004586771596223116, -0.005152739584445953, -0.04299968108534813, -0.01673598773777485, 0.0069339764304459095, -0.0037083060014992952, 0.006420091725885868, -0.010395756922662258, 0.022777609527111053, -0.023916490375995636, -0.003808999666944146, 0.020347071811556816, -0.02409704402089119, -0.039055269211530685, -0.011117973364889622, -0.052416279911994934, 0.01217352133244276, 0.016180437058210373, -0.014722113497555256, -0.007604110520333052, -0.0061735655181109905, 0.005770790856331587, 0.009638817980885506, 0.005409682169556618, 0.006197870709002018, -0.030999770388007164, 0.017027651891112328, -0.005878428928554058, -0.005520792677998543, -0.018110977485775948, -0.03349975124001503, 0.020805401727557182, -0.004340245854109526, -0.0007321994635276496, 0.00027755871997214854, 0.00421524653211236, 0.011492970399558544, -0.029805336147546768, 0.016527656465768814, -0.0069860597141087055, 0.021749839186668396, -0.036221954971551895, 0.022263724356889725, -0.005392321385443211, 0.0012343658600002527, 0.01113880705088377, -0.03438863530755043, 0.009319375269114971, 0.018110977485775948, 0.003868027124553919, -0.0180693119764328, -0.01708320714533329, 0.01647210121154785, 0.015555440448224545, -0.0053541273809969425, -0.022416500374674797, -0.013701288029551506, -0.02033318392932415, 0.0019999851938337088, 0.010333256796002388, -0.0067152283154428005, -0.02348593808710575, 0.023124828934669495, 0.013166569173336029, 0.0006840227288194001, 0.009486041031777859, 0.017388761043548584, -0.010694365948438644, -0.03636084124445915, -0.022652611136436462, 0.014291561208665371, 0.0018090144731104374, -0.004493022337555885, 0.013166569173336029, -0.017610982060432434, 0.03974970802664757, 0.00036826985888183117, 0.026152584701776505, -0.003305531106889248, 0.013701288029551506, 0.0004952220479026437, -0.009861038066446781, 0.02072206884622574, 0.017708202823996544, -0.021972060203552246, -0.01909708045423031, -0.0004307694034650922, 0.006843699607998133, 0.016360990703105927, 0.004253441002219915, -0.016166547313332558, 0.001987832598388195, -0.010194369591772556, -0.006649256683886051, 0.008138828910887241, 0.007867997512221336, 0.022222058847546577, 0.00863882526755333, 0.028833121061325073, -0.000420786818722263, 0.02247205562889576, -0.016485989093780518, 0.0010086731053888798, 0.01131241675466299, 0.0020954706706106663, -0.006399258505553007, 0.008388826623558998, 0.007972163148224354, -0.012298520654439926, 0.001677070977166295, 0.02786090597510338, 0.02186094969511032, -0.009472152218222618, 0.002878451021388173, -0.025888698175549507, -0.0012170049594715238, 0.00843049306422472, -0.013263790868222713, -0.0034895576536655426, -0.017069319263100624, 0.0016440850449725986, -0.022388724610209465, -0.01791653409600258, -0.017069319263100624, 0.010798531584441662, 0.0011319360928609967, -0.017708202823996544, 0.018860971555113792, 0.03238864988088608, -0.010819364339113235, 0.014888779260218143, -0.025569256395101547, -0.02604147419333458, -0.015819327905774117, 0.027277575805783272, 0.003972192760556936, 0.0013828022638335824, 0.006597173400223255, 0.007097169756889343, -0.0034652522299438715, -0.011145750992000103, 0.028472011908888817, -0.015222109854221344, -0.011340194381773472, -0.007826331071555614, 0.0027465075254440308, 0.010715198703110218, -0.027555352076888084, -0.01959707774221897, 0.0009852357907220721, 0.010173535905778408, 0.01848597452044487, 0.03855527192354202, -0.029888669028878212, 0.056138474494218826, 0.023305384442210197, -0.0032308788504451513, 0.01934707909822464, 0.01177769061177969, 0.025888698175549507, -0.01324295811355114, 0.012562407180666924, 0.01161796972155571, -0.010590199381113052, 0.027652572840452194, -0.029749780893325806, 0.008104106411337852, -0.03172198683023453, -0.01427767239511013, 0.0020381794311106205, 0.012770739383995533, 0.0029131730552762747, -0.023097051307559013, -0.007222169078886509, 0.01233324222266674, -0.0036909449845552444, 0.006996476091444492, 0.03344419598579407, -0.0316108763217926, -0.001580717507749796, -0.0010607560398057103, -0.013854064047336578, -0.021874839439988136, -0.0419996902346611, 0.0007886226521804929, 0.015055444091558456, -0.05580514296889305, 0.0014392255106940866, -0.0015815855003893375, -0.014680447056889534, 0.011708246544003487, 0.006399258505553007, 0.0044305226765573025, 0.017222095280885696, -0.010840198025107384, 0.0036110845394432545, -0.02141650952398777, -0.004979129880666733, 0.0031978930346667767, -0.009416596964001656, -0.016610989347100258, 0.015180443413555622, -0.0044305226765573025]} +{"id": "test:10000082", "text": "\"My husband is being posted to Darwin for work and we have decided to drive there. We are going in January and I know this is the wet season. My question is, do you think it will be safe travelling the Stuart Hwy to Darwin in a Holden Sedan at the start of January?\\nWe plan on stopping at all the main towns for the night. I just worry about flooding on the Hwy and we are travelling with a child and want to make sure we are doing the right thing.\\nJust use common sense. As long as you don't drive into any water deeper than 200mm, and you stick to the road and sealed parking bays (i.e. don't pull off the road into any soft shoulders), there is no problem at all. It's a normal highway.\\nOf course there is always a chance of major flooding with the highway being cut by water deeper than that, which means you would have to wait for the water to go down before you can continue your journey.\\nIt does happen nearly every year, sometimes more than once, but I really can't predict when and where or for how long. Only Petrus can tell you :-).\\nKeep an eye on the weather forecast. Big rains do not come out of nowhere, they come out of big, monsoonal depressions. If one of those is heading for the region, then you may have to reconsider your driving plans.\\nAnd of course, always check the road reports.\\nHow did the travel go?\\nThat must have been a long journey. I am thinking of traveling around Australia one day. How did the travel go?\\nIn my opinion traveling from Melbourne to Darwin in January might be a bit uneasy, especially when you are planning to do it without any rescue baggage. There can be flash floods, and within just a matter of seconds the entire highway can become dangerous to travel on. Furthermore, while stopping at major cities for the night is a good idea, it doesn't take you out from the risk of facing floods. And of course, you are not going to like the idea of being stranded on the terrace of a roadside motel till rescue workers pull you guys out. Think over it. I guess you can come up with a better alternative than this.\\nEditor's note: I removed your totally unrelated spam link, mate. If your comment wasn't a dead give away that you have never been anywhere near the north, the link certainly was.\\nWell, we did it and thankfully had not one problem. There was rain but that stopped our journey because we were lucky enough that the water was off the Ed by the time we reached the apparent flooded areas.... It was a very long drive and all I can say is prepare\u2026 prepare\u2026 prepare. :-) Make sure you have everything you need in case of emergency and you should be fine.... Wishing you safe travels!\\nI hope you have a safe trip to Darwin.\\nI hope you have a safe trip!\\nThat's a long trip you're planning mate, this is what you need: axe, sleeping bag, one huge sack of food, around 30-40 litres of water, some things your kid would like to do, phone, tent, 30-40 litres of oil, torch, 3 knifes, extra tyres (8 tyres), building tools.\\nJoin in and write your own page! It's easy to do. How? Simply click here to return to Australian Outback Travel Questions.\"", "vector_field": [0.010146673768758774, -0.005514205899089575, -0.0069403513334691525, 0.009004421532154083, -0.015390345826745033, 0.022537773475050926, -0.008436635136604309, -0.010614261962473392, 0.0059350356459617615, -0.0173942968249321, 0.005631103180348873, 0.028135476633906364, -0.004599068313837051, 0.002473209984600544, 0.003513594623655081, 0.03265104815363884, 0.04138827323913574, -0.005657822825014591, -0.0022611250169575214, -0.023606546223163605, -0.007554896641522646, 0.00956552755087614, -0.004983158782124519, 0.008657069876790047, -0.029177531599998474, 0.012725090608000755, 0.003797487821429968, -0.01283864863216877, -0.0062890672124922276, -0.008042524568736553, -4.4671414798358455e-05, 0.019665442407131195, 0.0035670334473252296, -0.027547651901841164, -0.009912879206240177, -0.02320575714111328, -0.013446513563394547, 0.0002607224159874022, 0.014936117455363274, -0.029498163610696793, 0.016993507742881775, -0.0029090694151818752, -0.01225082203745842, -0.018142439424991608, -0.012992284260690212, 0.028562987223267555, 0.005173534154891968, -0.0039477841928601265, -0.009144698269665241, 0.02040022425353527, 0.02026662789285183, -2.31902572522813e-06, 0.006636418867856264, -0.016218645498156548, -0.025957848876714706, 0.006509501952677965, -0.014802520163357258, -0.01295220572501421, 0.001394416089169681, -0.018944019451737404, 0.018850501626729965, 0.02003951370716095, -0.01906425692141056, 0.01772828958928585, -0.01981239952147007, -0.025917770341038704, 0.0050132181495428085, 0.002661915263161063, 0.008316397666931152, 0.04395333305001259, 0.023112239316105843, 0.024167653173208237, 0.01174983475357294, -0.009558848105370998, 0.025116190314292908, -0.009939598850905895, -0.005711261183023453, 0.00048721066559664905, 0.014602125622332096, 0.006793395150452852, 0.011789913289248943, -0.018436351791024208, -0.025957848876714706, 0.04232345148921013, 0.03249073028564453, 0.006329146213829517, -0.00835647713392973, 0.013513311743736267, -0.014495247974991798, -0.030753973871469498, 0.0019488426623865962, 0.021629314869642258, 0.00020540501282084733, -0.005220293067395687, 0.026532314717769623, 0.00038075074553489685, -0.008109322749078274, 0.026532314717769623, 0.007788690738379955, -0.01041386742144823, -0.01589801348745823, -0.007681813556700945, 0.012651612982153893, -9.993663843488321e-05, -0.03371982276439667, 0.01327951718121767, 0.02538338303565979, -0.00716078607365489, -0.008416595868766308, -0.0042617362923920155, -0.0029441385995596647, -0.001915443455800414, 0.00023358558246400207, -0.035830650478601456, -0.015029635280370712, -0.023713424801826477, 0.009826041758060455, 0.0016941739013418555, -0.0119101507589221, -0.002989227417856455, 0.015123153105378151, 0.015243389643728733, 0.024889076128602028, -0.0014979536645114422, 0.02313895896077156, 0.01054078433662653, -0.00971248373389244, 0.001805226202122867, 0.0038475864566862583, -0.008009125478565693, -0.009932918474078178, 0.010841377079486847, 0.013419793918728828, -0.030941007658839226, -0.01050738524645567, 0.02669263258576393, -0.04780092090368271, -0.02821563556790352, -0.013593469746410847, -0.007574935909360647, 0.01965208351612091, 0.013386394828557968, -0.014802520163357258, -0.0017968763131648302, 0.001845305203460157, 0.023940538987517357, 0.01402765978127718, -0.010306989774107933, -0.0013059083139523864, 0.002760442905128002, 0.008510113693773746, -0.03203650191426277, 0.021843070164322853, 0.00483954232186079, -0.014415089972317219, 0.010280270129442215, 0.018356194719672203, -0.012972244992852211, -0.01782180741429329, 0.007548216730356216, 0.004131479654461145, 0.01425477396696806, 0.027066702023148537, 0.000511425081640482, 0.010133313946425915, 0.008396555669605732, -0.005370589438825846, -0.003129503922536969, -0.020507102832198143, -0.01091485470533371, -0.016445759683847427, -0.0004963954561389983, -0.008249599486589432, 0.021549155935645103, 0.03144867718219757, 0.0220568235963583, -0.010888135991990566, 0.0024381408002227545, -0.006292407400906086, -0.0029140792321413755, -0.016859911382198334, 0.017915325239300728, 0.01677975244820118, 0.04665198549628258, -0.02427452988922596, -0.033773258328437805, 0.02479555830359459, 0.02023990824818611, 0.002005621325224638, 0.005146814975887537, 0.008069244213402271, 0.023940538987517357, -0.014521967619657516, -0.005183554254472256, -0.6126213073730469, -0.012197383679449558, -0.0011172029189765453, -0.0029341187328100204, 0.016993507742881775, 0.01827603578567505, 0.015256749466061592, 0.031047886237502098, -0.02545018121600151, 0.04026606306433678, -0.00017357140313833952, 0.01470900233834982, 0.004034622106701136, -0.007882208563387394, -0.0007744436734355986, -0.003189622424542904, -0.007494777906686068, -0.005133455153554678, 0.027788124978542328, 0.009966317564249039, 0.00578473974019289, 0.04323190823197365, -0.022016745060682297, -0.016539277508854866, 0.0010863086208701134, -0.01675303280353546, 0.011322325095534325, -0.02134876139461994, -0.009939598850905895, 0.02522306703031063, -0.03965151682496071, 0.011709755286574364, 0.011582838371396065, 0.0074814180843532085, 0.049350641667842865, -0.02866986393928528, 0.0031311740167438984, 0.044140368700027466, 0.003844246733933687, 0.04245704784989357, 0.0017150483326986432, 0.002989227417856455, 0.01295220572501421, 0.009051180444657803, 0.037647563964128494, 0.011095210909843445, 0.04953767731785774, -0.02476883865892887, 0.004946419969201088, -0.027520932257175446, 0.011342364363372326, 0.04507554695010185, -0.025997929275035858, -0.023539748042821884, -0.010313669219613075, -0.016512559726834297, 0.04373957961797714, -0.010547463782131672, 0.004158198833465576, -0.007394580636173487, -0.009605607017874718, 0.033906854689121246, -0.017714930698275566, -0.005567644722759724, -0.025637216866016388, 0.0372200571000576, -0.03874306008219719, 0.0018837142270058393, -0.009892839938402176, -0.006439363583922386, 0.0009994707070291042, 0.04050653800368309, -0.0008404070977121592, 0.0121840238571167, 0.017875246703624725, 0.03933088481426239, 0.03489547222852707, 0.026759430766105652, -0.0036071124486625195, 0.013219399377703667, 0.021562516689300537, -0.010053155943751335, -0.0008249599486589432, -0.019598644226789474, 0.038956813514232635, -0.025116190314292908, -0.0011414173059165478, 0.0009677414782345295, 0.006339165847748518, -0.013025683350861073, -0.014268133789300919, 0.0482284277677536, -0.010099914856255054, -0.014268133789300919, 0.010514064691960812, -0.012558095157146454, -0.003907705191522837, -0.012664972804486752, 0.007875529117882252, -0.02388709969818592, -0.02078765444457531, -0.01983911730349064, 0.019197853282094002, -0.007454698905348778, 0.033185433596372604, -0.006866873241961002, -0.01935816928744316, 0.008603631518781185, 0.0008616990526206791, -0.007635054644197226, 0.0018887241603806615, 0.0015981511678546667, -0.011556119658052921, -0.008463354781270027, -0.028883619233965874, -0.024448206648230553, 0.0015121483011171222, -0.0015188280958682299, 0.020480383187532425, -0.03323887288570404, 0.019077615812420845, -0.0054106684401631355, 0.0003344093856867403, -0.015684258192777634, 0.008189480751752853, 0.02522306703031063, 0.008924263529479504, -0.023513028398156166, 0.017621412873268127, -0.0094519704580307, -0.004889641422778368, 0.0008024155395105481, 0.00731442216783762, 0.0011447571450844407, -0.022444255650043488, -0.004285115748643875, 0.0031194842886179686, -0.0002494501823093742, -0.012037067674100399, -0.009739203378558159, -0.018289396539330482, -0.0032580909319221973, 0.012631572782993317, -0.02199002541601658, 0.007040549069643021, -0.02293856255710125, 0.003334908979013562, -0.005003198515623808, -0.04168218746781349, -0.01723398081958294, -0.012210743501782417, -0.014054378494620323, -0.025142909958958626, -0.006873553153127432, -0.030085990205407143, -0.007668453734368086, 0.021482357755303383, 0.001965542323887348, -0.0030493459198623896, -0.015123153105378151, 0.0011923509882763028, 0.030246306210756302, -0.03187618404626846, 0.002317903796210885, -0.016592716798186302, 0.0007794535486027598, 0.02166939340531826, 0.019465047866106033, 0.0022945243399590254, -0.03893009573221207, 0.0027253737207502127, -0.023085519671440125, -0.0018803743878379464, 0.03299839794635773, -0.035109225660562515, -0.003059365786612034, -0.013346316292881966, -0.0258776918053627, -0.013773825950920582, 0.015002915635704994, -0.010246871039271355, -0.011101890355348587, 0.012397779151797295, -0.013586790300905704, 0.023606546223163605, -0.02108156867325306, -0.001805226202122867, 0.0188638623803854, -0.031208202242851257, 0.04379301518201828, 0.013286197558045387, -0.005116755608469248, 0.00961228646337986, -0.010467305779457092, -0.005975115112960339, -0.02909737266600132, 0.001935482956469059, 0.026959825307130814, 0.010133313946425915, 0.013760466128587723, 0.02280496619641781, 0.0006312446785159409, 0.022364096716046333, -0.005804779008030891, 0.025169629603624344, -0.0018519849982112646, -0.0022243859712034464, -0.015163231641054153, 0.049056727439165115, -0.01990591734647751, -0.006389264948666096, -0.05074004828929901, -0.02570401504635811, -0.013459873385727406, -0.006309106945991516, 0.010681060142815113, -0.0119101507589221, 0.0217629112303257, -0.009184776805341244, 0.025530340149998665, 0.022818325087428093, 0.012097186408936977, 0.00977260246872902, 0.007180825807154179, -0.01827603578567505, -0.04515570402145386, 0.02939128689467907, 0.02101477049291134, -0.027520932257175446, -0.009859440848231316, -0.003744048997759819, 0.008009125478565693, -0.017514534294605255, 0.010487345047295094, 0.01612512767314911, 0.01935816928744316, -0.008703828789293766, -0.01661943644285202, 0.03131508082151413, -0.013787184841930866, 0.052156172692775726, -0.014642204158008099, 0.023446230217814445, -0.025690656155347824, 0.030620375648140907, 0.014962836168706417, 0.028135476633906364, 0.010073195211589336, -0.006913632154464722, 0.020774295553565025, 0.02619832381606102, 0.011362403631210327, -0.010360428132116795, -0.004655846860259771, -0.01327951718121767, -0.011890111491084099, 0.004425392486155033, -0.0074814180843532085, 0.03884993493556976, 0.005016558337956667, 0.005464107263833284, 0.005951735656708479, 0.006025213748216629, -0.016659514978528023, 0.024755477905273438, -0.00983940064907074, -0.01961200311779976, -0.021722832694649696, 0.008870824240148067, 0.0034968950785696507, -0.01645912043750286, -0.023993976414203644, 0.021816350519657135, -0.0020290007814764977, -0.0006746636354364455, 0.003326559206470847, 0.009598926641047001, 0.0014520296826958656, 0.00819616112858057, -0.004068021196871996, 0.01967880129814148, -0.03855602443218231, 0.016472479328513145, 0.015617460943758488, -0.021201804280281067, -0.011756514199078083, -0.012083826586604118, -0.0011731465347111225, 0.00953212846070528, 0.011482641100883484, -0.01750117540359497, 0.005487486720085144, -0.012518015690147877, 0.016178566962480545, 0.016699593514204025, -0.006993790157139301, 0.036311596632003784, -0.022858405485749245, 0.017581332474946976, -0.005731300916522741, -0.022604571655392647, -0.022885125130414963, -0.008156081661581993, -0.022818325087428093, 0.019625363871455193, -0.010286950506269932, 0.001718388288281858, 0.0007351996609941125, 0.009719164110720158, 0.007701852824538946, 0.022564491257071495, -0.02420773170888424, -0.01332627609372139, 0.016539277508854866, -0.024234451353549957, 0.015310187824070454, -0.005637783091515303, -0.012558095157146454, 0.03751396760344505, -0.01072113960981369, -0.023272555321455002, 0.011916830204427242, -0.009952958673238754, -0.014989555813372135, 0.1011594608426094, -0.009064540266990662, 0.01156947948038578, 0.022604571655392647, 0.029605040326714516, 0.018356194719672203, -0.003329899162054062, -0.04454115778207779, 0.03759412840008736, -0.011455921456217766, 0.013419793918728828, -0.011095210909843445, -0.021175086498260498, 0.0008825735421851277, 0.007387900725007057, 0.0035002350341528654, -0.0011823312379419804, -0.014695643447339535, -0.015523943118751049, -0.003914385102689266, 0.004892981145530939, 0.0042049577459692955, 0.0035403140354901552, 0.038315549492836, 0.02538338303565979, -0.00599515438079834, 0.012237463146448135, 0.0011472621699795127, 0.016873270273208618, 0.015123153105378151, -0.010514064691960812, 0.022738168016076088, 0.003964483737945557, 0.023566467687487602, 0.011262206360697746, 0.0017952064517885447, -0.007795370649546385, -0.014668923802673817, 0.02909737266600132, -0.006502822041511536, 0.023272555321455002, 0.03088757023215294, 0.00010061505599878728, -0.02955160290002823, 0.03150211647152901, -0.016993507742881775, 0.01446852833032608, 0.007661773823201656, -0.020186468958854675, -0.010627621784806252, 0.009425250813364983, -0.012865367345511913, -0.012083826586604118, -0.011402483098208904, -0.0009335073409602046, 0.014575405977666378, -0.012711731716990471, -0.0020072911866009235, -0.01961200311779976, 0.01691334880888462, -0.025102829560637474, -0.012818608433008194, 0.02361990697681904, -0.006853513419628143, -0.05132787302136421, 0.003358288435265422, -0.018008843064308167, 0.028429388999938965, -0.036739107221364975, 0.007741931825876236, -0.0033232192508876324, -0.011008372530341148, -0.03150211647152901, -0.0028239013627171516, 0.030059270560741425, 0.009993037208914757, -0.004098080564290285, -0.009178097359836102, 0.01883714273571968, 0.016819830983877182, 0.0013418124290183187, -0.013352995738387108, -0.007822089828550816, -0.012404458597302437, -0.011576158925890923, 0.010600902140140533, 0.010387147776782513, -0.0024181012995541096, -0.008015804924070835, 0.014682283625006676, 0.048014674335718155, 0.009618966840207577, 0.009131338447332382, -0.0019622023683041334, -0.008556872606277466, -0.013747106306254864, -0.02495587430894375, 0.021749552339315414, 0.020667418837547302, 0.008142721839249134, 0.016886629164218903, -0.017193902283906937, -0.018129080533981323, 0.0032146719750016928, 0.028856899589300156, -0.01808900013566017, 0.004993178881704807, -0.016218645498156548, -0.0002955828094854951, -0.010861416347324848, 0.006382585037499666, -0.020640699192881584, 0.005981794558465481, -0.006993790157139301, -0.005347209982573986, 0.04379301518201828, 0.01867682673037052, 0.015083073638379574, -0.004923040512949228, 0.002459850162267685, -0.005360569804906845, -0.03770100325345993, 0.02450164407491684, 0.012030388228595257, 0.016111768782138824, 0.02261793054640293, 0.03665895015001297, -0.02581089362502098, -0.023900460451841354, 0.0013459873152896762, 0.01528346911072731, 0.011349044740200043, 0.00927161518484354, -0.023513028398156166, 0.0006496142013929784, -0.022511053830385208, -0.0011714765569195151, -0.010260230861604214, -0.011021732352674007, -0.01402765978127718, -0.002211026381701231, -0.009278294630348682, 0.014147896319627762, -0.022203780710697174, 0.01057418342679739, -0.0194516871124506, -0.007508137729018927, -0.018422992900013924, 0.0028940397314727306, 0.025436822324991226, -0.005253692623227835, -0.03353278711438179, 0.011362403631210327, -0.005828158464282751, 0.008670429699122906, -0.036258161067962646, -0.014508607797324657, 0.002676944946870208, -0.012003668583929539, 0.005854877643287182, 0.027160219848155975, -0.03754068911075592, 0.03991870954632759, 0.00461242813616991, 0.002673604991286993, -0.008810706436634064, -0.009826041758060455, -0.0026602454017847776, -0.04315175116062164, -0.009618966840207577, 0.02065405808389187, 0.04365941882133484, -0.0046525071375072, 0.0319296233355999, 0.015817856416106224, 0.04507554695010185, -0.001611510873772204, -0.016085049137473106, -0.019237933680415154, -0.03700629994273186, -5.6152384786400944e-05, 0.02173619158565998, 0.0043318746611475945, 0.021455638110637665, -0.037193335592746735, -0.023406151682138443, 0.049217045307159424, -0.0017400977667421103, 0.011963589116930962, -0.003374987980350852, 0.008817385882139206, -0.0007289372733794153, 0.004104760475456715, -0.0018720244988799095, 0.005343870259821415, -0.002937458688393235, -0.005894956644624472, -0.02221713960170746, -0.007701852824538946, 0.008369836956262589, -0.00861699040979147, -0.009652365930378437, 0.005110075697302818, 0.02955160290002823, -0.024381408467888832, -0.007327781990170479, -0.008029164746403694, -0.03040662221610546, 0.002095799194648862, 0.0013134231558069587, -0.002802192000672221, -0.024007337167859077, -0.020587259903550148, -0.023566467687487602, 0.020373504608869553, -0.02203010581433773, -0.012831968255341053, 0.006426003761589527, -0.014802520163357258, -0.009592247195541859, -0.050686608999967575, -0.0031679130624979734, 0.01528346911072731, 0.02017311006784439, 0.03909040987491608, 0.02228393964469433, -0.01987919770181179, -0.03321215510368347, 0.01802220195531845, 0.01064098160713911, 0.005400648806244135, 0.012858687900006771, -0.008062563836574554, 0.010781258344650269, -0.03636503592133522, -0.01410781778395176, 0.013660267926752567, 0.0025984568055719137, -0.021295322105288506, 0.029898954555392265, 0.03147539496421814, -0.0121840238571167, 0.003957803826779127, -0.022243859246373177, 0.0033048498444259167, 0.03115476295351982, -0.008610310964286327, 0.03262432664632797, -0.007508137729018927, -0.013192679733037949, -0.01593809202313423, 0.002509949030354619, -0.011836672201752663, 0.010661020874977112, -0.006279047578573227, 0.014201334677636623, -0.015684258192777634, -0.02065405808389187, 0.010066515766084194, -0.004315175116062164, 0.022364096716046333, 0.002099138917401433, -0.01587129384279251, 0.007668453734368086, 0.005734640639275312, -0.00853683240711689, -0.007828770205378532, -0.015924733132123947, -0.00019976889598183334, 0.025984568521380424, -0.019932635128498077, 0.002665255218744278, 0.00477274414151907, -0.00969244446605444, -0.03131508082151413, -0.01838291436433792, -0.011342364363372326, -0.025957848876714706, -0.014348291791975498, -0.006319126579910517, -0.006332486402243376, 0.014909397810697556, -0.012497976422309875, -0.022110262885689735, -0.01772828958928585, -0.027948440983891487, 0.01698014698922634, 0.007828770205378532, -0.0020106311421841383, -0.01967880129814148, -0.016312163323163986, -0.004405353218317032, -0.021749552339315414, -0.020186468958854675, -0.00837651640176773, -0.030540218576788902, 0.01711374521255493, 0.004739345051348209, -0.01156947948038578, 0.03401373326778412, 0.03208994120359421, 0.04109436273574829, -0.012417818419635296, 0.014842599630355835, 0.0011455921921879053, -0.02701326459646225, -0.000776113651227206, 0.02981879562139511, -0.025009311735630035, -0.028536267578601837, -0.0030209566466510296, 0.022885125130414963, -0.04336550831794739, -0.003984523005783558, 0.05004534497857094, 0.00483954232186079, 0.0023730124812573195, -0.004385313484817743, -0.03665895015001297, 0.014668923802673817, -0.01072113960981369, 0.010033116675913334, 0.013266157358884811, 0.009512089192867279, 0.030647095292806625, -0.005945055745542049, 0.006305766757577658, -0.021068207919597626, -0.01528346911072731, -0.010647661052644253, 0.002676944946870208, 0.016044970601797104, -0.010467305779457092, -0.020974690094590187, -0.03396029397845268, -0.018756983801722527, -0.016352243721485138, 0.0165526382625103, -0.01772828958928585, -0.026612473651766777, 0.03679254651069641, 0.006422664038836956, -0.011302285827696323, -0.004518910311162472, -0.013853983953595161, -0.008576911874115467, 0.003513594623655081, -0.004792783409357071, -0.0034801955334842205, -0.0001522794336779043, 0.008590271696448326, 0.05132787302136421, -0.008810706436634064, -0.02694646641612053, -0.021455638110637665, 0.016873270273208618, -0.03652535378932953, 0.0005322995712049305, 0.00035820630728267133, 0.014829239808022976, 0.007381220813840628, 0.0063458457589149475, 0.010213471949100494, 0.017514534294605255, 0.025102829560637474, 0.02320575714111328, 0.002202676609158516, -0.0027537632267922163, -0.0220568235963583, -0.0033699781633913517, 0.004351914394646883, -0.008316397666931152, -0.041014205664396286, -0.017327498644590378, 0.022738168016076088, -0.01766149140894413, -0.004492191132158041, 0.0028973796870559454, -0.019157774746418, -0.010614261962473392, 0.0020490402821451426, 0.019104335457086563, 0.013219399377703667, -0.005711261183023453, 0.0018703545210883021, -0.028242353349924088, -0.008850784972310066, 3.1755324016558006e-05, 0.003096104832366109, -0.011208768002688885, 0.0008529318147338927, 0.013747106306254864, 0.025597138330340385, -0.01648584008216858, -0.015376986935734749, -0.01580449566245079, -0.010487345047295094, -0.02522306703031063, 0.009585566818714142, 0.026171604171395302, -0.022764887660741806, -0.0021926567424088717, 0.017674850299954414, -0.011015052907168865, -0.00011324412480462343, -0.005397309083491564, -0.012297580949962139, 0.015978172421455383, 0.010473985224962234, 0.008483394049108028, 0.02802859991788864, -0.011970269493758678, 0.011676356196403503, -0.02880346029996872, -0.017407657578587532, 0.003269780660048127, -0.017835166305303574, -0.002788832178339362, 0.05921008065342903, -0.004098080564290285, -0.009472009725868702, 0.004956439603120089, -0.004238356836140156, -0.010400507599115372, 0.015390345826745033, -0.010253551416099072, -0.008530152961611748, -0.026078086346387863, 0.0045456294901669025, 0.0061588105745613575, -0.00800244603306055, 0.009231535717844963, 0.018075641244649887, 0.019478406757116318, -0.006195549387484789, -0.013573430478572845, 0.20477710664272308, 0.0047994633205235004, -0.0031061244662851095, 0.03967823460698128, -0.010180072858929634, 0.03144867718219757, 0.03684598580002785, 0.00954548828303814, -0.0117631945759058, 0.003301509888842702, -0.014495247974991798, 0.0008675439166836441, -0.018596667796373367, 0.004138159565627575, 0.01454868633300066, 0.005654482636600733, -0.043712858110666275, -0.04128139838576317, 0.004191598389297724, 0.014668923802673817, 0.029872234910726547, -0.008643710054457188, -0.013773825950920582, -0.017848527058959007, 0.030326463282108307, 0.008637030608952045, -7.49916143831797e-05, -0.009264934808015823, -0.008256279863417149, -0.006395944859832525, -0.028990495949983597, -0.017447736114263535, 0.00073937454726547, 0.011355724185705185, -0.006295747123658657, -0.0003473515680525452, 0.008890864439308643, -0.014081098139286041, -0.00971248373389244, -0.007147426251322031, 0.015991531312465668, 0.013880702666938305, -0.037166617810726166, -0.011455921456217766, -0.022043464705348015, 0.03147539496421814, 0.0036071124486625195, -0.034067172557115555, -0.019758960232138634, 0.0331052765250206, -0.009732523933053017, -0.027066702023148537, -0.024755477905273438, -0.009952958673238754, -0.03150211647152901, -0.0372200571000576, 0.02463524229824543, -0.01440173014998436, 0.0007698513218201697, -0.005230313166975975, -0.0013869013637304306, 0.03500235080718994, -0.003724009497091174, 0.024327969178557396, -0.011402483098208904, 0.02182970941066742, -0.01623200625181198, 0.01716718263924122, 0.019171133637428284, -0.008416595868766308, -0.002890699775889516, -0.02808203734457493, -0.015604101121425629, 0.022243859246373177, -0.004913020879030228, -0.0077486117370426655, -0.0019588624127209187, 0.001955522457137704, 0.02424781024456024, 0.011609558016061783, -0.01929137110710144, -0.011983629316091537, 0.012885406613349915, -0.010420546866953373, 0.011302285827696323, -0.026438798755407333, 0.018289396539330482, -0.009745883755385876, 0.004044641740620136, -0.015056353993713856, -0.004398673307150602, 0.0022544453386217356, 0.0035469939466565847, -0.02189650759100914, 0.003062705509364605, -0.0017233981052413583, 0.025944489985704422, 0.013005644083023071, 0.0014161255676299334, 0.02078765444457531, -0.033746540546417236, 0.003997882828116417, 0.011636277660727501, 0.0044721513986587524, -0.024621881544589996, -0.005357230082154274, 0.010280270129442215, 0.0378078818321228, 0.0013351325178518891, -0.022537773475050926, -0.0002607224159874022, -0.024581803008913994, 0.011335684917867184, 0.003637171583250165, -0.009993037208914757, 0.01279188971966505, -0.00857023149728775, -0.03532298281788826, 0.0035703734029084444, -0.007040549069643021, -0.002588437171652913, -0.027814844623208046, 0.006689857691526413, 0.004131479654461145, -0.022364096716046333, -0.013987580314278603, -0.032891519367694855, 0.006526201497763395, 0.005434047896414995, -0.035242825746536255, 0.026893027126789093, -0.02169611304998398, -0.003067715559154749, -0.009578887373209, 0.012998964637517929, -0.009438610635697842, -0.006719916593283415, -0.04261736571788788, -0.010921535082161427, 0.006786715239286423, 0.009004421532154083, -0.008216200396418571, 0.020279986783862114, -0.0060285534709692, 0.01938488893210888, -0.02936456725001335, 0.014562046155333519, -0.009819361381232738, -0.02760108932852745, 0.0009760913089849055, 0.012925486080348492, -0.01564417965710163, 0.00835647713392973, 0.0009259924991056323, 0.01906425692141056, 0.007928967475891113, -0.028750021010637283, -0.026959825307130814, 0.013827264308929443, 0.001597316237166524, -0.002546688076108694, 0.015029635280370712, 0.030192866921424866, -0.010841377079486847, -0.009325053542852402, -0.00933841336518526, -0.16726313531398773, 0.031047886237502098, 0.03564361482858658, -0.0058882771991193295, 0.017354218289256096, 0.03174258768558502, 0.05178210139274597, 0.025356663390994072, -0.021615954115986824, -0.019264651462435722, 0.011656316928565502, 0.001324277836829424, -0.020079592242836952, 0.0002674022689461708, -0.01054078433662653, 0.007848809473216534, -0.034521400928497314, 0.02515626884996891, 0.036605510860681534, -0.013853983953595161, 0.013787184841930866, -0.01583121530711651, -0.0024782198015600443, -0.02209690399467945, 0.009538807906210423, 0.014655563980340958, 0.007615014910697937, 0.004061341285705566, -0.02320575714111328, -0.01401429995894432, 0.004084720741957426, 0.008657069876790047, 0.009585566818714142, 0.00832307804375887, 0.011475961655378342, -0.00043961682240478694, 0.0007869683904573321, -0.017688211053609848, -0.025142909958958626, 0.041201237589120865, 0.0036605510395020247, 0.006058612838387489, 0.009986357763409615, -0.019344810396432877, -0.00031812727684155107, 0.04438084363937378, -0.010888135991990566, -0.006222269032150507, 0.01736757904291153, -0.029872234910726547, -0.009438610635697842, -0.0005022403202019632, -0.017915325239300728, 0.004281776025891304, 0.00025779998395591974, -0.006449383217841387, 0.0022895142901688814, 0.015323547646403313, 0.017781728878617287, 0.007027189247310162, -0.014909397810697556, -0.0032631007488816977, -0.002443150617182255, -0.03575049340724945, -0.0021425578743219376, -0.02997911162674427, 0.00019945578242186457, 0.007521497551351786, -0.016565997153520584, 0.016766393557190895, -0.012751810252666473, -0.02665255218744278, 0.015417065471410751, -0.009665724821388721, 0.012685012072324753, -0.0165526382625103, -0.01877034455537796, -0.02307215891778469, -0.0025984568055719137, -0.012518015690147877, -0.032864801585674286, 0.052931033074855804, -0.043686140328645706, -0.011703075841069221, -0.005427367985248566, -0.007267663720995188, 0.012190704233944416, 0.015189951285719872, 0.0017400977667421103, -0.0015572372358292341, 0.011415842920541763, -0.004963119514286518, 0.0022327357437461615, -0.006699877325445414, 0.022978641092777252, 0.005303791258484125, 0.018289396539330482, 0.00964568555355072, 0.023993976414203644, -0.009552167728543282, 0.005781399551779032, -0.02710678242146969, -0.04272424057126045, 0.0034968950785696507, -0.028295792639255524, 0.019665442407131195, 0.023018721491098404, 0.016138488426804543, 0.031261641532182693, -0.0214689988642931, 0.01641904190182686, -0.012357699684798717, 0.008349796757102013, 0.004054661374539137, 0.01083469670265913, 0.03208994120359421, -0.00015593245916534215, -0.015497223474085331, 0.013747106306254864, 0.03217009827494621, 0.007367860991507769, -0.013707026839256287, -0.004338554572314024, -0.02212362363934517, 0.0014010960003361106, -0.03580392897129059, -0.11596198379993439, -0.0217629112303257, 0.013506632298231125, 0.007073948159813881, 0.002666925312951207, 0.03454812243580818, 0.00979264173656702, 0.0035670334473252296, -0.031128043308854103, 0.009705804288387299, -0.0010337048443034291, -0.017033586278557777, 0.01064098160713911, -0.011529400013387203, 0.0365520715713501, -0.021268604323267937, 0.02821563556790352, -0.00294079864397645, -0.011121929623186588, 0.030326463282108307, -0.012424497865140438, -0.03027302399277687, -0.009244895540177822, -0.008055884391069412, -0.013987580314278603, 0.005951735656708479, -0.0281889159232378, 0.014281493611633778, 0.007681813556700945, 0.015149871818721294, -0.010861416347324848, 0.013787184841930866, -0.001771826995536685, -0.01319935917854309, -0.018329475075006485, -0.011589518748223782, -0.030353182926774025, -0.0008500093827024102, 0.024675320833921432, -0.019278012216091156, 0.004228337202221155, 0.012077147141098976, 0.022404175251722336, -0.034040454775094986, 0.0005849032895639539, -0.014869319275021553, -0.02893705666065216, 0.02531658485531807, -0.017287420108914375, -0.03818195313215256, -0.01179659366607666, 0.004238356836140156, 0.004291795659810305, -0.011496000923216343, 0.005778059829026461, -0.03580392897129059, 0.007267663720995188, -0.003724009497091174, -0.006906952243298292, 0.0165526382625103, -0.01587129384279251, 0.0006333321216516197, 0.013493272475898266, 0.015390345826745033, 0.029658479616045952, 0.0120571069419384, -0.00716078607365489, 0.01971888169646263, 0.0120571069419384, -0.008055884391069412, -0.0023396131582558155, 0.006522861775010824, 0.0024264510720968246, 0.0061588105745613575, -0.034494683146476746, 0.009899519383907318, -0.011429202742874622, 0.018195878714323044, -0.0267193503677845, 0.007815410383045673, -0.01528346911072731, -0.00461242813616991, 0.009538807906210423, -0.030807411298155785, 0.011642957106232643, 0.023833660408854485, -0.013666948303580284, -0.023152317851781845, -0.01084805652499199, 0.0033332391176372766, 0.0037674284540116787, 0.035082507878541946, 0.01564417965710163, 0.015564021654427052, 0.027788124978542328, 0.008449994958937168, -0.012117225676774979, 0.02721365913748741, -0.015604101121425629, -0.0012382749700918794, 0.009411890991032124, -0.016739673912525177, -0.04435412213206291, -0.0010921534849330783, 0.01691334880888462, -0.00018557424482423812, -0.01062094233930111, 0.017474455758929253, 0.004094740375876427, 0.006506161764264107, 0.006913632154464722, 0.003854266367852688, -0.005520885810256004, 0.005317151080816984, -0.024942513555288315, -0.018262676894664764, -0.030834130942821503, 0.005965095013380051, -0.01528346911072731, -0.008429955691099167, -0.001295888563618064, 0.018422992900013924, -0.018195878714323044, -0.01727406121790409, -0.002306214068084955, -0.00919145718216896, -0.003035986330360174, 0.02395389787852764, 0.023312633857131004, -0.0019488426623865962, 0.006041913293302059, 0.006783375050872564, 0.027039984241127968, -0.0072943828999996185, 0.0030777351930737495, 0.0019187834113836288, -0.02134876139461994, 0.005434047896414995, 0.00016031610721256584, 0.008657069876790047, 0.018129080533981323, 0.046678707003593445, -0.023833660408854485, -0.0032063222024589777, 0.01639232225716114, -0.021816350519657135, -0.01652591861784458, -0.0060285534709692, -0.013446513563394547, 0.011489320546388626, 0.003984523005783558, 0.021094927564263344, 0.012584814801812172, 0.020867813378572464, -0.005968435201793909, -0.0018269356805831194, -0.04323190823197365, -0.012457897886633873, -0.019986074417829514, -0.0065562608651816845, -0.01795540377497673, -0.009558848105370998, 0.037166617810726166, 0.012564774602651596, 0.027841564267873764, 0.0020440304651856422, 0.010447266511619091, -0.008556872606277466, -0.015096433460712433, 0.0074814180843532085, 0.0035169345792382956, -0.022016745060682297, -0.04031950235366821, 0.005263712257146835, 0.030967727303504944, 0.027053343132138252, 0.026665912941098213, -0.010841377079486847, 0.01402765978127718, -0.01432157214730978, -0.016806472092866898, 0.01543042529374361, 0.011248846538364887, -0.014361651614308357, -0.04908344894647598, 0.016565997153520584, 0.020293347537517548, 0.0035403140354901552, 0.00727434316650033, 0.007875529117882252, -0.0074079399928450584, 0.021041488274931908, -0.01305240299552679, -0.008222879841923714, 0.011402483098208904, -0.013332956470549107, 0.005130115430802107, 0.029898954555392265, -0.005313810892403126, -0.012190704233944416, 0.004158198833465576, 0.027814844623208046, 0.016565997153520584, 0.02088117226958275, -0.0002945390879176557, -0.010921535082161427, -0.006212248932570219, 0.008022485300898552, 0.00491970032453537, -0.01083469670265913, -0.0061487904749810696, -0.00977260246872902, -0.007414619904011488, 0.010580862872302532, 0.015376986935734749, 0.011188728734850883, 0.016245365142822266, 0.00023108064488042146, 0.0054941666312515736, -0.013553391210734844, -0.013059082441031933, 0.024822277948260307, -0.005113415885716677, -0.00842327531427145, 0.002580087399110198, 0.016565997153520584, 0.025329945608973503, -0.012471256777644157, 0.04454115778207779, 0.011576158925890923, -0.0005953405634500086, -0.005113415885716677, -0.00700714997947216, -0.0189039409160614, -0.011576158925890923, -0.018102360889315605, -0.009739203378558159, -0.027293818071484566, 0.013339635916054249, 0.040880605578422546, -0.015457144938409328, 0.048014674335718155, 0.04034622013568878, -0.03692614287137985, 0.02395389787852764, -0.011996988207101822, 0.015376986935734749, 0.011783233843743801, -0.01675303280353546, -0.010373787954449654, -0.006739956326782703, 0.022524412721395493, -0.004649166949093342, 0.022070184350013733, -0.026959825307130814, -0.037647563964128494, -0.021001409739255905, 0.004111439920961857, -0.019932635128498077, 0.00235464284196496, -0.0029090694151818752, 0.001975561957806349, 0.00604525301605463, -0.00482284277677536, 0.027788124978542328, -0.014041018672287464, -0.04085388779640198, 0.008603631518781185, -0.01913105510175228, -0.03353278711438179, -0.024835636839270592, -0.02420773170888424, 0.018877221271395683, -0.005911656655371189, -0.008189480751752853, 0.006933671422302723, 0.012538055889308453, -0.000949371955357492, -0.004796123597770929, 0.03401373326778412, 0.01935816928744316, 0.00230955402366817, 0.007594975642859936, -0.002334603341296315, -0.03249073028564453, -0.013840624131262302, -0.014147896319627762, -0.036151282489299774, -0.01752789504826069, 0.011516040191054344]} +{"id": "test:10000083", "text": "\"JessicaaDivine SarahKeller jasmin. AmyNicollete. LilyRay. DouglasCox.\\nMoanSweetviking28MishaRayNr1GoodBoy .milksquirtingtitLilyRaymilksquirtingtitdayreth .JURISEX89RomeoHotFighterJoonMaliSamaraStyle21 .VictoriaKisJUSEFFMOREDIVAextasygirl69 .MarySexxxAngelJessica28MENTATOOSHOTXmariochase33 .Ashley707DouglasCoxsouthernlisaLillyJ .MENTATOOSHOTXmatureandyounggAlisacandyyLittleAlica .davidmaxxNicolee1x0NightMan0BigCumsAriana .davetest4126MillkyBunnyTranSensualDollmariochase33 .SamaraStyle21ScarletLove4uBigCumsArianapenetrating .PrettyPocahontasSnowchaniJURISEX89LillyJ .\\nTatyStarTANYAPRINCEBigeyesGreenMoanSweet .JoonMaliSnowchanilillyflower1990BellaBestBoobs .LillyJAshley707faberxxMoanSweet .CamChaseCuteLilBabemilksquirtingtitdavidmaxx .LolaAssU5CuteMariusForUCamChaseBigCumsAriana .TearMyClothesOffBastetXLionelWooddavetest4126 .TranSensualDollMOREDIVAReginaCuteVictoriaKis .MarisMuscleMoanSweetMilfKate4uVioletNimfa .penetratingRomeoHotFighterBastetXLionelWood .AliceAustinLaMaliciosaantonellarobertsLittleAlica .RomeoHotFighterMarySexxxMELANYFOXHOT23MarySexxx .\"", "vector_field": [-0.03433440253138542, -0.014247424900531769, -0.0018839988624677062, -0.02284454181790352, -0.020735815167427063, 0.027643248438835144, -0.006880397908389568, -0.009279750287532806, -0.008306492120027542, -0.038957379758358, 0.007272404618561268, 0.0046871863305568695, -0.000592657073866576, -0.0036125467158854008, -0.0057314117439091206, 0.01134116668254137, 0.0251154787838459, 0.00695474399253726, 0.027981186285614967, -0.047527462244033813, -0.001904275151900947, 0.013524239882826805, -0.02781897597014904, -0.004758153110742569, -0.031333521008491516, -0.008705257438123226, 0.00438642269000411, -0.012152215465903282, -0.028305605053901672, 0.012463117018342018, -0.027616214007139206, 0.004930501338094473, 0.008144281804561615, -0.02564266137778759, -0.021722592413425446, -0.00580237852409482, 0.004190418869256973, -0.009773138910531998, 0.003903172677382827, -0.0012317803921177983, 0.021357620134949684, 0.005048779305070639, 0.009793414734303951, -0.004619598854333162, -0.030819855630397797, -0.011334408074617386, -0.012550980783998966, -0.004538494162261486, 0.0037139279302209616, 0.027737870812416077, 0.03514545038342476, 0.010151628404855728, -0.0012419185368344188, -0.01461239717900753, -0.01754569076001644, 0.018518948927521706, -0.008286215364933014, 0.028765199705958366, 0.01180076040327549, 0.00548133859410882, 0.02527768909931183, 0.011685862205922604, -0.028873339295387268, -0.012165732681751251, -0.009374372661113739, -0.013044369406998158, -0.029684387147426605, -0.0008845501579344273, -0.007569789420813322, -0.0017657208954915404, 0.03309079259634018, 0.004062002990394831, 0.011361442506313324, 0.0024365258868783712, 0.022222738713026047, -0.018870403990149498, -0.008935054764151573, -0.005221127066761255, -0.011516893282532692, -0.0018569639651104808, 0.03514545038342476, -0.008178075775504112, -0.01896502636373043, 0.007644135504961014, 0.01603173278272152, 0.02373669669032097, -0.011273578740656376, 0.03325300291180611, 0.00018407007155474275, -0.006586392410099506, 0.026075219735503197, 0.02898147888481617, 0.010922124609351158, 0.000511129735969007, -0.018235081806778908, 0.00682970741763711, -0.025156032294034958, 0.02784601040184498, -0.016504844650626183, -0.026615919545292854, 0.005535408388823271, -0.0022117977496236563, -0.011111369356513023, -0.007508960552513599, 0.003889655228704214, -0.014490739442408085, -0.001432008109986782, -0.002287833718582988, 0.024547744542360306, 0.010462529957294464, 0.004261385649442673, 0.0017293926794081926, 0.02077636867761612, -0.041093140840530396, 0.002145900158211589, 0.007779310457408428, -0.009921831078827381, -0.021222444251179695, -0.01845136098563671, -0.01435556448996067, -0.0004154513298999518, 0.01639670506119728, 0.017897143959999084, -0.017734935507178307, 0.014139285311102867, 0.028900373727083206, -0.018018802627921104, -0.022587710991501808, -0.011564204469323158, -0.0027322208043187857, 0.04858182743191719, -0.014085214585065842, 0.03011694736778736, 0.012010281905531883, 0.00047268939670175314, 0.0063329399563372135, -0.00917836930602789, 0.002635908778756857, -0.015274753794074059, 0.0072250934317708015, 0.0020310014951974154, 0.035605043172836304, -0.004176901653409004, -0.002845429815351963, 0.02514251507818699, 0.018897438421845436, 0.001348368707112968, -0.0301439817994833, -0.00241793948225677, 0.012807813473045826, -0.008576842024922371, -0.018126942217350006, 0.004481045063585043, 0.0002433146582916379, 0.01678871177136898, 0.01678871177136898, -0.005207609385251999, 0.022141633555293083, 0.002598735736683011, 0.006941226311028004, 0.006941226311028004, 0.021614450961351395, -0.021776661276817322, -0.01390948798507452, 0.0234393123537302, 0.02795414999127388, 0.024939751252532005, -0.02768380008637905, -0.013645896688103676, -0.0050386409275233746, -0.005231264978647232, 0.029954737052321434, -0.030846891924738884, 0.01153717003762722, -0.03384777158498764, 0.021181892603635788, -0.001326402765698731, -0.0067959134466946125, -0.015207165852189064, -0.023790765553712845, -0.023615039885044098, 0.007900967262685299, 0.027007926255464554, 0.02191183716058731, -0.02502085641026497, 0.010942400433123112, 0.015558620914816856, -0.034550681710243225, -0.009570376016199589, -0.020979130640625954, -0.000695305410772562, 0.024142220616340637, -0.0019752418156713247, -0.032901547849178314, -0.6341320872306824, -0.009820450097322464, 0.013693207874894142, 3.157598985126242e-05, 0.010300320573151112, 0.02038436196744442, 0.0010999849764630198, 0.0025717008393257856, 0.008576842024922371, 0.007644135504961014, -0.01666705310344696, 0.007535995449870825, -0.0032441953662782907, 0.013321476988494396, -0.010225974023342133, -0.015112543478608131, 0.0013796278508380055, -0.011692620813846588, -0.002201659604907036, 0.005494855809956789, -0.008860709145665169, 0.007907725870609283, 0.030684681609272957, 0.018045837059617043, 0.010111075825989246, -0.014950334094464779, 0.009894795715808868, -0.016707606613636017, 0.0017141855787485838, -0.008299733512103558, -0.024385536089539528, 0.009502789005637169, 0.005525270476937294, -0.002720393007621169, 0.050717588514089584, 0.0036024085711687803, -0.00827945675700903, 0.0353887639939785, 0.042877450585365295, 0.0326041653752327, -0.030846891924738884, 0.025561556220054626, 0.004893328063189983, 0.00397075992077589, 0.0019566554110497236, 0.021736109629273415, 0.02039787918329239, -0.013808107003569603, -0.025831906124949455, 0.01538289338350296, 0.00871877558529377, -0.015409927815198898, -0.019046129658818245, -0.03190125524997711, -0.021830732002854347, -0.005021743942052126, 0.026697024703025818, -0.017843075096607208, -0.013240372762084007, 0.011442547664046288, -0.014896264299750328, -0.00027710836729966104, -0.016085801646113396, -0.014166319742798805, -0.032901547849178314, 0.022263290360569954, -0.0021543484181165695, -0.00795503705739975, 0.008948571979999542, 0.008624153211712837, -0.017478102818131447, 0.00847546011209488, -0.002794739091768861, 0.0012520566815510392, -0.008806638419628143, 0.03349631652235985, -0.013044369406998158, 0.0035348213277757168, -0.0034739926923066378, -0.0033405073918402195, -0.009408166632056236, -0.021506311371922493, 0.019005578011274338, -0.016072284430265427, 0.026602402329444885, -0.008252422325313091, 0.00029147067107260227, 0.015545102767646313, 0.018654122948646545, 0.0003820800338871777, 0.016640018671751022, 0.01108433399349451, 0.009962383657693863, 0.0003149150579702109, -0.003842344041913748, -0.011956212110817432, -0.013632379472255707, -0.0030329846777021885, 0.003747721668332815, -0.042012330144643784, 0.005619892850518227, -0.010111075825989246, 0.035578008741140366, -0.005268438253551722, 0.0018113424303010106, 0.020857473835349083, -0.033658526837825775, 0.006018658168613911, 0.023628557100892067, -0.012496910989284515, -0.008576842024922371, -0.018370255827903748, 0.009658239781856537, -0.027224207296967506, -0.012571257539093494, -0.01695092022418976, 0.019776074215769768, -0.007711722981184721, -0.002328386064618826, 0.0018012042855843902, -0.0009048263891600072, 0.007319715805351734, -0.01781604066491127, -0.011692620813846588, 0.023912424221634865, 0.026075219735503197, -0.0004887414397671819, -0.0021796938963234425, -0.006525564007461071, -0.0006150453700684011, 0.02730531059205532, -0.004832499194890261, 0.018951507285237312, -0.02498030476272106, 0.00839435588568449, -0.02717013657093048, 0.013490445911884308, -0.010482806712388992, 0.03149573132395744, -4.3324580474291e-05, 0.01910020038485527, -0.008441667072474957, 0.00326954061165452, -0.0029704663902521133, -0.0013035921147093177, -0.006191006395965815, -0.029170723631978035, 0.008042900823056698, -0.00657625449821353, -0.013760795816779137, -0.012611809186637402, 0.01577490009367466, -0.002428077394142747, 0.021709073334932327, -0.022357912734150887, -0.009259474463760853, -0.025493968278169632, -0.002983983838930726, 0.002657874720171094, -0.017153684049844742, 0.008117247372865677, 0.013172784820199013, -0.02873816341161728, -0.010665292851626873, 0.003103951457887888, 0.007934761233627796, -0.01935703307390213, 0.0015815452206879854, -0.007056124974042177, -0.019965318962931633, 0.009056712500751019, -0.008630911819636822, -0.01539641059935093, 0.019830144941806793, -0.006934467703104019, 0.02449367567896843, -0.0040721409022808075, 0.00871877558529377, 0.0024838370736688375, 0.00975286215543747, 0.0035415799356997013, -0.008874226361513138, -0.023358207195997238, 0.013416099362075329, 0.02498030476272106, 0.010577429085969925, -0.00030921236611902714, 0.022885095328092575, -0.026737576350569725, -0.0052582998760044575, -0.011672344990074635, -0.008468701504170895, -0.004430354572832584, 0.01340258214622736, 0.008617393672466278, -0.010563910938799381, 0.020235668867826462, 0.00265956437215209, -0.015477515757083893, 0.010449012741446495, 0.024534227326512337, 0.012037316337227821, 0.031117241829633713, -0.009131058119237423, 0.005048779305070639, -0.03644312918186188, -0.009732586331665516, -0.03268526867032051, 0.03479399532079697, -0.021060235798358917, 0.022736402228474617, -0.0016609604936093092, -0.009455477818846703, 0.016991473734378815, 0.002906258450821042, 0.03733528032898903, 0.0068601216189563274, 0.006714808754622936, -0.00013760372530668974, 0.0007079780334606767, 0.015180131420493126, 0.014166319742798805, 0.013443134725093842, -0.004481045063585043, -0.010354390367865562, 0.016329117119312286, 0.014477222226560116, -0.01588303968310356, 0.008948571979999542, 0.0001108855867641978, -0.011138403788208961, 0.003957242704927921, 0.016004696488380432, 0.011334408074617386, 0.004714221227914095, -0.03476696088910103, 0.05009578540921211, -0.0074548907577991486, 0.01058418769389391, -0.011510134674608707, 0.011435789056122303, 0.026440192013978958, 0.02256067469716072, 0.005809137597680092, 0.032631199806928635, -0.0013052817666903138, -0.004842637572437525, 0.009705550968647003, -0.01860005408525467, -0.011429030448198318, -0.028386710211634636, 0.01758624240756035, -0.00955685880035162, -0.007049366366118193, 0.0193705502897501, -0.016896851360797882, 0.007306198589503765, 0.0027693938463926315, 0.016477808356285095, 0.03590242937207222, 0.006065969355404377, 0.01108433399349451, 0.025940045714378357, 0.015666760504245758, 0.011070816777646542, 0.01973552256822586, -0.01793769747018814, 0.0020783126819878817, -0.0038727582432329655, -0.021965906023979187, -0.0010763293830677867, -0.015801934525370598, 0.007387303281575441, 0.029522178694605827, -0.0009073609253391623, -0.007353509776294231, -0.008556565269827843, -0.006052452139556408, 0.008955331519246101, 0.004633116535842419, 0.010361148975789547, -0.022885095328092575, 0.02731882967054844, 0.030360262840986252, 0.00731295719742775, 0.002250660676509142, 0.011009988375008106, -0.023182479664683342, -0.022371429949998856, -0.009827208705246449, 0.01732940971851349, 0.041498664766550064, 0.006417423952370882, 0.006001761648803949, -0.0005474579520523548, 0.01155068725347519, 0.019194822758436203, -0.00968527514487505, -0.016193941235542297, 0.01922185719013214, 0.019965318962931633, -0.007623859215527773, -0.005133263301104307, 0.0008097816025838256, 0.01835673861205578, -0.015301788225769997, -0.007089918479323387, -0.008218628354370594, -0.011151921935379505, -0.024196291342377663, -0.0006585547816939652, -0.0241151861846447, -0.03836261108517647, 0.010989711619913578, 0.009996176697313786, 0.015436963178217411, -0.018924472853541374, -0.02384483627974987, 0.00022324964811559767, -0.010394942946732044, -0.03279341012239456, -0.006657359190285206, -0.014382599852979183, 0.011854831129312515, 0.052231546491384506, 0.020695263519883156, -0.007258887402713299, 0.020965613424777985, -0.011794001795351505, 0.00010961831867462024, -0.016342634335160255, 0.009327061474323273, 0.02024918608367443, -0.01949220709502697, -0.0028944306541234255, 0.004369525704532862, 0.010029970668256283, 0.007711722981184721, 0.02257419377565384, -0.009097264148294926, 0.0011346235405653715, 0.006383630447089672, -0.008813397958874702, -0.00470746261999011, -0.003977518528699875, 0.010198939591646194, -0.0004137616488151252, 0.010198939591646194, 0.0032374367583543062, -0.014125767163932323, 0.024723472073674202, 0.00503188231959939, -0.0029772252310067415, -0.0031140896026045084, -0.012104904279112816, 0.05139346048235893, 0.016572430729866028, 0.010165145620703697, 0.0030228467658162117, 0.007231852039694786, 0.029170723631978035, -0.004001174122095108, 0.01847839541733265, -0.016964439302682877, 0.013294442556798458, 0.029197758063673973, -0.0007992210448719561, 0.002509182319045067, 0.011807519942522049, -0.03000880777835846, -0.02806228958070278, 0.021073753014206886, 0.020059941336512566, -0.01397707499563694, 0.03384777158498764, 0.01065177470445633, 0.008320009335875511, -0.03522655367851257, 0.01058418769389391, 0.0284678153693676, -0.005542166996747255, -0.0026629436761140823, -0.03257712721824646, -0.0006277179927565157, -0.01428797747939825, -0.0024365258868783712, -0.010935641825199127, -0.024182774126529694, -0.04347221925854683, -0.028251534327864647, -0.018140459433197975, 0.007164265029132366, -0.013233614154160023, -0.02973845787346363, -0.02027622051537037, -0.012469875626266003, -0.0033168517984449863, 0.027102548629045486, 0.006643841974437237, 0.00048620690358802676, 0.025183066725730896, 0.001797824981622398, 0.014125767163932323, 0.018424326553940773, -0.008009107783436775, -0.019384067505598068, 0.025953562930226326, -0.0011785553069785237, -0.017396997660398483, 0.00898236595094204, 0.0065965307876467705, -0.006049072835594416, -0.02088450826704502, 0.031603869050741196, 0.020830437541007996, -0.008610635064542294, 0.008353803306818008, -0.02694034017622471, 0.025980597361922264, 0.013693207874894142, 0.021181892603635788, 0.004383043386042118, -0.023952975869178772, -0.017140164971351624, 0.007164265029132366, -0.03027915768325329, -0.005210988689213991, -0.009664998389780521, -0.006133556831628084, 0.010225974023342133, -0.006893915124237537, 0.023790765553712845, -0.035821326076984406, -0.006042313762009144, -0.029900668188929558, -0.0032577128149569035, 0.014125767163932323, 0.0009724138071760535, -0.01166558638215065, 0.025088444352149963, 0.00941492523998022, 0.03154980018734932, -0.013030851259827614, -0.013950040563941002, -0.006359974853694439, -0.034442540258169174, 0.024534227326512337, 0.012483393773436546, -0.013017334043979645, 0.03330707177519798, 0.009225680492818356, -0.03171201050281525, -0.0036902723368257284, 0.005342784337699413, 0.016356151551008224, 0.009070229716598988, -0.03436143696308136, -0.039281800389289856, -0.004967674147337675, -0.015977662056684494, 0.01295650564134121, 0.02049250155687332, -0.005778722930699587, -0.000983396777883172, -0.017843075096607208, 0.014869228936731815, -0.026994409039616585, 0.004927121568471193, -0.00042242128984071314, -0.03522655367851257, -0.0026865992695093155, 0.009509547613561153, 0.005940932780504227, 0.03601057082414627, 0.00409917626529932, -0.004572287667542696, -0.011625033803284168, 0.005359681323170662, -0.019938284531235695, -0.025615626946091652, -0.03292858228087425, -0.029414039105176926, 0.0320364311337471, 0.028116360306739807, 0.034415505826473236, -0.02001938968896866, 0.0053022317588329315, 0.007421096786856651, 0.01613987237215042, 0.031333521008491516, -0.0008000658708624542, 0.0036868928000330925, -0.03355038911104202, 0.01681574620306492, 0.010205698199570179, 0.016248011961579323, 0.01436908170580864, -0.022871578112244606, 0.011611515656113625, 0.02922479435801506, -0.01718071848154068, -0.011780484579503536, -0.00917836930602789, -0.024561263620853424, -0.016748158261179924, -0.0073670269921422005, -0.006711429450660944, 0.00010718940029619262, -0.02976549230515957, -0.015572138130664825, 0.035686150193214417, 0.001386386575177312, 0.013632379472255707, -0.015801934525370598, 0.023804284632205963, -0.013287683948874474, -0.009583894163370132, 0.0013610413298010826, 0.026980891823768616, -0.024399053305387497, 0.00630928436294198, 0.00032632044167257845, -0.015342340804636478, 0.010219215415418148, -0.006518804933875799, 0.017991766333580017, 0.0029180862475186586, 0.011712897568941116, -0.03463178500533104, 0.018424326553940773, -0.015423445962369442, -0.010050247423350811, 0.0047513945028185844, -0.03630795329809189, -0.020330291241407394, 0.007968555204570293, -0.029441073536872864, -0.020046424120664597, 0.0036226848606020212, -0.004278282634913921, -0.008543048053979874, 0.02475050650537014, -0.025453416630625725, -0.025967080146074295, -0.013186302967369556, -0.0013162647373974323, 0.030360262840986252, 0.005589478183537722, 0.041228316724300385, 0.0464460626244545, 0.019154271110892296, 0.0006674256292171776, -0.0037071690894663334, -0.0069479853846132755, -0.009955625049769878, 0.013476928696036339, 0.038389645516872406, -0.004602702334523201, -0.004342490807175636, 0.005971346981823444, -0.0016863057389855385, -0.020722297951579094, -0.030089912936091423, 0.026250947266817093, 0.0026697025168687105, 0.000591812189668417, -0.02127651497721672, 0.007272404618561268, -6.8841468419122975e-06, -0.011118127964437008, -0.023669108748435974, 0.004558770451694727, -0.02257419377565384, 0.004068761598318815, -0.016896851360797882, 0.026967374607920647, 0.004798705689609051, 0.01695092022418976, -0.00011891158646903932, -0.010320596396923065, 0.01500440388917923, -0.006745222955942154, -0.013037609867751598, -0.01365941483527422, -0.02604818530380726, 0.04436437040567398, -0.015828970819711685, -0.011043781414628029, 0.0030279157217592, 0.01768086478114128, -0.001637304900214076, 0.0007316336268559098, 0.003717307234182954, 0.030873926356434822, 0.022263290360569954, 0.016491327434778214, 0.0125036695972085, -0.008705257438123226, 0.0024821474216878414, 0.008387597277760506, -0.0008338596089743078, -0.00019938284822274, -0.007867174223065376, -0.007576548028737307, 0.029900668188929558, -0.009840725921094418, -0.001966793555766344, -0.012537463568150997, -0.021357620134949684, -0.004856154788285494, 0.019978836178779602, -0.015937110409140587, 0.012145456857979298, -0.04258006438612938, -0.018099907785654068, 0.0013052817666903138, -0.015017921105027199, 0.022019976750016212, 0.013003816828131676, 0.00016389945812989026, 0.0193705502897501, 0.020465465262532234, -0.03479399532079697, 0.0202627032995224, -0.03920069336891174, -0.029846597462892532, -0.04168790951371193, 0.01263208594173193, 0.00986776128411293, 0.0001545005798107013, 0.00752923684194684, -0.013625620864331722, -0.010658533312380314, -0.011543928645551205, 0.014504256658256054, -0.02050601877272129, 0.01430149469524622, 0.004467527382075787, 0.023155445232987404, -0.01050984114408493, -0.005336025729775429, -0.0060389344580471516, -0.005592857487499714, 0.0037984121590852737, -0.03336114436388016, -0.03011694736778736, -0.00035884688259102404, 0.0028488091193139553, 0.02600763365626335, 0.001880619558505714, 0.003370921825990081, 0.017532173544168472, -0.017505137249827385, -0.012469875626266003, -0.00047311183880083263, 0.017653830349445343, 0.0026865992695093155, -0.0012106593931093812, -0.027981186285614967, -0.016004696488380432, -0.02284454181790352, 0.031603869050741196, -0.013774313032627106, -0.004923742264509201, -0.006471493747085333, 0.04139052703976631, -0.005880104377865791, 0.01757272519171238, -0.0038795170839875937, 0.002289523370563984, 0.00871877558529377, -0.028900373727083206, -0.022587710991501808, -0.009651481173932552, -0.006403906736522913, 0.024939751252532005, -0.0012562809279188514, -0.01770789921283722, -0.03836261108517647, 0.014436669647693634, -0.036362022161483765, -0.01755920797586441, -0.029197758063673973, 0.006468114443123341, 0.020695263519883156, 0.04282337799668312, -0.010949159041047096, 0.021371137350797653, 0.012165732681751251, 0.0002682375197764486, -0.029332933947443962, -0.0030819857493042946, 0.02078988589346409, -0.007022331468760967, 0.03179311379790306, 0.0062247999012470245, -0.04220157489180565, -0.010212456807494164, 0.017843075096607208, -0.0035415799356997013, 0.002103657927364111, 0.0015697174239903688, -0.009597411379218102, -0.015572138130664825, 0.011591239832341671, 0.01563972607254982, 0.00519409216940403, -0.023182479664683342, 0.0397413931787014, -0.03549690544605255, -0.015247718431055546, 0.019302962347865105, 0.006387009751051664, -0.01808638870716095, 0.013666173443198204, -0.0015088887885212898, -0.011401995085179806, -0.003953862935304642, -0.004210695158690214, 0.0011287096422165632, 0.0021898318082094193, 0.004349249415099621, 0.01985717937350273, 0.002980604534968734, -0.029332933947443962, 0.01846487820148468, 0.0037950328551232815, 0.012807813473045826, 0.017843075096607208, -0.009712309576570988, -0.014571844600141048, -0.001878929790109396, 0.01847839541733265, -0.018262116238474846, 0.0046533928252756596, -0.031225381419062614, -0.0034030259121209383, -0.013375547714531422, -0.0003542002523317933, 0.0035855118185281754, -0.014139285311102867, -0.01588303968310356, 0.011652068234980106, 0.008171317167580128, -0.03633498772978783, 0.015153096057474613, 0.0036294437013566494, -0.0038051707670092583, -0.013936522416770458, -0.008610635064542294, -0.012909194454550743, -3.2764051866251975e-05, -0.012794295325875282, 0.005021743942052126, -0.0064444588497281075, -0.011456064879894257, -0.00806993618607521, 0.024899199604988098, -0.0024973545223474503, 0.007468407973647118, 0.21130526065826416, -0.014436669647693634, 0.00859035924077034, 0.07299439609050751, -0.0011498307576403022, 0.013328236527740955, 0.004957536235451698, 0.00042643427150323987, 0.011625033803284168, 0.028440779075026512, -0.004967674147337675, 0.007826621644198895, 0.004893328063189983, -0.013936522416770458, -0.005126504693180323, -0.016869816929101944, -0.027264758944511414, -0.01103026419878006, 7.4082134233322e-05, -0.022628262639045715, -0.013111956417560577, 0.011712897568941116, -0.006485011428594589, -0.00110505404882133, 0.022858060896396637, -0.01795121468603611, -0.001490302267484367, 2.4381628463743255e-05, 0.004308696836233139, -0.008151041343808174, -0.005221127066761255, 0.008313250727951527, 0.00682970741763711, -0.005724653135985136, -0.007508960552513599, -0.02334468998014927, 0.02976549230515957, -0.005525270476937294, 0.03452364727854729, 0.0036497197579592466, -0.00903643574565649, -0.008272698149085045, -0.007765792775899172, -0.014855711720883846, 0.012578016147017479, 0.020195117220282555, -0.002493975218385458, -0.020411396399140358, -0.006694532465189695, 0.01365941483527422, -0.010374666191637516, -0.004545252770185471, 0.010415218770503998, 0.03984953463077545, 0.00682632764801383, 0.01000969484448433, 0.030684681609272957, 0.009915072470903397, -0.02280399017035961, 0.019140752032399178, -0.007698205299675465, 0.030225086957216263, -0.016856297850608826, 0.029332933947443962, 0.006495149340480566, 0.020979130640625954, -0.015153096057474613, -0.03644312918186188, -0.015085509046912193, -0.014220389537513256, -0.01148310024291277, -0.02731882967054844, -0.007765792775899172, -0.0004549477016553283, -0.020303256809711456, -0.023615039885044098, 0.020952096208930016, 0.02062767557799816, -0.0011827795533463359, 0.04014691710472107, -0.025453416630625725, -0.007684688083827496, -0.010057006031274796, 0.00871877558529377, -0.0069040535017848015, -0.0351724848151207, 0.006961502600461245, -0.003757859580218792, 0.0014007489662617445, -0.022479571402072906, -0.03511841595172882, -0.00265956437215209, -0.018545983359217644, -0.00589362159371376, 0.002983983838930726, -0.014788123778998852, 0.025575073435902596, -0.023493381217122078, -0.0016837712610140443, 0.015558620914816856, -0.02653481438755989, 0.018910955637693405, 0.01353099849075079, -0.012206285260617733, -0.03528062626719475, -0.021763144060969353, 0.009029677137732506, 0.001165882684290409, 0.0024635607842355967, -0.01090184785425663, -0.01668057218194008, -0.00877284538000822, -0.007083159871399403, 0.003865999635308981, -0.009448719210922718, 0.007853656075894833, 0.02293916419148445, -0.03806522488594055, 0.016234494745731354, 0.03765970095992088, -0.0001788954104995355, -0.03733528032898903, -0.0037071690894663334, -0.002412870293483138, -0.008434908464550972, -0.02795414999127388, 0.0025869079399853945, -0.002140830969437957, 0.002862326567992568, -0.04963618889451027, 0.012929470278322697, -0.010631498880684376, 0.028440779075026512, -0.00833352655172348, -0.03836261108517647, -0.015585655346512794, 0.00580237852409482, 0.0030312950257211924, -0.007576548028737307, 0.0012757122749462724, 0.012578016147017479, 0.011510134674608707, 0.013963557779788971, 0.007934761233627796, 0.026845717802643776, -0.00859035924077034, 0.009901554323732853, 0.010678810067474842, -0.02835967391729355, -0.015612690709531307, -0.04460768774151802, -0.012138698250055313, 0.012219802476465702, 0.0203708428889513, 0.04385070875287056, 0.0071372296661138535, -0.023777248337864876, -0.03206346556544304, -0.014544809237122536, 0.004274903330951929, -0.017626795917749405, 0.0001439400512026623, 0.025291206315159798, -0.0021982803009450436, -0.02283102460205555, -0.002478768117725849, -0.1734563112258911, 0.017234787344932556, 0.0008828604477457702, -0.01873522810637951, 0.003947104327380657, -0.012091387063264847, 0.04831147566437721, -0.014571844600141048, -0.019059648737311363, -0.017640313133597374, 0.023006752133369446, 0.006153833121061325, -0.015274753794074059, -0.025034373626112938, -0.017748452723026276, 0.00954334158450365, -0.03154980018734932, 0.019424619153141975, 0.032874513417482376, -0.00282177422195673, 0.034172192215919495, -0.0014252493856474757, 0.002066484885290265, 0.00884043239057064, 0.0027085652109235525, 0.002696737414225936, -0.006086245644837618, 0.013963557779788971, -0.0039166901260614395, 0.009779897518455982, -0.003328679595142603, -0.02562914416193962, 0.023642074316740036, -0.023723179474473, -0.012591533362865448, 0.00954334158450365, -0.01896502636373043, 0.007468407973647118, -0.013375547714531422, 0.013064645230770111, 0.005190712865442038, 0.01532882358878851, 0.002252350328490138, 0.02011401206254959, 0.007204817142337561, 0.01691036857664585, 0.0255074854940176, 0.0036936516407877207, 0.011692620813846588, -0.017775487154722214, 0.010739638470113277, -0.0189785435795784, 0.030549507588148117, -0.0023266964126378298, 0.013416099362075329, 0.012902435846626759, -0.002531148260459304, 0.014315011911094189, 0.021060235798358917, -0.015436963178217411, -0.03373963385820389, -0.016883334144949913, 0.009637963958084583, 0.0032577128149569035, -0.0005635099951177835, -0.0015849246410652995, -0.011307372711598873, 0.015355858020484447, -0.03217160329222679, 0.0014607327757403255, -0.01678871177136898, 0.004051865078508854, -0.00208507152274251, 0.0017910662572830915, -0.00024225859669968486, 0.004447251092642546, -0.014260942116379738, -0.001532544381916523, 0.01962738297879696, -0.008705257438123226, -0.035064347088336945, 0.04433733597397804, -0.021438725292682648, 0.019843662157654762, -0.004021450411528349, 0.0047311182133853436, 0.01743755117058754, -0.0066742561757564545, 0.012983540073037148, 0.004977812524884939, 0.02553452178835869, -0.022533640265464783, -0.01155068725347519, 0.0031732285860925913, -0.0006061745225451887, 0.0013432996347546577, -0.008610635064542294, 0.013591826893389225, -0.0025717008393257856, -0.005863207392394543, -0.00025831061066128314, -0.023696143180131912, -0.016315599903464317, 0.015801934525370598, 0.0014244045596569777, -0.015869522467255592, 0.029819563031196594, 0.021100787445902824, 0.02616984210908413, -0.002840360626578331, -0.008874226361513138, -0.005011606030166149, 0.02231736108660698, 0.014544809237122536, -0.0077522750943899155, 0.010753155685961246, 0.0004541028756648302, -0.009076988324522972, 0.033523350954055786, 0.02396649308502674, 0.09429794549942017, -0.020587123930454254, 0.020316774025559425, 0.005504994187504053, 0.005842931102961302, -0.008921537548303604, -0.07818511128425598, -0.019465172663331032, 0.01882985047996044, 0.049446944147348404, -0.028765199705958366, 0.03901144862174988, 0.007265646010637283, 0.015234201215207577, -0.018897438421845436, 0.013591826893389225, 0.013368788175284863, -0.013733760453760624, 0.0017724796198308468, 0.0008258335874415934, 0.005822654813528061, 0.007556271739304066, -0.020870991051197052, -0.02319599688053131, -0.029549213126301765, 0.042093437165021896, -0.00903643574565649, -0.0034368194174021482, -0.008779603987932205, -0.020560087636113167, -0.005028503015637398, 0.010340873152017593, -0.02050601877272129, 0.016477808356285095, 0.013754037208855152, -0.01121275033801794, -0.0007823241758160293, -0.018153976649045944, 0.005474579520523548, -0.03857889026403427, -0.03471289202570915, -0.011077575385570526, -0.012706431560218334, -0.015261235646903515, 0.02050601877272129, -0.03168497607111931, 0.006123418919742107, -0.0037713772617280483, 0.0074886842630803585, -0.0012385392328724265, -0.015315305441617966, -0.00548133859410882, -0.01653187908232212, 0.040849827229976654, 0.013416099362075329, -0.017910663038492203, -0.01950572431087494, -0.020952096208930016, -0.03011694736778736, -0.010935641825199127, 0.033523350954055786, 0.0284678153693676, 0.04398588091135025, 0.008563323877751827, -0.031603869050741196, -0.04087686166167259, -0.010401701554656029, 0.008867467753589153, -0.0012959884479641914, 0.020722297951579094, -0.010043487884104252, -0.005498235113918781, -0.002985673490911722, -0.014450186863541603, 0.005268438253551722, -0.005586098879575729, -0.007380544673651457, 0.01390948798507452, -0.009468995034694672, 0.01313899178057909, -0.03546987101435661, 0.0024990441743284464, -0.014558326452970505, -0.0320364311337471, 0.014139285311102867, -0.02077636867761612, 0.006032175850123167, -0.024196291342377663, 0.005535408388823271, 0.002509182319045067, 0.03509138152003288, -0.0061673508025705814, -0.005508373491466045, -0.009313544258475304, -0.03563208132982254, -0.02717013657093048, -0.020181598141789436, 0.014734053984284401, -0.014977368526160717, -0.02193887159228325, -0.030873926356434822, 0.0035888911224901676, -0.009009401313960552, -0.038011156022548676, 0.0071034361608326435, 0.0013956798939034343, -0.008630911819636822, -0.018681159242987633, -0.06126122176647186, 0.017032025381922722, -0.009259474463760853, 0.0065661161206662655, -0.008455184288322926, -0.03633498772978783, -0.0030566402710974216, 0.009793414734303951, -0.014179836958646774, 0.03971435874700546, 0.010955918580293655, 0.01742403209209442, 0.001592528191395104, -0.0023250067606568336, -0.011848071590065956, -0.016342634335160255, 0.020560087636113167, 0.005805757828056812, 0.0021121064200997353, -0.00560637516900897, -0.02101968228816986, 0.009658239781856537, 0.03847074881196022, 0.006272111088037491, 0.013537757098674774, 0.03933586925268173, -0.021992940455675125, -0.00769144669175148, -0.019167788326740265, 0.01835673861205578, -0.0010526737896725535, -0.03103613667190075, -0.0039944155141711235, 0.002597046084702015, -0.01282808929681778, -0.001066191354766488, 0.010949159041047096, 0.009097264148294926, 0.0001360196474706754, 0.006866880226880312, -0.01642373949289322, -0.02652129717171192, -0.008556565269827843, 0.0008194972760975361, -0.004119452089071274, 0.021371137350797653, 0.008272698149085045, 0.007394061889499426, -0.009658239781856537, -0.004393181297928095, 0.024196291342377663, 0.018654122948646545, -0.008536289446055889, -0.023925941437482834, -0.0074751670472323895, -0.006613427307456732, 0.05139346048235893, -0.006457976531237364, 0.025291206315159798, -0.008705257438123226, 0.01435556448996067, 0.01076667383313179, 0.02604818530380726, -0.015572138130664825, 0.003026226069778204, 0.007056124974042177, 0.0004359387676231563, 0.008117247372865677, 0.020424913614988327, -0.031820148229599, -0.01691036857664585, 0.010841019451618195, 0.01578841730952263, 0.05742225795984268, 0.008110488764941692, 0.006927709095180035, -0.0038355852011591196, -0.00593079486861825, -0.02653481438755989, 0.03065764717757702, 0.02154686488211155, -0.0009935348061844707, -0.011293855495750904, 0.00968527514487505, -0.020749332383275032, 0.005748308729380369, -0.022601228207349777, 0.006768878549337387, -0.0015215614112094045, 0.018667640164494514, -0.0002177581627620384, 0.002928224392235279, 0.01754569076001644, -0.009022918529808521, 0.009529824368655682, -0.01101674698293209, -0.00808345340192318, -0.02242550067603588, 0.008009107783436775, 0.03065764717757702, 0.015612690709531307, -0.011848071590065956, -0.0013821624452248216, -0.001350058359093964, -0.02654833160340786, -0.012740225531160831, -0.033280037343502045, -0.01000969484448433, 0.008367320522665977, 0.004332352429628372, 0.018897438421845436, -0.009604169987142086, 0.0047176009975373745, 0.012645603157579899, -0.02565617859363556, 0.021317066624760628, 0.001858653617091477, -0.013105197809636593, -0.02756214328110218, 0.05144753307104111, -0.00954334158450365, -0.007731999270617962, 0.042769309133291245, -0.02345282956957817, 0.017748452723026276, 0.0151260606944561, 0.0016170286107808352, -0.029008513316512108, 0.011476341634988785, 0.007684688083827496, -0.01618042401969433, -0.00808345340192318, -0.0146259143948555, 0.01217924989759922, -0.01024625077843666, -0.006528943311423063, 0.012645603157579899, 0.052907418459653854, -0.024426087737083435, 0.07521126419305801, 0.004082279279828072, -0.041985295712947845, -0.01372700184583664, 0.004775050096213818, 0.02614280767738819, 0.020073458552360535, 0.022276807576417923, -0.017802521586418152, -0.008468701504170895, -0.013226854614913464, -0.015964144840836525, 0.01757272519171238, -0.02707551419734955, -0.01545048039406538, 0.01173317339271307, -0.020086975768208504, 0.012165732681751251, -0.006312663666903973, -0.021776661276817322, 0.00102479406632483, -0.011672344990074635, 0.016721123829483986, -0.03065764717757702, 0.0072250934317708015, -0.013841900043189526, 0.008110488764941692, -0.004585805349051952, 0.01090184785425663, 0.0027085652109235525, -0.003838964505121112, -0.010597704909741879, 0.012530704960227013, -0.007002055179327726, 0.0047446358948946, -0.0029924323316663504, 0.003906551748514175, 0.0036226848606020212, 0.021195409819483757, 0.009854243136942387, -0.01155068725347519, 0.00781986303627491, -0.027521591633558273, -0.02487216517329216, 0.016491327434778214, -0.004552011843770742, -0.013578309677541256, -0.017518654465675354, -0.008502495475113392]} +{"id": "test:10000084", "text": "\"The Arlington County Board plans to vote Saturday afternoon on giving Amazon $23 million and other incentives to build a headquarters campus in Crystal City, but only after hearing scores of northern Virginia residents and advocates testify for or against the project.\\nThe five-member board is expected to support the plan, which was announced amid much hoopla on Nov. 13. The proposed county incentives are part of an agreement in which Amazon would occupy significant office space and bring at least 25,000 high-paying jobs to Arlington in coming years.\\nOpponents hope to postpone the vote until after additional public hearings, where they want representatives of the online retail giant to answer questions directly from anyone in the community.\\nThe Saturday hearing was scheduled to begin no earlier than 1 p.m. and last several hours before the vote. Ninety-one people signed up in advance to speak on the topic.\\nIn the four months since Arlington won a much-publicized, nationwide contest to attract the facility known as HQ2, Arlington residents have been asking questions about its impact on their community.\\nPeople have looked at the county\u2019s five online Q&A sessions 14,000 times, and about 400 attended community events to discuss the provisions in the Amazon agreement. Board members and county staff also met with scores of civic organizations, served on multiple panels and appeared on television, online and in news articles to discuss the deal.\\nMost Arlingtonians, northern Virginians and residents of the Washington region support Amazon\u2019s arrival, several surveys have found. Business organizations, universities and nonprofit groups came out strongly for the deal.\\nBut a small, vocal group of activists has sought to block the project, saying that the county and commonwealth should not give any incentives to one of the world\u2019s most valuable companies. They also have demanded housing and job protections for existing residents.\\nThese opponents \u2014 including left-wing organizations and immigrants groups \u2014 felt empowered after Amazon canceled plans last month to build a headquarters facility in New York City, also with 25,000 jobs. The company withdrew after criticism of the plan from some elected leaders, unions and community activists.\\nIn Virginia, however, such opposition did not appear to catch fire among the broader public.\\nOfficials estimate that the Amazon project\u2019s net fiscal impact on Arlington could be worth additional revenue of $162 million over 12 years and $392.5 million over 16 years.\\nThe incentives agreement promises the world\u2019s largest online retailer cash grants estimated at about $23 million if it occupies 6.05 million square feet of office space in Crystal City and Pentagon City through 2035.\\nThe money would come from an expected increase in the hotel, motel and lodging tax paid by visitors; Amazon would get up to 15 percent of that increase, pegged to how much floor space is in active use by the company each year from 2020 to 2035.\\nAmazon\u2019s offices will be located within an already-established special tax district where a portion of the property tax revenue goes toward infrastructure improvements such as parks and wider sidewalks.\\nThe incentive agreement says that half of any new revenue from that district starting in 2021 will go specifically toward improvements around the Amazon buildings for the following 10 years. That grant is worth an estimated $28 million but the county says it\u2019s not a grant just for Amazon, because the improvements will benefit other companies in the immediate area. Amazon will have a chance to express its opinion on how the county uses the money, although the board will make the decision.\\nThe county also offered Amazon the possibility of using its fast, fiber-optic network connection, which would be the subject of a separate agreement if the company chooses to use it.\\nIt\u2019s not yet clear whether Amazon will pay the local business license tax because that tax is levied only on certain types of business, and Amazon has not yet announced which of its business units will be based in Arlington. If the company does pay the license tax, then some of its operations could be eligible for a discount of up to 72 percent under an existing program designed to attract technology companies.\\nWhile Arlington pored over the details, the Virginia General Assembly passed, and Democratic Gov. Ralph Northam signed, an incentives package worth up to $750 million for Amazon.\"", "vector_field": [0.0020527797751128674, -0.014285274781286716, -0.008748597465455532, -0.01839083433151245, -0.016111407428979874, 0.019854329526424408, -0.02425776980817318, -0.03299989178776741, -0.008586706593632698, -0.018882984295487404, 0.0051092845387756824, 0.005128711462020874, -0.02282017655670643, -0.017458342015743256, 0.003210301510989666, -0.0061939554288983345, 0.0076218354515731335, -0.01657765358686447, 0.0030824074055999517, -0.026226365938782692, -0.04126281663775444, 0.00973937101662159, 0.0009608239633962512, -0.007213869597762823, -0.007155588828027248, 0.014129859395325184, 0.01279587671160698, -0.03084997646510601, 0.01723816990852356, 0.019077252596616745, 0.010904987342655659, 0.010548827238380909, -0.0002885709691327065, 0.012860632501542568, -0.04025261476635933, 0.0053294566459953785, -0.001071719452738762, -0.0016019128961488605, 0.035072099417448044, -0.023480692878365517, 0.044008489698171616, -0.007731921039521694, 0.00179375393781811, -0.014557251706719398, -0.01780802570283413, -0.000622066727373749, 0.0096681397408247, -0.010969744063913822, -0.006883611436933279, 0.022625906392931938, 0.020657310262322426, 0.03318120911717415, -0.024801723659038544, -0.011248196475207806, -0.02875186689198017, -0.012446191161870956, 0.005070430692285299, -0.0055204881355166435, -0.012666363269090652, -0.008133411407470703, 0.026576051488518715, 0.0075376518070697784, -0.013074329122900963, 0.006563066970556974, -0.022625906392931938, -0.0047952160239219666, -0.028881380334496498, 0.028052497655153275, -0.025436338037252426, -0.003412665333598852, 0.015515647828578949, 0.049111299216747284, 0.023195764049887657, 0.016927339136600494, 0.025941437110304832, 0.0005998067208565772, -0.005112522281706333, -0.0023377081379294395, -0.008981720544397831, -0.00496682059019804, 0.016940290108323097, -0.012148311361670494, -0.0305650494992733, 0.0003057719150092453, 0.015839429572224617, 0.013611807487905025, -0.0016059601912274957, 0.005607909522950649, -0.02831152454018593, 0.01604665070772171, 0.003056504763662815, 0.013041950762271881, -0.011979945003986359, 0.025190262123942375, -0.006491835229098797, 0.01315851230174303, -0.0011801865184679627, 0.03204149752855301, 0.009344357065856457, -0.017056850716471672, 0.012737595476210117, -0.0009324930142611265, -0.04413800314068794, 0.00020044147095177323, 0.001460258150473237, -0.004626849200576544, -0.010736620984971523, 0.008081605657935143, 0.029969289898872375, -0.0063299438916146755, -0.004150888882577419, -0.018611006438732147, 0.024477941915392876, -0.018973642960190773, 0.008994672447443008, 0.001214183634147048, -0.001962120644748211, -0.012899486348032951, 0.010542351752519608, -0.005177279002964497, 0.017872782424092293, 0.025824876502156258, 0.0019491694401949644, 0.02473696693778038, 0.024853529408574104, 0.002090014750137925, -0.015101206488907337, -0.018701665103435516, -0.004892350640147924, -0.012938340194523335, -0.002577307168394327, 9.936069545801729e-05, 0.019323326647281647, -0.01675897091627121, -0.05271175876259804, 0.009214844554662704, -0.011941091157495975, 0.0200485996901989, -0.01828722469508648, -0.0027311036828905344, 0.023040348663926125, 0.011662637814879417, -0.009344357065856457, 0.0009664901299402118, -0.018222467973828316, 0.05084677040576935, -0.007919714786112309, 0.010296276770532131, 0.02373971790075302, 0.00599321024492383, 0.001230372698046267, -0.02678327076137066, 0.008547852747142315, 0.029140407219529152, 0.013404587283730507, -0.009642236866056919, -0.0315234437584877, 0.010924414731562138, -0.01306137815117836, -0.024633357301354408, 0.01096326857805252, 0.013754271902143955, -0.003976046573370695, -0.009311978705227375, 0.007660689298063517, 0.020838627591729164, -0.0006787286256439984, -0.006572780665010214, 0.006592207588255405, -0.010056678205728531, 0.006890087388455868, -0.008405388332903385, -0.038232214748859406, 0.016733068972826004, -0.004591233097016811, 0.018507396802306175, -0.002439699601382017, -0.007097308058291674, -0.0027699575293809175, -0.00754412729293108, 0.003279914613813162, 0.010218569077551365, -0.002361991908401251, 0.022185562178492546, -0.00442934175953269, -0.012860632501542568, 0.00952567532658577, 0.010820804163813591, 0.009447967633605003, -0.010037250816822052, -0.0012918913271278143, 0.003603697055950761, 0.003723496338352561, -0.014984644949436188, -0.6452851891517639, -0.001207707915455103, 0.018636908382177353, -0.009415589272975922, 0.010134385898709297, 0.0013032237766310573, 0.0026372068095952272, 0.028777770698070526, -0.023908084258437157, 0.0045200008898973465, -0.01370246708393097, 0.013961492106318474, -0.017536049708724022, -0.017846880480647087, -0.00012617390893865377, -0.01341753825545311, 0.006941892206668854, 0.009985445998609066, 0.004503811709582806, 0.022030146792531013, -0.016499945893883705, -0.0016423857305198908, -0.0017095705261453986, -0.014738569967448711, 0.027249518781900406, 0.022859029471874237, -3.966332951677032e-05, -0.0076736402697861195, -0.03924241289496422, -0.003626361722126603, -0.0006839900743216276, 0.013780174776911736, 0.0027197713498026133, 0.019413987174630165, 0.044630151242017746, 0.013624759390950203, -0.003088883124291897, 0.006818855181336403, 0.005977021064609289, 0.018559200689196587, -0.014544300734996796, -0.015502695925533772, 0.008113984018564224, -0.003066218225285411, -0.016486994922161102, -0.011300002224743366, -0.009214844554662704, 0.022509345784783363, 0.0032880092039704323, -0.017795074731111526, -0.011979945003986359, -0.0056305741891264915, -0.018559200689196587, -0.02205605059862137, 0.012588655576109886, -0.016784874722361565, 0.009376735426485538, -0.0002792622253764421, 0.006378511432558298, -0.012672838754951954, 0.005967307835817337, -0.002837951760739088, 0.00130403321236372, -0.04273926466703415, -0.020074501633644104, 0.036988887935876846, -0.0025416910648345947, -0.0130160478875041, -0.002200100803747773, 0.014906937256455421, -0.006689342204481363, 0.048204708844423294, -0.024374330416321754, -0.025319775566458702, -0.0032944849226623774, -0.004105559550225735, 0.03647083789110184, -0.01026389840990305, -0.017419487237930298, 0.010911463759839535, 0.004950631409883499, -0.011280574835836887, -0.033543847501277924, -0.0063590845093131065, 0.004050516523420811, -0.0041379379108548164, 0.008580231107771397, -0.029684361070394516, 8.069261821219698e-05, -0.024853529408574104, 0.01236848346889019, 0.018455591052770615, -0.000943015911616385, -0.030487341806292534, -0.017989344894886017, 0.015424988232553005, 0.007239772006869316, -0.003155258484184742, 0.01330097671598196, 0.0007285101455636322, -0.018546249717473984, -0.02903679572045803, -0.021369632333517075, 0.001627815538085997, 0.05177926644682884, 0.006236047018319368, -0.004934442229568958, -0.005760087165981531, 0.039138805121183395, -0.019129058346152306, -0.029140407219529152, -0.007466419599950314, -0.006818855181336403, -0.0015889615751802921, -0.0009130660910159349, -0.013767222873866558, 0.019025446847081184, 0.0007960997172631323, -0.016020748764276505, -0.010445216670632362, 0.021460290998220444, 0.013352781534194946, 0.024296622723340988, -0.036082297563552856, -0.003946905955672264, 0.005332694388926029, -0.0003169019182678312, -0.020165162160992622, -0.014984644949436188, -0.01339163538068533, -0.006485359277576208, 0.004079657141119242, 0.024244818836450577, 0.004613897763192654, 0.015645161271095276, 0.0022826651111245155, 0.03318120911717415, -0.012238970957696438, -0.01604665070772171, -0.028622355312108994, -0.019465791061520576, 0.010522924363613129, 0.009480345994234085, -0.021706365048885345, 0.00703255133703351, -0.02511255443096161, -0.0195823535323143, 0.0016739544225856662, -0.047116801142692566, -0.004312780220061541, -0.013203841634094715, -0.005650001112371683, -0.0033446711022406816, -0.0014092624187469482, -0.03955324366688728, -0.0028411897365003824, -0.002284283982589841, -0.012782924808561802, -0.014078054577112198, -0.032119203358888626, 0.001437593367882073, -0.008379485458135605, -0.009914213791489601, -0.00795209314674139, -0.009033526293933392, 0.0015752008184790611, 0.0096681397408247, 0.003210301510989666, 0.006417365279048681, -0.009706993587315083, 0.02469811402261257, -0.010004873387515545, 0.0037817771080881357, 0.025682412087917328, -0.020281722769141197, -0.006608396768569946, -0.0019070777343586087, 0.014052151702344418, -0.002141819801181555, -0.0026226365007460117, 0.00897524505853653, 0.010781950317323208, -0.0052744136191904545, 0.005747135728597641, 0.041288718581199646, 0.003396476386114955, 0.024089403450489044, 0.031497541815042496, -0.00017777671746443957, -0.009778224863111973, -0.0056564765982329845, 0.015424988232553005, 0.008327680639922619, 0.006443267688155174, 0.005773038137704134, -0.004937679972499609, 0.0056014335714280605, -0.004082894884049892, -0.008366534486413002, 0.002844427479431033, 0.004730459302663803, 0.03165295720100403, 0.009331406094133854, 0.00299012940376997, 0.008586706593632698, -0.01972481794655323, 0.006404413841664791, -0.02740493416786194, 0.005368310492485762, -0.0021531523671001196, 0.012031749822199345, -0.025941437110304832, -0.011533125303685665, -0.0035227513872087, -0.005776276346296072, 0.023532496765255928, -0.013870833441615105, 0.021032897755503654, -0.007000173442065716, 0.011507222428917885, 0.008904012851417065, 0.006349370814859867, 0.02859645150601864, 0.015269572846591473, -0.010736620984971523, -0.026679661124944687, 0.01627977378666401, 0.0034903730265796185, 0.0067281960509717464, -0.013987394981086254, -0.012232495471835136, 0.002713295631110668, -0.02191358618438244, 0.0023684673942625523, 0.02287198044359684, 0.0012781305704265833, -0.007259198930114508, -0.018921837210655212, 0.016150260344147682, 0.0167201180011034, 0.010866133496165276, 0.013365733437240124, 0.023856280371546745, -0.020955190062522888, 0.012724644504487514, -0.018403785303235054, 0.03566785901784897, 0.027353128418326378, 0.00973937101662159, -0.0067540984600782394, -0.006100058555603027, -0.006605159025639296, 0.01537318341434002, 0.020553700625896454, 0.01871461607515812, 0.007958568632602692, -0.005400688853114843, 0.011228770017623901, 0.028777770698070526, 0.024801723659038544, 0.002115917392075062, 0.014129859395325184, 0.014194616116583347, -0.012731119990348816, -0.020139258354902267, 0.006485359277576208, 0.022030146792531013, -0.012362007983028889, 0.008392437361180782, -0.036030493676662445, 0.01714751124382019, -0.009823555126786232, -0.0008284779614768922, 0.00536183500662446, -0.015865333378314972, 0.015826478600502014, -0.03968275710940361, 0.0016059601912274957, 0.0026873929891735315, 0.002985272789373994, 0.0029464189428836107, -0.03372516483068466, 0.00876154936850071, 0.013229744508862495, 0.00894934218376875, 0.002716533374041319, 0.011623783968389034, 0.0011809959542006254, 0.005174041260033846, 0.02168046310544014, -0.011312953196465969, 0.007013124413788319, -0.01771736703813076, 0.0040116626769304276, -0.01275054644793272, -0.017406536266207695, 0.02440023422241211, 0.0018763183616101742, -0.027534445747733116, -0.02425776980817318, -0.011539600789546967, -0.014893985353410244, 0.0001408452953910455, 0.007459944114089012, 0.03737742826342583, -0.006462694611400366, -0.012090031057596207, -0.025656508281826973, 0.02239278331398964, -0.034735362976789474, -0.01031570415943861, 0.017302926629781723, -0.00897524505853653, 0.016111407428979874, -0.006498310714960098, 0.004319255705922842, -0.01055530272424221, 0.002465602243319154, 0.016888484358787537, 0.008994672447443008, 0.0030095563270151615, -0.031212612986564636, -0.02898499183356762, 0.005627336446195841, 0.12267463654279709, 0.00188603182323277, -0.012912438251078129, 0.03160115331411362, -0.019737768918275833, -0.0021677224431186914, 0.0018245131941512227, -0.029969289898872375, -0.010594156570732594, 0.011164013296365738, 0.0010684815933927894, -0.01785983145236969, -0.008088082075119019, -0.01901249587535858, 0.025099603459239006, 0.023377081379294395, 0.013164987787604332, -0.02406349964439869, -0.027249518781900406, -0.014039200730621815, -0.004911777563393116, -0.021382583305239677, 0.03170476108789444, 0.025630606338381767, 0.013819028623402119, 0.0029707024805247784, 0.012018798850476742, 0.02111060544848442, -0.003116404637694359, -0.016266822814941406, 0.006556591484695673, -0.012575704604387283, 0.013883784413337708, 0.0013007953530177474, -0.0016367194475606084, -0.013041950762271881, -0.003778539365157485, 0.004396963398903608, 0.006446505431085825, 0.009706993587315083, -0.002640444552525878, 0.012219543568789959, -0.0002028698509093374, -0.03385467827320099, 0.012258397415280342, -0.00521613284945488, 0.008431291207671165, 0.02020401507616043, 0.020786823704838753, -0.0010271993232890964, 0.02712000533938408, -0.009810603223741055, 0.002182292751967907, 0.011526649817824364, -0.020618457347154617, 0.007401663344353437, 0.013262122869491577, -0.013352781534194946, 0.0023377081379294395, -0.017639659345149994, -0.02125306986272335, -0.019737768918275833, -0.0017289974493905902, -0.011267623864114285, -0.009273124858736992, -0.01713455840945244, 0.007874385453760624, 0.007026075851172209, -0.009616333991289139, -0.008489571511745453, 0.022094903513789177, -0.039993587881326675, -0.0022826651111245155, -0.01227134931832552, 0.03854304552078247, -0.02153799869120121, 0.009188941679894924, 0.015153011307120323, 0.011157537810504436, 0.013313927687704563, -0.019310375675559044, -0.015554501675069332, -0.009402638301253319, -0.013236219994723797, -0.0153990862891078, 0.005539915058761835, -0.015023498795926571, 0.014893985353410244, -0.009855933487415314, -0.002847665222361684, -0.010756047442555428, 0.020074501633644104, 0.039993587881326675, -0.029295822605490685, 0.015010546892881393, 0.027715764939785004, -0.0039307172410190105, -0.0010385316563770175, 0.005293840542435646, -0.0024963614996522665, 0.015321378596127033, -0.01026389840990305, 0.0073045287281274796, -0.0037040694151073694, 0.014803326688706875, 0.01963415928184986, 0.008651463314890862, -0.01148132048547268, -0.02779347263276577, -0.0003697189094964415, 0.009810603223741055, -0.020981092005968094, -0.017924588173627853, -0.0029108028393238783, -0.0007005839142948389, -0.007142637390643358, -0.005271175876259804, 0.015049400739371777, 0.01093736570328474, 0.0017257595900446177, 0.003007937455549836, -0.03367335721850395, 0.02038533240556717, 0.01871461607515812, -0.0006002114387229085, 0.026278171688318253, 0.003006318584084511, 0.010976219549775124, -0.012005847878754139, 0.006890087388455868, -0.03792138397693634, 0.007822580635547638, -0.0017289974493905902, -0.004484384786337614, -0.014129859395325184, -0.03825811669230461, 0.016150260344147682, 0.002145057776942849, -0.037040695548057556, 0.008929915726184845, -0.009370259940624237, 0.0029188974294811487, -0.007434041704982519, -0.02950304187834263, 0.002553023397922516, -0.019893184304237366, -0.000548406271263957, -0.004112035036087036, -0.020799774676561356, 0.015748770907521248, -0.004199456423521042, -0.0202946737408638, -0.020165162160992622, -0.00011635926057351753, 0.01810590550303459, -0.008185216225683689, 0.02560470439493656, -0.027715764939785004, 0.012854157015681267, 0.00548163428902626, 0.057141099125146866, -0.009558053687214851, -0.009778224863111973, 0.011999371461570263, 0.027586251497268677, -0.020450089126825333, 0.006443267688155174, 0.008483096025884151, -0.0039015766233205795, 0.00716206431388855, 0.019569402560591698, 0.036315422505140305, -0.00782905612140894, 0.0025384530890733004, -0.002821762813255191, 0.027016393840312958, -0.001864986028522253, -0.012841206043958664, -0.030824074521660805, -0.0027586251962929964, 0.0028832813259214163, -0.030772268772125244, -0.011772723868489265, 0.02192653715610504, -0.03463175520300865, -0.01485513150691986, 0.03952734172344208, 0.009279600344598293, 0.0434645339846611, -0.024387283250689507, 0.016875533387064934, -0.018934788182377815, 0.011992895975708961, -0.002520645270124078, -0.0007005839142948389, -0.0036555021069943905, -0.006414127070456743, -0.01914200931787491, 0.022846078500151634, 0.019375132396817207, 0.0064108893275260925, 0.012070603668689728, 0.005841032601892948, 0.0012886535841971636, 0.005368310492485762, 0.018701665103435516, -0.02320871502161026, -0.0226906631141901, -0.00377206364646554, -0.021175362169742584, -0.00033794777118600905, -0.04064115509390831, -0.0026048284489661455, -0.017030948773026466, 0.025863729417324066, -0.004053754266351461, 0.014388885349035263, 0.004723983816802502, -0.0073304311372339725, -0.034605853259563446, 0.02560470439493656, -0.03136802837252617, 0.009855933487415314, -0.006935416720807552, 0.013003096915781498, 0.007984471507370472, -0.0031779231503605843, -0.02516436018049717, 0.0010377222206443548, -0.0009292552131228149, -0.004455244168639183, 0.02487943135201931, 0.016357481479644775, -0.02893318608403206, 0.002761862939223647, 0.0169532410800457, 0.007602408062666655, -0.024128256365656853, -0.027145907282829285, -0.004024614114314318, 0.031057197600603104, 0.0012943197507411242, -0.007900288328528404, -0.011623783968389034, -0.017069801688194275, 0.009674615226686, 0.005692092701792717, -0.004319255705922842, -0.022755419835448265, -0.029062699526548386, 0.0006180194322951138, 0.011235245503485203, 0.005831318907439709, 0.011118683964014053, 0.027094103395938873, -0.010535876266658306, -0.012698741629719734, -0.005727708805352449, 0.01429822575300932, 0.032973989844322205, -0.005125473719090223, 0.01929742470383644, -0.022859029471874237, 0.03566785901784897, -0.009616333991289139, 0.025203213095664978, -0.01327507384121418, -0.006407651584595442, -0.01795049011707306, 0.02373971790075302, -0.003684642491862178, -0.011831005103886127, -0.00320058804936707, 0.024244818836450577, 0.015153011307120323, -0.020566651597619057, -0.018028197810053825, -0.012200117111206055, -0.030098801478743553, 0.002444556215777993, 0.031730666756629944, 0.018727567046880722, -0.006543640047311783, 0.006116247735917568, 0.020566651597619057, 0.007369284983724356, 0.0017047137953341007, 0.019077252596616745, 0.024995993822813034, 0.025915535166859627, -0.022949689999222755, 0.026860978454351425, -0.01207707915455103, 0.009862408973276615, 0.006851233541965485, 0.00575684942305088, -0.012886535376310349, 0.03520161285996437, -0.009901262819766998, 0.01406510267406702, 0.009687566198408604, 0.0053844996728003025, -0.036134105175733566, 0.028000693768262863, 0.0035907456185668707, -0.004649513866752386, 0.030772268772125244, -0.02999519184231758, -0.016927339136600494, -0.036419034004211426, 0.0045750439167022705, 0.019375132396817207, -0.023428887128829956, 0.003872436238452792, -0.007026075851172209, -0.012193641625344753, -0.010820804163813591, 0.027197713032364845, -0.04240252822637558, 0.018351981416344643, 0.00624252250418067, -0.00873564649373293, 0.02449089288711548, -0.020851580426096916, 0.014272323809564114, -0.005115760490298271, 0.016888484358787537, 0.01896069198846817, -0.008126935921609402, -0.006540402304381132, 0.020644359290599823, 0.014738569967448711, -0.024918286129832268, -0.01795049011707306, -0.0063040414825081825, -0.0022179086226969957, -0.010153812356293201, 0.0030014619696885347, -0.03634132444858551, -0.008845732547342777, 0.0114618930965662, 0.016875533387064934, 0.014699716120958328, 0.010626534931361675, -0.008852208033204079, -0.005604671314358711, 0.0003887411148753017, -0.02372676692903042, -0.05377376452088356, -0.019517596811056137, -0.0009235889883711934, 0.022198515012860298, -0.0056305741891264915, -0.03527931869029999, -0.009324929676949978, -0.013080804608762264, -0.028959088027477264, -0.023105105385184288, 0.0015832954086363316, -0.0057309465482831, 0.046002987772226334, 0.010114958509802818, -0.0007892193389125168, 0.01031570415943861, 0.006695817690342665, 0.029787970706820488, -0.01728997379541397, 0.007887336425483227, 0.015023498795926571, -0.014505446888506413, 0.013430489227175713, 0.01972481794655323, -0.0353311225771904, 0.003992235753685236, 0.02468516305088997, 0.03768825903534889, -0.021745219826698303, 0.025047797709703445, 0.003548654029145837, 0.0015598211903125048, 0.010484070517122746, 0.004882636945694685, -0.023908084258437157, 0.008787451311945915, 0.0169532410800457, -0.017924588173627853, 0.008120459504425526, 0.00757003016769886, 0.003571318695321679, 0.006187479477375746, 0.0016164830885827541, -0.015334329567849636, -0.011047451756894588, 0.01848149299621582, -0.003017650917172432, -0.022742468863725662, -0.012621033936738968, -0.006773525848984718, 0.00821111910045147, 0.016396336257457733, 0.0064659323543310165, 0.009046477265655994, 0.0101473368704319, -0.028881380334496498, 0.0030824074055999517, 0.01406510267406702, -0.026006193831562996, -0.00018617481691762805, 0.013113182969391346, -0.005060717463493347, 0.015243670903146267, 0.021797023713588715, 0.0012125646462664008, -0.029451237991452217, 0.013126133941113949, -0.0017921349499374628, -0.0005403116811066866, -0.043956685811281204, 0.012705217115581036, 0.004419628530740738, 0.0004901254433207214, 0.01289301086217165, -0.0057924650609493256, -0.013313927687704563, 0.005883124191313982, -0.006922465283423662, -0.0013469343539327383, 0.0008126935572363436, 0.00287842471152544, 0.013028999790549278, -0.02706819958984852, -0.03950143977999687, -0.005403926596045494, 0.001632672268897295, -0.018921837210655212, -0.004160602577030659, 0.2086712121963501, 0.024413185194134712, -0.017056850716471672, 0.039423733949661255, -0.02120126411318779, 0.013456392101943493, 0.031057197600603104, 0.022263269871473312, -0.009940116666257381, -0.005484872031956911, -0.022846078500151634, 0.00574066024273634, -0.022703614085912704, 0.0029172785580158234, 0.02201719582080841, -0.04260975122451782, -0.02015220932662487, -0.041521839797496796, -0.013430489227175713, 0.031730666756629944, 0.025514045730233192, 0.004235072527080774, -0.011183440685272217, -0.009150087833404541, 0.0167071670293808, -0.002580544911324978, -0.027638057246804237, 0.003318768460303545, -0.005135187413543463, 0.03147163987159729, -0.030098801478743553, -0.014686765149235725, 0.025423385202884674, 0.004455244168639183, -0.001913553336635232, -0.0010870990809053183, -0.011656162329018116, 0.015865333378314972, 0.003707307390868664, 0.02612275630235672, -0.005960831884294748, -0.03017650917172432, -0.037040695548057556, -0.00357455643825233, 0.0016820490127429366, 0.02712000533938408, 0.010231520980596542, 0.008865159004926682, -0.02054074965417385, 0.01752309873700142, -0.03255954757332802, -0.014596105553209782, 0.019789574667811394, 0.026446538046002388, 0.01876642182469368, -0.0019815475679934025, 0.013806076720356941, -0.006015874911099672, -0.003045172430574894, -0.012109457515180111, -0.019206766039133072, 0.04846373572945595, -0.010768999345600605, 0.02072206698358059, -0.013106707483530045, 0.001648861332796514, -0.006009399425238371, 0.031393930315971375, 0.014984644949436188, 0.011772723868489265, 0.012996621429920197, 0.014116908423602581, -0.03950143977999687, -0.008392437361180782, -0.0057309465482831, -0.03887977823615074, 0.05320390686392784, 0.018416736274957657, 0.020890433341264725, 0.007311004213988781, 0.0027035821694880724, -0.000253562000580132, 0.03413960337638855, -0.00714911287650466, 0.009059428237378597, -0.028907284140586853, 0.022574102506041527, -0.010768999345600605, -0.017017997801303864, -0.0033220064360648394, 0.02105879969894886, -0.025552898645401, -0.014168713241815567, -0.057814568281173706, 0.024724015966057777, 0.006903038360178471, -0.0022648570593446493, 0.008632035925984383, -0.030875880271196365, 0.020942239090800285, -0.009130660444498062, 0.041003789752721786, 0.0029221351724117994, 0.0015233956510201097, -0.008644986897706985, 0.004788740072399378, 0.03219691291451454, 0.020903384312987328, 0.007751347962766886, 0.014725618995726109, 0.012828254140913486, -0.0037202585954219103, 0.01096326857805252, 0.011377709917724133, 0.013754271902143955, -0.003281533485278487, 0.014388885349035263, -0.052608147263526917, 0.01537318341434002, -0.0023004731629043818, -0.00821759458631277, -0.012426764704287052, 0.00038914583274163306, 0.003275057999417186, 0.008936391212046146, -0.02727542072534561, -0.020463041961193085, 0.008632035925984383, 0.016512896865606308, -0.031963787972927094, -0.008178740739822388, 0.00156386848539114, -0.006232809275388718, -0.026109803467988968, -0.0017419487703591585, -0.000880687846802175, -0.0070455027744174, -0.010943841189146042, -0.015865333378314972, 0.008826305158436298, 0.007103783544152975, 0.003681404748931527, 0.03600459173321724, -0.005313267465680838, -0.009655187837779522, 0.004840545356273651, 0.023959890007972717, -0.018921837210655212, -0.016966192051768303, -0.009532150812447071, -0.026731466874480247, 0.008411863818764687, 0.0022923785727471113, -0.006041777785867453, 0.025798972696065903, -0.01435003150254488, -0.028725964948534966, -0.03017650917172432, 0.007667164783924818, 0.028570549562573433, -0.036315422505140305, -0.022315075621008873, 0.036030493676662445, -0.014712667092680931, -0.02392103523015976, 0.005106046795845032, -0.16246099770069122, 0.023713815957307816, 0.008392437361180782, -0.0034871352836489677, 0.02722361497581005, 0.012400861829519272, 0.005977021064609289, 0.008204643614590168, -0.0013105088146403432, -0.01652584783732891, 0.01143599022179842, 0.00663753692060709, -0.013093755580484867, -0.007466419599950314, 0.019180864095687866, -0.0038465335965156555, -0.03610820323228836, 0.006909514311701059, 0.024193013086915016, 0.00268091750331223, 0.003840058110654354, -0.03165295720100403, 0.02034647949039936, -0.00663753692060709, 0.02740493416786194, 0.002200100803747773, -0.012951292097568512, 0.01661650650203228, 0.019685963168740273, -0.02143438719213009, -0.000826859031803906, -0.006741147488355637, -0.004196218680590391, 0.003116404637694359, 0.03289628028869629, -0.011416563764214516, -0.002564355731010437, 0.03810270130634308, 0.003085645381361246, 0.0033122929744422436, 0.014790374785661697, -0.01031570415943861, -0.013210318051278591, 0.00384977157227695, -0.026420636102557182, -0.001193947158753872, 0.006430316250771284, -0.0044940984807908535, -0.021693414077162743, -0.005313267465680838, 0.011707968078553677, -0.016642410308122635, -0.0002907969756051898, 0.00612272322177887, 0.0026420634239912033, 0.022548198699951172, 0.0036393129266798496, 0.0016885246150195599, -0.01186338346451521, -0.017730318009853363, 0.01871461607515812, 0.0035130379255861044, 0.03250774368643761, -0.014622008427977562, 0.0010927652474492788, -0.02116241119801998, -0.014751520939171314, 0.011546076275408268, -0.03154934570193291, 0.0007952902815304697, 0.004827593918889761, -0.016253871843218803, 0.025177311152219772, 0.00599321024492383, -0.0019524071831256151, -0.0052193705923855305, 0.0035583674907684326, 0.007097308058291674, 0.014375933445990086, 0.007446992676705122, 0.004500573966652155, 0.010373984463512897, -0.0019216479267925024, -0.014660862274467945, -0.039475537836551666, 0.005591720342636108, -0.01609845645725727, 0.007265674881637096, 0.018805276602506638, 0.005374785978347063, 0.0038530093152076006, -0.025203213095664978, -0.02999519184231758, 0.007317479699850082, 0.004134699702262878, 0.013547051697969437, 0.03626361861824989, 0.005798941012471914, -0.019219717010855675, -0.00830825325101614, -0.008651463314890862, 0.008677365258336067, -0.016357481479644775, 0.02072206698358059, -0.007459944114089012, 0.010989171452820301, 0.002038209466263652, 0.05162384733557701, 0.013624759390950203, -0.0032621065620332956, -0.012659887783229351, -0.0009583955979906023, 0.022457540035247803, -0.00717501575127244, -0.012789400294423103, 0.009201892651617527, 0.0054395426996052265, -0.02139553427696228, -0.007395187392830849, 0.00850252341479063, 0.05139072611927986, 0.015981893986463547, -0.02048894390463829, -0.019517596811056137, -0.0202946737408638, -0.005552866496145725, -0.09687566012144089, -0.012083555571734905, 0.0011097638634964824, 0.018468542024493217, -0.024866480380296707, 0.021045848727226257, -0.014531349763274193, 0.029425334185361862, -0.03862075135111809, 0.037273816764354706, 0.009972495026886463, -0.028207913041114807, 0.007265674881637096, 0.00010775879491120577, 0.016176164150238037, 0.0022891408298164606, -0.0053553590551018715, 0.013806076720356941, -0.008327680639922619, 0.02329937368631363, 0.0028670921456068754, -0.005811891984194517, 0.023364130407571793, -0.036419034004211426, -0.0063590845093131065, -0.039320122450590134, -0.018792323768138885, 0.012530375272035599, 0.028907284140586853, -0.0004492479201871902, 0.0362895205616951, -0.019129058346152306, 0.0017354730516672134, 0.0071102590300142765, -0.0022988542914390564, -0.014142810367047787, -0.0193362794816494, -0.010348082520067692, 0.001957264030352235, -0.03830992057919502, 0.00127165496814996, -0.0038562470581382513, 0.006721720565110445, -0.024801723659038544, -0.0076218354515731335, -0.00228104623965919, -0.019504645839333534, -0.0035939833614975214, -0.0014756377786397934, -0.005316505208611488, 0.007375760469585657, 0.003198969177901745, -0.007718970067799091, -0.011312953196465969, -0.005151376128196716, -0.009778224863111973, 0.019413987174630165, 0.0033932384103536606, -0.025850778445601463, 0.0076218354515731335, -0.017730318009853363, 0.029218114912509918, 0.001515301177278161, -0.01043226569890976, 0.007369284983724356, 0.011895761825144291, 0.0009211606229655445, 0.008994672447443008, -0.008107508532702923, -0.027871180325746536, -0.017199315130710602, 0.008703268133103848, -0.002768338657915592, -0.007803153246641159, -0.02253524772822857, -0.005144900642335415, -0.0353570282459259, -0.006151863839477301, 0.0059996857307851315, 0.013547051697969437, -0.028777770698070526, -0.015243670903146267, 0.01738063432276249, 0.0024850291665643454, 0.012821778655052185, -0.025954388082027435, -0.001621339819394052, -0.017833929508924484, 0.011805102229118347, -0.030435536056756973, -0.005041290540248156, 0.0284151341766119, 0.02278132177889347, 0.013637710362672806, -0.023765619844198227, 0.019452840089797974, 0.00507366843521595, 0.017160462215542793, -0.0038141554687172174, 0.0007159636006690562, -0.04260975122451782, -0.0033543845638632774, -0.06216619908809662, 0.004646276123821735, -0.013495245948433876, -0.03243003413081169, -0.014272323809564114, -0.00666343979537487, 0.007298052776604891, 0.002750530606135726, -0.013210318051278591, -0.0023538973182439804, -0.024995993822813034, 0.0021644847001880407, -0.014906937256455421, 0.004345158580690622, -0.023856280371546745, -0.007123210467398167, 0.017082754522562027, -0.007485846523195505, 0.012763498350977898, -0.002888138173148036, 0.0222114659845829, 0.01131942868232727, 0.022612955421209335, 0.014013297855854034, -0.013430489227175713, 0.018882984295487404, -0.008470145054161549, 0.016733068972826004, -0.006922465283423662, -0.02368791215121746, -0.013171464204788208, -0.015515647828578949, -0.0260191448032856, 0.004649513866752386, -0.016409287229180336, -0.004662464838474989, -0.016111407428979874, 0.030590951442718506, -0.0017694702837616205, -0.007854958064854145, -0.025721265003085136, -0.009914213791489601, 0.006190717685967684, -0.017160462215542793, -0.01972481794655323, -0.011261148378252983, 0.016797825694084167, -0.0035065622068941593, 0.002494742628186941, 0.009661663323640823, 0.0018520347075536847, 0.00624252250418067, -0.0036619778256863356, -0.0231569092720747, -0.019854329526424408, -0.019944990053772926, 0.019219717010855675, -0.0006439220160245895, -0.007013124413788319, -0.006077393889427185, 0.055276114493608475, 0.017445389181375504, 0.005692092701792717, -0.007090832106769085, 0.017820976674556732, 0.008755072951316833, -0.016266822814941406, -0.0006888468051329255, 0.016461091116070747, -0.03499438986182213, -0.019465791061520576, -0.04279106855392456, 0.02831152454018593, 0.003160115098580718, -0.0198154766112566, 0.014026248827576637, 0.012731119990348816, -0.02377857267856598, -0.022949689999222755, 0.017406536266207695, -0.0023765619844198227, -0.02230212464928627, 0.026045048609375954, 0.00717501575127244, 0.004183267243206501, 0.02039828523993492, -0.0047919778153300285, -0.008496046997606754, 0.004532952327281237, 0.0284151341766119, 0.0037429232615977526, -0.0037429232615977526, 0.006365559995174408, -0.014622008427977562, 0.02397284097969532, 0.0070455027744174, 0.014272323809564114, 0.006336419377475977, 0.016124358400702477, 0.01072367001324892, 0.03162705525755882, 0.012297251261770725, 0.0072203450836241245, -0.0016642409609630704, -0.017639659345149994, -0.010710718110203743, -0.030306022614240646, -0.020191064104437828, -0.010451692156493664, 0.029762068763375282, 0.005446018185466528, 0.023895133286714554, 0.03413960337638855, 0.015865333378314972, -0.024944188073277473, 0.0226906631141901, 0.008683840744197369, -0.014634959399700165, -0.010335130617022514, 0.01435003150254488, 0.014272323809564114, -0.002860616659745574, 0.00391776580363512, -0.012983670458197594, 0.04260975122451782, 0.018028197810053825, 0.007628310937434435, 0.008288826793432236, 0.005539915058761835, 0.010684816166758537, -0.012316678650677204, 0.00639793835580349, -0.027042297646403313, -0.017976393923163414, -0.006041777785867453, 0.009363783523440361, -0.0015954372938722372, -0.008172265253961086, 0.006197193171828985, 0.049214910715818405, 0.03986407443881035, 0.004196218680590391, 0.006462694611400366, 0.002417034935206175, 0.02768986113369465, 0.005254986695945263, 0.013844930566847324, 0.004620373249053955, -0.029684361070394516, 0.029062699526548386, -0.0006423031445592642, 0.021473241969943047, -0.0260191448032856, -0.0205277968198061, 0.013611807487905025, 0.004332207143306732, 0.04014900326728821, -0.015645161271095276, 0.007939142175018787, 0.005559341982007027, -0.012990145944058895, 0.01613730937242508, 0.020605504512786865, -0.035305220633745193, -0.004095845855772495, 0.027327226474881172, 0.006171290762722492, 0.002389513421803713, -0.014129859395325184, 0.0044099148362874985, -0.013754271902143955, -0.026278171688318253, -0.004999198485165834, 0.004830831661820412, -0.013857882469892502, -0.02024286985397339, -0.008923440240323544, 0.01986728236079216, -0.013935590162873268, -0.005297078285366297, 0.010736620984971523, -0.008916964754462242, -0.024711064994335175, 0.028881380334496498, 0.0260191448032856, -0.0010401506442576647, -0.057089295238256454, 0.003171447664499283]} +{"id": "test:10000085", "text": "\"To find out more information about Dr. Leigh Ricketts, please review the contact information below in order speak with Dr. Leigh Ricketts.\\nDr. Leigh Ricketts is a chiropractic practice that delivers chiropractic services to patients in Knoxville, TN.\\nAre you Dr. Leigh Ricketts in Knoxville, TN?\"", "vector_field": [-0.002312218304723501, 0.028374429792165756, 3.3183021059812745e-06, -0.05509195849299431, -0.0008778015035204589, 0.03795338794589043, -0.009281233884394169, -0.013294041156768799, -0.008336283266544342, -0.010387992486357689, 0.008465728722512722, 0.009928461164236069, -0.0071518574841320515, 0.00019912341667804867, 0.006562881171703339, -0.008213309571146965, 0.018433023244142532, -0.013630599714815617, 0.01927441731095314, -0.02353317104279995, -0.0018009089399129152, -0.0022863291669636965, 0.015145109966397285, -0.00731366453692317, -0.01126174908131361, -0.016012394800782204, 0.00017040272359736264, -0.021526768803596497, -0.015986505895853043, -0.018420077860355377, -0.00010274727537762374, 0.0038898338098078966, -0.005944779608398676, -0.030134886503219604, -0.025526629760861397, 0.0034464835189282894, -0.005876820534467697, -0.009462458081543446, 0.021915104240179062, -0.0022183703258633614, -0.0011358832707628608, -0.018044687807559967, 0.0008858918445184827, -0.020064035430550575, -0.003773333039134741, 0.0030387304723262787, 0.005446414463222027, -0.03262023627758026, -0.0002303723304066807, 0.027105864137411118, 0.01372121088206768, 0.0005432661273516715, -0.023727338761091232, 0.0018996110884472728, 0.015054498799145222, -0.020128758624196053, 0.005941543262451887, 0.006465797312557697, 0.007488415576517582, -0.032775573432445526, -0.013863600790500641, -0.01722918078303337, -0.01086046826094389, -0.02499590441584587, 0.0076955282129347324, -0.0027037905529141426, -0.02767542377114296, 0.0025516923051327467, -0.023261336609721184, 0.030393777415156364, 0.02609618939459324, 0.024012118577957153, 0.008187420666217804, -0.010465659201145172, 0.008750508539378643, -0.006074224598705769, -0.017099736258387566, 0.017449237406253815, -0.001747512724250555, 0.014044824056327343, -0.007520777173340321, -0.0043655456975102425, -0.0031196337658911943, 0.017475128173828125, 0.023429615423083305, -0.0032555514480918646, -0.003779805265367031, 0.014446104876697063, -0.020530037581920624, 0.012983372434973717, 0.0007172083714976907, -0.006685854401439428, -0.0005489293253049254, -0.0016601370880380273, -0.032697904855012894, -0.00010203936835750937, -0.02396034076809883, 0.024866458028554916, -0.019287362694740295, -0.05612751841545105, -0.0069318003952503204, 0.022109271958470345, -0.003407649928703904, -0.015572279691696167, -0.03549392521381378, -0.011747169308364391, 0.016646677628159523, -0.007611388806253672, 0.029435880482196808, -0.008012669160962105, -0.018886081874370575, 0.008388061076402664, 0.009876683354377747, 0.0043655456975102425, 0.006540228147059679, -0.018510689958930016, 0.029927773401141167, -0.0002497891546227038, -0.03722849488258362, -0.03655537590384483, 0.027261199429631233, -0.01955919712781906, -0.006627603899687529, -0.029073433950543404, 0.019714532420039177, 0.03161056339740753, -0.0221222173422575, 0.013785934075713158, 0.005003064405173063, 0.017850518226623535, -0.028374429792165756, -0.0022830930538475513, 0.019895756617188454, 0.02273060940206051, 0.005731194745749235, 0.02480173669755459, -0.012497952207922935, 0.02163032442331314, -0.014873274601995945, -0.017915241420269012, 0.005656763445585966, 0.010938134975731373, 0.007947946898639202, -0.0029853342566639185, 0.011209970340132713, 0.0271835308521986, 0.018536578863859177, -6.543059862451628e-05, -0.0061130584217607975, 0.003397941356524825, -0.0027474784292280674, -0.013345818966627121, 0.01599944941699505, 0.012193755246698856, -0.019727477803826332, -0.002875305712223053, 0.012355562299489975, 0.0049448139034211636, -0.005540262442082167, -0.039429064840078354, -0.005572624038904905, -0.004164905287325382, 0.027727201581001282, 0.007132440805435181, 0.020530037581920624, 0.017151514068245888, 0.0057085417211055756, 0.008485144935548306, -0.008530450984835625, 0.02197982743382454, 0.010006128810346127, 0.04649678245186806, -0.01616772823035717, 0.008744035847485065, 0.011455916799604893, -0.003142286790534854, -0.025099460035562515, 0.02692463994026184, -0.018976693972945213, 0.010232658125460148, 0.004692395217716694, 0.010653355158865452, 0.03702137991786003, 0.026342136785387993, 0.008756980299949646, -0.0010282817529514432, 0.019598031416535378, -0.0271835308521986, 0.019494475796818733, -0.003757152473554015, 0.03463958576321602, 0.01680201105773449, -0.007986780256032944, -0.009947878308594227, -0.6412206888198853, -0.030704446136951447, 0.014925053343176842, 0.004491754807531834, 0.026510415598750114, 0.03492436558008194, 0.0220057163387537, -0.014653217978775501, -0.004689159337431192, 0.039972733706235886, 0.020698318257927895, 0.005109856836497784, 0.009844321757555008, -0.013268152251839638, -0.025215961039066315, -0.018044687807559967, -0.01694440096616745, -0.0019465349614620209, 0.016879677772521973, -0.0015549627132713795, -0.0147697189822793, 0.015235722064971924, 0.008465728722512722, -0.00368919363245368, 0.005223121494054794, 0.016866734251379967, 0.006090405397117138, 0.0029141393024474382, 0.0289439894258976, 0.013656488619744778, -0.051363930106163025, 0.022161049768328667, 0.026846973225474358, -0.0050257169641554356, 0.04113774374127388, -0.006239267531782389, -0.010290908627212048, 0.025177128612995148, 0.014161325059831142, 0.026303302496671677, -0.03381113335490227, -0.03080800175666809, 0.03567514941096306, 0.0068670776672661304, -0.016478396952152252, -0.004996591713279486, -0.0036956658586859703, -0.012271422892808914, -0.011779529973864555, 0.009378318674862385, 0.012174339033663273, 0.007223052438348532, -0.007941474206745625, 0.003045202698558569, 0.0014190450310707092, -0.0274165328592062, 0.00977959856390953, -0.010983441025018692, -0.0006071797688491642, 0.0026876097545027733, -0.03409591317176819, 0.025500740855932236, 0.00034545737435109913, -0.003059765323996544, -0.027571868151426315, 0.01800585351884365, -0.005368747282773256, -0.005867111962288618, 0.043804317712783813, -0.02056887187063694, 0.014446104876697063, 0.046652115881443024, -0.007960891351103783, 0.031222227960824966, 0.008977037854492664, 0.02056887187063694, 0.021384378895163536, -0.0007669639308005571, 0.011378249153494835, -0.002893104450777173, -0.019714532420039177, -0.021397322416305542, 0.0010258547263219953, -0.0029465006664395332, 0.015158054418861866, -0.02543601766228676, -0.008148587308824062, -0.0038607085589319468, 0.03885950520634651, -0.00710655190050602, 0.021733880043029785, -0.005362275056540966, -0.008472200483083725, -0.022976556792855263, -0.005834750831127167, -0.0029189935885369778, -0.007488415576517582, 0.0002352265437366441, 0.00662436755374074, -0.005080731585621834, -0.004841257352381945, -0.01324873510748148, 0.027312977239489555, -0.0017863463144749403, -0.01369532197713852, -0.0031406686175614595, -0.029176989570260048, -0.005080731585621834, 0.020879540592432022, -0.002558164531365037, -0.03311213105916977, -0.030600888654589653, -0.006375185213983059, 0.007650222163647413, 0.02974654920399189, -0.027571868151426315, 0.028063759207725525, 0.0037959860637784004, 0.0009231074363924563, -0.03339691087603569, -0.0008802286465652287, -0.0004927014815621078, 0.022562330588698387, -0.004786243196576834, -0.0027685132808983326, -0.006750577129423618, 0.003022549906745553, 0.0015703343087807298, -0.014925053343176842, -0.01344937551766634, -0.006957689765840769, -0.03513147681951523, 0.021824492141604424, -0.03192123398184776, 0.02249760739505291, 0.014821496792137623, 0.03958439826965332, -0.030471444129943848, -0.00031026441138237715, -0.02487940341234207, -0.0014109547482803464, -0.017617518082261086, -0.0010994767071679235, -0.016581954434514046, -0.005617929622530937, 0.007268358487635851, 0.0013316693948581815, -0.006226323079317808, -0.015598169527947903, 0.0025160948280245066, 0.006614659447222948, 0.0033170380629599094, -0.01438138261437416, 0.02873687632381916, -0.014937997795641422, -0.021966882050037384, -0.013268152251839638, -0.012064309790730476, 0.0027151170652359724, -0.022484663873910904, 0.0036633044946938753, 0.028348539024591446, -0.019196750596165657, -0.010970496572554111, -0.0012167866807430983, -0.016504287719726562, 0.007223052438348532, 0.00725541403517127, 0.005805625580251217, -0.015080387704074383, -0.008640479296445847, -0.00648844987154007, -0.026743417605757713, 0.0008591937366873026, 0.02661397121846676, -0.025940855965018272, 0.008543395437300205, 0.011611251160502434, 0.002590525895357132, -0.012459118850529194, 0.0014319895999506116, -0.006261920556426048, 0.011319998651742935, 0.016361897811293602, 0.002893104450777173, 0.004621200263500214, 0.019610976800322533, 0.03106689266860485, 0.008394533768296242, 0.0018640136113390326, -0.03784983232617378, 0.016413675621151924, -0.021410267800092697, 0.027753090485930443, 0.02577257715165615, -0.00046276726061478257, 0.012400868348777294, 0.0137082664296031, 0.006511102896183729, 0.01654312014579773, 0.036581266671419144, 0.015844115987420082, 0.023779118433594704, -0.02131965570151806, 0.003692429745569825, 0.005702069494873285, 0.012329673394560814, -0.03337102010846138, 0.01631011813879013, 1.12316629383713e-05, 0.010232658125460148, -0.0065369922667741776, -0.001007246901281178, 0.027468310669064522, -0.001682789996266365, -0.007818501442670822, -0.00045063174911774695, 0.021526768803596497, -0.013475264422595501, -0.0031018350273370743, 0.009184150025248528, -0.008349227719008923, 0.031973011791706085, 3.8150992622831836e-05, -0.011675974354147911, 0.0050645507872104645, 0.0011415465269237757, 0.023623783141374588, 0.011650084517896175, 0.01094460766762495, -0.002093779155984521, 0.014148380607366562, 0.034510139375925064, 0.030859779566526413, 0.026769306510686874, 0.002760422881692648, -0.009449513629078865, -0.03135167434811592, 0.02614796906709671, -0.002969153458252549, 0.03593403846025467, -0.001729713985696435, 0.025164183229207993, 0.0042619891464710236, -0.0058282786048948765, 0.00724894180893898, 0.002032292541116476, -0.007410748396068811, -0.005905945785343647, 0.013682377524673939, -0.028555652126669884, 0.009870210662484169, -0.014692051336169243, 0.005960959941148758, 0.027727201581001282, -0.013669433072209358, -0.00698357867076993, -0.017073847353458405, 0.03233545646071434, -0.005391400307416916, 0.017047956585884094, 0.006585534196346998, 0.013915379531681538, -0.012866871431469917, 0.03233545646071434, -0.009734293445944786, 0.002630977425724268, -0.004093710333108902, -0.016374841332435608, -0.005404344759881496, -0.010938134975731373, -0.02339078113436699, 0.03495025634765625, -0.007475471124053001, -0.0008094381773844361, -0.0022572039160877466, -0.001970805926248431, -0.027105864137411118, -0.006543464493006468, 0.014018935151398182, -0.018200021237134933, -0.020439427345991135, 0.016750233247876167, 0.0005242538172751665, 0.007540193852037191, -0.012931594625115395, 0.0004878473118878901, 0.011572417803108692, -0.020698318257927895, 0.016103006899356842, 0.04509877413511276, -0.010161463171243668, 0.010666299611330032, -0.001468396163545549, 0.017190346494317055, 0.0059836129657924175, 0.028089648112654686, -0.037357937544584274, 0.014290770515799522, -0.02327428013086319, -0.002378558972850442, 0.004753882065415382, -0.0012111234245821834, -0.010711605660617352, 0.02252349816262722, 0.01918380707502365, 0.008588701486587524, -0.014976831153035164, -0.002263676142320037, -0.021966882050037384, 0.011824836023151875, -0.012485007755458355, -0.01866602525115013, -0.03362991288304329, 0.012679175473749638, 0.015844115987420082, 0.02148793451488018, -0.019287362694740295, 0.0028445622883737087, -0.012951010838150978, -0.007540193852037191, -0.02485351450741291, -0.0204782597720623, -0.030600888654589653, 0.08885131031274796, 0.009443040937185287, 0.0032506973948329687, 0.031222227960824966, 0.009947878308594227, -0.005223121494054794, 0.02428395487368107, -0.041759081184864044, 0.015170999802649021, -0.02315777912735939, 0.017280958592891693, 0.00673115998506546, -0.018769580870866776, 0.035027921199798584, -0.008226254023611546, 0.0028494165744632483, 0.008465728722512722, 0.029332324862480164, 0.02226460725069046, -0.00731366453692317, 0.006281337235122919, 0.006469033192843199, 0.008401005528867245, 0.025824354961514473, 0.009112955071032047, 0.006032154895365238, 0.021241987124085426, 0.00031329828198067844, 0.012387923896312714, -0.02951354905962944, 0.00798030849546194, 0.015468724071979523, 0.0005007918225601315, -0.0072813029401004314, -0.026419803500175476, 0.0148862199857831, -0.01596061699092388, -0.034794919192790985, 0.018251799046993256, 0.0012184047373011708, 0.006132475100457668, -0.0017345681553706527, -0.0026374496519565582, -0.014549661427736282, 0.021397322416305542, -0.03748738393187523, 0.006077460944652557, -0.012342617847025394, -0.010122628882527351, -0.032956793904304504, 0.02821909449994564, 0.0194168072193861, 0.010303853079676628, -0.01570172607898712, -0.0046503255143761635, -0.002821909496560693, -0.005533790215849876, 0.0015112749533727765, -0.0012200227938592434, 0.007358970120549202, -0.011999587528407574, -0.019170861691236496, -0.0003486935165710747, -2.2046167941880412e-05, -0.006223087199032307, -0.005524082109332085, -0.023999175056815147, 0.026303302496671677, -0.021241987124085426, -0.0020630359649658203, 0.013022205792367458, 0.0009384790319018066, 0.0020242021419107914, -0.00744958221912384, 0.016698455438017845, 0.005316969472914934, 0.017501017078757286, -0.0012855544919148088, 0.01048507634550333, 0.022627053782343864, 0.0012912177480757236, -0.0053590391762554646, 0.007844390347599983, -0.0031406686175614595, -0.03544214740395546, 0.041293077170848846, 0.02416745387017727, -0.028141427785158157, 9.253322787117213e-06, 0.013552932068705559, 0.0144719947129488, 0.006540228147059679, -0.0034011774696409702, -0.019598031416535378, 0.0028332360088825226, 0.01884724758565426, 0.029047545045614243, 0.013041622936725616, -0.007889696396887302, -0.01324873510748148, 0.004721520468592644, -0.009378318674862385, 0.015158054418861866, 0.006970634218305349, -0.0007726271287538111, -0.006892967037856579, 0.015248666517436504, 0.029642993584275246, -0.007889696396887302, -0.00671821553260088, 0.02565607614815235, -0.019144972786307335, 0.028141427785158157, -0.00790911354124546, 0.006003029644489288, 0.0012596654705703259, 0.026316247880458832, 0.003033876186236739, 0.020879540592432022, -0.011527111753821373, 0.0065920064225792885, -0.013067511841654778, 0.0026002342347055674, 0.014963886700570583, -0.0017539849504828453, -0.0013818294974043965, 0.010433298535645008, -0.0271835308521986, -0.027494199573993683, -0.030419666320085526, 0.012743898667395115, 0.0194168072193861, -0.008964093402028084, -0.02687286213040352, -0.033215686678886414, 0.011811891570687294, -0.039998624473810196, 0.017449237406253815, 0.00671821553260088, -0.009578959085047245, 0.0035079699009656906, -0.0052198851481080055, 0.0016342480666935444, 0.006815299857407808, 0.005261954851448536, -0.0204782597720623, -0.01022618543356657, 0.004905980080366135, 0.01449788361787796, 0.003066237550228834, -0.005795917008072138, 0.005873584654182196, -0.017151514068245888, 0.0011819981737062335, -0.0007843581261113286, -0.02186332643032074, -0.0029853342566639185, 0.012523841112852097, 0.020814817398786545, -0.023753227666020393, 0.01094460766762495, -0.0014853858156129718, -0.014290770515799522, -0.029047545045614243, 0.004941577557474375, 0.008051503449678421, -0.004854201804846525, 0.0007301528821699321, -0.04139663651585579, 0.014834441244602203, 0.0068023549392819405, -0.008711674250662327, 0.026251524686813354, 0.008226254023611546, -0.023377837613224983, 0.034303028136491776, -0.01372121088206768, -0.01680201105773449, -0.0024481359869241714, -0.03751327469944954, -0.02569490857422352, -0.009248873218894005, -0.0030613834969699383, 0.021164320409297943, -0.015572279691696167, 0.008122698403894901, 0.04030929505825043, -0.003040348645299673, -0.003611526219174266, -0.011197025887668133, 0.017669295892119408, -0.00353062292560935, 0.003747443901374936, -0.013889489695429802, 0.0016746997134760022, -0.01502860989421606, -0.0017216237029060721, -0.032775573432445526, 0.016737287864089012, -0.007591972127556801, 0.019831033423542976, -0.003253933507949114, 0.030704446136951447, 0.002050091279670596, -0.012057838030159473, -0.025940855965018272, -0.004899507854133844, -0.016387786716222763, 0.0038833615835756063, -0.0205947607755661, -0.026562193408608437, 0.010892828926444054, -0.022678831592202187, 0.00711302412673831, -0.021500878036022186, -0.0017378042684867978, -0.012944539077579975, 0.01873074844479561, -0.02770131267607212, -0.01425193715840578, 0.0020630359649658203, -0.024232177063822746, 0.010439770296216011, -0.01344937551766634, 0.011688918806612492, 0.005151926539838314, 0.015429889783263206, -0.012951010838150978, 0.02972066029906273, -0.006944744847714901, -0.012763314880430698, 0.015714669600129128, 0.01106110867112875, -0.02002520114183426, -0.02603146806359291, -0.005614693742245436, 0.017332738265395164, -0.03005721978843212, -0.0435454286634922, -0.0030468208715319633, 0.025902021676301956, -0.0076308054849505424, -0.01835535652935505, -0.008614590391516685, -0.023144835606217384, 0.014989775605499744, 0.0019805144984275103, -0.011837780475616455, -0.027261199429631233, -0.015546390786767006, -0.021526768803596497, 0.014925053343176842, -0.04349365085363388, 0.0449434369802475, -0.0004449685220606625, 0.02350728213787079, -0.03707315772771835, -0.020038146525621414, -0.00785086303949356, 0.011319998651742935, 0.007993252947926521, 0.008232726715505123, 0.003938375972211361, 0.01434254925698042, -0.010433298535645008, 0.0031325784511864185, 0.014264881610870361, 0.0009077357826754451, -0.0012135504512116313, 0.02821909449994564, -0.013371708802878857, -0.006229559425264597, -0.016504287719726562, -0.005802389699965715, -0.015921782702207565, -0.0016010776162147522, -0.0020064034033566713, 0.007210107985883951, -0.0020711261313408613, 0.006520811468362808, 0.042302753776311874, 0.01840713433921337, -0.022432886064052582, -0.011112886480987072, 6.209333514561877e-05, -0.022225772961974144, -0.018277687951922417, -0.009158261120319366, 0.016245396807789803, -0.00362447090446949, -0.020089924335479736, -0.008394533768296242, 0.024348678067326546, -0.024516956880688667, 0.01331993006169796, -0.019740421324968338, 0.005158398766070604, 0.01291864924132824, -0.02715764194726944, -0.009468929842114449, -0.024724068120121956, 0.007158329710364342, -0.029306435957551003, -0.018743691965937614, 0.013158123940229416, -0.041784971952438354, 0.0195333082228899, -0.00196271575987339, -0.04162963479757309, -0.04605666920542717, 0.01434254925698042, -0.005601749289780855, -0.006970634218305349, -0.0230671688914299, 0.02065948396921158, 0.012239061295986176, 0.014588494785130024, -0.007093607448041439, -0.03748738393187523, 0.02534540742635727, -0.017876407131552696, -0.012743898667395115, 0.01962392032146454, -0.004666506312787533, 0.022691776975989342, 0.0024093021638691425, 0.020607706159353256, 0.009242400527000427, 0.002331634983420372, -0.03544214740395546, -0.010698661208152771, -0.002863979199901223, -0.014523772522807121, -0.011928392574191093, 0.006210142280906439, -0.008989982306957245, -0.02002520114183426, 0.029099322855472565, -0.01032974198460579, -0.005032189656049013, -0.0033202741760760546, 0.007579027209430933, -0.01313223410397768, 0.036089375615119934, 0.019960477948188782, 0.00032179313711822033, -0.007475471124053001, -0.014976831153035164, -0.034303028136491776, -0.008582228794693947, 0.0015048026107251644, 0.01602533832192421, 0.027235308662056923, -0.008653423748910427, -0.030937448143959045, 0.005003064405173063, -0.05172637850046158, 0.00012398441322147846, -0.02569490857422352, 0.02033586986362934, 0.00783791858702898, 0.018510689958930016, 0.011941337026655674, 0.000685656035784632, -0.009410679340362549, 0.05457417666912079, -0.004342892672866583, -0.025539575144648552, 0.04131896793842316, 0.0072359973564744, 0.004692395217716694, 0.022885944694280624, -0.018342411145567894, 0.007591972127556801, 0.005151926539838314, -0.001538782031275332, 0.005954487714916468, 0.012103144079446793, -0.04608255624771118, -0.030937448143959045, 0.0015452542575076222, 0.018575413152575493, -0.025215961039066315, 0.014070713892579079, 0.0147697189822793, -0.00042797881178557873, 0.0021665922831743956, 0.0009643681114539504, 0.007708472665399313, -0.006342824082821608, 0.019636865705251694, -0.0028348539490252733, 0.008653423748910427, 0.028115538880228996, 0.00751430494710803, -0.006899439264088869, -0.000428787840064615, 0.01694440096616745, 0.016090061515569687, 0.008206837810575962, 0.01120349857956171, 0.0141224917024374, -0.011423555202782154, -0.010362103581428528, 0.021643267944455147, 0.010776328854262829, -0.021953938528895378, -0.006032154895365238, 0.008498089388012886, -0.007546666078269482, 0.00343030272051692, 0.006647020578384399, -0.01720329187810421, -0.005909182131290436, -0.011248803697526455, -0.009889627806842327, 0.021953938528895378, -0.017591627314686775, 0.015041554346680641, 0.0028736875392496586, -0.008368643932044506, -0.018536578863859177, -0.013028678484261036, -0.021306710317730904, -0.026484526693820953, -0.016931457445025444, 0.017022067680954933, -0.014031879603862762, 0.0030791822355240583, 0.009993183426558971, -0.0056341104209423065, 0.0005622783792205155, 0.021940993145108223, -0.004009570926427841, -0.023028334602713585, 0.01462732907384634, 0.20007078349590302, -0.009145316667854786, 0.009093538857996464, 0.02339078113436699, -0.014743829146027565, 0.007468998897820711, 0.03023844212293625, -0.0005715822917409241, -0.024918237701058388, 0.007734362035989761, 0.02275649830698967, -0.018031742423772812, 0.004770062398165464, 0.005931834690272808, -0.01628422923386097, -0.018717803061008453, -0.02537129633128643, -0.01797996461391449, 0.008083864115178585, -0.027312977239489555, 0.0009943023324012756, 0.02798609249293804, 0.03280146047472954, 0.005080731585621834, 0.007669639308005571, 0.019261473789811134, -0.02522890642285347, 0.0338888019323349, 0.032775573432445526, -0.011747169308364391, -0.008989982306957245, 0.004685922991484404, -0.008970565162599087, -0.017967019230127335, 0.0009109718957915902, 6.21944636804983e-05, 0.01927441731095314, 0.008161531761288643, 0.046729784458875656, 0.0030468208715319633, 0.02249760739505291, 0.0008907460724003613, -0.01100285816937685, -0.019598031416535378, -0.004747409373521805, 0.008905842900276184, 0.011307054199278355, -0.017138568684458733, 0.005116329062730074, 0.02534540742635727, -0.035856373608112335, -0.03282735124230385, 0.011980170384049416, 0.016646677628159523, -0.012031949125230312, -0.0021309945732355118, 0.03293090686202049, 0.009805488400161266, -0.015222777612507343, 0.0022151342127472162, -0.01720329187810421, 0.023779118433594704, 0.01140413898974657, -0.004482046701014042, -0.017915241420269012, 0.029409991577267647, -0.0037668608129024506, -0.0003474799741525203, 0.012362034060060978, -0.005122801288962364, 0.028400318697094917, -0.03003132902085781, 0.008750508539378643, 0.002673047361895442, -0.027778979390859604, -0.04266519844532013, 0.012737425975501537, 0.021656213328242302, 0.024322789162397385, 0.032982684671878815, -0.011773058213293552, 0.007494887802749872, 0.014044824056327343, 0.012284367345273495, -0.03184356540441513, -0.017539849504828453, 0.01901552826166153, 0.01788935251533985, -0.010510965250432491, -0.0038574724458158016, 0.011559473350644112, 0.01502860989421606, -0.025850243866443634, -0.004197266884148121, 0.017992908135056496, 0.024516956880688667, 0.00698357867076993, -0.001336523680947721, -0.026588082313537598, 0.010834579356014729, -0.029358213767409325, 0.06280690431594849, -0.013876545242965221, 0.013811822980642319, -0.023377837613224983, 0.011669501662254333, -0.013643544167280197, 0.018976693972945213, 0.0031649398151785135, -0.021293766796588898, -0.0025889077223837376, -0.02902165614068508, 0.00698357867076993, 0.0034853171091526747, 0.023947397246956825, 0.008433367125689983, 0.01720329187810421, -0.015844115987420082, 0.0021714463364332914, 0.01119055412709713, 0.02588907815515995, -0.023688506335020065, 0.01734568178653717, 0.012303784489631653, 0.002613178687170148, -0.017358627170324326, -0.03150700777769089, -0.014187214896082878, 0.01668551005423069, -0.02091837488114834, 0.0022021897602826357, -0.0005258718738332391, 0.001776637975126505, 0.02350728213787079, -0.028270872309803963, -0.01344937551766634, 0.02419334277510643, 0.011430027894675732, 0.009598375298082829, -0.010795745067298412, 0.013041622936725616, 0.003027403960004449, -0.0025711089838296175, -0.008711674250662327, -0.006549936719238758, 0.008070919662714005, 0.0032005372922867537, 0.01111935917288065, -0.021785657852888107, 0.010523909702897072, 0.004339656792581081, -0.018225910142064095, -0.005592040717601776, 0.01053038239479065, -0.005579096265137196, 0.009261817671358585, -0.000735816138330847, -0.01310634519904852, -0.006375185213983059, 0.011514167301356792, -0.008284504525363445, 0.008129170164465904, -0.0049253967590630054, -0.03857472538948059, -0.019572142511606216, -0.008731091395020485, -0.1621691882610321, 0.019313251599669456, 0.004967466928064823, -0.021798603236675262, 0.023455504328012466, 0.0276495348662138, 0.022678831592202187, -0.022368162870407104, -0.013604709878563881, 0.015404000878334045, -0.008769924752414227, -0.003362343879416585, -0.017734017223119736, -0.01119055412709713, 0.0015986505895853043, 0.012148449197411537, 0.0006007074844092131, 0.009320068173110485, 0.02207043766975403, 0.006210142280906439, 0.01324873510748148, -0.01127469353377819, 0.003207009518519044, -0.015041554346680641, 0.015158054418861866, 0.0031843564938753843, -0.029384102672338486, -0.017280958592891693, 0.007753778714686632, 0.008996454067528248, -0.013928323984146118, 0.00342383049428463, 0.00358887342736125, -0.01036857534199953, 0.02459462359547615, -0.02574668824672699, 0.023805007338523865, 0.0016941165085881948, -0.03212834522128105, 0.026950528845191002, -0.005986849311739206, 0.028918098658323288, 0.0282449834048748, 0.012420284561812878, 0.008776397444307804, 0.033215686678886414, -0.010640410706400871, -0.008465728722512722, -0.014329604804515839, -0.014562605880200863, 0.01369532197713852, -0.011494750156998634, 0.01691851206123829, 0.0007333889952860773, 0.021669158712029457, 0.0011318380711600184, -0.0022151342127472162, 0.0021682102233171463, -0.0007912349537946284, -0.006375185213983059, -0.004757117945700884, -0.0010145282140001655, -0.014795607887208462, -0.016374841332435608, 0.023468447849154472, -0.007268358487635851, -0.0017928186571225524, -0.010620993562042713, 0.0011884705163538456, 0.005821806378662586, -0.007203635759651661, -0.029099322855472565, -0.0204782597720623, 0.0018899026326835155, 0.01078927330672741, 0.01107405312359333, -0.017022067680954933, 0.03285323828458786, 0.013617654331028461, -0.015805281698703766, -0.004533824510872364, 0.01383771188557148, -0.04326064884662628, 0.0067376322112977505, -0.019869867712259293, -0.0063978382386267185, 0.03523503616452217, 0.02692463994026184, 0.02459462359547615, -0.01614183932542801, 0.006957689765840769, -0.04924102500081062, 0.007326608989387751, -0.0194168072193861, 0.008983509615063667, 0.025164183229207993, 0.013488209806382656, 0.024296898394823074, -0.01993458904325962, -0.0009020725265145302, 0.020633595064282417, -0.014653217978775501, -0.011591834016144276, 0.005291080102324486, 0.006129239220172167, 0.0026374496519565582, -0.011041691526770592, 0.02951354905962944, -0.0021309945732355118, 0.008012669160962105, -0.02896987833082676, -0.014679106883704662, 0.017643406987190247, 0.007210107985883951, -0.0185883566737175, 0.001703824964351952, -0.008057975210249424, -0.006841188762336969, 0.015041554346680641, 0.00019750534556806087, 0.03979151323437691, -0.0008559576235711575, 0.014536716975271702, 0.01950741931796074, 0.008362172171473503, 0.004689159337431192, -0.07849568128585815, 0.005012772511690855, 0.012031949125230312, -0.01111935917288065, -0.011054635979235172, 0.019455641508102417, -0.01852363534271717, 0.008996454067528248, 0.012504423968493938, 0.03466547653079033, -0.02160443551838398, -0.006326643284410238, 0.016439564526081085, 0.01324873510748148, -0.02367556095123291, -0.012672703713178635, 0.0031018350273370743, -0.03140345215797424, -0.013164595700800419, 0.0271835308521986, -0.014277826063334942, -0.021617379039525986, -0.005003064405173063, -0.022847110405564308, -0.019468586891889572, -0.008983509615063667, -0.009475402534008026, -0.017565738409757614, 0.01645250804722309, -0.010465659201145172, -0.002346197608858347, -0.0008373498567380011, 0.00803855899721384, -0.01835535652935505, 2.2855201677884907e-05, 0.0142389927059412, -0.055247291922569275, 0.005617929622530937, 0.001021809526719153, -0.03052322193980217, -0.017280958592891693, -0.004886563401669264, 0.016245396807789803, -0.02278238907456398, -0.007714944891631603, -0.013345818966627121, -0.037150826305150986, 0.011339415796101093, 0.015870004892349243, -0.01642661914229393, 0.006815299857407808, 0.005294316448271275, -0.03259434923529625, -0.01395421288907528, 0.003271732246503234, -0.009281233884394169, -0.012413812801241875, 0.005744139198213816, -0.027338866144418716, 0.009255344979465008, 0.00019720195268746465, 0.003520914586260915, -0.011999587528407574, 0.018057631328701973, -0.003391469130292535, -0.01113230362534523, -0.011099942028522491, 0.0010493167210370302, -0.004407615400850773, -0.004317003767937422, 0.01614183932542801, 0.007494887802749872, -0.005724722053855658, 0.01729390397667885, -0.01788935251533985, 0.029384102672338486, -0.01422604825347662, -0.008679313585162163, 0.0020986334420740604, -0.019598031416535378, -0.011177608743309975, 0.007708472665399313, -0.01094460766762495, -0.0222387183457613, 0.03515736758708954, 0.011384721845388412, -0.03668482229113579, 0.008180948905646801, -0.005637346766889095, -0.04709223285317421, 0.010666299611330032, 0.025539575144648552, 0.004187558311969042, 0.008536923676729202, 0.023261336609721184, 0.01570172607898712, -0.021306710317730904, -0.011960754171013832, -0.014834441244602203, 0.013235790655016899, -0.04320887103676796, -0.03049733303487301, -0.068243607878685, 0.012077254243195057, -0.01939091831445694, -0.03438069298863411, -0.002601852174848318, -0.0009587049135006964, 0.006077460944652557, 2.0440740627236664e-05, 0.01034268643707037, 0.023339003324508667, 0.00039723553345538676, 0.02522890642285347, 0.006294282153248787, 0.012465590611100197, -0.02131965570151806, -0.012620924971997738, 0.0010841051116585732, 0.0006096068536862731, 0.021669158712029457, 0.027468310669064522, -0.004465865902602673, -0.01864013634622097, -0.0029028127901256084, 0.026743417605757713, 0.0029092850163578987, 0.029125211760401726, -0.004045168403536081, 0.00990904401987791, -0.024840569123625755, -0.033008575439453125, 0.00724246958270669, -0.022173995152115822, 0.013760044239461422, 0.021643267944455147, 0.012672703713178635, -0.009177678264677525, -0.02313189022243023, 0.044865772128105164, -0.008944676257669926, 0.003022549906745553, -0.01357882097363472, 0.005119564943015575, 0.019002582877874374, 0.009268289431929588, -0.008692258037626743, -0.011630668304860592, -0.02313189022243023, 0.020115813240408897, -0.0020565635059028864, 0.026018522679805756, 0.02948765829205513, -0.005650291219353676, -0.042069751769304276, 0.008116225712001324, 0.004776534624397755, -0.012705064378678799, -0.0029998968821018934, -0.020931318402290344, 0.013080456294119358, -0.002381795085966587, 0.038988951593637466, 0.013294041156768799, -0.0007790994131937623, 0.004514407832175493, 0.006443144287914038, -0.008595173247158527, -0.004579130560159683, 0.0016746997134760022, 0.004977175034582615, -0.0336557999253273, -0.0449434369802475, -0.016387786716222763, 0.0003195682947989553, 0.018821358680725098, 0.012536785565316677, -0.009087066166102886, -0.009410679340362549, -0.008284504525363445, -0.008698729798197746, 0.011889559216797352, 0.0026374496519565582, 0.0031859746668487787, -0.024491067975759506, -0.011041691526770592, 0.0029141393024474382, 0.009559541940689087, -0.02214810624718666, 0.01449788361787796, 0.013928323984146118, 0.008996454067528248, -0.014018935151398182, 0.01979219913482666, -0.004666506312787533, -0.02790842577815056, 0.027287088334560394, -0.027209419757127762, 0.0038251110818237066, -0.019028471782803535, 0.04139663651585579, 0.043830208480358124, 0.015235722064971924, 0.0003106689255218953, -0.011811891570687294, -0.019973423331975937, -0.015494612976908684, 0.0021698283962905407, -0.00649492209777236, -0.0489562451839447, -0.006080697290599346, -0.008368643932044506, 0.0031406686175614595, 0.009663098491728306, -0.00685413321480155, 0.009352428838610649, -0.004067821428179741, -0.02419334277510643, 0.017967019230127335, -0.011863669380545616, -0.012284367345273495, 0.03797927498817444, 0.008595173247158527, -0.002147175371646881, 0.03344868868589401, 0.0022248425520956516, 0.015520501881837845, -0.000535984814632684, 0.012478535063564777, -0.025021793320775032, -0.015895893797278404, 0.009786071255803108, -0.02494412660598755, 0.03784983232617378, -0.05250304937362671, -0.0018316522473469377, -0.011229387484490871, 0.019598031416535378, -0.00405487697571516, 0.019028471782803535, -0.01892491616308689, 0.053901057690382004, -0.015196888707578182, -0.01656900905072689, 0.012167866341769695, -0.01786346361041069, 0.01448493916541338, 0.024245120584964752, -0.002889868337661028, -0.02393445186316967, -0.0220057163387537, 0.011889559216797352, 0.012607980519533157, 0.011727752164006233, -0.006750577129423618, -0.023170724511146545, -0.01910613849759102, 0.015235722064971924, 0.00970840360969305, 0.019610976800322533, -0.0012256860500201583, 0.03497614338994026, 0.028840431943535805, -0.003375288564711809, -0.01906730607151985, -0.020271148532629013, 0.01962392032146454, 0.030393777415156364, -0.01932619698345661, -0.005996557418256998, -0.03189534321427345, 0.02099604159593582, 0.008336283266544342, -0.022652942687273026, -0.0009409061749465764, 0.007572554983198643, -0.035623371601104736, -0.029642993584275246, 0.0006055617122910917, 0.02816731669008732, 0.0028477986343204975, -0.020206425338983536, 0.03186945617198944, -0.013281096704304218, -0.02583729848265648, -0.029617104679346085, 0.024504011496901512, -0.012517369352281094, -0.00042393364128656685, -0.049008022993803024]} +{"id": "test:10000086", "text": "\"Pre-Owned, AWD Titanium 4dr Crossover, Ebony interior, Gas, 20(city)/27(highwa\u00ady) mpg, Auto 6 speed, I4, 2.00L, All wheel drive. EPA 27 MPG Hwy/20 MPG City! Nav System, Heated Leather Seats, All Wheel Drive, Power Liftgate, iPod/MP3 Input, Premium Sound System AND MORE KEY FEATURES INCLUDE: All ... 4/19/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, AWD SEL 4dr Crossover, Gas, 18(city)/25(highwa\u00ady) mpg, Auto 6 speed, V6, 3.50L, All wheel drive. ONLY 34,000 Miles! CD Player, iPod/MP3 Input, Dual Zone A/C, Onboard Communications System, Alloy Wheels, All Wheel Drive AND MORE KEY FEATURES INCLUDE: All Wheel Drive, Satellite Radio, iPod/MP3 ... 4/18/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, XLT 4dr SuperCab 4WD SB, GasAuto, V6, 4.00L, Four wheel drive. EPA 20 MPG Hwy/17 MPG City! Running Boards, iPod/MP3 Input, CD Player, CLOTH FRONT 60/40 SPLIT BENCH SEAT, Alloy Wheels CLICK NOW KEY FEATURES INCLUDE: 4x4, Running Boards, iPod/MP3 Input, CD Player, Aluminum ... 4/18/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, 4x4 LTZ 4dr Crew Cab SB, Ebony interior, DieselAuto 6 speed, V8, 6.60L. GREAT MILES 52,060! Heated Leather Seats, 4x4, Turbo, Alloy Wheels, Onboard Communications System, Satellite Radio, Tow Hitch CLICK ME KEY FEATURES INCLUDE: Leather Seats, 4x4, Heated Driver Seat, Premium Sound System, ... 4/17/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, 4x4 XLT 4dr SuperCrew 5.5 ft. SB, Medium Earth Gray interior, Flexible Fuel, 15(city)/21(highwa\u00ady) mpg, Auto 6 speed, V8, 5.00L. EPA 21 MPG Hwy/15 MPG City! Bluetooth, iPod/MP3 Input, Smart Device Integration, CD Player, 4x4, ENGINE: 5.0L V8 FFV, Alloy Wheels CLICK ME KEY ... 4/17/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, Base 4dr Sedan, Ebony interior, Gas, 22(city)/30(highwa\u00ady) mpg, Auto 4 speed, I4, 2.40L, Front wheel drive. FUEL EFFICIENT 30 MPG Hwy/22 MPG City! G6 trim. AUDIO SYSTEM, AM/FM STEREO WITH CD PL Onboard Communications System. CLICK NOW KEY FEATURES INCLUDE: Onboard Communications System MP3 ... 4/17/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, 4x4 Trailhawk 4dr SUV, Gas, 19(city)/26(highwa\u00ady) mpg, Auto 9 speed, V6, 3.20L. EPA 26 MPG Hwy/19 MPG City! ONLY 41,158 Miles! Heated Leather Seats, iPod/MP3 Input, Satellite Radio, Smart Device Integration CLICK NOW KEY FEATURES INCLUDE: Leather Seats, 4x4, Back-Up Camera, Satellite Radio, iPod/MP3 ... 4/17/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, AWD XLT 4dr SUV, Camel interior, Flexible Fuel, 18(city)/23(highwa\u00ady) mpg, Auto 6 speed, V6, 3.00L, All wheel drive. Clean CARFAX. 2012 Ford Escape XLT Features: AWD. Odometer is 72570 miles below market average! Awards 2012 KBB.com Total Cost of Ownership Awards * 2012 KBB.com ... 4/16/2019 ~ Torrington, Wyoming ~ 29.2 mi.\\n$20,995 ~ 96,077 mi ~ Summit White ~ Good deal!\\nPre-Owned, 4x4 LT 4dr SUV, Light Titanium interior, Flexible Fuel, 15(city)/21(highwa\u00ady) mpg, Auto 6 speed, V8, 5.30L. Clean CARFAX. Summit White 2013 Chevrolet Tahoe LT Features: 4WD, light titanium dark titanium Cloth. Odometer is 12699 miles below market average! Awards JD Power Initial Quality Study ... 4/16/2019 ~ Torrington, Wyoming ~ 29.2 mi.\\n$20,995 ~ 52,957 mi ~ Deep Cherry Red ~ Good deal!\\nPre-Owned, Limited 4dr Mini-Van, Black interior, Flexible Fuel, 17(city)/25(highwa\u00ady) mpg, Auto 6 speed, V6, 3.60L, Front wheel drive. Clean CARFAX. Deep Cherry Red Crystal 2016 Chrysler Town Country Limited Features: black Leather. Recent Arrival! Awards JD Power Initial Quality Study (IQS 2016 KBB.com Brand Image ... 4/16/2019 ~ Torrington, Wyoming ~ 29.2 mi.\\n$17,995 ~ 74,241 mi ~ Silver Ice Metallic ~ Good deal!\\nPre-Owned, AWD LTZ 4dr SUV, Brownstone interior, Gas, 20(city)/29(highwa\u00ady) mpg, Auto 6 speed, I4, 2.40L, All wheel drive. Silver Ice Metallic 2015 Chevrolet Equinox LTZ Features: AWD. Odometer is 17063 miles below market average! 20/29 City/Highway MPG Awards JD Power Initial Quality Study * 2015 ... 4/16/2019 ~ Torrington, Wyoming ~ 29.2 mi.\\nPre-Owned, EX 4dr Sedan, Black interior, Gas, 31(city)/41(highwa\u00ady) mpg, Continuously Variable, I4, 2.00L, Front wheel drive. FUEL EFFICIENT 41 MPG Hwy/31 MPG City! LOW MILES - 19,906! EX trim. Moonroof, Onboard Communications System, Remote Engine Start. AND MORE KEY FEATURES INCLUDE: Sunroof, Back-Up Camera, iPod/MP3 ... 4/14/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, LT 4dr Sedan, Jet Black / Dark Titanium interior, Gas, 22(city)/30(highwa\u00ady) mpg, Auto 6 speed, I4, 2.50L, Front wheel drive. FUEL EFFICIENT 30 MPG Hwy/22 MPG City! Onboard Communications System, Satellite Radio, Remote Engine Start, WiFi Hotspot, Rear Air AND MORE KEY FEATURES INCLUDE: ... 4/13/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, AWD SLT-2 4dr SUV, Brownstone interior, Flexible Fuel, 16(city)/23(highwa\u00ady) mpg, Auto 6 speed, V6, 3.00L, All wheel drive. EPA 24 MPG Hwy/17 MPG City! Moonroof, Heated Leather Seats, All Wheel Drive, Onboard Communications System, Power Liftgate AND MORE KEY FEATURES INCLUDE: Leather Seats, Sunroof, ... 4/13/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\\nPre-Owned, 4x4 RTL 4dr Crew Cab, Beige interior, Gas, 15(city)/20(highwa\u00ady) mpg, Auto 5 speed, V6, 3.50L. EPA 20 MPG Hwy/15 MPG City! Moonroof, Heated Leather Seats, Multi-CD Changer, Dual Zone A/C, Consumer Guide Recommended Pickup READ MORE KEY FEATURES INCLUDE: Leather Seats, Sunroof, 4x4, Heated ... 4/11/2019 ~ Scottsbluff, Nebraska ~ 0.0 mi.\"", "vector_field": [-0.012471460737287998, -0.003985140938311815, 0.023198381066322327, -0.011033214628696442, -0.010647018440067768, 0.018537398427724838, -0.0063455961644649506, -0.009255382232367992, 0.012771095149219036, -0.028258878737688065, 0.0030612675473093987, 0.033399276435375214, -0.019722620025277138, -0.011099799536168575, 0.0013683316064998507, 0.013650023378431797, -0.004338044207543135, -0.02688721753656864, 0.004444580990821123, -0.011399433948099613, -0.04336046427488327, -0.0002611399395391345, -0.0015106580685824156, 0.011426068842411041, -0.003515713382512331, 0.0012692856835201383, 0.012977510690689087, -0.01058709155768156, -0.003812018781900406, -0.016047101467847824, 0.006505401339381933, 0.001289261388592422, -0.04240163415670395, -0.014009585604071617, -0.003375883912667632, -0.02988356351852417, -0.0036022744607180357, -0.02836541458964348, 0.018990179523825645, -0.01680617593228817, 0.009455137886106968, 0.0050971186719834805, -0.0033375972416251898, 0.002871498931199312, -0.006628584582358599, 0.00867608841508627, -0.007837111130356789, 0.018364274874329567, -0.005649778060615063, 0.0036855062935501337, -0.0031994322780519724, 0.018923593685030937, -0.003515713382512331, -0.0009063948527909815, -0.0009421845315955579, 0.0014224323676899076, -0.01998896151781082, 0.02836541458964348, -0.00467430055141449, -0.00572968041524291, -0.004161592572927475, -0.00233881501480937, -0.03576971963047981, 0.0020225339103490114, -0.03148161619901657, -0.03345254436135292, -0.013277145102620125, 0.033958595246076584, -0.005010557360947132, 0.0013741578441113234, -0.010513847693800926, 0.010087700560688972, 0.008682746440172195, 0.007184573449194431, 0.017711738124489784, -0.015381245873868465, -0.01893691159784794, 0.0065719871781766415, -0.007630696054548025, 0.004174909554421902, 0.03161478787660599, -0.020987743511795998, -0.03891255334019661, 0.0273533146828413, 0.02504945732653141, 0.010453920811414719, -0.009921236895024776, 0.029031269252300262, -0.026687460020184517, -0.012657900340855122, -0.0012326637515798211, 0.00691823149099946, 0.020468376576900482, 0.011505970731377602, 0.004747544880956411, 0.018923593685030937, 0.0081300875172019, 0.01816451922059059, -4.515327964327298e-05, -0.02077466994524002, -0.0020541620906442404, 0.010194237343966961, 0.008609502576291561, -0.004647666588425636, -0.03670191764831543, -0.008063501678407192, -0.00043322177953086793, 0.021467158570885658, 0.0008606173214502633, -0.0037321161944419146, -0.01054714061319828, 0.009601626545190811, 0.02932424657046795, -0.026221362873911858, -0.013317096047103405, -0.00447787344455719, 0.008815918117761612, 0.0003990967234130949, -0.0281523410230875, -0.005772961303591728, 0.018097933381795883, 0.000973812653683126, 0.00465432507917285, -0.027859365567564964, 0.0003993048158008605, 0.020015595480799675, -0.00887584500014782, -0.01042728591710329, 0.008689405396580696, -0.00667852396145463, 0.003545676823705435, 0.008502965793013573, 0.02559545822441578, 0.004894033074378967, -0.034677717834711075, 0.0071379635483026505, -0.029617222025990486, 0.007051402237266302, -0.027699559926986694, -0.024796433746814728, 0.024037359282374382, 0.02379765175282955, 0.019363058730959892, 0.01594056375324726, 0.002245595445856452, 0.01727227307856083, 0.031348444521427155, 0.006448803935199976, 0.023970773443579674, -0.004810800775885582, 0.005460009444504976, -0.018097933381795883, 0.02157369628548622, 0.018257739022374153, 0.008669429458677769, -0.014675440266728401, 0.023424772545695305, 0.01958944834768772, -0.0182177871465683, 0.000486906326841563, -0.030123271048069, 0.0027000412810593843, -0.001617194851860404, -0.02108096331357956, 0.004717581439763308, 0.02441023662686348, -0.004251482896506786, -0.02298530749976635, -0.016073735430836678, -0.005233618896454573, -0.007950306870043278, 0.015967197716236115, -0.015354611910879612, 0.0221862830221653, 0.019163301214575768, 0.020428424701094627, 0.02718019299209118, 0.01550109963864088, -0.017032565549016, -0.022212916985154152, 0.01696597971022129, 0.004111653659492731, 0.009614943526685238, 0.024996189400553703, -0.002973041730001569, -0.03917889669537544, 0.01775169000029564, -0.0026584253646433353, 0.016473248600959778, -0.018976861611008644, 0.0010670323390513659, 0.03720796853303909, -0.012318314053118229, -0.00016490310372319072, -0.6243054866790771, -0.01720568910241127, -0.010993262752890587, 0.0016846127109602094, -0.00035040610237047076, 0.02718019299209118, 0.016313442960381508, -0.0027832731138914824, -0.022332770749926567, 0.017698420211672783, 0.005982705391943455, 0.02504945732653141, 0.024623310193419456, -0.017578566446900368, -0.003505725646391511, -0.026048239320516586, -0.012657900340855122, -0.012071947567164898, -0.00346910348162055, 0.005420058034360409, -0.01990905962884426, 0.02876492775976658, -0.016380028799176216, 0.011073165573179722, 0.011798947118222713, -0.0031428346410393715, 0.015887295827269554, 0.0031295176595449448, -0.00832984410226345, 0.0060925716534256935, -0.005606497637927532, 0.01887032575905323, 0.012025337666273117, 0.005150387063622475, 0.04415949061512947, -0.03204093500971794, -0.02195989154279232, 0.03462444990873337, 0.009874626994132996, 0.04104328900575638, -0.022878771647810936, -0.03140171244740486, -0.0006163318757899106, -0.006924889981746674, -0.0015589325921609998, 0.007983598858118057, 0.013603413477540016, 0.0033775484189391136, 0.021440524607896805, -0.01815120130777359, 0.022958673536777496, 0.03941860422492027, 0.01958944834768772, 0.03321283683180809, -0.00548997288569808, -0.02615477703511715, 0.04056387394666672, 0.0024403578136116266, 0.02608819119632244, -0.015141538344323635, 0.007284451741725206, 0.009268699213862419, 0.005976046901196241, -0.004397971089929342, -0.016619736328721046, 0.02520926296710968, -0.014049536548554897, -0.0004107491986360401, 0.0202020350843668, -0.00011475591600174084, -0.021507110446691513, 0.03369225189089775, 0.012318314053118229, 0.019296472892165184, 0.0412030965089798, -0.011485995724797249, 0.037927091121673584, 0.011059848591685295, 0.003725457703694701, 0.011013238690793514, 0.008289892226457596, -0.020974425598978996, 0.01137945894151926, -0.013030778616666794, 0.03537020832300186, -0.00568639999255538, 0.010087700560688972, -0.01648656465113163, 0.014542268589138985, 0.01640666276216507, 0.021187499165534973, 0.013323755003511906, -0.020495010539889336, -0.027166876941919327, -0.00652204779908061, 0.008063501678407192, -0.011732361279428005, 0.024849701672792435, 0.026421118527650833, -0.04495851695537567, -0.027126925066113472, -0.02251921035349369, 0.018031347543001175, -0.00036517975968308747, 0.03382542356848717, 0.024596676230430603, -0.0029563952703028917, -0.02379765175282955, 0.02093447558581829, -0.022226233035326004, -0.018271055072546005, 0.004631020128726959, -0.012684534303843975, 0.0010487213730812073, -0.0004344702756498009, -0.023171747103333473, 0.00035956158535555005, -0.007850428111851215, 0.019709302112460136, -0.05694390460848808, 0.013650023378431797, 0.0019026801455765963, 0.006964841391891241, -0.007337720133364201, -0.021840037778019905, -0.007430939469486475, 0.007410963997244835, -0.018484128639101982, 0.00253524212166667, -0.011266263201832771, 0.005543241277337074, 0.020415108650922775, 0.02559545822441578, -0.008396429009735584, 0.003305969061329961, -0.006392206065356731, 0.01869720220565796, -0.009641577489674091, 0.019536180421710014, -0.02442355453968048, -0.03337264433503151, 0.014568903483450413, -0.011885507963597775, -0.016153637319803238, 0.007670647464692593, -0.007297768723219633, 0.0060759251937270164, 0.01172570325434208, -0.01018757838755846, -0.019243203103542328, -0.004321397747844458, -0.014728708192706108, -0.0029364197980612516, 0.004557776264846325, -0.013077388517558575, -0.010760214179754257, -0.008416404947638512, -0.023478040471673012, -0.017978079617023468, -0.014409097842872143, 0.01058043260127306, 0.02852522023022175, -0.02441023662686348, 0.010946652851998806, -0.0015447831247001886, 0.017938127741217613, 0.0060925716534256935, 0.027885999530553818, -0.011146409437060356, -0.03337264433503151, -0.012198460288345814, -0.014289244078099728, -0.003635567147284746, 0.0037421039305627346, -0.030469516292214394, -0.02283881977200508, -0.007564110681414604, -0.022292818874120712, 0.03257361799478531, 0.003712140489369631, 0.01808461733162403, 0.040697045624256134, -0.008769308216869831, 0.009881285019218922, 0.013756560161709785, -0.003588957479223609, 0.0074043055064976215, 0.01879042200744152, -0.02298530749976635, 0.012151850387454033, -0.035876255482435226, 0.010081041604280472, -0.015274709090590477, 0.03917889669537544, -0.013410315848886967, 0.013849779963493347, -0.0016729602357372642, -0.0023504674900323153, 0.012544704601168633, 0.01832432486116886, 0.015447831712663174, 0.022066429257392883, 0.004421276040375233, -0.02823224477469921, 0.02767292596399784, -0.015341294929385185, 0.007630696054548025, -0.013050754554569721, 0.025462288409471512, 0.009028990752995014, 0.02369111403822899, -0.02638116665184498, 0.003422493813559413, -0.03129517659544945, 0.01526139210909605, -0.0006450468208640814, 0.002032521879300475, 0.0186173003166914, 0.010220871306955814, -0.023211698979139328, 0.014555586501955986, 0.014675440266728401, 0.02434365265071392, -0.00014409514551516622, -0.01751198060810566, 0.005589851178228855, 0.01145936083048582, -0.008815918117761612, -0.011512629687786102, -0.013010802678763866, -0.004154934082180262, 0.00588615657761693, 0.025901751592755318, 0.003725457703694701, 0.027646292001008987, 0.014662123285233974, 0.012185143306851387, -0.024143895134329796, 0.03161478787660599, -0.016313442960381508, 0.03148161619901657, 0.023105161264538765, 0.004910679068416357, -0.019869107753038406, 0.0013733255909755826, 0.02021535113453865, 0.02506277523934841, 0.03265351802110672, -0.01856403239071369, 0.005946083460003138, -0.009301992133259773, 0.009222089312970638, -0.012338289991021156, 0.005643119569867849, -0.0019542837981134653, -0.016752906143665314, 0.005483314394950867, 0.0023920834064483643, 0.019616082310676575, 0.009914577938616276, 0.006468779407441616, 0.006628584582358599, -0.002776614623144269, -0.015354611910879612, 0.005619814619421959, -0.009082259610295296, -0.02647438645362854, -0.019056765362620354, -0.006152498535811901, 0.019775887951254845, 0.0021956562995910645, -0.02211969718337059, 0.012231753207743168, -0.020881207659840584, 0.01300414465367794, 0.024623310193419456, -0.0027649621479213238, -0.027486486360430717, 0.017245639115571976, 0.007131305057555437, 0.009195455349981785, -0.026674143970012665, 0.027566388249397278, 0.024942921474575996, -0.00731108570471406, -0.014928464777767658, -0.0073044272139668465, 0.011279580183327198, 0.010946652851998806, 0.018497446551918983, -0.004018433857709169, 0.029510684311389923, 0.006964841391891241, 0.007058061193674803, 0.01383646298199892, -0.004181568045169115, 0.045704275369644165, -0.015447831712663174, -0.003625579411163926, -0.0206015482544899, 0.009721480309963226, -0.009488430805504322, -0.02266569808125496, 0.0030296393670141697, 0.017538616433739662, -0.00990791991353035, 0.00669849943369627, -0.01320390123873949, -0.008529599756002426, -0.0011078158859163523, 0.006145840045064688, -0.008496307767927647, 0.013130656443536282, -0.016273491084575653, -0.012285021133720875, -0.00975477322936058, 0.002958060009405017, -0.02195989154279232, 0.015141538344323635, 0.01181226409971714, -0.01335704792290926, -0.021360622718930244, -0.0158473439514637, 0.001638835179619491, 0.07596071809530258, 0.021613646298646927, 0.004664313048124313, 0.0018876984249800444, 0.008669429458677769, 0.0019709302578121424, -0.011639142408967018, -0.029377514496445656, -0.005243606399744749, 0.005133740603923798, -0.00306293205358088, 0.002355461474508047, -0.027220144867897034, -0.02394413948059082, 0.007024768274277449, -0.021120915189385414, -0.015607636421918869, -0.025582142174243927, -0.0027166875079274178, -0.026700777933001518, -0.013703292235732079, -0.0008206660277210176, 0.004058385267853737, 0.016047101467847824, 0.0010820140596479177, 0.019469594582915306, 0.024290382862091064, 0.0182311050593853, 0.02719351090490818, 0.00568972947075963, 0.00034041827893815935, 0.004624361637979746, 0.0014890178572386503, 0.012191801331937313, 0.013630048371851444, -0.013450266793370247, 0.010746896266937256, -0.017099151387810707, 0.019549496471881866, 0.00875599030405283, 0.027619658038020134, 0.015327977947890759, -0.006552011240273714, -0.010094358585774899, 0.0043680076487362385, -0.03393195942044258, 0.01038067601621151, 0.028791561722755432, 0.0011877184733748436, -0.012378240935504436, 0.01121965330094099, -0.003931872546672821, -0.022319452837109566, 0.01192545983940363, -0.02210637927055359, -0.0026334556750953197, -0.0077372328378260136, -0.006778401788324118, -0.008949088864028454, -0.013996267691254616, -0.006911573000252247, -0.02149379253387451, 0.03654211014509201, 0.004594398196786642, -0.020508328452706337, 0.0037853845860809088, -0.001414109137840569, 0.02108096331357956, -0.004970605950802565, 0.01610036939382553, -0.004095007199794054, -0.024383602663874626, -0.013303779065608978, 0.011026555672287941, 0.04621032252907753, 0.011312873102724552, 0.002390418667346239, 0.004021762870252132, -0.00023346533998847008, 0.01225172821432352, -0.006292327772825956, 0.013969633728265762, 0.028445318341255188, -0.03212083503603935, 0.03116200491786003, -0.007677305955439806, -0.007790501229465008, 0.009867968037724495, 0.00016833642439451069, -0.002140723168849945, 0.034118399024009705, -0.012884290888905525, 0.026181410998106003, -0.016766224056482315, 0.006132522597908974, -0.02014876715838909, 0.012964192777872086, 0.009102235548198223, 0.004777508322149515, 0.020907841622829437, 0.018111251294612885, -0.011066506616771221, 0.0015714173205196857, -0.02157369628548622, 0.008602844551205635, -0.011885507963597775, 0.008343161083757877, -0.010393993929028511, 0.005566546227782965, -0.017778323963284492, -0.024516774341464043, -0.01950954645872116, -0.01109314151108265, -0.012331631034612656, -0.019216569140553474, 0.0033692254219204187, -0.02823224477469921, -0.02036184072494507, -0.0030879017431288958, -0.0007549129077233374, 0.003415835089981556, -0.037527576088905334, 0.03204093500971794, 0.006568657699972391, -0.0012667888076975942, 0.013097364455461502, -0.010260822251439095, -0.028631756082177162, -0.026620876044034958, 0.03819343075156212, -0.0041382876224815845, 0.017072517424821854, -0.005206984467804432, 0.0002788267156574875, 0.011039872653782368, -0.022892087697982788, -0.017312224954366684, 0.024609994143247604, -0.006818353198468685, -0.008602844551205635, -0.01217182632535696, 0.008889161981642246, -0.017099151387810707, -0.014515634626150131, 0.0027066997718065977, -0.025635410100221634, -0.015567685477435589, 0.01161250751465559, -0.011406092904508114, 0.019709302112460136, 0.0020341863855719566, -0.007430939469486475, -0.027539754286408424, 0.0027649621479213238, -0.006838328670710325, -0.025142677128314972, -0.01720568910241127, -0.01856403239071369, 0.03358571603894234, 0.020175401121377945, 0.026461070403456688, -0.009901260957121849, 0.006055949721485376, -0.006631914060562849, -0.013969633728265762, 0.025022823363542557, 0.01256468053907156, -0.002911450108513236, -0.03829997032880783, 0.0013158955844119191, 0.015620953403413296, 0.0166463702917099, -0.02847195230424404, 0.003525701118633151, -0.009421845898032188, 0.0428011454641819, -0.022878771647810936, -0.012431508861482143, -0.012751119211316109, -0.024796433746814728, 0.02117418311536312, -0.009734797291457653, -0.02353130839765072, 0.00731108570471406, -0.06786391884088516, -0.024223798885941505, 0.03590289130806923, 0.0071579390205442905, 9.883782331598923e-05, 0.0035822989884763956, 0.031348444521427155, -0.004401300102472305, 0.0297237578779459, 0.014329195953905582, -0.008010233752429485, -0.005979376379400492, -0.024676579982042313, -0.033718887716531754, -0.008749332278966904, -0.00527357030659914, -0.015674222260713577, 0.01113975141197443, 0.014142756350338459, 0.012398216873407364, 0.009854651056230068, 0.0019609425216913223, -0.0150616355240345, -0.03337264433503151, 0.022865453734993935, -0.0018527409993112087, 0.004993910901248455, -0.013583438470959663, -0.04186895117163658, -0.026594240218400955, -0.00836313609033823, 0.004724239930510521, 0.005373448133468628, 0.020974425598978996, -0.047195788472890854, -0.023571260273456573, 0.002886480651795864, 0.021600330248475075, 0.01760520040988922, 0.0008606173214502633, 0.025249214842915535, 0.008223307318985462, -0.003001340664923191, -0.01502168457955122, 0.0016471582930535078, -0.0021540403831750154, -0.007038085255771875, 0.019389692693948746, 0.03004336915910244, -0.0052802287973463535, -0.007577427662909031, -0.006631914060562849, 0.014688757248222828, 0.023504674434661865, -0.029936831444501877, 0.020894523710012436, 0.012544704601168633, 0.004817459732294083, -0.007417622487992048, -0.020388474687933922, -0.03518376871943474, 0.023970773443579674, 0.006844987627118826, -0.00032397996983490884, -0.006202437449246645, -0.0008631143136881292, -0.023438088595867157, 0.03933870047330856, 0.0005439201486296952, -0.008589526638388634, 0.008829235099256039, -0.007284451741725206, 0.0026018277276307344, -0.017405444756150246, -0.01600714959204197, -0.007184573449194431, -0.005383436102420092, 0.06573318690061569, 0.002005887683480978, 0.028898099437355995, -0.0032543654087930918, -0.03537020832300186, -0.007044743746519089, 0.0025019494350999594, 0.01926983892917633, 0.021547062322497368, -0.007284451741725206, -0.018097933381795883, 0.02323833294212818, 0.015820709988474846, -0.01245148479938507, -0.018297690898180008, -0.011232970282435417, -0.020987743511795998, -0.03044288232922554, -0.019243203103542328, -0.0046842885203659534, 0.01618027128279209, -0.0077372328378260136, -0.0029613892547786236, -0.010966628789901733, -0.02860512211918831, -0.009634919464588165, -0.013250511139631271, 0.019482912495732307, -0.026434436440467834, 0.004864069167524576, -0.0023521322291344404, 0.009961187839508057, 0.029191074892878532, 0.012045313604176044, -0.006292327772825956, 0.011239629238843918, 0.012018679641187191, -0.048314426094293594, 0.01418270729482174, 0.03153488412499428, 0.026674143970012665, -0.014742025174200535, -0.012964192777872086, 0.014142756350338459, -0.012498094700276852, 0.014156073331832886, -0.0010295780375599861, -0.02996346727013588, -0.01299748569726944, -0.0025136019103229046, -0.006139181554317474, 0.00817003846168518, -0.019283154979348183, 0.00775054981932044, 0.006338937673717737, 0.008895820006728172, -0.003375883912667632, -0.005340155679732561, 0.018244421109557152, 0.02607487514615059, 0.002272229641675949, 0.02607487514615059, 0.030096637085080147, 0.018577348440885544, -0.0075707691721618176, 0.00896240584552288, -0.011878849938511848, -0.01791149377822876, -0.010593749582767487, -0.00489736208692193, 0.004021762870252132, -0.007257817313075066, -0.023904187604784966, -0.026261312887072563, 0.009461796842515469, -0.02345140650868416, 0.01157921552658081, 0.0003959755413234234, 0.0037520918995141983, 0.014835244975984097, 0.002057491336017847, -0.014808611012995243, 0.010314091108739376, -0.020814621821045876, -0.03632903844118118, -0.02591506950557232, -0.009215430356562138, -0.0202020350843668, -0.014049536548554897, 0.007217865902930498, 0.03449127823114395, 0.017698420211672783, -0.030149905011057854, -0.0025019494350999594, 0.0014856884954497218, -0.048074718564748764, -0.03124190680682659, -0.015714174136519432, 0.042215194553136826, 0.02511604316532612, -0.0009621601784601808, -0.021214134991168976, 0.027726193889975548, 0.015794076025485992, 0.02940414845943451, -0.010873408988118172, 0.01216516736894846, -0.008070160634815693, 0.013496876694262028, -0.001284267520532012, 3.222633313271217e-05, -0.04083021730184555, -0.0037620796356350183, -0.0010487213730812073, 0.014555586501955986, 0.009814700111746788, 0.0036488843616098166, -0.004840764682739973, 0.0037221284583210945, 0.017219005152583122, -0.0007549129077233374, 0.02506277523934841, -0.0040550557896494865, 0.01712578535079956, -0.05452019348740578, 0.0038319944869726896, 0.00832984410226345, 0.007617379073053598, 0.011998703703284264, -0.015634270384907722, 0.022226233035326004, 0.0044545684941112995, 0.013037437573075294, 0.0007823794148862362, -0.03347918018698692, -0.007723915856331587, -0.022386038675904274, -0.013403656892478466, 0.005539911799132824, -0.022133013233542442, 0.03068258985877037, 0.014675440266728401, -0.014528951607644558, 0.008429721929132938, 0.015913929790258408, 0.0182177871465683, -0.021134231239557266, -0.00017072934133466333, -0.0021889975760132074, 0.011299556121230125, 0.003588957479223609, 0.015594319440424442, -0.015101587399840355, -0.006445474456995726, -0.0002886064467020333, 0.0014831916196271777, -0.028951367363333702, 0.03185449540615082, 0.010314091108739376, -0.022892087697982788, 0.004224848933517933, -0.020428424701094627, -0.002864840207621455, -0.0029347550589591265, -0.003802031045779586, 0.027566388249397278, -0.03108210302889347, -0.021307354792952538, -0.016153637319803238, 0.004161592572927475, -0.05183013901114464, 0.016539832577109337, 0.007084695156663656, -0.01856403239071369, 0.009588309563696384, 0.2054028958082199, -0.024356968700885773, -0.00153146602679044, 0.03361235186457634, -0.007990257814526558, 0.022252866998314857, 0.02783273160457611, 0.008862527087330818, -0.000613002572208643, 0.014489000663161278, -0.005167033523321152, 0.007044743746519089, 0.00489736208692193, 0.0026950472965836525, 0.012318314053118229, -0.008263258263468742, -0.03156151622533798, -0.007517500780522823, -0.020481694489717484, 0.012571338564157486, 0.010127651505172253, 0.016686322167515755, 0.012577997520565987, -0.02314511314034462, 0.02535575069487095, -0.020481694489717484, -0.018590666353702545, 0.02275891788303852, 0.011306215077638626, -0.005719692911952734, -0.01145936083048582, 0.008409745991230011, 0.008642795495688915, 0.010413968935608864, 0.0043713366612792015, 0.0057596443220973015, 0.018417544662952423, 0.007823794148862362, -0.00042822788236662745, -0.0008056843071244657, -0.008689405396580696, -0.009495089761912823, -0.009481772780418396, -0.027007071301341057, -0.0005967723554931581, 0.01837759278714657, -0.013077388517558575, 0.02157369628548622, -0.010320749133825302, 0.01280438806861639, -0.014795294031500816, -0.020255303010344505, -0.009009015746414661, 0.028978001326322556, -0.0012717826757580042, -0.015567685477435589, 0.02498287335038185, 0.02520926296710968, 0.02005554735660553, -0.002901462372392416, 0.021347304806113243, 0.024836383759975433, -0.01161916647106409, 0.01990905962884426, -0.012677875347435474, 0.03675518557429314, 0.01054048165678978, -0.013476901687681675, 0.05068486928939819, -0.012944217771291733, -0.006185790989547968, -0.013363705947995186, -0.010393993929028511, 0.0013325419276952744, -0.01379651203751564, -0.017804957926273346, 0.009009015746414661, -0.005942753981798887, 0.007244500331580639, 0.005616485141217709, 0.022093063220381737, -0.03140171244740486, 0.0014332524733617902, 0.005326838698238134, 0.022971991449594498, -0.04413285851478577, -0.018417544662952423, -0.008443038910627365, 0.012837680988013744, 0.002705035265535116, -0.007837111130356789, 0.013017461635172367, -0.018177837133407593, -0.02496955543756485, -0.008582868613302708, 0.008343161083757877, 0.0037221284583210945, 0.00689825601875782, -0.01672627218067646, 0.0013891395647078753, -0.020654816180467606, 0.03691498935222626, -0.002698376541957259, 0.0038053602911531925, -0.0029181085992604494, 0.0030862370040267706, 0.005636461079120636, 0.007657330483198166, 0.007504183799028397, 0.002816565800458193, 0.026021605357527733, -0.04146943613886833, 0.005819571204483509, -0.0029514015186578035, 0.024197163060307503, 0.011892166920006275, -0.005736339371651411, -0.02290540561079979, 0.026061557233333588, -0.020561596378684044, 0.007910354994237423, -0.026833949610590935, 0.017778323963284492, 0.010214212350547314, -0.012884290888905525, -0.024929603561758995, -0.03060268610715866, 0.025382384657859802, -0.003808689536526799, -0.007410963997244835, 0.010900042951107025, 0.008216648362576962, 0.01808461733162403, -0.0018993509002029896, 0.012398216873407364, -0.016513198614120483, -0.011326190084218979, -0.0028615109622478485, -0.018177837133407593, 0.014755342155694962, 0.03036297857761383, 0.011279580183327198, 0.016286808997392654, 0.009974504821002483, 0.017858225852251053, -0.02836541458964348, 0.02362452819943428, -0.021999843418598175, -0.02639448456466198, -0.024596676230430603, -0.03425157070159912, 0.007823794148862362, 0.014595537446439266, 0.0016821157187223434, 0.025568824261426926, 0.009068942628800869, -0.0305227842181921, -0.015913929790258408, -0.019482912495732307, 0.003067926038056612, -0.013410315848886967, 0.01917661912739277, 0.01634007692337036, -0.01893691159784794, -0.048314426094293594, 0.01078684814274311, -0.16918039321899414, 0.00487405713647604, -0.0028248890303075314, -0.021826719865202904, 0.007084695156663656, 0.012245070189237595, 0.025741947814822197, 0.021826719865202904, -0.007810476701706648, -0.017152419313788414, 0.023997407406568527, -0.03510386496782303, -0.040776949375867844, 0.018510764464735985, 0.0002711277629714459, 0.011499312706291676, -0.017032565549016, 0.016353394836187363, 0.009681529365479946, 0.0019609425216913223, 0.04336046427488327, -0.007837111130356789, 0.012304997071623802, -0.02028193697333336, 0.0166463702917099, 0.02648770436644554, 0.011818923056125641, 0.02282550372183323, -0.015341294929385185, -0.01359675545245409, 0.013277145102620125, -0.005380107089877129, 0.017258957028388977, -0.00669516995549202, 0.016127003356814384, 0.014156073331832886, -0.03521040081977844, -0.004141617100685835, -0.0024253760930150747, 0.059873662889003754, 0.010074383579194546, 0.003176127327606082, -0.0014815269969403744, 0.016033783555030823, 0.0007678138208575547, 0.009448479861021042, 0.002911450108513236, 0.012291680090129375, 0.005956071428954601, -0.03353244811296463, 0.005147057585418224, 0.0053501431830227375, -0.015740808099508286, 0.0003655959153547883, 0.014875196851789951, 0.009388552978634834, -0.012265045195817947, 0.01240487489849329, 0.009601626545190811, -0.027246778830885887, -0.03156151622533798, -0.008263258263468742, -0.010433944873511791, -0.0055965096689760685, -0.004278117325156927, -0.039924655109643936, -0.002746651181951165, -0.0012393222423270345, -0.02203979343175888, 0.01261129043996334, -0.0005851199384778738, -0.015288026072084904, -0.004494519904255867, -0.029350880533456802, 0.0029330903198570013, 0.011539263650774956, 0.00711798807606101, 0.023744381964206696, 0.0154611486941576, -0.025409018620848656, -0.00447787344455719, 0.024836383759975433, -0.025968337431550026, -0.011665776371955872, 0.002608486218377948, -0.00731108570471406, -0.0035123841371387243, -0.004644337110221386, -0.001171904499642551, -0.007197890430688858, -0.0046343496069312096, -0.0011111452477052808, -0.013343730010092258, -0.00014243050827644765, 0.008256599307060242, 0.029856929555535316, 0.014941781759262085, -0.004690947011113167, 0.021773451939225197, 0.00689825601875782, 0.0013150632148608565, -0.02315843105316162, -0.02433033473789692, -0.0009555016295053065, 0.030149905011057854, -0.004817459732294083, 0.016193589195609093, 0.012917582876980305, 0.02149379253387451, -0.011878849938511848, -0.016992615535855293, 0.0029497367795556784, 0.022292818874120712, 0.01689939573407173, 0.004730898421257734, -0.014156073331832886, -0.021600330248475075, -0.017938127741217613, 0.022133013233542442, 0.025009507313370705, 0.02868502587080002, -0.015154855325818062, -0.024117261171340942, -0.0015431185020133853, -0.001127791590988636, -0.03446464613080025, -0.07516168802976608, -0.013756560161709785, 0.011705727316439152, 0.01872383803129196, 0.009734797291457653, 0.02441023662686348, -0.026061557233333588, 0.0190168134868145, -0.006199108436703682, 0.03877938538789749, 0.00263678515329957, -0.013570120558142662, -0.0048540816642344, 0.009728138335049152, 0.056251414120197296, -0.014555586501955986, 0.005326838698238134, -0.027779461815953255, -0.030948931351304054, 0.02480974979698658, -0.012005362659692764, -0.023744381964206696, -0.004411288071423769, -0.004487861413508654, -0.015581002458930016, -0.000640885264147073, -0.05364126339554787, 0.025715311989188194, 0.015554368495941162, 0.014009585604071617, 0.008423062972724438, -0.010314091108739376, 0.027153559029102325, -0.0009671541047282517, -0.02482306770980358, 0.00019934028387069702, -0.03670191764831543, -0.022146331146359444, 0.00234880275093019, -0.034198302775621414, -0.012484777718782425, 0.005796266254037619, -0.02639448456466198, -0.03955177590250969, -0.015314660966396332, 0.00012141446495661512, -0.019562814384698868, 0.001864393474534154, -0.01344360876828432, -0.03654211014509201, -0.018204471096396446, -0.01438246387988329, -0.026301264762878418, -0.018670568242669106, 0.006382218562066555, 0.016233541071414948, -0.001260130200535059, 0.005147057585418224, -0.025901751592755318, -0.01640666276216507, -0.017338858917355537, 0.00043904801714234054, 0.006272352300584316, 0.015594319440424442, -0.014302561059594154, 0.02077466994524002, -0.0034358107950538397, 0.0025918397586792707, -0.009927894920110703, -0.01478197705000639, -0.008742673322558403, 0.01542119774967432, -0.021467158570885658, 0.02165359817445278, -0.025741947814822197, 0.014928464777767658, -0.018444178625941277, -0.02037515677511692, 0.015128221362829208, -0.014901830814778805, -0.02290540561079979, -0.006288998760282993, 0.007171256467700005, -0.026913851499557495, -0.0065719871781766415, 0.005576534196734428, -0.010660335421562195, -0.0031844505574554205, 0.01869720220565796, -0.027326680719852448, 0.014568903483450413, 0.016220223158597946, 0.021999843418598175, -0.030869029462337494, 0.0033941948786377907, -0.009501747786998749, 0.006142510566860437, 0.018537398427724838, 0.003615591675043106, -0.02306521125137806, -0.01367665734142065, -0.0008556234533898532, -0.08512287586927414, 0.04583744332194328, 0.00954835768789053, -0.041336268186569214, 0.003305969061329961, -0.00025073596043512225, 0.004318068269640207, -0.029191074892878532, -9.519642844679765e-06, 0.018843691796064377, -0.04589071497321129, 0.0015397892566397786, -0.01113309245556593, -0.01688607782125473, -0.015541051514446735, -0.007390988525003195, -0.0012126880465075374, 0.005503289867192507, 0.00035331922117620707, 0.006681852973997593, -0.025728629902005196, 0.003380877897143364, -0.006288998760282993, 0.004581081215292215, -0.021347304806113243, 0.006761755794286728, -0.000729111023247242, 0.01299748569726944, 0.0010645353468135, -0.028258878737688065, 0.0388592854142189, -0.007297768723219633, 0.01018757838755846, 0.03632903844118118, -0.03201429918408394, -0.041256364434957504, 0.007284451741725206, 0.021666916087269783, 0.009894602932035923, -0.022612430155277252, -0.004474544432014227, -0.011426068842411041, 0.006884938571602106, -0.006385547574609518, -0.010400651954114437, 0.004967276938259602, 0.006685182452201843, 0.0085229417309165, 0.013130656443536282, 0.02964385598897934, 0.03747430816292763, 0.01038733497262001, -0.017485346645116806, -0.017485346645116806, -0.02695380337536335, -0.013650023378431797, -0.008889161981642246, -0.018484128639101982, 0.008376453071832657, -0.015674222260713577, 0.043467000126838684, 0.026740729808807373, 0.02671409584581852, -0.018137885257601738, 0.016393344849348068, -0.011006579734385014, -0.01502168457955122, -0.010820141062140465, -0.010114334523677826, -0.015581002458930016, -0.01982915587723255, -0.023837601765990257, 0.02623467892408371, 0.017418762668967247, 0.0029347550589591265, -0.012318314053118229, 0.0037154697347432375, 0.006944865453988314, -0.02132067084312439, 0.03217410296201706, -0.0015206459211185575, -0.024370286613702774, -0.015154855325818062, 0.005756314843893051, -0.00887584500014782, 0.008243282325565815, -0.021200817078351974, -0.0020974427461624146, 0.009588309563696384, 0.00265676062554121, -0.010174261406064034, 0.020122133195400238, -0.005020545329898596, -0.008003574796020985, -0.0024753152392804623, 0.01845749467611313, 0.006392206065356731, -0.00546666793525219, 0.015594319440424442, 0.04258807376027107, 0.002493626205250621, 0.017019249498844147, -0.018643934279680252, -0.034091766923666, -0.010134310461580753, 0.022212916985154152, -0.015953881666064262, -0.020947791635990143, -0.018111251294612885, 0.017152419313788414, 0.027539754286408424, 0.0038619579281657934, 0.01438246387988329, 0.002838206011801958, -0.015194807201623917, 0.0031644750852137804, -0.02410394325852394, -0.005999351851642132, -0.03941860422492027, 0.03614259883761406, 0.007663988973945379, 0.04344036802649498, 0.03345254436135292, -0.0020192046649754047, 0.027166876941919327, -0.011099799536168575, 0.007683964446187019, 0.0077372328378260136, 0.015128221362829208, -0.01618027128279209, 0.014875196851789951, -0.003708811243996024, -0.03315956890583038, -0.0006429660716094077, -0.024929603561758995, 0.007038085255771875, 0.013476901687681675, 0.010640359483659267, 0.0008522941498085856, 0.07542803138494492, 0.018843691796064377, -0.007517500780522823, 0.018497446551918983, 0.0054466924630105495, 0.02068145014345646, 0.00036018583341501653, 0.0029181085992604494, 0.02037515677511692, -0.034517914056777954, 0.0014765330124646425, -0.002272229641675949, 0.008629478514194489, -0.022958673536777496, -0.02672741189599037, -0.016513198614120483, 0.014662123285233974, 0.02202647738158703, -0.021507110446691513, 0.002671742346137762, 0.020561596378684044, -0.022226233035326004, 0.03790045529603958, -0.0013824810739606619, -0.04136290028691292, -0.0035223718732595444, 0.01855071447789669, 0.019709302112460136, 0.0024919616989791393, -0.028871463611721992, -0.01098660472780466, 0.007051402237266302, -0.03678182139992714, -0.0011019897647202015, -0.0021423879079520702, -0.013263828121125698, -0.02948405034840107, -0.009335284121334553, 0.01153260562568903, -0.004514495376497507, 0.010220871306955814, 0.05089794099330902, -0.008529599756002426, -0.01121965330094099, 0.04868730530142784, -0.014968415722250938, -0.012704509310424328, -0.007344378624111414, -0.04487861320376396]} +{"id": "test:10000087", "text": "\"J Brand can do no wrong! They always have such beautiful pieces that belong in every women closet. I just wished my wallet agreed with me \ud83d\ude0b Lol!\"", "vector_field": [-0.027255350723862648, -0.01003695372492075, -0.010443945415318012, -0.035657770931720734, -0.014362886548042297, -0.002998285461217165, 0.007142057176679373, -0.005914515815675259, -0.0038762728217989206, -0.02606063149869442, 0.004362037405371666, -0.02189880795776844, -0.01268240250647068, -0.02833191119134426, -0.009643089957535267, 0.017146188765764236, 0.02168874628841877, 0.005510806106030941, 0.01915489137172699, -0.030590061098337173, -0.014494175091385841, 0.0005103813600726426, -0.014546689577400684, -0.007063284516334534, 0.004427681677043438, -0.005674915853887796, 0.02486591227352619, -0.013758962973952293, -0.019509369507431984, 0.005596143193542957, 0.005307309795171022, 0.02788553200662136, -0.024104442447423935, -0.026415107771754265, -0.004493325483053923, -0.02629694901406765, -0.011238236911594868, 0.0029572579078376293, 0.012393569573760033, 0.0030524416361004114, 0.024301374331116676, -0.000989581923931837, 1.6616113498457707e-05, 0.02401254139840603, -0.03662930056452751, 0.0014367810217663646, -0.0024567232467234135, 0.016253432258963585, -0.012564243748784065, -0.009039166383445263, 0.0062361713498830795, 0.030064908787608147, -0.031062696129083633, 0.009721863083541393, -0.006997640710324049, 0.012432956136763096, -0.0003626825928222388, 0.02258150465786457, 0.013457001186907291, -0.04006903991103172, 0.0001003120923996903, -0.004201210103929043, -0.02951350063085556, 0.0033872255589812994, -0.027150319889187813, -0.03158784657716751, -0.019036732614040375, 0.00875033251941204, 0.0170411579310894, 0.021951323375105858, 0.018170233815908432, -0.0035775930155068636, 0.005960466805845499, 0.008901313878595829, 0.0041191549971699715, -0.011159463785588741, -0.003682623151689768, 0.003636672394350171, -0.010667135007679462, 0.015386931598186493, 0.025272903963923454, -0.0054221865721046925, -0.006807273253798485, 0.020375868305563927, 0.01186185423284769, 0.014008409343659878, -0.010056646540760994, 0.03739076852798462, 0.01709367334842682, -0.02699277549982071, -0.009991002269089222, 0.005921080242842436, -0.00554362777620554, -0.0012931849341839552, -0.03649801388382912, 0.014218470081686974, 0.009964745491743088, 0.0029227950144559145, 0.019194278866052628, -0.008960393257439137, 0.010693392716348171, -0.00040268435259349644, -0.026362592354416847, -0.0056059896014630795, -0.032218027859926224, -0.013772091828286648, -0.005133353639394045, -0.030379999428987503, 0.03179790824651718, 0.010877195745706558, -0.035368937999010086, 0.026887744665145874, -0.00955118890851736, -0.0447428859770298, 0.031981710344552994, -0.001580377109348774, 0.0007360323215834796, 0.0036662123166024685, -0.006305097136646509, -0.007457147818058729, 0.011074126698076725, 0.011960320174694061, 0.054773274809122086, -0.0015606839442625642, -0.007417761720716953, 0.03360968083143234, -0.015295030549168587, -0.018301520496606827, -0.020441513508558273, 0.016279688104987144, 0.046055763959884644, 0.039543889462947845, 0.011192286387085915, 0.021977579221129417, 0.008054506964981556, 0.0196275282651186, -0.018393423408269882, 0.033767227083444595, 0.015124356374144554, -0.009341128170490265, 0.005921080242842436, 0.032638151198625565, -0.00644951406866312, -0.00518258661031723, 0.00581933232024312, 0.023881252855062485, -0.0010872272541746497, 0.0031000336166471243, -0.005097249522805214, -0.008993214927613735, 0.008691253140568733, -0.007279909215867519, 0.018747899681329727, -0.010535847395658493, 0.005556756630539894, 0.006853223778307438, 0.01887918822467327, 0.01847219467163086, -0.0003485281195025891, -0.0038959658704698086, 0.0036760587245225906, 0.003830322064459324, 0.010227320715785027, -0.0041191549971699715, 0.023671193048357964, 0.02487904019653797, 0.012144123204052448, 0.012636451981961727, -0.018380293622612953, -0.004965961445122957, 0.01797330193221569, 0.03224428743124008, -0.027833016589283943, 0.01383773609995842, -0.006846659351140261, 0.021964451298117638, 0.02189880795776844, 0.008888185024261475, -0.0024468766059726477, 0.010581797920167446, -0.015505091287195683, 0.006728500593453646, -0.005464855115860701, 0.005927644670009613, -0.007050155662000179, -0.011815903708338737, 0.01570202223956585, -0.02079598978161812, -0.00026811432326212525, 0.009859714657068253, -0.00420777453109622, 0.025706153362989426, 0.01244608499109745, -0.010522718541324139, -0.6373235583305359, -0.02191193588078022, 0.0022302516736090183, 0.01453356072306633, 0.0223714429885149, 0.034686241298913956, 0.022423958405852318, -0.0017970019252970815, -0.02378935180604458, 0.010115725919604301, -0.021150466054677963, 0.0318504236638546, -0.007745981216430664, -0.005254794843494892, 0.0009666065452620387, -0.01014198362827301, 0.02146555855870247, -0.01797330193221569, -0.01979820244014263, 0.0142972432076931, -0.02353990450501442, 0.01588582620024681, -0.007128928322345018, -0.0005066888988949358, 0.0001568274019518867, 0.0186691265553236, 0.014310372062027454, 0.001778949867002666, -0.0043029580265283585, 0.023644935339689255, -0.02510222978889942, 0.037495799362659454, -0.015163742937147617, -0.007273344788700342, 0.051517337560653687, -0.010089468210935593, -0.017855143174529076, 0.020887890830636024, 0.021793777123093605, 0.029198409989476204, -0.0336359366774559, 0.009984438307583332, 0.003218192607164383, -0.019982004538178444, 0.007266780361533165, 0.035605255514383316, 0.01706741563975811, 0.023474261164665222, 0.008592787198722363, 0.006035957485437393, 0.015767667442560196, 0.0159252118319273, 0.0032674255780875683, 0.01732999086380005, 0.018301520496606827, -0.0051202247850596905, 0.01416595559567213, -0.007896961644291878, 0.002199070993810892, -0.00194634182844311, 0.0019840870518237352, 0.01212442945688963, -0.022778436541557312, -0.026231305673718452, 0.003397072199732065, 0.020887890830636024, -0.00959057454019785, -0.02378935180604458, 0.009262355044484138, -0.01396902371197939, -0.01130388118326664, -0.0006047444767318666, -0.04568815976381302, 0.000303603068459779, 0.006265711039304733, 0.03710193559527397, 0.01566263660788536, 0.02284407988190651, -0.013667061924934387, 0.01705428771674633, -0.017842013388872147, -0.023592419922351837, -0.01118572149425745, -0.006958254147320986, 0.035867832601070404, -0.0010658929822966456, 0.011133207008242607, -0.032848212867975235, -0.013575159944593906, -0.0010437380988150835, 0.018642868846654892, 0.01442853081971407, 0.015623250044882298, -0.01258393656462431, 0.0003526308573782444, -0.01026670727878809, -0.004657435230910778, 0.0066530099138617516, -0.005609272047877312, -0.025968730449676514, 0.0036333901807665825, 0.016699809581041336, 0.005773381795734167, 0.013174732215702534, -0.017461279407143593, 0.015807053074240685, 0.013680189847946167, 0.023342972621321678, 0.0020185501780360937, -0.03261189162731171, 0.004125719424337149, -0.021990709006786346, -0.00710267061367631, 0.010706521570682526, -0.020205194130539894, -0.025706153362989426, 0.026415107771754265, -0.005290898960083723, 0.01887918822467327, -0.01752692274749279, 0.00785757601261139, 0.005107095930725336, -0.00785757601261139, -0.016739197075366974, -0.010555540211498737, 0.006101601291447878, -0.0009305023704655468, -0.021308012306690216, 0.02003451995551586, 0.0425109937787056, 0.028489455580711365, 0.012669273652136326, 0.018839800730347633, -0.008041379041969776, -0.008894748985767365, -0.008376162499189377, -0.01638471893966198, -0.018524710088968277, 0.04056793451309204, -0.03161410614848137, 0.025023456662893295, -0.010752472095191479, 0.00691886804997921, 0.008008556440472603, 0.0014088823227211833, -0.03830978274345398, -0.015859568491578102, 0.00484452024102211, -0.02673019841313362, -0.030274970456957817, -0.02701903134584427, -0.010936275124549866, -0.024852782487869263, 0.034922558814287186, -6.5438769524917e-05, -0.005258076824247837, 0.009748119860887527, 0.0035710285883396864, -0.010023824870586395, -0.007673772983253002, -0.0021055282559245825, 0.012964671477675438, -0.010253578424453735, -0.015386931598186493, -0.0013276480603963137, -0.00842211302369833, 0.019680043682456017, -0.0022827668581157923, -0.017343120649456978, -0.02838442474603653, 0.02512848749756813, -0.002202352974563837, -0.010292964987456799, -0.0002633961848914623, -0.007614693138748407, -0.011231672950088978, -0.005842307582497597, -0.013200989924371243, 0.016752324998378754, -0.014152826741337776, -0.0006954151322133839, -0.007660644128918648, -0.022660275921225548, 0.015413189306855202, 0.028226880356669426, 0.01823587715625763, 0.007588435895740986, 0.02027083933353424, -0.010791858658194542, -0.008559965528547764, 0.004903600085526705, 0.00691886804997921, -0.013214118778705597, -0.03665555641055107, -0.018839800730347633, 0.003249373519793153, 0.0005690506077371538, 0.015098098665475845, 0.01557073462754488, 0.0031016746070235968, 0.002066141925752163, -0.004076486453413963, 0.01338479295372963, -0.011481119319796562, -0.00016780223813839257, -0.029408469796180725, 0.02995987981557846, -0.004641024395823479, -0.005435315426439047, 0.011559892445802689, 0.0024304657708853483, 0.019955748692154884, 0.0023631807416677475, -0.0011060998076573014, 0.017421893775463104, 0.019325565546751022, 0.01589895412325859, 0.017881400883197784, 0.00955118890851736, -0.00011651793465716764, -0.010299528948962688, 0.005799639038741589, 0.01420534122735262, -6.369510083459318e-05, 0.019299307838082314, -0.013680189847946167, 0.002988438820466399, -0.00679414439946413, 0.0016361743910238147, -0.008047943003475666, 0.002799712587147951, 0.011172592639923096, 0.01085093803703785, 0.024787139147520065, -0.0035546175204217434, -0.01685735583305359, 0.027150319889187813, -0.013371664099395275, 0.017395636066794395, 6.969536480028182e-05, 0.024327632039785385, 0.015505091287195683, 0.0017855142941698432, 0.010608055628836155, 0.029592273756861687, -0.013548902235925198, 0.03899247944355011, 0.00806107185781002, 0.0044178348034620285, -0.005244947969913483, 0.001145486137829721, -0.013575159944593906, -0.027833016589283943, 0.01209160778671503, 0.004696821793913841, -0.03521139174699783, 0.019771944731473923, -0.0008550118654966354, -0.0039058125112205744, 0.02146555855870247, -0.008382727392017841, 0.010050082579255104, 0.0031262910924851894, 0.007069848943501711, 0.03179790824651718, 0.02465585246682167, 0.0005645375931635499, -0.01915489137172699, -0.018367165699601173, 0.010227320715785027, -0.009958180598914623, -0.019483111798763275, 0.006646445486694574, -0.02764921449124813, 0.001171743730083108, -0.009774377569556236, 0.015426318161189556, 0.01639784872531891, 0.007929784245789051, 0.021504944190382957, -0.00519899744540453, -0.0306688342243433, 0.025482965633273125, 0.012065350078046322, 0.014901166781783104, 0.00714862160384655, -0.016660423949360847, -0.004985654726624489, 0.01166492234915495, 0.04813011363148689, 0.010430816560983658, 0.04513674974441528, -0.003175524063408375, 0.004709950182586908, 0.023408617824316025, 0.00023221531591843814, 0.03153533488512039, -0.02233205735683441, 0.013181296177208424, -0.0026979646645486355, 0.011251365765929222, -0.015308159403502941, 0.004732925910502672, -0.015386931598186493, 0.032900724560022354, 0.013411050662398338, -0.00026975543005391955, 0.010201063007116318, -0.0016320716822519898, -0.010338915511965752, 0.006410127505660057, -0.019535627216100693, -0.032874468713998795, -0.0005620759329758584, 0.0009181941277347505, 0.012636451981961727, -0.013535773381590843, -0.0010667134774848819, 0.023119784891605377, 0.011264494620263577, -0.01039799489080906, -0.012203202582895756, -0.035158876329660416, 0.023395488038659096, 0.11595340073108673, 0.005494394805282354, -0.011467990465462208, 0.00710267061367631, 0.0006921329186297953, 0.005632247310131788, -0.011920933611690998, 0.020664701238274574, 0.02925092540681362, 0.004289829172194004, 0.023592419922351837, -0.005793074611574411, -0.0017247936921194196, -0.000895218807272613, 0.010082904249429703, -0.002228610683232546, 0.004837955813854933, 0.0020989638287574053, -0.006295250728726387, -0.009426465258002281, -0.00980720017105341, 0.019299307838082314, -0.017907658591866493, 0.030091166496276855, 0.004942986182868481, -0.0038171932101249695, 0.011271058581769466, 0.020152678713202477, 0.007975734770298004, -0.019036732614040375, -0.0027537618298083544, 0.02259463258087635, -0.02055967226624489, 0.015426318161189556, -0.004998783580958843, -0.0013908302644267678, -0.018170233815908432, -0.018537839874625206, 0.0248921699821949, -0.005658505018800497, 0.014546689577400684, 0.008113587275147438, 0.02397315576672554, -0.008481192402541637, -0.007522791624069214, -0.013122216798365116, -0.011829031631350517, 0.023408617824316025, 0.019207406789064407, 0.001418728963471949, 0.0027849427424371243, -0.012327926233410835, -0.016214044764637947, -0.014034667052328587, 0.02906712144613266, 0.01013541966676712, -0.023815609514713287, -0.004549122881144285, -0.018564097583293915, -0.007791931740939617, 0.016502877697348595, -0.03269066661596298, 0.01232136134058237, 0.014467917382717133, -0.016095886006951332, -0.03731199726462364, -0.027491668239235878, -0.0012316438369452953, -0.021084822714328766, -0.010358608327805996, -0.03326833248138428, -0.0058685652911663055, 0.0008804489043541253, -0.003840168472379446, 0.014769879169762135, 0.005714301951229572, 0.010732779279351234, 0.01778949797153473, 0.027754243463277817, 0.029408469796180725, -0.0021120926830917597, -0.00991223007440567, -0.0012217971961945295, -0.020887890830636024, -0.0006896713166497648, 0.013279762119054794, 0.017172446474432945, -0.009406771510839462, -0.043692585080862045, 0.022515859454870224, 0.03213925659656525, 0.013890250585973263, 0.0364454984664917, 0.0031312142964452505, -0.006498747039586306, 0.00032411678694188595, 0.005586296319961548, 0.009150760248303413, 0.0014589357888326049, -0.008159537799656391, 0.031955454498529434, 0.017684468999505043, -0.009216404519975185, 0.020231451839208603, 0.008599352091550827, -0.015846438705921173, -0.009629961103200912, 0.0032625021412968636, -0.01141547504812479, 0.00959057454019785, -0.0028768442571163177, -0.011907804757356644, 0.010936275124549866, -0.01085093803703785, -0.016975514590740204, 0.030485030263662338, -0.0034922559279948473, 0.016739197075366974, 0.007444018963724375, -0.025062844157218933, -0.01916802115738392, -0.02699277549982071, 0.005215408280491829, 0.024537691846489906, -0.0058685652911663055, -0.00455240486189723, 0.004952832590788603, -0.02424885891377926, -0.023881252855062485, 0.028752030804753304, -0.013627675361931324, 0.006761322263628244, -0.03439740836620331, -0.02372370846569538, -0.03303201496601105, 0.02374996617436409, -0.03114146925508976, 0.01662103645503521, -0.015137485228478909, -0.009347692131996155, -0.02676958590745926, 0.02192506566643715, -0.02054654248058796, -0.01613527163863182, -0.006442949641495943, -0.0302224550396204, -0.01130388118326664, -0.026966517791152, -0.028673259541392326, 0.05353917181491852, -0.017435021698474884, 0.024104442447423935, -0.018787287175655365, -0.0081857955083251, 0.0003965302312280983, -0.04453282803297043, -0.023618677631020546, -0.019286179915070534, 0.00479200528934598, 0.03920254111289978, -0.006183655932545662, -0.00010164548439206555, 0.013443872332572937, -0.0011151258368045092, -0.016975514590740204, -0.015807053074240685, -0.0016033524880185723, -0.017014900222420692, -0.013667061924934387, 0.022883465513586998, 0.013404485769569874, 0.0029293594416230917, -0.014257856644690037, 0.009012908674776554, -0.013916508294641972, 0.004732925910502672, -0.006945125292986631, -0.013758962973952293, -0.019089248031377792, 0.012209766544401646, 0.009085116907954216, -0.02073034644126892, -0.004624613095074892, -0.0033297871705144644, -0.014546689577400684, 0.006781015545129776, -0.001632892177440226, -1.7975147784454748e-05, 0.008303954266011715, 0.003062288276851177, 0.01543944701552391, -0.01199970580637455, 0.0025437013246119022, 0.010004131123423576, -0.004765747580677271, -0.02649388089776039, -0.010233884677290916, -0.009111374616622925, -0.012400134466588497, 0.0089997798204422, 0.007043591234833002, 0.018866058439016342, -0.008815976791083813, 0.013863992877304554, 0.01082468032836914, 7.672131323488429e-05, -0.004043664783239365, -0.01960127055644989, 0.0010388147784397006, -0.018065202981233597, -0.016240302473306656, -0.006708807311952114, -0.02561425231397152, -0.02141304314136505, -0.001925007556565106, 0.021321140229701996, -0.022279541939496994, 0.003872990608215332, -0.002670065965503454, -0.0191155057400465, -0.01960127055644989, 0.005674915853887796, 0.011743694543838501, -0.010647442191839218, 0.021058565005660057, 0.018550967797636986, -0.000374170282157138, -0.01680484041571617, -0.00377780687995255, -0.010076339356601238, -0.005609272047877312, -0.0057897926308214664, 0.016936128959059715, -0.0061705270782113075, -0.013319148682057858, 0.01776324212551117, -0.013653933070600033, -0.03859861567616463, -0.01912863366305828, 0.031955454498529434, 0.002432106761261821, 0.012603630311787128, -0.013043444603681564, -0.011172592639923096, -0.01500619761645794, 0.007352117449045181, -0.021609975025057793, -0.007713159080594778, -0.0030967514030635357, -0.023251071572303772, 0.011166028678417206, 0.025299161672592163, 0.0070238979533314705, 0.01571515202522278, -0.01303031574934721, -0.015872696414589882, -0.008573094382882118, -0.005146482493728399, 0.02076973207294941, 0.012780868448317051, -0.009183582849800587, 0.02564051002264023, 0.0023172299843281507, 0.009308306500315666, 0.02583744190633297, 0.0066530099138617516, -0.007634386420249939, -0.01497993990778923, -0.004056793637573719, 0.015767667442560196, -0.03844107314944267, -0.001434319419786334, -0.0028538687620311975, -0.002323794411495328, -4.0078681195154786e-05, -0.018537839874625206, -0.018550967797636986, -0.04981059581041336, 0.022778436541557312, -0.008566529490053654, -0.002747197402641177, -0.00010595336789265275, -0.0019348541973158717, -0.0018084896728396416, -0.016883613541722298, -0.006800708826631308, -0.025220388546586037, -0.023434875532984734, 0.02103230729699135, -0.01681796833872795, -0.01725121960043907, -0.035158876329660416, -0.007404632866382599, 0.01280712615698576, 0.0040830508805811405, -0.014507303945720196, -0.006091754417866468, 0.02213512547314167, -0.013417614623904228, 0.02398628368973732, -0.014467917382717133, -0.004657435230910778, -0.027045289054512978, 0.009465851821005344, 0.005563321057707071, -0.011822467669844627, 0.0038762728217989206, 0.031482819467782974, -0.02515474520623684, -0.023815609514713287, -0.002202352974563837, -0.007004204671829939, 0.0069057391956448555, -0.007069848943501711, 0.016240302473306656, -0.005143200047314167, -0.0008550118654966354, -0.008152972906827927, -0.03116772696375847, 0.010286400094628334, 0.0063871522434055805, 0.021176723763346672, 0.019890103489160538, -0.01462546270340681, 0.004857649095356464, 0.0007466994575224817, 0.013601417653262615, 0.0042635719291865826, -0.016988642513751984, -0.0022220462560653687, -0.0033363515976816416, 0.023618677631020546, 0.006114730145782232, -0.024288246408104897, -0.021820034831762314, 0.0027652496937662363, -0.013562031090259552, 0.0223714429885149, -0.00391565915197134, -0.007883833721280098, -0.0014310372062027454, 0.016739197075366974, 0.02378935180604458, 0.028909577056765556, -0.012767739593982697, -0.012912156991660595, -0.016043370589613914, -0.012649580836296082, -0.005041452124714851, -0.0336359366774559, -0.011605842970311642, 0.04038413241505623, 0.021820034831762314, -0.035841573029756546, -0.006672703195363283, 0.003449587384238839, -0.010982225649058819, -0.024275116622447968, -0.023474261164665222, 0.007037026807665825, 0.022555246949195862, 0.016909871250391006, 0.005599425174295902, 0.023815609514713287, 0.014664849266409874, 0.01521625742316246, -0.006091754417866468, 0.009702169336378574, -0.0025880110915750265, -0.010739343240857124, 0.000277550658211112, 0.0019332130905240774, -0.054563216865062714, -0.011428603902459145, -0.006078626029193401, 0.029907364398241043, 0.010430816560983658, -0.00489047123119235, 0.003935352433472872, -0.000507509452290833, 0.00022688174794893712, 0.00851401500403881, 0.033110786229372025, 0.012400134466588497, 0.03326833248138428, -0.011218544095754623, -0.007910090498626232, 0.011855289340019226, 0.02720283530652523, -0.011671486310660839, -0.020480899140238762, 0.001962752779945731, 0.006003135349601507, -0.0004221723647788167, -0.0302224550396204, 0.0018084896728396416, -0.009209840558469296, 0.022660275921225548, -0.0004689436755143106, -0.011395782232284546, -0.017605695873498917, 0.04343000799417496, 0.0018396704690530896, -0.011290752328932285, -0.01801268756389618, 0.010759036056697369, 0.032401833683252335, -0.02373683638870716, 0.027150319889187813, -0.0232642013579607, -0.01268240250647068, -0.011855289340019226, 0.001841311575844884, 0.004102744162082672, -0.016529135406017303, -0.024314504116773605, -0.022502731531858444, -0.024498306214809418, 0.0018232595175504684, 0.0019282897701486945, -0.008605916053056717, 0.022752178832888603, -0.01664729416370392, 0.00851401500403881, 0.0075556137599051, -0.0322968028485775, -0.01687048375606537, -0.04873403534293175, -0.012872770428657532, 0.015741409733891487, -0.03710193559527397, -0.012984365224838257, -0.016489749774336815, -0.005796357057988644, 0.02396002598106861, -0.009616832248866558, 0.23988910019397736, -0.017014900222420692, -0.010942839086055756, 0.024445790797472, 0.012524857185781002, 3.6693920264951885e-05, 0.0020858352072536945, 0.0025092384312301874, -0.021268626675009727, -8.502937271259725e-05, -0.009433029219508171, 0.008894748985767365, -0.004988937173038721, -0.006938560865819454, 0.01085093803703785, -0.04190707206726074, -0.03166662156581879, -0.005550192203372717, 0.0016886894591152668, -0.018091460689902306, 0.0018790567992255092, 0.018866058439016342, -0.020480899140238762, -0.017881400883197784, 0.027386637404561043, 0.0016919716726988554, -0.021951323375105858, 0.012866205535829067, 0.006807273253798485, 0.02627069130539894, -0.010443945415318012, 0.015360673889517784, 0.012157252058386803, -0.02396002598106861, 0.00750309880822897, -0.018800415098667145, -0.022975368425250053, 0.011015047319233418, 0.027386637404561043, 0.002553547965362668, -0.0028998195193707943, 0.009721863083541393, -0.023631807416677475, 0.0038762728217989206, 0.024393275380134583, 0.010522718541324139, -0.013929637148976326, -0.021084822714328766, 0.0036136971320956945, 0.015295030549168587, 9.987617886508815e-06, 0.00031591131119057536, 0.03405606001615524, 0.02465585246682167, -0.008960393257439137, -0.006662856787443161, 0.016502877697348595, -0.0008566529722884297, -0.02906712144613266, -0.004847802687436342, 0.0009362462442368269, 0.01361454650759697, -0.0036038504913449287, 0.014323500916361809, -0.008894748985767365, 0.015426318161189556, -0.005330285057425499, -0.018078332766890526, 0.01163866464048624, -0.006567672826349735, -0.0028423811309039593, -0.005077556241303682, -0.008671560324728489, 0.0059440559707582, -0.019522497430443764, -0.010923146270215511, 0.020388998091220856, -0.015491962432861328, -0.004404705949127674, 0.009892537258565426, 0.010883759707212448, -0.0032149103935807943, -0.03557899594306946, -0.015636378899216652, -0.010706521570682526, -0.026914002373814583, -0.014507303945720196, -0.001098714885301888, 0.00875033251941204, -0.018642868846654892, 0.013411050662398338, 0.021964451298117638, 0.011205415241420269, -0.017697596922516823, 0.004923292901366949, 0.02610001713037491, -0.018301520496606827, 0.015203128568828106, -0.02535167708992958, 0.011966884136199951, 0.004929857328534126, -0.026441365480422974, -0.0186691265553236, 0.005632247310131788, -0.006761322263628244, 0.021321140229701996, -0.014454788528382778, 0.014796136878430843, 0.0015467345947399735, -0.01752692274749279, -0.0055698854848742485, -0.017343120649456978, 0.0003799141268245876, 0.0071748788468539715, -0.005176022183150053, 0.0060195461846888065, 0.003280554199591279, -0.005917798262089491, 0.011093820445239544, -0.01186185423284769, -0.016791710630059242, -0.005395929329097271, 0.007910090498626232, -0.009282048791646957, 0.0195618849247694, -0.017802627757191658, 0.004929857328534126, 0.014835523441433907, 0.017119931057095528, -0.032874468713998795, 0.010791858658194542, 0.0035907216370105743, 0.0014310372062027454, 0.0008275235304608941, -0.006931996438652277, 0.013811478391289711, -0.0038467328995466232, -0.014940553344786167, -0.0010437380988150835, -0.009833456948399544, -0.02585056982934475, 0.022410830482840538, 0.02812184952199459, -0.019785074517130852, 0.023014754056930542, -0.040489163249731064, 0.0021006050519645214, -0.03437114879488945, -0.010082904249429703, 0.012387005612254143, 0.0009124503121711314, -0.00404694676399231, -0.020809117704629898, -0.02076973207294941, 0.03752205893397331, 0.010332350619137287, -0.05162236839532852, -0.02694026008248329, -0.01754005253314972, 0.022489601746201515, -0.020612187683582306, -0.00348569150082767, 0.049442991614341736, -0.022227026522159576, -0.005894822999835014, -0.02306726947426796, -0.1659477949142456, 0.04639711230993271, 0.019496239721775055, 0.01475675031542778, 0.013233811594545841, 0.015373802743852139, -0.008356469683349133, 0.00500534800812602, -0.05051955208182335, -0.0006359253311529756, 0.01570202223956585, -0.0031115212477743626, -0.01618778705596924, 0.009242662228643894, -0.001658329158090055, -0.013404485769569874, -0.010896888561546803, 0.008907877840101719, -0.007398068439215422, 0.014783008024096489, 0.025036586448550224, -0.008815976791083813, -0.003682623151689768, 0.005504241678863764, -0.0065348511561751366, 0.011356395669281483, 0.013069702312350273, 0.015137485228478909, 0.011651793494820595, -0.012439520098268986, -0.005550192203372717, -0.009984438307583332, 0.029644789174199104, -0.00126692745834589, 0.015623250044882298, 0.03339961916208267, -0.012951542623341084, -0.017185574397444725, 0.012019399553537369, 0.021728133782744408, 0.007916655391454697, 0.014769879169762135, -0.002742274198681116, 0.004929857328534126, -0.005064427386969328, 0.020165808498859406, 0.020677831023931503, -0.00783131830394268, -0.0010018901666626334, -0.03245434910058975, 0.0075556137599051, -0.025653639808297157, -0.00391565915197134, 0.02028396725654602, 0.015596992336213589, 0.01037173718214035, 0.00024883143487386405, 0.017802627757191658, -0.0034659982193261385, -0.030064908787608147, 0.005976877640932798, -0.01641097664833069, 0.0023467696737498045, -0.0014523714780807495, -0.009826892986893654, -0.00032842467771843076, 0.001815053983591497, -0.025679895654320717, -0.012400134466588497, 0.014572947286069393, -0.01872164197266102, 0.006666138768196106, 0.02235831506550312, -0.006623470224440098, 0.01533441711217165, 0.004444092512130737, -0.005468137562274933, 0.004535994026809931, 0.005888258572667837, -0.03397728502750397, -0.009032601490616798, 0.0179995596408844, -0.013653933070600033, 0.017907658591866493, -0.001549196196720004, 0.02581118419766426, 0.01186841819435358, -0.007759109605103731, 0.014257856644690037, -0.006951689720153809, 0.014546689577400684, -0.022922853007912636, -0.017605695873498917, -0.022279541939496994, -0.01616152934730053, 0.020244581624865532, 0.014270985499024391, -0.007713159080594778, -0.0010864066425710917, 0.0022220462560653687, 0.0075424849055707455, 0.01731686294078827, -0.025312291458249092, 0.035185135900974274, 0.02904086373746395, -0.01732999086380005, 0.01658165082335472, 0.00023980540572665632, 0.019207406789064407, 0.007713159080594778, -0.01752692274749279, -0.0037712424527853727, 0.012255717068910599, 0.01635846123099327, 0.008047943003475666, 0.03479127213358879, -0.003836886491626501, -0.025036586448550224, 0.05949963629245758, -0.026388850063085556, 0.04841894656419754, 0.004401423968374729, -0.020139550790190697, 0.023198556154966354, -0.00554362777620554, -0.00935425702482462, -0.10539785772562027, -0.024130700156092644, 0.013601417653262615, 0.012130994349718094, -0.009570881724357605, 0.003406918840482831, -0.004644306376576424, 0.009203275665640831, -0.014769879169762135, 0.010220756754279137, 0.00968247652053833, -0.024590207263827324, -0.010483331978321075, -0.0036399546079337597, 0.00040924872155301273, -0.0036793409381061792, -0.0018117718864232302, 0.01939121074974537, -0.023592419922351837, 0.01488803792744875, 0.001433498808182776, -0.0018774156924337149, -0.02239770069718361, -0.037023164331912994, 0.012643016874790192, 0.004460503347218037, -0.0428260862827301, 0.007089541759341955, 0.014664849266409874, -0.011678051203489304, -0.013483258895576, -0.0034528695978224277, -0.002694682450965047, -0.01039799489080906, -0.0027833017520606518, -0.016726067289710045, 0.004342344589531422, -0.007910090498626232, 0.018104590475559235, -0.03024871274828911, 0.014336629770696163, 0.011172592639923096, -0.024143829941749573, -0.018038945272564888, 0.0013481617206707597, 0.01500619761645794, -0.020415255799889565, 0.021124210208654404, 0.019758816808462143, 0.0075556137599051, -0.010811551474034786, 0.02053341455757618, -0.009288612753152847, 0.0013670343905687332, 0.021609975025057793, -0.007279909215867519, 0.02217451110482216, -0.009505237452685833, 0.03418734669685364, -0.009748119860887527, 0.006183655932545662, 0.022673405706882477, -0.008382727392017841, 0.02213512547314167, -0.010109161958098412, 0.005031605716794729, -0.008822540752589703, 0.00558957876637578, 0.013109087944030762, -0.024065056815743446, 0.01521625742316246, 0.031088953837752342, -0.015583863481879234, 0.04568815976381302, -0.03129901364445686, 0.023828737437725067, -0.03557899594306946, -0.005783228203654289, 0.0138508640229702, -0.012925284914672375, -0.007798496168106794, -0.02147868648171425, -0.0016410977113991976, -0.025286033749580383, 0.02469523809850216, 0.0030163375195115805, -0.018747899681329727, -0.006334636826068163, -0.008671560324728489, -0.02694026008248329, -0.013509515672922134, 0.009734991006553173, 0.011133207008242607, -0.021097952499985695, -0.015596992336213589, -0.000978094176389277, -0.004033817909657955, -0.01221633143723011, 0.021334270015358925, -0.03350464999675751, -0.02510222978889942, -0.02054654248058796, -0.04406018927693367, 0.03253312036395073, -0.017579438164830208, -0.019207406789064407, -0.0008102919673547149, -0.015295030549168587, -0.008842234499752522, -0.004860931541770697, 0.006177091505378485, -0.022555246949195862, -0.02167561836540699, -0.012472342699766159, -0.01683109812438488, -0.020205194130539894, -0.01062118448317051, -0.025942472741007805, 0.0041519771330058575, -0.006781015545129776, 0.011881547048687935, 0.014270985499024391, -0.004286547191441059, 0.0033051706850528717, 0.026677682995796204, -0.010227320715785027, -0.004227467812597752, 0.0383622981607914, -0.03739076852798462, 0.03479127213358879, 0.0018708513816818595, -0.027281608432531357, 0.0016090963035821915, -0.023933768272399902, -0.02124236896634102, 0.0138508640229702, -0.006039239466190338, -0.04358755424618721, -0.005914515815675259, 0.028752030804753304, 0.008454935625195503, -0.0391237698495388, -0.00011385115067241713, -0.010030388832092285, 0.004657435230910778, -0.01244608499109745, 0.0014975016238167882, -0.015242515131831169, -0.008408984169363976, -0.013273198157548904, 0.021399913355708122, 0.010942839086055756, -0.0041519771330058575, 0.0383622981607914, 0.013076266273856163, -0.03476501256227493, -0.022712791338562965, -0.02470836602151394, 0.011940626427531242, -0.004279982764273882, 0.005655222572386265, -0.020126422867178917, 0.03991149738430977, -0.007299602497369051, -0.00010903042857535183, -0.03450243920087814, 0.022515859454870224, -0.0006232068408280611, -0.0006917226710356772, 0.022423958405852318, 0.010109161958098412, -0.0008127536275424063, -0.022935980930924416, 0.004329215735197067, -0.007319295778870583, 0.004962679464370012, 0.0089997798204422, -0.010634313337504864, -0.005041452124714851, -0.005146482493728399, -0.0184459388256073, 0.04088302701711655, -0.03431863337755203, 0.0009132708655670285, -0.026113146916031837, 0.001700177206657827, -0.017579438164830208, 0.011953755281865597, -0.005471419543027878, -0.01290559209883213, 0.013627675361931324, 0.04125063121318817, -0.029145894572138786, 0.005038169678300619, -0.00541234016418457, -6.390023918356746e-05, 0.01013541966676712, 0.010450510308146477, 0.00037704218993894756, 0.007916655391454697, -0.0031476253643631935, 0.036576785147190094, 0.006508593447506428, -0.00239600264467299, -0.01108725555241108, -0.0006506952340714633, -0.008901313878595829, -0.0063543301075696945, -0.02171500399708748, -0.006886045914143324, -0.008507450111210346, 0.020940406247973442, 0.024957813322544098, -0.0019036732846871018, 0.01085093803703785, 0.011592714115977287, -0.002437029965221882, 0.012590501457452774, 0.011120078153908253, -0.00502175884321332, -0.008888185024261475, 0.018997346982359886, 0.0114089110866189, 0.0019529062556102872, 0.019719429314136505, 0.01289902813732624, 0.02629694901406765, 0.030064908787608147, 0.01521625742316246, 0.009085116907954216, -0.0008254721178673208, 0.0024944685865193605, 0.03337336331605911, -0.030353743582963943, -0.003718727268278599, 0.006643163505941629, -0.009767813608050346, 0.011448297649621964, -0.005724148824810982, 0.0451892651617527, -0.023185428231954575, 0.051989976316690445, -0.0070567200891673565, -0.022227026522159576, 0.0008599351858720183, -0.005586296319961548, 0.023119784891605377, 0.0211898535490036, -0.011697744019329548, -0.0032313212286680937, 0.010443945415318012, -0.0211898535490036, -0.010555540211498737, 0.006377305369824171, 0.002574882237240672, -0.018537839874625206, 0.012853076681494713, -0.012380440719425678, -0.0008250618702732027, -0.021557459607720375, 0.01462546270340681, 0.01618778705596924, 0.0054221865721046925, 0.015189999714493752, 0.01267583854496479, -0.005310592241585255, -0.03360968083143234, 0.0012029246427118778, 0.011093820445239544, 0.0016156606143340468, -0.024721495807170868, 0.029408469796180725, -0.025561736896634102, -0.024275116622447968, -0.00030729552963748574, 0.020809117704629898, -0.0179995596408844, -0.01957501284778118, 0.012833383865654469, -0.02493155561387539, 0.018117718398571014, 0.0037088808603584766, 0.008507450111210346, -0.03024871274828911, -0.04999439790844917, 0.046055763959884644, -0.008133280090987682, 0.003449587384238839, 0.03132527321577072, -0.038152240216732025]} +{"id": "test:10000088", "text": "\"I received the brass that you sent me, and all looks well. Thank you very much for all your trouble and the extra 3 pieces. I feel that you have an outstanding company and are striving the best that you can to achieve customer satisfaction. I will be certain to tell my friends about US Reloading Supply. ~ Michelle Admin Note: By accident we shorted them two shells, so we sent them replacements.\"", "vector_field": [-0.03073452040553093, 0.012099354527890682, -0.0005684231873601675, -0.05020691826939583, 0.007332532666623592, 0.037297338247299194, -0.006856525782495737, -0.019958533346652985, -0.01979648880660534, -0.020012548193335533, 0.0012600680347532034, -0.006670849397778511, -0.009472875855863094, -0.008709914982318878, 0.0033826876897364855, 0.001660116482526064, 0.01929684914648533, -0.002722692210227251, 0.007555344607681036, -0.035892948508262634, -0.024968421086668968, 0.018959255889058113, -0.00837907288223505, 0.010424889624118805, -0.0035413566511124372, -0.03754040598869324, 0.020107073709368706, -0.01516470406204462, 0.0141114117577672, -0.0011587899643927813, -0.014084404334425926, 0.00014442675455939025, -0.025589594617486, 0.010667957365512848, -0.015826387330889702, -0.0014280208852142096, -0.00775790074840188, 0.0011511940974742174, 0.0038823261857032776, -0.021970590576529503, 0.013753563165664673, 0.007332532666623592, 0.019026774913072586, -0.010377626866102219, -0.03378636762499809, -0.003548108506947756, -0.005151678342372179, -0.0006844709860160947, -0.013152645900845528, 0.005627685226500034, 0.025197984650731087, -0.00873017031699419, -0.01049916073679924, 0.009729446843266487, 0.003413071157410741, 0.0017841821536421776, 0.01891874521970749, 0.005995661951601505, 0.005860624834895134, -0.019350863993167877, -0.004294190555810928, -0.0016668683383613825, -0.024320242926478386, 0.019607435911893845, 0.004520378075540066, -0.017905963584780693, -0.010992047376930714, -0.02591368369758129, -0.005428505130112171, -0.010026529431343079, 0.013787321746349335, 0.010391131043434143, -0.004169280640780926, 0.009067763574421406, 0.0442112572491169, -0.005590549670159817, -0.016798656433820724, 0.022686289623379707, 0.007778156083077192, -0.004652039613574743, 0.009452620521187782, -0.037972528487443924, -0.01926984265446663, 0.0096821840852499, 0.02430673874914646, 0.005070655606687069, 0.005155053921043873, 0.022645777091383934, -0.014070901088416576, -0.02617025561630726, 0.0024087303318083286, 0.034434545785188675, 0.008244035765528679, -0.0023817226756364107, -0.009850980713963509, -0.001620449242182076, -0.006883533205837011, 0.04699302837252617, -0.007393299601972103, -0.031220655888319016, -0.018351588398218155, 0.004365085158497095, 0.0017656144918873906, -0.008048230782151222, -0.025251999497413635, -0.009351341985166073, -0.002754763700067997, -0.044724397361278534, 0.008500606752932072, -0.033678337931632996, -0.022699793800711632, 0.005583798047155142, 0.016744641587138176, -0.00795370526611805, 0.011876542121171951, 0.006117195822298527, -0.01620449312031269, -0.02403666265308857, 0.003418134991079569, -0.03327322378754616, 0.010006274096667767, 0.007035450544208288, 0.013659036718308926, -0.02143044024705887, -0.010242589749395847, 0.010181822814047337, -0.012004828080534935, -0.0038046797271817923, 0.022267673164606094, 0.030248384922742844, 0.00560067780315876, 0.03424549475312233, 0.02634580433368683, -0.0006950207753106952, -0.005610805470496416, 0.02070123888552189, -0.04132145643234253, 0.005668196361511946, -0.003953220788389444, -0.024954918771982193, 0.01463805790990591, 0.02318592742085457, 0.019485902041196823, -0.0035616124514490366, 0.005165182054042816, 0.034731630235910416, 0.03146372362971306, -0.004368461202830076, -0.014232945628464222, -0.0212548915296793, 0.0013951055007055402, -0.02862793579697609, 0.024522798135876656, 0.008034727536141872, 0.015502297319471836, -0.006505428347736597, -0.002783459145575762, 0.029384145513176918, 0.01463805790990591, -0.034461554139852524, 0.008352065458893776, 0.002839162014424801, 0.02731807343661785, -0.021484455093741417, 0.015434779226779938, 0.027115516364574432, 0.009182545356452465, -0.02860092930495739, -0.0006114664138294756, -0.022416213527321815, 0.013307939283549786, 0.014759591780602932, -0.010458649136126041, -0.0034012554679065943, -0.010600438341498375, 0.00775790074840188, 0.01806800812482834, -0.00215722294524312, 0.003865446662530303, -0.016420552507042885, 0.02371257357299328, 0.0058167376555502415, 0.020930802449584007, 0.011937309056520462, -0.010121055878698826, 0.009250064380466938, 0.02629178948700428, -0.019864005967974663, 0.038296617567539215, -0.0005198941216804087, 0.026210766285657883, 0.015070177614688873, -0.006424406077712774, -0.042536791414022446, -0.6529330015182495, -0.03700025752186775, -0.008385824970901012, -0.011910301633179188, -0.0030248386319726706, 0.019877510145306587, 0.008075238205492496, -0.00920955277979374, -0.02190307155251503, 0.0185136329382658, -0.013854840770363808, 0.0014086092123761773, -0.0010110927978530526, 0.00039625048520974815, -0.004905235022306442, -0.02970823645591736, 0.014894628897309303, -0.004425852093845606, 0.006701232865452766, 0.0016904999502003193, -0.024954918771982193, 0.014584043063223362, 0.005090911407023668, -0.0115051893517375, 0.010870513506233692, 0.010127807967364788, 0.01502966694533825, -0.026332300156354904, -0.009229809045791626, 0.009580906480550766, -0.02930312417447567, 0.010701716877520084, -0.022294681519269943, -0.0037844241596758366, 0.044697392731904984, 0.024468783289194107, -0.02962721325457096, -0.008716666139662266, 0.0036426347214728594, 0.021065840497612953, -0.01871618814766407, -0.001963106682524085, 0.02017459273338318, -0.0335162915289402, -0.01901327073574066, 0.025197984650731087, 0.04153751581907272, 0.02491440623998642, 0.0035852438304573298, -0.0008895591017790139, 0.016217995434999466, -0.005860624834895134, 0.0056513166055083275, 0.00410176208242774, 0.014611050486564636, -0.004837715998291969, 0.04726310074329376, 0.0060800607316195965, -0.013692796230316162, -0.0015976616414263844, -0.01713624969124794, 0.01804100163280964, -0.03781048208475113, -0.018000490963459015, -0.02315892092883587, -0.0008161325240507722, -0.012761037796735764, -0.01959393173456192, 0.05193539708852768, 0.0017056915676221251, 0.006849773693829775, 0.019256338477134705, -0.03068050555884838, 0.024171700701117516, -0.008000968024134636, 0.03740536794066429, -0.00227369274944067, 0.014084404334425926, 0.0007621175027452409, 0.03961998224258423, -0.0009022188605740666, -0.014232945628464222, 0.02137642540037632, -0.020215103402733803, 0.014192434027791023, -0.012727278284728527, -0.029384145513176918, -0.024144694209098816, 0.0036595144774764776, 0.01085025817155838, -0.012767789885401726, -0.00511116674169898, 0.015637334436178207, -0.006988187320530415, -0.002138655399903655, 0.03240898624062538, 0.01080299448221922, 0.0018398850224912167, 0.013233669102191925, -0.020890291780233383, 0.005590549670159817, -0.009999522008001804, 0.015043170191347599, 0.012112857773900032, -0.0049423701129853725, 0.0017284791683778167, -0.0005300219054333866, 0.018378594890236855, 0.015002659521996975, -0.000576441059820354, 0.025549082085490227, 0.020039554685354233, -0.006653969641774893, -0.016353033483028412, -0.018446113914251328, -0.028411876410245895, 0.019350863993167877, 0.018932247534394264, -0.003283097641542554, -0.010661205276846886, 0.017757423222064972, 0.02395564131438732, 0.0132606765255332, -0.010971792042255402, 0.005259708035737276, 0.013962870463728905, 0.017379317432641983, -0.013382209464907646, 0.013476735912263393, 0.016839168965816498, 0.011160844005644321, -0.003267905907705426, 0.00020804205269087106, -0.03875574469566345, 0.011410662904381752, -0.005371113773435354, 0.021362923085689545, -0.0281958170235157, 0.03462359681725502, -0.046776968985795975, 0.008446591906249523, 0.006161083001643419, 0.020255615934729576, -0.028789980337023735, 0.0220516137778759, -0.02117387019097805, -0.01964794658124447, -0.004891731310635805, -0.02222716249525547, 0.0034231990575790405, 0.015502297319471836, -0.013780570589005947, 0.0037844241596758366, 0.0247658658772707, -0.005918015725910664, -0.0017006277339532971, -0.023226438090205193, -0.013537502847611904, 0.004564265254884958, -0.009837477467954159, -0.004422476049512625, 0.021106351166963577, -0.02725055441260338, -0.006431157700717449, 0.0030822295229882, -0.024698346853256226, 0.022875340655446053, 0.008892214857041836, 0.004827588330954313, -0.02272680029273033, 0.02094430662691593, -0.02325344644486904, -0.004901858977973461, -0.00944586843252182, 0.013294435106217861, 0.02134941890835762, -0.004388716537505388, 0.006039549130946398, 0.015623831190168858, -0.000440981617430225, 0.0016305770259350538, 0.009695688262581825, -0.02433374524116516, 0.016704130917787552, 0.010310107842087746, 0.017824942246079445, 0.02363155037164688, 0.017919467762112617, -0.009459372609853745, 0.03205788880586624, -0.021335914731025696, -0.011444422416388988, -0.017325302585959435, -0.00919604953378439, -0.007420307025313377, -0.007778156083077192, -0.007629615254700184, 0.000865927548147738, 0.024522798135876656, -0.0011216546408832073, 0.02220015414059162, -0.0318688340485096, 0.03246299922466278, 0.0068092625588178635, -0.004662167280912399, -0.024077175185084343, 0.02152496762573719, -0.0072582620196044445, 0.03848566859960556, 0.009074515663087368, -0.004257054999470711, -0.02661587856709957, -0.006583075039088726, -0.020282622426748276, 0.005094287451356649, 0.012754285708069801, 0.025427548214793205, 0.022848334163427353, 0.006299496162682772, -0.008892214857041836, -0.02328045479953289, 0.012274903245270252, 0.005354234483093023, -0.025751639157533646, -0.013841337524354458, -0.02059320919215679, 0.007703885901719332, 0.017703408375382423, -0.002356403274461627, 0.024846887215971947, 0.016974205151200294, 0.017878957092761993, 0.02529251202940941, 0.025832660496234894, 0.024374257773160934, 0.014557035639882088, 0.016839168965816498, -0.0030991090461611748, 0.01551580149680376, -0.014003382064402103, 0.029195092618465424, 0.0003103750932496041, -0.00025488316896371543, 0.00017438818758819252, 0.02107934281229973, -0.0004367617075331509, 0.018608158454298973, -0.017217272892594337, -0.00740680331364274, -4.351792449597269e-05, 0.0009553898707963526, 0.013665788806974888, -0.017973482608795166, 0.0025116961915045977, 0.007562096230685711, 0.0038857022300362587, 0.0063636391423642635, 0.033057164400815964, 0.04688499867916107, 0.023212935775518417, 0.012369428761303425, 0.021160366013646126, 0.03837763890624046, -0.01921582780778408, 0.012423443607985973, 0.010134559124708176, 0.0013562822714447975, -0.015502297319471836, 0.003500845516100526, 0.005708707496523857, 0.005985534284263849, 0.0106139425188303, 0.001110682962462306, -0.017149753868579865, 0.017028219997882843, -0.003865446662530303, -0.010971792042255402, -0.00862889178097248, -0.022335192188620567, -0.006181338801980019, -0.020458171144127846, -0.00222642975859344, 0.012720526196062565, 0.0115051893517375, -0.0034873418044298887, 0.00954714696854353, -0.004074754659086466, 0.009850980713963509, -0.001373161911033094, 0.0352177619934082, 0.00943236518651247, 0.02220015414059162, -0.001270195934921503, 0.023050889372825623, 0.023820603266358376, 0.014165426604449749, 0.017973482608795166, -0.008682907558977604, 0.02591368369758129, -0.014975651167333126, 0.01979648880660534, -0.009729446843266487, -0.00643453374505043, -0.005512903444468975, 0.024725355207920074, 0.005101039074361324, 0.0071502323262393475, -0.019701961427927017, -0.016096461564302444, -0.00661008246243, 0.0008566437172703445, -0.027736689895391464, -0.03737836331129074, 0.013659036718308926, 0.049585744738578796, -0.008952981792390347, -0.021443944424390793, -0.02024211175739765, 0.04383315145969391, -0.009162290021777153, -0.00413889717310667, -0.01255848165601492, -0.0015377388335764408, -0.0015394267393276095, 0.0811845064163208, 0.029519183561205864, -0.012565233744680882, 0.015083681792020798, 0.00291680870577693, 0.0027665793895721436, -0.013854840770363808, -0.016339529305696487, 0.01643405668437481, -0.02792574092745781, -0.0015394267393276095, 0.014205938205122948, 0.009594409726560116, 0.014340975321829319, 0.02097131311893463, -0.006069932598620653, 0.006508804392069578, -0.010013026185333729, -0.00634000776335597, -0.03910684213042259, 0.007859178818762302, 0.013814330101013184, 0.02395564131438732, 0.0110393101349473, -0.010586935095489025, 0.02789873443543911, 0.020336637273430824, 0.008311553858220577, 0.021741027012467384, -0.01763588935136795, 0.0026653013192117214, -0.003625755198299885, -0.01801399327814579, 0.018081512302160263, -0.017406325787305832, 0.009574154391884804, 0.007663374301046133, -0.007528337184339762, -0.019634442403912544, 0.001155414036475122, 0.0042975666001439095, -0.0013436224544420838, 0.007548592519015074, -0.016690626740455627, 0.024995429441332817, -0.006913916673511267, -0.02263227477669716, 0.019458893686532974, -0.0017976858653128147, -0.002901616971939802, 0.024347249418497086, -0.005755970720201731, 0.010640949942171574, -0.018999766558408737, 0.015299741178750992, -0.0021538471337407827, -0.020026052370667458, -0.008581629022955894, 0.0031598759815096855, 0.014921636320650578, 0.009959010407328606, -0.00802797544747591, 0.0013275868259370327, -0.013618525117635727, -0.04202365130186081, -0.03313818573951721, -0.02365855872631073, 0.010944783687591553, -0.002805402735248208, 0.00855462159961462, -0.024995429441332817, 0.003119364846497774, -0.012160121463239193, -0.002921872539445758, 0.03961998224258423, 0.014435501769185066, 0.006967931520193815, 0.01184278354048729, 0.019337361678481102, 0.016096461564302444, -0.005296843592077494, -0.018000490963459015, 0.01813552714884281, -0.021187374368309975, -0.0011199667351320386, 0.023928632959723473, -0.003195323282852769, -0.004473114851862192, -0.024171700701117516, -0.0031143007799983025, 0.013632029294967651, -0.010006274096667767, 0.008514109998941422, -0.0023766588419675827, 0.02433374524116516, 0.006502052303403616, 0.014313967898488045, 0.03284110501408577, -0.0002745057863648981, 0.016744641587138176, -0.01261924859136343, 0.0106139425188303, -0.010377626866102219, -0.021835552528500557, 0.026399819180369377, 0.016150476410984993, -0.011322888545691967, 0.0076971338130533695, 0.011937309056520462, -0.00448999460786581, 0.00713672861456871, -0.023050889372825623, 0.006383894942700863, -0.0060901883989572525, -0.0281958170235157, 0.0028087785467505455, 0.009202801622450352, 0.012538226321339607, -0.02272680029273033, -0.004368461202830076, -0.0007747772615402937, -0.04129444807767868, 0.03173379600048065, -0.007514833472669125, -0.01244369987398386, 0.024900903925299644, 0.00016046245582401752, -0.017055228352546692, 3.18076454277616e-05, 0.004000484012067318, -0.022389207035303116, 0.017649393528699875, -0.017824942246079445, 0.017001213505864143, 0.0013444664655253291, -0.03732434660196304, -0.01999904401600361, -0.00032809877302497625, -0.01327417977154255, -0.008088742382824421, -0.017703408375382423, -0.0026045346166938543, -0.011248618364334106, 0.005755970720201731, -0.01588040217757225, -0.02117387019097805, -0.021943584084510803, 0.024171700701117516, 0.0020913921762257814, 0.008298050612211227, -0.01551580149680376, -0.025954194366931915, -0.024279730394482613, 0.007926697842776775, -0.022537747398018837, -0.02787172608077526, 0.01450302079319954, 0.013064871542155743, 0.006066556554287672, 0.02892501838505268, 0.034083448350429535, 0.0018449489725753665, 0.0009992769919335842, 0.00873017031699419, -0.0013740059221163392, 0.009466123767197132, 0.0013925735838711262, -0.006684353109449148, -0.016312522813677788, 0.004986257292330265, 0.015218718908727169, 0.016123469918966293, -0.005759346764534712, 0.01625850796699524, -0.00039772744639776647, 0.01813552714884281, -0.00027366180438548326, -0.012349173426628113, -0.013618525117635727, -0.02170051634311676, 0.018608158454298973, -0.017905963584780693, -0.005067279562354088, -0.0010136247146874666, -0.011289129965007305, -0.003939717076718807, 0.02097131311893463, 0.00522594852373004, 0.006637089885771275, -0.010006274096667767, 0.04013312608003616, -0.017176762223243713, 0.012362677603960037, -0.0011807335540652275, 0.005259708035737276, -0.019310353323817253, 0.018972760066390038, -0.04140247777104378, -0.008352065458893776, -0.008601884357631207, 0.010242589749395847, 0.016150476410984993, 0.012956841848790646, 0.003153124125674367, -0.010769235901534557, 0.0052900915034115314, -0.017554866150021553, -0.023550529032945633, -0.029897287487983704, -0.0006772971246391535, -0.0012448763009160757, -0.007906441576778889, -0.008298050612211227, -0.0023715950082987547, 0.015664342790842056, 0.015569816343486309, -0.00856812484562397, 0.01261924859136343, -0.004584521055221558, -0.026102736592292786, -0.0007338440627790987, -0.013517247512936592, 0.028006764128804207, 0.007575599942356348, 0.0015664342790842056, 0.016326025128364563, 0.0041220178827643394, -0.03435352444648743, -0.0211198553442955, 0.016312522813677788, -0.00873017031699419, 0.01886473037302494, 0.035811927169561386, -0.016650116071105003, -0.0008342781802639365, -0.01185628678649664, 0.00937159825116396, 0.0019647947046905756, -0.02260526642203331, 0.012139865197241306, -0.010728724300861359, -0.0017875580815598369, -0.026683397591114044, 0.007798411883413792, -0.006792382802814245, 0.00873017031699419, -0.017581874504685402, 0.005398121662437916, -0.0005253800190985203, 0.008122501894831657, -0.0247658658772707, 0.038323625922203064, -0.014016886241734028, 0.024104181677103043, -0.004095010459423065, -0.033408261835575104, -0.005121294874697924, -0.01357126235961914, -0.014259953051805496, -0.0010516040492802858, 0.0061644590459764, 0.014827110804617405, -0.028519906103610992, 0.004935618489980698, -0.016596101224422455, 0.0076093594543635845, -0.016717635095119476, -0.00962141714990139, 0.011147340759634972, 0.023415490984916687, -0.008655899204313755, -0.0005511215422302485, -0.01989101432263851, -0.0011233426630496979, -0.0018027498153969646, -0.014408494345843792, -0.016569092869758606, -0.005634436849504709, -0.016501573845744133, 0.004226671531796455, -0.005516279023140669, 0.010080544278025627, 0.0042097922414541245, -0.014584043063223362, -0.008925974369049072, 3.761003608815372e-05, -0.0025893428828567266, 0.00480395695194602, 0.004807332530617714, -0.017919467762112617, -0.02117387019097805, -0.0159074105322361, -0.0025268879253417253, 0.01092452835291624, 0.008824696764349937, -0.007143480237573385, -0.003342176554724574, 0.017406325787305832, -0.007656622678041458, 0.030977588146924973, -0.0019985539838671684, 0.013226917013525963, -0.05590549856424332, -0.0034518945030868053, 0.015191711485385895, -0.03273307532072067, 0.0012685079127550125, -0.008561373688280582, -0.015299741178750992, -0.018702683970332146, 0.006029421463608742, -0.01428696047514677, 0.01290957909077406, 0.0008093806100077927, 0.027628660202026367, 0.018257061019539833, 0.01432747207581997, 0.0001989692245842889, -0.024698346853256226, 0.025373533368110657, -0.011525444686412811, 0.02865494415163994, -0.006086812354624271, -0.00028716554515995085, -0.019337361678481102, 0.014057396911084652, 0.00011435982742113993, 0.022348696365952492, -0.01078949123620987, -0.010681461542844772, 0.010067041032016277, 0.004192912485450506, -0.011079821735620499, -0.023996151983737946, -0.020431164652109146, 0.000645647756755352, -0.02415819652378559, 0.043536070734262466, 0.008352065458893776, -0.02195708639919758, 0.0033168569207191467, 0.00991174764931202, 0.029060056433081627, 0.011410662904381752, -0.005769474431872368, 0.0035075973719358444, -0.00661008246243, -0.001402701367624104, -0.023739581927657127, -0.010040033608675003, -0.0005891008186154068, 0.009965762495994568, -0.015502297319471836, -0.019377872347831726, -0.03421848639845848, -0.00731227733194828, -0.03813457116484642, 0.005809985566884279, -0.014975651167333126, 0.003087293356657028, 0.017325302585959435, 0.003517725272104144, -0.02102532796561718, 0.01369954738765955, -0.003777672303840518, 0.007751148659735918, -0.007535088807344437, -0.02172752283513546, 0.0033539922442287207, 0.005796481855213642, -0.010546423494815826, -0.005614181514829397, -0.03597397357225418, 0.010256092995405197, 0.027790704742074013, 0.011289129965007305, -0.02749362215399742, 0.023915130645036697, -0.0110393101349473, -0.0034839657600969076, -0.002986015286296606, 0.0035211010836064816, 0.01979648880660534, -0.0030383423436433077, -0.0005388837889768183, -0.03294913470745087, -0.014935140497982502, 0.013659036718308926, 0.012038587592542171, -0.0025589594151824713, -0.0053812419064342976, 0.008088742382824421, 0.021457448601722717, -0.001683747977949679, -0.005448760464787483, 0.005219196900725365, 0.025954194366931915, 0.0019901141058653593, -0.011761760339140892, -0.0037067777011543512, -0.01625850796699524, 0.0066303382627666, 0.006252233404666185, 0.001046540099196136, 0.009135282598435879, 0.020188096910715103, -0.0032982893753796816, 0.0003331626649014652, 0.0038586948066949844, 0.0011064630234614015, -0.02784471958875656, -0.009189297445118427, 0.013908855617046356, -0.006684353109449148, 0.005479143932461739, -0.014584043063223362, 0.004587897099554539, -0.046749960631132126, 0.010553175583481789, 0.012423443607985973, -0.015799380838871002, 0.003320232965052128, -0.006920668762177229, -0.022389207035303116, 0.006191466469317675, -0.01725778356194496, -0.01137690432369709, -0.03781048208475113, -0.030275393277406693, -0.0012448763009160757, -0.0010321923764422536, -0.035487838089466095, 0.004051123280078173, 0.010276349261403084, -0.018824217841029167, 0.003370872000232339, 0.2083897739648819, -0.01080299448221922, -0.0042199199087917805, 0.038701727986335754, 0.0020272494293749332, 0.0024897526018321514, 0.004176032729446888, 0.0037202814128249884, 0.016339529305696487, 0.016852671280503273, 0.003727033268660307, 0.0071502323262393475, -0.006701232865452766, 0.004905235022306442, 0.012160121463239193, 0.013753563165664673, -0.0037506648804992437, -0.00978346262127161, 0.01339571364223957, 0.013429473154246807, 0.007069209590554237, 0.007400051690638065, 0.005539910867810249, -0.012308661825954914, 0.02082277275621891, 0.0028999289497733116, -0.013112135231494904, 0.030248384922742844, 0.010094048455357552, 0.018608158454298973, -0.03154474496841431, -0.018621662631630898, 0.012598992325365543, -0.019175315275788307, -0.0012862315634265542, 0.0012600680347532034, 0.015583319589495659, 0.009310831315815449, 0.004969377536326647, 0.020390652120113373, 0.008291298523545265, -0.021538471803069115, 0.007717389613389969, -0.010843506082892418, 0.0018854602240025997, 0.02661587856709957, 0.004456235561519861, 0.002550519537180662, 0.0014221130404621363, 0.018054505810141563, -0.017730414867401123, 0.0034940936602652073, 0.009351341985166073, 0.03859369829297066, 0.022470228374004364, 0.010566679760813713, 0.01889173686504364, -0.012916330248117447, -0.013267427682876587, -0.01570485346019268, -0.013125638477504253, 0.030572475865483284, -0.01567784696817398, 0.0335162915289402, -0.01303111296147108, 0.02933013066649437, 0.008959733881056309, -0.0281958170235157, -0.008480350486934185, -0.033057164400815964, 0.017946474254131317, 0.0010397882433608174, 0.00937159825116396, -0.002440801588818431, 0.00039772744639776647, -0.026858946308493614, -0.012450451962649822, 0.01228165440261364, 0.023145416751503944, -0.007778156083077192, -0.007481073960661888, 0.004098386038094759, -0.01786545291543007, 0.003299977397546172, -0.01185628678649664, -0.03662215173244476, 0.015569816343486309, 0.011545700952410698, 0.002560647437348962, -0.0193643681704998, -0.0015276109334081411, 0.0035379808396101, -0.013280931860208511, 0.009317583404481411, -0.003689897945150733, 0.006086812354624271, -0.0025893428828567266, 0.036163024604320526, -0.011768512427806854, 0.002903304761275649, -0.025279007852077484, 0.021741027012467384, -0.011518693529069424, 0.015488794073462486, -0.0006806730525568128, -0.013193157501518726, -0.012700270861387253, 0.04121342673897743, -0.0022990123834460974, -0.005128046497702599, 0.010944783687591553, -0.017770927399396896, 0.007447314448654652, -0.0005013264599256217, 0.020539194345474243, -0.007048954255878925, 0.019229330122470856, -0.026021713390946388, 0.0010929591953754425, -0.006647217553108931, 0.023442499339580536, -0.019431887194514275, 0.0071704876609146595, 0.024954918771982193, 0.025454556569457054, -0.007352788466960192, -0.022915853187441826, 0.014084404334425926, 0.004567641299217939, -0.007541840896010399, 0.02052569016814232, 0.016393544152379036, 0.002805402735248208, -0.03359731286764145, -0.022416213527321815, -0.02789873443543911, 0.002587654860690236, 0.008196772076189518, 0.000261635024799034, 0.017122747376561165, 0.0014288648962974548, 0.008210276253521442, 0.013240420259535313, -0.01997203752398491, 0.017122747376561165, -0.024576812982559204, 0.004925490356981754, -0.002217989880591631, -0.02927611581981182, -0.019175315275788307, 0.008392577059566975, 0.002420546021312475, -0.012011580169200897, -0.004381964914500713, 0.006856525782495737, -0.011106829158961773, -0.053555846214294434, -0.044238265603780746, -0.02342899516224861, 0.0020255616400390863, -0.04321197792887688, -0.00897998921573162, 0.037243325263261795, -0.01156595628708601, 0.0043279496021568775, 0.002675429219380021, -0.17252382636070251, 0.045345570892095566, -0.0032527141738682985, 0.008723418228328228, 0.024806376546621323, -0.016366537660360336, 0.00937159825116396, 0.012578736990690231, -0.019553421065211296, -0.008588381111621857, 0.011714497581124306, 0.02371257357299328, -0.017824942246079445, -0.011998075991868973, -0.006751871667802334, -0.012524722144007683, -0.027682675048708916, -0.005094287451356649, 0.03492068126797676, 0.011012302711606026, 0.02064722403883934, 0.00956065021455288, 0.026858946308493614, -0.0041321455501019955, 0.00046883311006240547, 0.005597301758825779, -0.008534366264939308, -0.0023209559731185436, -0.017824942246079445, -0.01866217330098152, -0.008514109998941422, 0.003397879423573613, 0.021160366013646126, -0.010998799465596676, 0.02430673874914646, -0.00831830594688654, -0.007481073960661888, -0.012598992325365543, 0.003046782221645117, 0.039755020290613174, 0.00855462159961462, 0.0014060772955417633, -0.0037675444036722183, 0.0016904999502003193, 0.0036190033424645662, 0.020201601088047028, 0.009749703109264374, -0.0005253800190985203, -1.1083883691753726e-05, -0.001543646678328514, -0.007352788466960192, -0.048937566578388214, -0.00125922414008528, -0.00028505557565949857, 0.0002647999790497124, 0.0011224986519664526, -0.014935140497982502, 0.03637908399105072, -0.0016322650481015444, -0.0019732345826923847, 0.009337838739156723, -0.015826387330889702, 0.006329879630357027, -0.014840614050626755, -0.008547869510948658, -0.02313191257417202, -0.021984094753861427, -0.017824942246079445, -0.024927910417318344, 0.013638781383633614, 0.012227639555931091, -0.02965422160923481, -0.01886473037302494, 0.001245720311999321, 0.015772372484207153, -0.009830725379288197, -0.026953471824526787, 0.022146139293909073, -0.004959249868988991, -0.019688457250595093, -0.00622184993699193, 0.028438884764909744, -0.006674225442111492, -0.00856812484562397, -0.01273403037339449, 0.00625898502767086, -0.010951535776257515, 0.020863283425569534, 0.01678515411913395, 0.004638535901904106, 0.010742227546870708, -0.023212935775518417, -0.030086340382695198, -0.011511941440403461, 0.0024796249344944954, 0.009277071803808212, -0.0007338440627790987, -0.00015698101196903735, -0.00672486424446106, 0.00325777824036777, 0.01610996574163437, -0.00010111983283422887, -0.025603096932172775, 0.04091634228825569, 0.00583361741155386, 0.00723125459626317, -0.03483965992927551, -0.0008013627957552671, 0.00019010738469660282, -0.031652774661779404, -0.0027800831012427807, 0.028681950643658638, 0.026750916615128517, 0.029087062925100327, 0.004375212825834751, 0.022767310962080956, 0.012625999748706818, -0.021997598931193352, 0.022915853187441826, -0.0033691839780658484, 0.03419147804379463, 0.008406080305576324, -0.026737412437796593, 0.000613576325122267, 0.018999766558408737, 0.021335914731025696, -0.1234782338142395, -0.01979648880660534, -0.009877988137304783, 0.021443944424390793, 0.0050200168043375015, 0.003652762621641159, 0.006086812354624271, 0.029870280995965004, -0.00731227733194828, 0.013463232666254044, -0.003185195615515113, -0.01085025817155838, -0.01754136197268963, -0.003009646898135543, 0.03208489343523979, -0.006734991911798716, -0.0006848929915577173, -0.021835552528500557, -0.004695926792919636, 0.0010777675779536366, -0.014692072756588459, -0.028384868055582047, -0.00915553793311119, -0.018783707171678543, 0.015083681792020798, -0.040079109370708466, -0.015664342790842056, 0.005080783739686012, 0.02962721325457096, 0.001861828612163663, 0.007156983949244022, 0.002982639241963625, -0.004901858977973461, -0.01976948045194149, -0.010316859930753708, -0.004486618563532829, -0.038620706647634506, 0.007737644948065281, 0.04637185484170914, -0.016515078023076057, -0.013551007024943829, -0.003375935833901167, -0.009358094073832035, -0.02260526642203331, 0.011903549544513226, -0.00645478954538703, -0.01462455466389656, 0.022740304470062256, 0.0105734309181571, 0.0011435982305556536, -0.0422937236726284, 0.01678515411913395, -0.004324574023485184, -0.009013748727738857, 0.017365815117955208, 0.023928632959723473, 0.007386547513306141, 0.014719080179929733, -0.028789980337023735, -0.01868918165564537, 0.010060288943350315, 0.0010389442322775722, -0.02079576440155506, 0.01696070097386837, 0.0281958170235157, 0.00767687801271677, 0.013908855617046356, -0.006063180975615978, 0.013463232666254044, 0.0031379323918372393, 0.017271287739276886, 0.012666511349380016, -0.028492899611592293, -0.002850977936759591, -0.010269597172737122, 0.003740536980330944, -0.03448856249451637, -0.03673018142580986, -0.0012744157575070858, -0.01608295924961567, 0.015866898000240326, -0.001291295513510704, -0.012173624709248543, -0.034461554139852524, 0.027088509872555733, 0.012531474232673645, 0.002661925507709384, 0.01883772201836109, 0.017298296093940735, -0.04440030828118324, -0.017379317432641983, -0.005867376457899809, 0.018270565196871758, -0.022618770599365234, -0.028411876410245895, 0.010823250748217106, -0.009182545356452465, -0.018297571688890457, 0.00614082720130682, -0.03489367291331291, -0.06071282923221588, -0.017554866150021553, -0.05466315522789955, 0.024495791643857956, -0.0029471919406205416, -0.014773095026612282, -0.01245720311999321, -0.01314589474350214, -0.010019777342677116, -0.021619493141770363, 0.015691349282860756, -0.009540394879877567, -0.016015440225601196, 0.023266950622200966, -0.00627924082800746, -0.014894628897309303, -0.024644332006573677, -0.011032558046281338, 0.010357371531426907, 0.010377626866102219, 0.018257061019539833, -0.005566918291151524, 0.004648663569241762, -0.012943338602781296, 0.008757177740335464, -0.0110393101349473, -0.01432747207581997, 0.029492175206542015, -0.019877510145306587, 0.027507126331329346, -0.0352177619934082, -0.00927031971514225, 0.00802797544747591, -0.019836999475955963, -0.010013026185333729, 0.0229023490101099, -0.006606706418097019, -0.008345313370227814, 0.00989824440330267, 0.021687012165784836, 0.013287683948874474, -0.0388367660343647, -0.012119609862565994, -0.01921582780778408, 0.00024475535610690713, 0.014003382064402103, -0.01121485885232687, 0.010141311213374138, -0.01567784696817398, 0.014084404334425926, 0.019310353323817253, 0.025197984650731087, -0.020215103402733803, 0.017622385174036026, -0.00880444049835205, -0.04140247777104378, -0.012079098261892796, 0.0035852438304573298, -0.0017470468301326036, -0.014759591780602932, 0.004135521594434977, -0.033759359270334244, 0.0354338213801384, 0.007879434153437614, 0.005148302298039198, -0.0027648916002362967, 0.03343527019023895, 0.029924295842647552, -0.016272010281682014, 0.009641672484576702, 0.004311070311814547, -0.01467856951057911, -0.013429473154246807, -0.0005646252539008856, 0.002574151149019599, 0.0110393101349473, 0.0010330363875254989, -0.010316859930753708, -0.01263275183737278, -0.01210610568523407, -0.029816266149282455, 0.003838439006358385, -0.0070624579675495625, -0.012355925515294075, -0.013523999601602554, 0.005360986106097698, 0.01778442971408367, 0.013166150078177452, -0.014921636320650578, 0.012578736990690231, -0.009263567626476288, 0.030950579792261124, -0.03397541865706444, 0.017041724175214767, -0.0014617802808061242, 0.00433470169082284, 0.010782739147543907, 0.009101523086428642, 0.014070901088416576, 0.004040995147079229, 0.028384868055582047, 0.0101480633020401, 0.0009739574743434787, -0.0028442260809242725, 0.005820113699883223, -0.0068902852945029736, 0.003279721597209573, 0.012322166003286839, -0.038647715002298355, -0.024954918771982193, -0.004851219709962606, 0.022240664809942245, 0.01344972848892212, -0.013659036718308926, 0.023753084242343903, 0.03211190178990364, -0.016393544152379036, 0.00598891032859683, -0.00919604953378439, -0.008952981792390347, -0.0018652045400813222, 0.034056439995765686, 0.014165426604449749, 0.008291298523545265, -0.005556790623813868, 0.0011984572047367692, 0.01643405668437481, 0.012943338602781296, 0.0003656560438685119, -0.007129976525902748, -0.0229833722114563, 0.01585339568555355, 0.02342899516224861, -0.005911263637244701, -0.030437437817454338, -0.004169280640780926, -0.001920907525345683, 0.01344972848892212, -0.028411876410245895, 0.01713624969124794, -0.02167350798845291, 0.06465592235326767, 0.00713672861456871, 0.003060285933315754, 0.012160121463239193, -0.0026568614412099123, 0.023496514186263084, -0.0057390909641981125, 0.0033320486545562744, -0.0034839657600969076, -0.0336243212223053, 0.00147697189822793, -0.010013026185333729, 0.027129020541906357, -0.011133836582303047, -0.01964794658124447, -0.012274903245270252, 0.01696070097386837, 0.024981925264000893, -0.03710828721523285, -0.0033961914014071226, 0.008790937252342701, -0.03313818573951721, 0.007177239749580622, 0.01801399327814579, -0.02120087668299675, -0.01733880676329136, -0.005185437388718128, -0.018621662631630898, -0.019404878839850426, -0.035838935524225235, -0.008345313370227814, -0.0020863283425569534, -0.030950579792261124, -0.0020390653517097235, -0.0075215850956737995, -0.018810715526342392, -0.0014786599203944206, 0.0007178083760663867, 0.011322888545691967, 0.019904518499970436, -0.009587657637894154, 0.028816988691687584, -0.01856764778494835, -0.018284069374203682, 0.03489367291331291, 0.01467856951057911, -0.020377149805426598, -0.003801303915679455, -0.010411386378109455]} +{"id": "test:10000089", "text": "\"So you have your children writing every day. Great! Now, of course, you need to become their editor. Here is where some find a challenge in the natural approach to language arts. What if I don\u2019t feel competent to edit my child\u2019s writing? We\u2019ll address the underlying problem with that argument further down, but for now \u2014 here are 10 ways to become a better editor of your child\u2019s writing.\\n1. Read A Strong Start in Language.\\nThis Ruth Beechick title for beginning grades is recommended widely \u2014 from The Well-Trained Mind to Sonlight. Typically packaged as part of The Three Rs, A Strong Start in Language helps you become the editor from the beginning \u2014 from the time your child traces his first letters, to copywork, and through dictation. Read our full review (scroll down).\\n2. Read You Can Teach Your Child Successfully.\\nOver 100 pages are devoted to writing in this followup to the Three Rs by Ruth Beechick mentioned above. The book speaks to you as your child\u2019s tutor and therefore includes writing activities, writing lessons to get you started and that you can pattern your own lessons after, and suggestions for usage, grammar, mechanics, and spelling. Best of all it tells you what you can expect from your child at various levels on the writing scale. Very valuable reference to finding your way as you teach writing naturally to your children and we cannot recommend it highly enough. You can read our full review (scroll down).\\n3. Use Learning Grammar Through Writing.\\nThis handy reference published by Educators Publishing Service is now out of print. But it is worth finding. When you find a mistake in your child\u2019s writing, you can find the coordinating pithy rule with an example to have your child copy into a grammar notebook. A checklist at the back helps you keep track of the skills you have covered.\\nThe best way to improve your editing is to read good writing. While you work through the suggestions above, you can be building your foundation in good writing at the same time.\\n5. Make use of a dictionary.\\nBefore the days of Kindle we had electronic dictionaries handy that included a grammar reference. Although somewhat pricey, the better models not only include the definitions, but also the etymology of the word. Most also come with a built-in thesaurus and other tools. If nothing else, a good dictionary can shed some light on how words are properly used in sentences.\\nNot popular with some, but a valuable skill for understanding how words work in sentences. There are easy online courses that can help you quickly learn the basics. For simple and easy practice we like The Complete Book of Diagrams by Mary Daily published by The Riggs Institute Press, which also is a great help for Mom.\\n7. Use Learn to Write the Novel Way.\\nWhile your child writes his novel as he works through this book, you have a guide that helps you edit your child\u2019s writing. Learn by doing. Read our full review.\\n8. Read The Elements of Style.\\nThis widely recommended classic by Strunk and White holds the title for the most concise and simple usage, composition, and spelling reference. There is even an early version in the public domain.\\n9. Read How to Write Clearly by Ruth Beechick.\\nSubtitled, The Meaning Approach, this is another handy writing reference that covers finding a topic, getting organized, outlining, using abstract and concrete terms, sentences, comma usage, active and passive voice, choosing words, usage, and more! Read our full review.\\nIt\u2019s never too late. Having a working knowledge of how Latin works is probably one of the best ways to understand how the English language works.\\nYou will also want a grammar reference. Our favorite grammar reference is the Handbook of Grammar and Composition published by A Beka. We purchased it on a recommendation in The Well-Trained Mind and haven\u2019t looked back.\\nNow, back to the initial question: what if you don\u2019t feel competent to edit your child\u2019s writing? You do know how to write, yes? In this day of streaming output you may even write for a blog or other publication. Using the ideas above, you should be able to stay ahead of your child\u2019s work. Of course, at some point, if you feel you are losing ground there are many qualified and reputable online writing tutors available.\\nWhat to have your child do.\\nMore structure for those who need it.\\nWhere to put the writing.\\nYou\u2019ll also want to have your child keep one of these.\"", "vector_field": [0.024587394669651985, 0.026216212660074234, -0.006318133324384689, -0.018033342435956, 0.0011626334162428975, 0.03257635980844498, -0.02864651195704937, -0.01240357756614685, -0.009294603951275349, -0.005002798046916723, -0.004640838596969843, 0.023398099467158318, -0.033765655010938644, -0.004408149980008602, -0.013185668736696243, 0.02807771973311901, 0.04234926775097847, 0.016081346198916435, 9.266124106943607e-05, -0.009488510899245739, -0.022958576679229736, -0.003415993181988597, 0.012481140904128551, -0.02509155310690403, -0.017916997894644737, -0.00022279094264376909, 0.026604026556015015, -0.015370354056358337, -0.01917092874646187, 0.0025563393719494343, 0.01480155996978283, 0.0066962516866624355, -0.0006213100277818739, -0.04744255542755127, -0.011718439869582653, 0.01917092874646187, -0.011259526945650578, -0.02942213974893093, 0.0153574263677001, -0.011524533852934837, 0.049924563616514206, 0.03860040009021759, -0.005713790189474821, -0.020709257572889328, -0.0064183184877038, 0.007103456184267998, -0.019868994131684303, -0.011737830936908722, -0.028206991031765938, 0.0013064477825537324, 0.008202262222766876, 0.02162708342075348, -0.026940131559967995, 0.004627911373972893, -0.018821896985173225, 0.005536042153835297, 0.000533647951669991, 0.027715759351849556, -0.0021168170496821404, -0.008331533521413803, -0.004104362800717354, 0.0008758128387853503, 0.001006699982099235, 0.007439561188220978, 0.003907224163413048, 0.009365702979266644, -0.0059464783407747746, 0.00042942300206050277, -0.0024933195672929287, 0.0032915696501731873, 0.017723090946674347, -0.0032737948931753635, 0.006967721041291952, 0.016856973990797997, 0.00541969807818532, -0.010994520969688892, -0.019145075231790543, -0.012843100354075432, 0.008790446445345879, -0.004427541047334671, -0.008312142454087734, 0.0001588622690178454, -0.0339207798242569, 0.036118391901254654, 0.024858864024281502, 0.007478342857211828, 0.04498640075325966, 0.0307148527354002, -0.0072779725305736065, -0.01628817990422249, -0.003916919231414795, 0.0205799862742424, 0.027095258235931396, -0.0049123079515993595, -0.004298269748687744, 0.003955700900405645, -0.013909588567912579, 0.01098159421235323, -0.009940960444509983, -0.0038264296017587185, 0.010548535734415054, 0.01295298058539629, -0.007820911705493927, -0.002748630242422223, -0.010916958563029766, 0.012675047852098942, 0.0038716744165867567, -0.01980435848236084, 0.004931699018925428, -0.013263232074677944, -0.02753477916121483, 0.01062609814107418, 0.001392089994624257, -0.015473770909011364, 0.006153312511742115, -0.02418665401637554, -0.01449130941182375, -0.006987112108618021, -0.030611436814069748, 0.004317660350352526, 0.032653920352458954, 0.0010309383505955338, 0.038858942687511444, -0.032421234995126724, -0.0026629881467670202, 0.007226263638585806, -0.003496787743642926, 0.009339849464595318, -0.02763819694519043, -0.019481180235743523, 0.02215709537267685, 0.013676900416612625, 0.03335198760032654, 0.006579907611012459, 0.0021620618645101786, 0.023087847977876663, -0.021665863692760468, 0.0034418473951518536, -0.022519053891301155, -0.009908642619848251, 0.025789618492126465, 0.017361130565404892, -0.009204113855957985, -0.014374964870512486, 0.001725771464407444, 0.006903085857629776, 0.03149048238992691, -0.006114530842751265, 0.021665863692760468, -0.007336144335567951, 0.012345406226813793, -0.0101865753531456, -0.001400977373123169, 0.012830173596739769, 0.021859770640730858, 0.04113411530852318, 0.019119219854474068, 0.010878177359700203, -0.012713829055428505, -0.0075882235541939735, 0.016068417578935623, 0.01012194063514471, 0.032136835157871246, 0.0029393055010586977, -0.004841208923608065, 1.4921742149454076e-05, 0.007866156287491322, -0.004094667267054319, 0.007963109761476517, -0.005006029736250639, -0.0222605112940073, 0.04281464219093323, -0.017658455297350883, 0.0069483304396271706, 0.007872620597481728, 0.011459898203611374, 0.02192440629005432, 0.02461325004696846, 0.01211271807551384, -0.012183817103505135, 0.0034321520943194628, 0.023113703355193138, 0.02880163863301277, 0.04077215865254402, -0.036066684871912, -0.008124698884785175, -0.00675442349165678, -0.0022315452806651592, 0.011181964538991451, -0.02072218433022499, 0.02706940285861492, 0.029525557532906532, 0.017231859266757965, -0.01812383159995079, -0.6449084877967834, -0.012940053828060627, 0.006127458065748215, 0.0002660160243976861, 0.02735380083322525, -0.0076657859608531, -0.0061888620257377625, -0.013780317269265652, -0.013754462823271751, 0.022312220185995102, 0.02346273511648178, 0.008286288008093834, -0.005985259544104338, 0.0005178930005058646, -0.030249476432800293, -0.02360493317246437, 0.005164387170225382, -0.02201489731669426, -0.011065619997680187, -0.0016562880482524633, 0.011860638856887817, 0.01822724938392639, -0.019494106993079185, -0.015305718407034874, 0.005846293177455664, 0.005238717887550592, -0.0018114136764779687, -0.004996334668248892, 0.024160800501704216, 0.032472942024469376, -0.021795134991407394, 0.018485791981220245, -0.0025094784796237946, 0.014685215428471565, 0.04883868619799614, -0.00027874117949977517, -0.018007487058639526, 0.025789618492126465, 0.0008580380235798657, 0.021019507199525833, -0.023579079657793045, -0.0021539824083447456, 0.004010641016066074, 0.009604855440557003, 0.007749812211841345, -0.0035614233929663897, 0.0006386808236129582, 0.006651006639003754, 0.003968627657741308, 0.0127590736374259, -0.004104362800717354, 0.02095487155020237, 0.0028375042602419853, -0.010761832818388939, -0.0074718790128827095, -0.020347297191619873, 0.0031251327600330114, -0.009423875249922276, -0.0013234146172180772, -0.0021168170496821404, -0.017516257241368294, 0.01370275393128395, -0.028879201039671898, -0.0055489689111709595, -0.0016789105720818043, 0.018382374197244644, -0.002879517385736108, -0.0027308554854243994, -0.0031332122161984444, -0.015228155069053173, -0.019649231806397438, 0.007633468136191368, -0.020437786355614662, -0.01134355366230011, -0.00732968095690012, -0.0039104558527469635, 0.014129349030554295, -0.01999826543033123, 0.02077389322221279, -0.0012684742687270045, 0.016908681020140648, -0.030352894216775894, -0.02403152920305729, -0.009915105998516083, 0.030921686440706253, 0.007142237387597561, -0.00675442349165678, -0.022751742973923683, -0.02385054901242256, -0.008641784079372883, 0.004928466863930225, 0.012965908274054527, 0.002792259445413947, -0.032757338136434555, -0.0008370314608328044, 0.0019277577521279454, -0.006460331380367279, 0.0017015330959111452, 0.033041734248399734, 0.005461710970848799, -0.020941944792866707, -0.01113025564700365, 0.014077641069889069, 0.01946825347840786, 0.013767389580607414, 0.011776612140238285, 0.01845993660390377, -0.000810369267128408, 0.03567887097597122, -0.024445196613669395, 0.010923421941697598, 0.012326015159487724, -0.006857840809971094, -0.0017661686288192868, -0.012073935940861702, -0.013114569708704948, -0.017218932509422302, 0.017955778166651726, 0.029447995126247406, -0.02162708342075348, 0.004611752461642027, -0.007917865179479122, -0.021846843883395195, -0.006596066523343325, -0.014258620329201221, 0.005235486198216677, -0.012804319150745869, 0.004737792070955038, -0.005930319428443909, -0.02509155310690403, 0.006715642288327217, -0.008622393943369389, 0.029344577342271805, -0.010561462491750717, 0.018808968365192413, 0.011052693240344524, 0.020360223948955536, 0.006670397240668535, 0.012377724051475525, -0.007155164610594511, 0.004812122788280249, -0.0022590153384953737, 0.024975208565592766, 0.0013807787327095866, -0.004812122788280249, -0.03562716022133827, -0.013405430130660534, -0.016029637306928635, -0.02484593726694584, 0.02600937895476818, -0.012332478538155556, -0.007374925538897514, -0.026306701824069023, -0.01937776245176792, -0.02268710732460022, -0.015318645164370537, 0.002801954746246338, -0.0012369644828140736, -0.016663067042827606, -0.011724904179573059, 0.007155164610594511, 0.02908603474497795, -0.014413746073842049, 0.017619673162698746, -0.00400094548240304, -0.019248491153120995, 0.007129310164600611, -0.0011876797070726752, -0.006150080356746912, -0.024341778829693794, 0.005183777771890163, -0.009281677193939686, -0.011194891296327114, -0.002810034202411771, 0.015034248121082783, -0.000749773345887661, -0.02913774363696575, 0.017425766214728355, -0.014517162926495075, 0.0039007605519145727, -0.015538406558334827, 0.008855082094669342, 0.004372600466012955, -0.026216212660074234, 0.021549521014094353, 0.0008830843144096434, 0.025194969028234482, 0.008506049402058125, -0.012035154737532139, 0.00822165235877037, 0.012694437988102436, 0.024936426430940628, -0.009288140572607517, 0.009798762388527393, 0.0019406848587095737, 0.004879990126937628, 0.0034935560543090105, 0.02701769396662712, -0.0010616403305903077, 0.029835809022188187, 0.0064409407787024975, -0.012145035900175571, 0.028155282139778137, -0.025013990700244904, -0.0005994954844936728, -0.016184762120246887, 0.01739991270005703, -0.02422543615102768, 0.02701769396662712, -0.015098883770406246, -0.02523375116288662, -0.01783943548798561, -0.004944625776261091, -0.004537421278655529, -0.008234580047428608, 0.009824615903198719, -0.02427714318037033, 0.014607653021812439, -0.013599337078630924, 0.0008010778692550957, 0.012364796362817287, -0.005060970317572355, 0.009275213815271854, -0.02686256915330887, -0.011860638856887817, 0.03350711241364479, 0.0010729514760896564, -0.02417372725903988, 0.0011626334162428975, -0.027457216754555702, -0.0008847002172842622, 0.008984353393316269, 0.027276236563920975, 0.01696038991212845, 0.006857840809971094, 0.023139556869864464, 0.031025104224681854, -0.01577109470963478, 0.01331494003534317, -0.009953887201845646, -0.00400094548240304, 0.0005243565537966788, -0.010393409989774227, -0.02432885207235813, 0.042556099593639374, 0.005199936684221029, 0.04296977072954178, 0.0033481258433312178, 0.0027615574654191732, 0.015422062017023563, -0.00731675373390317, 0.021265123039484024, -0.017580892890691757, 0.002945768879726529, 0.015266936272382736, -0.029577266424894333, 0.01038048230111599, 0.006428013555705547, -0.0030330270528793335, 0.03978969529271126, -0.005545737221837044, -0.017516257241368294, -0.007368462160229683, -0.0027308554854243994, 0.01209332700818777, -0.020437786355614662, 0.005073897074908018, -0.01648208685219288, -0.030973395332694054, -0.019933629781007767, -0.05486272647976875, -0.01098159421235323, 0.015680605545639992, -0.013237377628684044, 0.002879517385736108, 0.00752358790487051, -0.028905054554343224, 0.011744294315576553, -0.004395223222672939, 0.036273516714572906, -0.02210538648068905, -0.009533755481243134, 0.04178047180175781, 0.0015819570980966091, -0.02201489731669426, -0.00420777965337038, -0.022325146943330765, -0.011808929964900017, 0.012326015159487724, 0.00838970486074686, -0.015939146280288696, 0.005041579250246286, -0.0016393212135881186, 0.009779371321201324, 0.007601150311529636, -0.007659322582185268, 0.016275253146886826, -0.01836944743990898, 0.01773601770401001, -0.00659929821267724, 0.012216134928166866, -0.009223504923284054, -0.01980435848236084, -0.013069325126707554, 0.011783075518906116, -0.005283962935209274, -0.015318645164370537, -0.006812595762312412, -0.004033263307064772, 0.015939146280288696, 0.0021927638445049524, -0.01259102113544941, -0.017761871218681335, 0.014245693571865559, 0.02235100232064724, -0.012170889414846897, -0.002111969282850623, -0.0010398257290944457, 0.022751742973923683, 0.008176407776772976, -0.017723090946674347, -0.0025401804596185684, -0.013405430130660534, 0.03712670877575874, 0.12048082053661346, -0.006402159575372934, -0.013521774671971798, 0.026888422667980194, 0.00970827229321003, -0.020269734784960747, -0.013754462823271751, -0.034773971885442734, 0.025776689872145653, -0.004692547023296356, 0.010548535734415054, -0.010968666523694992, 0.014698143117129803, -0.0014648049836978316, -0.0051126787438988686, 0.006596066523343325, 0.006279351655393839, -0.04596886411309242, -0.0010624482529237866, 0.0018146453658118844, 0.013405430130660534, 0.017477475106716156, -0.008169944398105145, -6.943281186977401e-05, 0.03916919231414795, 0.006689787842333317, 0.025931816548109055, -0.0029182988218963146, 0.007943719625473022, -0.023863475769758224, -0.02003704570233822, 0.014336183667182922, -0.003855515504255891, -0.0032511723693460226, 0.014168131165206432, 0.039246756583452225, 0.004265951924026012, 0.01860213465988636, 0.03373980149626732, -0.016378669068217278, 0.03257635980844498, 0.021420249715447426, 0.013379575684666634, -0.015305718407034874, -0.007575296331197023, -0.009863398037850857, -0.008422022685408592, 0.028491387143731117, -0.029189452528953552, -0.04630496725440025, 0.01764552854001522, 0.025440584868192673, -0.005733180791139603, -0.008932644501328468, 0.01300468947738409, -0.0021943796891719103, 0.004815354477614164, 0.023049067705869675, -0.001748393871821463, 0.012791391462087631, -0.011143183335661888, -0.014271548017859459, 0.002362432423979044, 0.0044889445416629314, -0.020993653684854507, 0.004446931648999453, -0.0007259389385581017, 0.020127536728978157, -0.017761871218681335, 0.012694437988102436, 0.004017104394733906, 0.007820911705493927, -0.015486697666347027, -0.024639103561639786, 0.026319630444049835, 0.025363022461533546, 0.045115672051906586, 0.003415993181988597, 0.014723997563123703, -0.0015431757783517241, -0.007717494387179613, -0.031800732016563416, 0.034153468906879425, -0.023488588631153107, 0.017567964270710945, 0.00592385558411479, -0.018253102898597717, -0.007711031008511782, -0.024910572916269302, 0.009423875249922276, 0.0018647380638867617, 0.029447995126247406, 0.01009608618915081, -0.022506127133965492, 0.02652646414935589, -0.00826689787209034, -0.0033901389688253403, 0.030120205134153366, 0.0341276153922081, -0.031464625149965286, -0.027224529534578323, -0.017076734453439713, -0.005390611942857504, -0.031076813116669655, 0.008189334534108639, 0.0016643675044178963, -0.015654750168323517, 0.016572576016187668, -0.01562889665365219, -0.03984140604734421, 0.019209710881114006, -0.006534662563353777, 0.03322271630167961, 0.022273439913988113, -0.013211523182690144, 0.03735939785838127, 0.0017839435022324324, 0.0028116500470787287, 0.029551411047577858, -0.016856973990797997, -0.013832025229930878, -0.07089236378669739, 0.026785006746649742, 0.028155282139778137, -0.002483624266460538, 0.004821818321943283, 0.015732312574982643, -0.017335277050733566, -0.01571938581764698, -0.00029045637347735465, -0.0037520984187722206, 0.009818152524530888, -0.002769636921584606, -0.016934536397457123, -0.04542592167854309, -0.01961045153439045, 0.0004746679333038628, 0.000878236663993448, -0.02970653772354126, 0.013095179572701454, 0.006812595762312412, 0.0010123555548489094, -0.026086941361427307, -0.02975824475288391, 0.03379150852560997, -0.029215306043624878, 0.007349071558564901, 0.01560304220765829, 0.016016710549592972, 0.029292868450284004, -0.023785913363099098, -0.014723997563123703, -0.03741110488772392, -0.0011618254939094186, 0.004408149980008602, -0.03937602788209915, 0.015706459060311317, 0.03673889487981796, 0.005183777771890163, -0.0038361249025911093, 0.02335931733250618, -0.00932692177593708, 0.03149048238992691, 0.0016490166308358312, 0.006657470017671585, -0.004899381194263697, -0.009792298078536987, 0.0024884718004614115, -0.0002969199267681688, 0.005988491233438253, -0.0006576675805263221, 0.001582765020430088, 0.008590076118707657, -0.007795057259500027, 0.007142237387597561, 0.029861662536859512, 0.0011513222707435489, 0.0021846843883395195, -0.001642553019337356, -0.03570472449064255, 0.0006810979684814811, 0.02947384864091873, 0.002868206240236759, 0.037850625813007355, -0.028155282139778137, -0.010858786292374134, 0.010509753599762917, 0.014930831268429756, 0.022700034081935883, -0.0014413746539503336, 0.009772907942533493, -0.003926614765077829, 0.020877309143543243, -0.028853347525000572, 0.004656997509300709, -0.01745162159204483, 0.01759381964802742, -0.0323953777551651, 0.01485326886177063, 0.009908642619848251, 0.025802545249462128, 0.005616836715489626, 0.0006956409779377282, -0.0004451779241207987, -0.008880935609340668, -0.011718439869582653, 0.009398020803928375, -0.006954794283956289, 0.001665983465500176, -0.0057558030821383, 0.006466795224696398, -0.015835730358958244, 0.011440507136285305, 0.015745241194963455, 0.01331494003534317, -0.005361525807529688, -0.002905371831730008, 0.011860638856887817, -0.023294681683182716, -0.038858942687511444, -0.011375871486961842, -0.02403152920305729, 0.01870555244386196, -0.00611776253208518, -0.001386434305459261, -0.01139526255428791, -0.0007166475988924503, 0.00124100421089679, -0.004059117753058672, -0.006305206101387739, -0.0009517596918158233, 0.00012169677938800305, 0.008829227648675442, -0.0008515744702890515, 0.026112796738743782, -0.006282583344727755, 0.015202300623059273, -0.015848657116293907, -0.015680605545639992, 0.027715759351849556, 0.006460331380367279, -0.014038859866559505, -0.014698143117129803, 0.013586410321295261, -0.0010034681763499975, 0.013857879675924778, 0.011427580378949642, 0.01888653263449669, -0.02591888979077339, 0.0002726815873757005, -0.02846553362905979, 0.028879201039671898, -0.016999172046780586, 0.010606707073748112, -0.0012886729091405869, 0.020166317000985146, -0.014232766814529896, -0.023346390575170517, 0.006324596703052521, 0.010425727814435959, 0.013819098472595215, 0.015939146280288696, -0.02903432585299015, 0.009966814890503883, 0.010503290221095085, 0.0007041244534775615, -0.005080360919237137, -0.012073935940861702, -0.008816299960017204, 0.024406414479017258, 0.008790446445345879, 0.0018388837343081832, -0.025026917457580566, 0.016999172046780586, -0.0037585620302706957, 0.0037682573311030865, -0.008764591999351978, -0.0017144602024927735, -0.0028908287640661, -0.00824104342609644, 0.012584557756781578, 0.007252118084579706, -0.0238376222550869, 0.02086438238620758, 0.0034418473951518536, -0.005752571392804384, 0.006014345679432154, -0.014271548017859459, 0.0017370826099067926, -0.02942213974893093, -0.01821432076394558, 0.019429471343755722, -0.010587316937744617, -0.0038361249025911093, -0.0014631891390308738, -0.01446545496582985, -0.0063019744120538235, 0.03570472449064255, -0.02279052510857582, 0.02259661816060543, -0.006024041213095188, -0.0005251645343378186, -0.0044889445416629314, -0.015422062017023563, 0.002641981467604637, -0.00449540838599205, 0.02668158896267414, -0.0014357189647853374, -0.00838970486074686, -0.0021749890875071287, 0.0011909115128219128, -0.011821857653558254, -0.012364796362817287, -0.01950703375041485, 0.001537520089186728, 0.0020020888186991215, -0.0049123079515993595, 0.00805359985679388, -0.021303905174136162, -0.013082251884043217, -0.008286288008093834, -0.006987112108618021, 0.0230232123285532, 0.013211523182690144, 0.0016562880482524633, -0.010361092165112495, 0.019015803933143616, -0.02841382473707199, -0.013857879675924778, -0.0058883060701191425, 0.0013823945773765445, 0.007892010733485222, 0.013133960776031017, -0.007174555212259293, 0.007627004757523537, -0.004879990126937628, -0.01831773854792118, 0.0037908798549324274, -0.020709257572889328, -0.00935277622193098, -0.0019956252072006464, 0.01835652068257332, 0.012487604282796383, 0.017089661210775375, -0.00935277622193098, -0.0043790643103420734, -0.007976037450134754, 0.004967248532921076, -0.017361130565404892, 0.004262720234692097, -0.004947857931256294, 0.05002798140048981, 0.021213414147496223, -0.02399274706840515, -0.03523934632539749, -0.02077389322221279, -0.04374539479613304, -0.00033448939211666584, -0.004572971258312464, 0.013573482632637024, 0.007943719625473022, 0.007342607714235783, 0.012196743860840797, 0.007284435909241438, -0.024858864024281502, 0.019209710881114006, -0.04229755699634552, -0.03826429322361946, -0.012513458728790283, 0.015977928414940834, -0.011052693240344524, 0.004592361859977245, -0.011337090283632278, -0.0009638788760639727, 0.034541282802820206, -0.005626531783491373, -0.0035775823052972555, 0.026552317664027214, 0.0054649426601827145, -0.0073878527618944645, -0.0056071411818265915, 0.02110999822616577, 0.03187829256057739, 0.01518937386572361, 0.005642690695822239, -0.007956646382808685, 0.01615890860557556, 0.012216134928166866, -0.009145942516624928, -0.019868994131684303, 0.009650100022554398, 0.02484593726694584, -0.016378669068217278, -0.010477435775101185, -0.006909549236297607, -0.011356480419635773, 0.012190280482172966, 0.02008875459432602, 0.03658377006649971, -0.0014930830802768469, -0.016016710549592972, 0.01552547886967659, 0.012022227980196476, 0.007413707207888365, -0.015111811459064484, 0.0052225589752197266, -0.015564260073006153, 0.0015173214487731457, 0.0015666061080992222, -0.035601306706666946, -0.005174082238227129, -0.011156110092997551, -0.012998226098716259, -0.019649231806397438, -0.010477435775101185, -0.003929846454411745, -0.006234106607735157, -9.422663424629718e-05, 0.009624245576560497, 0.016275253146886826, -0.009895715862512589, -0.010076695121824741, -0.0018760492093861103, -0.0067221056669950485, 0.007685177028179169, -0.04046190530061722, -0.0037585620302706957, -0.0179816335439682, 0.013114569708704948, -0.008738737553358078, 0.0015738776419311762, 0.0014930830802768469, -0.011434043757617474, 0.03200756385922432, -0.015202300623059273, 0.007122846785932779, 0.20249050855636597, -0.019817285239696503, 0.0020683403126895428, 0.03048216551542282, -0.009566073305904865, -0.00937216728925705, 0.04829574376344681, 0.027664050459861755, -0.0179299246519804, -0.0359632670879364, 0.014310329221189022, -0.005762266926467419, -0.03919504955410957, 0.011582705192267895, 0.01225491613149643, -0.029835809022188187, -0.04555519297719002, -0.028439678251743317, -0.007620541378855705, -0.009074842557311058, 0.000999428448267281, 0.01120135560631752, -0.017697235569357872, -0.00748480623587966, 0.02619035914540291, 0.020062901079654694, 0.002144287107512355, 0.02735380083322525, 0.008253970183432102, -0.0033448939211666584, -0.018808968365192413, 0.019778503105044365, 0.008357387036085129, 0.015887439250946045, -0.006657470017671585, -0.004330587573349476, 0.034153468906879425, 0.005190241150557995, 0.00913301482796669, 0.015939146280288696, 0.013547629117965698, 0.03547203540802002, -0.02503984421491623, -0.01701209880411625, -0.01081354171037674, 0.04671863466501236, 0.00879690982401371, 0.00287790154106915, -0.01552547886967659, 0.015615968964993954, -0.003988018725067377, -0.014310329221189022, 0.02908603474497795, -0.005506956018507481, 0.00807299092411995, 0.017904071137309074, -0.018485791981220245, 0.012965908274054527, -0.004805659409612417, -0.0016013478161767125, -0.019649231806397438, 0.020890235900878906, -0.013974224217236042, 0.032757338136434555, 0.0008822763920761645, 0.007187482435256243, 0.0013525006361305714, -0.02148488536477089, -0.006091908551752567, -0.01923556439578533, -0.0057784258387982845, 0.0010317462729290128, -0.013586410321295261, 0.004042958840727806, -0.007284435909241438, -0.02321711927652359, 0.0017451620660722256, 0.017619673162698746, 0.0055780550464987755, 0.04113411530852318, -0.00985693372786045, 0.026888422667980194, -0.005403539165854454, -0.026371337473392487, -0.007801521103829145, -0.03547203540802002, 0.00819579791277647, -0.02259661816060543, 0.003438615705817938, -0.011957592330873013, -0.00270984903909266, -0.008124698884785175, -0.004055886063724756, -0.015422062017023563, 0.013573482632637024, 0.013819098472595215, -0.014698143117129803, 0.020024118945002556, 0.001483387779444456, -0.013075788505375385, -0.018110904842615128, 0.02273881621658802, 0.017050880938768387, 0.032085128128528595, 0.014879122376441956, -0.003619595430791378, -0.001317758928053081, -0.01836944743990898, 0.004550348501652479, -0.012610412202775478, 0.010102549567818642, -0.04253024607896805, 0.004084972199052572, 0.010742442682385445, -0.007504196837544441, 0.011459898203611374, -0.0014276395086199045, -0.022312220185995102, -0.003619595430791378, 0.010632561519742012, 0.015073029324412346, -0.01480155996978283, -0.005070665385574102, 0.013767389580607414, 0.006651006639003754, 0.001181216211989522, -0.052561696618795395, 0.0019859299063682556, -0.008169944398105145, -0.018821896985173225, 0.010716588236391544, -0.01562889665365219, 0.014478381723165512, -0.020205099135637283, -0.004453395027667284, -0.03715256229043007, 0.0077562760561704636, 0.0028375042602419853, 0.028620658442378044, 0.01064548920840025, 0.013676900416612625, -0.02620328590273857, -0.015202300623059273, 0.0022380088921636343, 0.019881920889019966, -0.030404601246118546, -0.01888653263449669, -0.009701808914542198, 0.006285815499722958, -0.023824693635106087, -0.013896660879254341, 0.022027824074029922, -0.002376975491642952, -0.004556812345981598, -0.00785969290882349, -0.023372244089841843, -0.033817362040281296, -0.010367555543780327, -0.016029637306928635, 0.005283962935209274, -0.01917092874646187, 0.02662988007068634, 0.02394103817641735, -0.019390689209103584, 0.0019212942570447922, -0.04340929165482521, -0.16164079308509827, 0.013870807364583015, 0.002898908220231533, -0.033481258898973465, -0.0009654947789385915, 0.02162708342075348, 0.03299002721905708, -0.010554999113082886, -0.02452275902032852, 0.013780317269265652, -0.015111811459064484, -0.002688842359930277, -0.04147022217512131, -0.002701769582927227, -0.0178782157599926, 0.027793321758508682, -0.01763259992003441, 0.015693532302975655, 0.017865289002656937, 0.0010414416901767254, 0.01206747256219387, -0.028517240658402443, 0.0016255861846730113, 0.00749773345887661, 0.02153659239411354, 0.011278918012976646, -0.00035085028503090143, -0.007976037450134754, -0.020890235900878906, -0.018977021798491478, -0.00770456762984395, 0.019972410053014755, -0.01230662502348423, -0.008447877131402493, 0.014103495515882969, -0.004424308892339468, -0.007555905729532242, -0.023204192519187927, -0.016598431393504143, 0.026164503768086433, 0.011621487326920033, 0.027146965265274048, 0.010180111974477768, -0.009152405895292759, 0.025970596820116043, 0.026940131559967995, 0.007129310164600611, -0.009171796031296253, 0.0013274543453007936, -0.019830211997032166, -0.014866195619106293, -0.006447404623031616, -0.008085917681455612, -0.014426673762500286, 0.01184771116822958, -0.00910716038197279, -0.008215188980102539, 0.034256886690855026, -0.007426634430885315, -0.010968666523694992, 0.004724864847958088, -0.018007487058639526, -0.002905371831730008, 0.010031450539827347, -0.0009452961385250092, -0.025194969028234482, 0.015060102567076683, 0.014917904511094093, -0.04379710555076599, 0.02509155310690403, -0.01856335438787937, -0.021665863692760468, -0.008661175146698952, -0.025776689872145653, -1.9239199900766835e-05, -0.014245693571865559, -0.009824615903198719, 0.0012434279778972268, 0.007769203279167414, 0.012797855772078037, -0.009798762388527393, 0.00280033890157938, -0.019920703023672104, -0.007258581463247538, -0.02581547200679779, 0.0037682573311030865, 0.011621487326920033, 0.007271508686244488, -0.02341102622449398, -0.023475661873817444, 0.0007134157931432128, -0.02158830128610134, -0.02273881621658802, -0.01750332862138748, 0.011065619997680187, 0.018007487058639526, 0.007995427586138248, -0.0014938911190256476, -0.002764789154753089, -0.013353722169995308, 0.01913214661180973, -0.010367555543780327, -0.016895754262804985, -0.00934631284326315, 0.05310463532805443, -0.005051274783909321, 0.019972410053014755, -0.006289047189056873, 0.034205175936222076, -0.0055522010661661625, -0.025699127465486526, 0.019429471343755722, 9.685245458967984e-05, 0.010697197169065475, -0.0020877309143543243, 0.033377841114997864, -0.013276158832013607, 0.002294565085321665, 0.004967248532921076, 0.00714870123192668, 0.03384321928024292, -0.004223938565701246, -0.008945571258664131, 0.002325266832485795, -0.005506956018507481, -0.019597524777054787, -0.10331359505653381, -0.011278918012976646, 0.019920703023672104, -0.013444211333990097, -0.0070776017382740974, 0.018757261335849762, -0.015370354056358337, 0.007400779984891415, 0.013211523182690144, 0.019532889127731323, -0.020308515056967735, -0.012733220122754574, 0.00802774541079998, -0.004430772736668587, -0.014116422273218632, -5.948498073848896e-05, 0.019054584205150604, -0.015706459060311317, -0.02542765811085701, 0.03888479620218277, 0.011434043757617474, -0.006011113990098238, -0.01639159582555294, -0.005629763472825289, -0.009546683169901371, -0.012332478538155556, -0.033093445003032684, -0.0023899024818092585, 0.014814486727118492, -0.006282583344727755, 0.004934930708259344, -0.0015181294875219464, -0.0015132817206904292, -0.016326960176229477, 0.01918385550379753, -0.008363851346075535, -0.0020877309143543243, -0.003186536720022559, 0.0014753083232790232, -0.033429548144340515, -0.001105269300751388, -0.0014219839358702302, 0.004718401003628969, -0.04118582606315613, 0.022816378623247147, -0.022363929077982903, -0.011718439869582653, 0.017710164189338684, -0.006922476459294558, -0.020592913031578064, -0.027095258235931396, -0.00841555930674076, -0.028051864355802536, -0.015538406558334827, 0.01649501360952854, -0.0032220862340182066, -0.0005922240088693798, -0.025363022461533546, 0.010535608045756817, 0.009527292102575302, -0.02441934309899807, -0.01898994855582714, -0.017968706786632538, 0.0073619987815618515, 0.008693492971360683, -0.007807984482496977, 0.0038425885140895844, -0.009772907942533493, 0.004640838596969843, -0.01701209880411625, 0.0016805264167487621, 0.00807945430278778, 0.0049704802222549915, 0.030249476432800293, -0.012416505254805088, 0.013935442082583904, -0.017102587968111038, 0.002015015808865428, 0.015383280813694, 0.008118235506117344, -0.026423046365380287, -0.023682495579123497, 0.026707444339990616, -0.03844527527689934, -0.005435856990516186, 0.0023220351431518793, -0.004556812345981598, 0.01057438924908638, -0.006870768032968044, -0.03456713631749153, 0.00802774541079998, 0.04826989024877548, -0.002695305971428752, 0.01230662502348423, 9.629699343349785e-05, -0.004175461828708649, -0.008131162263453007, -0.005128837656229734, 0.030016787350177765, 0.0027470143977552652, -0.008376778103411198, -0.0072779725305736065, -0.030352894216775894, 0.003089583246037364, -0.008383241482079029, -0.01941654458642006, 0.0021232806611806154, -0.0036713038571178913, -0.005348598584532738, -0.010115476325154305, -0.008900326676666737, 0.0012919047148898244, -0.02706940285861492, 0.016430377960205078, 0.004747487138956785, -0.02330760844051838, -0.01482741441577673, 0.01683111861348152, -0.016223544254899025, -0.006518503651022911, -0.016262324526906013, 0.018873604014515877, -0.0030847357120364904, -0.006683324463665485, 0.01513766497373581, 0.00819579791277647, 0.019584596157073975, 0.018343592062592506, 0.00749773345887661, -0.002816497813910246, -0.0025013990234583616, -0.0061597758904099464, 0.006495881360024214, -0.029680682346224785, -0.0102899931371212, 0.024833010509610176, 0.012054545804858208, -0.001379162771627307, -0.013689827173948288, 0.009488510899245739, -0.005765498615801334, 0.08242335915565491, -0.0007740117143839598, -0.007025893311947584, 0.006786741316318512, -0.015124738216400146, 0.013088716194033623, 0.002897292375564575, -0.013392503373324871, 0.00838970486074686, 2.4288858185173012e-05, 0.010761832818388939, 0.04974358156323433, 0.009895715862512589, 0.010367555543780327, -0.004640838596969843, 0.014517162926495075, -0.034929096698760986, -0.01314688753336668, 0.008764591999351978, -0.024302998557686806, -0.003613131819292903, 0.018537499010562897, 0.010748906061053276, 0.012687974609434605, 0.0007934023742564023, -0.001485003624111414, -0.013961296528577805, 0.002680762903764844, 0.019287273287773132, -0.01763259992003441, -0.03156804293394089, -0.026461828500032425, 0.021316831931471825, 0.025350095704197884, 0.030611436814069748, 0.03697158396244049, 0.002633902011439204, -0.011000984348356724, 0.003325503319501877, -0.0011117329122498631, 0.027379654347896576, -0.00751066068187356, 0.003512946655973792, -0.02355322428047657, 0.0011876797070726752, 0.024923499673604965, 0.014193984679877758, 0.02523375116288662, -0.00802774541079998, -0.002129744039848447, 0.024483978748321533, -0.0004815354768652469, 0.005910928826779127, 0.011285381391644478, 0.02134268544614315, 0.009772907942533493, 0.019726796075701714, -0.011834784410893917, 0.004059117753058672, 0.024122018367052078, 0.010742442682385445, 0.011052693240344524, -0.02774161472916603, -0.009753516875207424, -0.005788120906800032, -0.019597524777054787, 0.019158001989126205, -0.02365664206445217, -0.03004264272749424, -0.017335277050733566, 0.004734559915959835, -0.026810860261321068, -0.00874520093202591, 0.025311313569545746, 0.005193473305553198, -0.021885626018047333, -0.003506483044475317, -0.019261417910456657, -0.018434083089232445, -0.024354707449674606, 0.005855988245457411, 0.021381467580795288, -0.0010697197867557406, 0.04516737908124924, -0.0073878527618944645, 0.01624939776957035, 0.01734820380806923, 0.02365664206445217, -0.030404601246118546, 0.003816734068095684, 0.005855988245457411, -0.012513458728790283, -0.002255783649161458, -0.03782477229833603, 0.0102382842451334, 0.0040785083547234535, -0.006318133324384689, 0.015098883770406246, 0.01701209880411625, -0.005413234233856201, 0.06075749546289444, 0.02258368954062462, -0.03234367072582245, 0.01302408054471016, -0.011020375415682793, 0.02509155310690403, -0.001961691537871957, -0.009759980253875256, -0.008783982135355473, -0.01336664892733097, 0.010270602069795132, -0.003165530040860176, 0.018447009846568108, -0.02110999822616577, -0.024975208565592766, -0.023074921220541, 0.01711551658809185, 0.010076695121824741, -0.011860638856887817, -9.599401528248563e-05, 0.018291885033249855, 0.0029877822380512953, 0.02624206803739071, 0.0023802071809768677, -0.019545815885066986, -0.017865289002656937, 0.023100774735212326, -0.012914199382066727, -0.016766482964158058, -0.018097978085279465, 0.0030233317520469427, -0.004505103453993797, -0.02980995364487171, -0.025944743305444717, -0.004627911373972893, -0.011227209120988846, -0.008868008852005005, -0.02547936700284481, 0.012190280482172966, 0.039298463612794876, 0.054293930530548096, -0.00579458475112915, -0.013263232074677944, -0.03345540538430214, -0.0375920832157135, 0.02489764615893364, -0.00791140180081129, 0.011330626904964447, -0.04702888801693916]} +{"id": "test:10000090", "text": "\"It comes on again on her grandson\u2019s christening day. A lost moment, a blank spot, one that Carole does not know how to measure. She is there one second, then she is not. She knows exactly where she is, then she does not. Her older church friends tell similar stories about their surgeries, how they count backward from ten with an oxygen mask over their faces, then wake up before reaching one, only to find that hours, and sometimes even days, have gone by. She feels as though she were experiencing the same thing.\\nDecisions that affect others in bad ways cannot always be traced to one individual. This makes it easier for coporations, as a whole, to engage in unethical behavior.\\nIndeed, some argue that whole purpose pf the coperation is to make the most money that it can for owners and investers.\\nI do what I can for you.\"", "vector_field": [-0.0027098015416413546, -0.013499738648533821, 0.025155404582619667, -0.03420684486627579, -0.022114794701337814, 0.004437739960849285, 0.0034576363395899534, -0.008896594867110252, -0.04116082936525345, -0.011810511350631714, 0.018004342913627625, 0.03187008202075958, -0.013936121948063374, -0.008312404155731201, 0.001267799991182983, 0.0025690326001495123, 0.038542527705430984, -0.013042239472270012, -0.0099242078140378, -0.009009210392832756, -0.023423945531249046, 0.018820803612470627, -0.021129412576556206, 0.015892809256911278, -0.005050084553658962, -0.009769362397491932, 0.01901787891983986, -0.01741311326622963, 0.034150537103414536, -0.01963726244866848, -0.010557668283581734, -0.011261512525379658, -0.01215539500117302, -0.011022205464541912, -0.02122795209288597, -0.004092855844646692, 0.000831416342407465, 0.003556174458935857, 0.0034611555747687817, -0.013584199361503124, 0.005996755324304104, 0.011683819815516472, -0.016272885724902153, 0.003397809574380517, -0.032123465090990067, 0.006735792383551598, 0.0022487833630293608, -0.025324326008558273, -0.020369261503219604, 0.003216569544747472, 0.01893341727554798, 0.027590706944465637, -0.01880672574043274, 0.004399028606712818, 0.013781276531517506, -0.007214406505227089, 0.011550089344382286, 0.029955623671412468, 0.022523025050759315, -0.00950893945991993, 0.031307004392147064, 0.0006629335694015026, -0.013527892529964447, 0.0013461026828736067, -0.013211161829531193, -0.02812562696635723, -0.02750624530017376, -0.00011063556303270161, -0.007636713329702616, 0.011768280528485775, 0.029505163431167603, 0.018623726442456245, -0.0010935984319075942, -0.002767868572846055, 0.00760855944827199, -0.012683278881013393, -0.01033947616815567, -0.003994317725300789, 0.0011147138429805636, 0.013021123595535755, 0.004018952138721943, -0.03913375735282898, -0.0031303486321121454, 0.03626207262277603, 0.008819172158837318, -0.003126829396933317, 0.0008578105480410159, 0.03581161051988602, -0.006936388090252876, 0.010121284052729607, 0.0074044447392225266, 0.004504605196416378, -0.004993777256458998, 0.006781542208045721, 0.0052295648492872715, 0.031307004392147064, -0.008101250976324081, 0.023691406473517418, 0.005511102732270956, -0.005306988023221493, -0.006380350794643164, 0.011599358171224594, -0.002509205834940076, -0.006946945562958717, -0.007657828740775585, -0.01319004688411951, 0.004254740197211504, -0.00329927122220397, 0.018567418679594994, -0.016272885724902153, -0.016455885022878647, 0.04574989527463913, -0.006046024616807699, -0.012443971820175648, 0.0013135499320924282, -0.030462391674518585, 0.0007597121875733137, 0.0014226457569748163, -0.02269194833934307, -0.012436932884156704, 0.00459258584305644, 0.01572388783097267, 0.011036282405257225, -0.014837043359875679, 0.02138279750943184, -0.007196810562163591, -0.010205745697021484, -0.008474288508296013, -0.0001070613507181406, -0.012303202413022518, 0.037078529596328735, 0.011099628172814846, 0.017145652323961258, 0.024902019649744034, 0.013119662180542946, 0.009248517453670502, -0.0007201209664344788, 0.03341853991150856, -0.0014367226976901293, -0.019792107865214348, 0.022255564108490944, 0.011803473345935345, -0.008115327917039394, -0.007742289919406176, -0.014400659129023552, 0.017919881269335747, -0.003137387102469802, 0.018060650676488876, -0.0025180038064718246, -0.012971854768693447, 0.015470502898097038, -0.017596114426851273, -0.0042864130809903145, -0.008298327215015888, 0.0360649935901165, 0.024634558707475662, 0.023170562461018562, 0.016118040308356285, -0.015583118423819542, 0.003969683311879635, 0.004828373435884714, 0.0324331559240818, 0.0041949134320020676, 0.02925177849829197, 0.029674086719751358, 0.014597735367715359, 0.0033379828091710806, 0.018018420785665512, -0.005109911318868399, -0.010768821462988853, -0.034009769558906555, 0.03924637287855148, -0.023311331868171692, 0.015020042657852173, 0.0027661090716719627, 0.024620482698082924, 0.0036494338419288397, 0.027182476595044136, -0.02618301659822464, -0.03823283687233925, -0.015203041955828667, 0.011148897930979729, 0.025042789056897163, 0.022593410685658455, -0.020017338916659355, 0.005099353846162558, 0.006179755087941885, 0.011134820990264416, 0.028998395428061485, 0.006471850443631411, -0.012472125701606274, 0.028505703434348106, -0.0010469687404111028, -0.0007957842317409813, -0.6702852249145508, -0.005317545495927334, 0.008579865097999573, 0.003482270985841751, 0.021622104570269585, 0.006243100855499506, 0.01824364997446537, -0.019130494445562363, -0.01872226409614086, 0.019412033259868622, -0.012472125701606274, 0.006309966091066599, 0.0003310268803033978, 0.0024177059531211853, -0.022382255643606186, -0.027196552604436874, 0.012824047356843948, 0.0038922603707760572, 0.0042336247861385345, -0.0017226595664396882, -0.013366008177399635, 0.02784409001469612, -0.007003253325819969, 0.006756907794624567, 0.0068132150918245316, 0.027604782953858376, 0.014231736771762371, -0.018482957035303116, -0.014337313361465931, 0.00036511936923488975, -0.0002826375712174922, 0.026633476838469505, 0.009142940863966942, 0.010255014523863792, 0.053632952272892, -0.004508124198764563, -0.001637318404391408, 0.0169204231351614, 0.009515978395938873, 0.023705484345555305, -0.0264645554125309, -0.014224697835743427, 0.006587984971702099, -0.024817558005452156, -0.005810236558318138, -0.011162974871695042, 0.009023287333548069, 0.012809970416128635, 0.0007227603928185999, 0.008523557335138321, -0.00431456696242094, -0.0169204231351614, -0.004268817137926817, 0.003695183899253607, 0.01054359134286642, -0.015343811362981796, 0.01535788830369711, -0.010093131102621555, -0.0065140812657773495, -0.010635090991854668, -0.001145506976172328, 0.01416839100420475, -0.0037796450778841972, -0.00940336287021637, -0.016695192083716393, 0.0302090086042881, -0.017652420327067375, 0.02067895233631134, 0.00908663310110569, -0.013415277004241943, 0.01907418668270111, 0.04797404259443283, -0.009079594165086746, -0.00750298285856843, -0.004986738786101341, 0.009332978166639805, 0.01033947616815567, -0.005630756262689829, -0.014541428536176682, 0.033249616622924805, 0.007284791208803654, -0.028998395428061485, -0.011571204289793968, 0.01148674264550209, 0.006260697264224291, 0.0037127798423171043, -0.015906887128949165, -0.003871144959703088, 0.007020849268883467, 0.009121824987232685, 0.007707097567617893, 0.021776949986815453, -0.008037904277443886, -0.02436709776520729, 0.0036599915474653244, 0.01914457231760025, -0.015090427361428738, 0.006144562736153603, 0.029082857072353363, -0.02443748340010643, 0.009987554512917995, -0.014098006300628185, -0.003248242661356926, 0.009340017102658749, 0.00866432674229145, 0.0013073912123218179, -0.011887934058904648, -0.0023807541001588106, 0.03184192627668381, -0.021833257749676704, 0.002088658744469285, -0.024944251403212547, 0.020383337512612343, 0.0009070796659216285, -0.003874664194881916, -0.02062264457345009, 0.011747165583074093, -0.001364578609354794, -0.00824905838817358, -0.021340565755963326, 0.003976721782237291, 0.01894749514758587, 2.4882005163817666e-05, -0.01831403560936451, 0.03412238508462906, 0.014006506651639938, -0.008551711216568947, -0.028773164376616478, 0.0071686566807329655, -0.01637142337858677, -0.0064366585575044155, -0.034854382276535034, 0.008347596041858196, -0.0054935067892074585, 0.019609108567237854, 0.007362213917076588, 0.022325949743390083, -0.008699518628418446, 0.009698977693915367, -0.02547917328774929, 0.008403903804719448, -0.009171094745397568, 0.01572388783097267, -0.012063895352184772, -0.004617220256477594, -0.03167300298810005, -0.020693030208349228, -0.01755388267338276, -0.029617778956890106, 0.001923255156725645, 0.0001589149032952264, -0.010086092166602612, -0.010712513700127602, 0.03217977285385132, -0.0020376299507915974, -0.004008394666016102, -0.02214294858276844, -0.04197729006409645, -0.03133516013622284, -0.013879814185202122, 0.004068221431225538, 0.03890852630138397, -0.02011587657034397, -0.021622104570269585, 0.008220904506742954, -0.014893350191414356, -0.004047106020152569, 0.006809696089476347, 0.006468331441283226, -0.016047654673457146, 0.004008394666016102, -0.007235521916300058, -0.010937743820250034, 0.010923666879534721, 0.010001630522310734, 0.01956687867641449, -0.008171634748578072, 0.0005608760984614491, -0.0019338128622621298, -0.0061867935582995415, 0.016329193487763405, -0.0020904182456433773, -0.017356805503368378, -0.0074044447392225266, 0.02764701284468174, -0.000614104385022074, 0.003228886751458049, -0.022649716585874557, -0.009431516751646996, 0.02709801495075226, -0.008741749450564384, 0.01277477853000164, -0.011887934058904648, 0.00867136474698782, 0.0026288593653589487, 0.012162433937191963, -0.00810828898102045, -0.00581375602632761, 0.0014912706101313233, 0.024958327412605286, 0.013147816061973572, -0.012141318060457706, 0.0017006644047796726, 0.009023287333548069, 0.01656850054860115, -0.0014376024482771754, 0.01152193546295166, -0.009762323461472988, 0.023508407175540924, 0.004867084790021181, -0.0143443513661623, -0.0007803876069374382, 0.009079594165086746, -0.013408238068223, 0.003095156280323863, 0.030828390270471573, 0.0025549556594341993, 0.01755388267338276, -0.0010302524315193295, 0.0007451954297721386, 0.014302121475338936, -0.0032183292787522078, 0.012317279353737831, -0.010972936637699604, 0.00929074827581644, -0.009579324163496494, 0.0202566459774971, 0.006098812911659479, 0.02639416977763176, -0.009600439108908176, 0.007031406741589308, 0.015400118194520473, 0.002919195219874382, 0.0323205403983593, -0.005894698202610016, -0.0009035604307428002, 0.02688686177134514, -0.01607580855488777, 0.018989725038409233, -0.01521711889654398, -0.00729182967916131, 0.019341647624969482, 0.03476991876959801, -0.0018599091563373804, -0.008101250976324081, -0.0024265041574835777, 0.037782374769449234, -0.011310781352221966, -0.007868981920182705, 0.002102735685184598, -0.006306447088718414, -0.014668120071291924, -0.014879273250699043, 0.002574311336502433, 0.016047654673457146, -0.01962318643927574, -0.004972661845386028, 0.021044950932264328, 0.02305794693529606, 0.03015270084142685, -0.004434220492839813, 0.0061727166175842285, 0.014639966189861298, -0.0035526552237570286, -0.0006550153484568, -0.005648352671414614, 0.001567813684232533, -0.00021566236682701856, -0.02387440763413906, -0.009874938987195492, 0.014808889478445053, -0.009009210392832756, -0.006549273617565632, -0.0032376849558204412, 0.0604180172085762, 0.008608018979430199, 0.0020288319792598486, -0.0011226320639252663, 0.016385501250624657, 0.017145652323961258, -0.004384951665997505, -0.0242967139929533, 0.020087722688913345, 0.005824313499033451, -0.005553333554416895, -0.003255280898883939, -0.019524646922945976, 0.014668120071291924, 0.010430975817143917, 0.008657287806272507, -0.01458365935832262, 0.00048521283315494657, -0.00628181267529726, 0.011493781581521034, 0.0010628052987158298, 0.020087722688913345, 0.012289125472307205, -0.014808889478445053, 0.012542509473860264, -0.028364935889840126, 0.0005028089508414268, 0.000504128634929657, 0.001335544977337122, -0.023775868117809296, 0.030518699437379837, 0.009530055336654186, -0.007791559211909771, -0.0217206422239542, -0.016835961490869522, -0.019130494445562363, 0.011261512525379658, -0.03215161710977554, 0.0018088804790750146, -0.017497574910521507, 0.007319983094930649, 0.005528698675334454, -0.012479163706302643, -0.010522475466132164, 0.010529514402151108, -0.005233084317296743, 0.02255117893218994, -0.020876029506325722, 0.0007764284964650869, 0.025324326008558273, 0.113347128033638, 0.03581161051988602, -0.01344343088567257, 0.013900930061936378, -0.012739586643874645, 0.014808889478445053, -0.011866819113492966, -0.019186802208423615, 0.016343269497156143, 0.0032412041909992695, 0.001079521607607603, -0.013028162531554699, 0.01746942102909088, -0.005197891965508461, 0.005577967967838049, 0.008741749450564384, 0.017652420327067375, 0.007228483445942402, -0.005623717792332172, -0.005718736909329891, -0.026915015652775764, -0.008537634275853634, 0.0045116436667740345, 0.01672334596514702, -0.018694110214710236, -0.01288739312440157, 0.00570817943662405, 0.025451019406318665, 0.0036107224877923727, -0.010473206639289856, 0.00023182880249805748, 0.005581487435847521, -0.016315115615725517, 0.024282636120915413, -0.030743930488824844, -0.013309700414538383, 0.01635734736919403, 0.021425027400255203, 0.029279932379722595, -0.004881161730736494, 0.014625889249145985, 0.02498648129403591, 0.0010487283580005169, -0.015920963138341904, 0.01406985241919756, -0.03260207921266556, 0.006243100855499506, 0.021917719393968582, 0.009621554985642433, -0.012950739823281765, 0.024676790460944176, -0.01713157631456852, -0.013394162058830261, -0.03274284675717354, 0.02089010551571846, -0.0104802455753088, 0.005655390676110983, 0.0074044447392225266, 0.0049163540825247765, 0.02944885566830635, -0.016272885724902153, -0.02549324929714203, -0.007960481569170952, -0.02145318128168583, 0.009558209218084812, -0.024958327412605286, -0.013879814185202122, -0.00234908121638, -0.033868998289108276, -0.020087722688913345, -0.0009281949605792761, -0.008530596271157265, -0.023860329762101173, -0.005993236321955919, 0.027196552604436874, 0.016807807609438896, 0.0013663382269442081, -0.01169789582490921, -0.008650249801576138, 0.006834330502897501, -0.01002274639904499, -0.02387440763413906, -0.011141858994960785, -0.019961031153798103, -0.004571470431983471, 0.007876020856201649, -0.004631297197192907, -0.02736547589302063, -0.039387140423059464, 0.023381715640425682, -0.005581487435847521, 0.006426100619137287, 0.01962318643927574, -0.010606937110424042, 0.005018411669880152, -0.006165678147226572, 0.006000274792313576, 0.017300499603152275, 0.0033766941633075476, -0.010972936637699604, 0.025451019406318665, 0.0010478486074134707, -0.0010988772846758366, -0.02415594458580017, 0.00750298285856843, -0.0005635155248455703, 0.00618327409029007, 0.001295953756198287, 0.010825129225850105, -0.013689775951206684, 0.0253524798899889, -0.008847326040267944, 0.002430023392662406, 0.00028417722205631435, -0.004036548547446728, 0.01644180901348591, 0.011043320409953594, 0.036036841571331024, 0.012338395230472088, -0.010874398052692413, -0.01096589770168066, -0.02138279750943184, 0.024634558707475662, 0.025366557762026787, -0.012190587818622589, 0.002343802247196436, -0.0024089079815894365, -0.018060650676488876, -0.006675965618342161, 0.00856578815728426, 0.014288044534623623, 0.00017079227836802602, 0.007017330266535282, -0.0205522608011961, -0.010142399929463863, -0.03147592768073082, -0.003987279254943132, -0.002035870449617505, -0.023564714938402176, -0.004638335667550564, -0.006834330502897501, 0.015020042657852173, 0.0037338952533900738, -0.029364394024014473, 0.050029270350933075, -0.027478091418743134, -0.01962318643927574, -0.022579332813620567, 0.009037364274263382, 0.022157026454806328, -0.011184089817106724, -0.01894749514758587, -0.01880672574043274, 0.005567410495132208, 0.008277212269604206, -0.03265838697552681, -0.009494862519204617, -0.013183007948100567, 0.018694110214710236, 0.02165025845170021, 0.032968077808618546, 0.006373312324285507, -0.005683544557541609, -0.0005701140617020428, 0.0006778902607038617, -0.0015273427125066519, 0.006267735734581947, -0.01935572549700737, -0.005863024853169918, 0.0272247064858675, -0.013478622771799564, 0.014851119369268417, 0.03814837336540222, -0.01614619418978691, 0.012282087467610836, 0.02110125869512558, -0.008516519330441952, -0.013485661707818508, -0.01033947616815567, -0.029955623671412468, -0.01594911701977253, 0.000504128634929657, -0.012451009824872017, 0.004782623611390591, -0.02485978975892067, -0.01762426644563675, 0.013373046182096004, 0.0029543875716626644, -0.0062114279717206955, -0.00543016055598855, 0.02785816788673401, -0.003737414488568902, -0.0005406406125985086, -0.0026922053657472134, 0.005095834378153086, -0.016118040308356285, -0.027830014005303383, -0.0077704438008368015, -0.007819713093340397, -0.012169471941888332, 0.002757311100140214, -0.0071440222673118114, -0.006260697264224291, -0.007355175446718931, -0.016596654430031776, 0.013654584065079689, -0.02660532295703888, -0.024564174935221672, 0.013668661005795002, -0.01329562347382307, 0.007024368736892939, -0.0074044447392225266, -0.012901470065116882, 0.00085957016563043, 0.020284799858927727, 0.026788324117660522, -0.014639966189861298, 0.020439645275473595, -0.01769465208053589, -0.022720102220773697, -0.014112083241343498, -0.008762864395976067, 0.037782374769449234, 0.007439636625349522, 0.050789423286914825, 0.005391449201852083, 0.0010276130633428693, -0.002632378600537777, 0.017948035150766373, -0.0029385508969426155, 0.007098271977156401, 0.016132116317749023, 0.009677862748503685, -0.015090427361428738, 0.014837043359875679, 0.015780193731188774, -0.010430975817143917, -0.016624808311462402, -0.032123465090990067, 0.011557127349078655, 0.004395509138703346, -4.888420153292827e-05, -0.007538175210356712, -0.02590147964656353, 0.007925289683043957, 0.02020033821463585, -0.00862913392484188, 0.0035684918984770775, -0.008924748748540878, -0.024043329060077667, -0.019383879378437996, -0.01739903725683689, -0.014499197714030743, 0.01837034337222576, 0.009269632399082184, -0.012077972292900085, -0.018609650433063507, 0.000505008443724364, 0.003316867398098111, 0.0224244873970747, -0.007182733621448278, 0.0013434633146971464, -0.018088804557919502, 0.0004957704804837704, -0.00711938738822937, -0.006267735734581947, 0.01725826784968376, -0.009227401576936245, -0.03643099218606949, 0.027252860367298126, -0.005088795907795429, -0.019890647381544113, -0.012992970645427704, -0.009473747573792934, 0.016329193487763405, -0.01517488807439804, -0.006770984269678593, -0.02401517517864704, -0.002035870449617505, -0.0029174357187002897, 0.021185720339417458, 0.02902654930949211, -0.008326481096446514, -0.013485661707818508, 0.004923392552882433, -0.01594911701977253, -0.006566869560629129, -0.005060642026364803, -0.004775585141032934, -0.010501360520720482, -0.024845711886882782, -0.03569899499416351, 0.00020136551756877452, 0.015583118423819542, -0.016413655132055283, -0.021636180579662323, 0.005028969142585993, 0.013211161829531193, -0.023438023403286934, 0.0169204231351614, -0.020974567160010338, 0.006345158442854881, -0.01915864832699299, 0.0027185995131731033, -0.0068026576191186905, -0.021565796807408333, 0.036459147930145264, -0.007228483445942402, -0.034995149821043015, -0.0015598954632878304, 0.028632396832108498, 0.006612619385123253, -0.022311871871352196, -0.010346515104174614, 0.004370874725282192, -0.0013962516095489264, -0.0031250696629285812, -0.019665416330099106, -0.035896070301532745, 0.00040998944314196706, -0.0028910415712743998, -0.007312944624572992, 0.012809970416128635, -0.02842124179005623, 0.005539256613701582, -0.01741311326622963, 0.009677862748503685, 0.0021731199230998755, -0.025014635175466537, 0.0027062823064625263, -0.004040067549794912, 0.03412238508462906, -0.018820803612470627, -0.005384410731494427, -0.021762873977422714, -0.029505163431167603, -0.015512733720242977, 0.019060110673308372, -0.003480511251837015, -0.02549324929714203, 0.007524098269641399, -0.01600542478263378, -0.012092049233615398, 0.02145318128168583, -0.00428289407864213, 0.00991716980934143, 0.007728212978690863, -0.0089880945160985, -0.004951546434313059, -0.009389285929501057, -0.022157026454806328, 0.04453928396105766, -0.002604224719107151, -0.011536012403666973, -0.017173806205391884, -0.0006035466794855893, -0.02032702974975109, -0.00950893945991993, -0.007319983094930649, 0.013879814185202122, 0.02549324929714203, 0.01782134361565113, 0.004050625488162041, 0.016864115372300148, -0.010621014051139355, 0.015203041955828667, -0.0070912339724600315, -0.008593942038714886, 0.020580414682626724, -0.009100710041821003, 0.02004549279808998, 0.012683278881013393, -0.03403792157769203, 0.007242560386657715, -0.0034330016933381557, -0.0016162029933184385, -0.013323777355253696, 0.011550089344382286, -0.016948577016592026, -0.0025408787187188864, -0.006454254500567913, 0.0006739311502315104, 0.01977803185582161, 0.0014815927715972066, -0.004057663958519697, -0.0169204231351614, -0.0018810245674103498, 0.020636722445487976, 0.024141868576407433, -0.0360649935901165, 0.00635571638122201, 0.0058313519693911076, 0.0041702790185809135, 0.016821885481476784, -0.013703852891921997, 0.00866432674229145, 0.010297245346009731, -0.006257177796214819, 0.027492167428135872, 0.012626971118152142, -0.0019267743919044733, 0.025239864364266396, 0.03412238508462906, 0.0018739860970526934, -0.013147816061973572, 0.01096589770168066, -0.019749877974390984, -0.022466717287898064, 0.011683819815516472, -0.026098554953932762, 0.007953443564474583, 0.006672446150332689, 0.007742289919406176, -0.025310249999165535, -0.000307052192511037, -0.013654584065079689, 0.008277212269604206, -0.040963754057884216, 0.018398497253656387, 0.016061732545495033, -0.024127790704369545, 0.01782134361565113, -0.012901470065116882, -0.023452099412679672, -0.0011067955056205392, -0.02186141163110733, -0.006946945562958717, -0.01997510902583599, -0.006939907092601061, 0.009114786982536316, -0.007826751098036766, -0.021059028804302216, -0.005662429146468639, 0.008030866272747517, -0.012127241119742393, 0.024493789300322533, 0.21047766506671906, -0.002539119217544794, -0.022579332813620567, 0.028745010495185852, -0.012788855470716953, -0.005275314673781395, 0.027632936835289, 0.02101679891347885, -0.021340565755963326, 0.019820261746644974, -0.008073097094893456, -0.01142339687794447, -0.003573770634829998, -0.008481326512992382, 0.0022417448926717043, -0.024212252348661423, -0.020833797752857208, -0.014541428536176682, -0.009368170984089375, -0.03167300298810005, 0.0016417173901572824, 0.004793181549757719, -0.0023385235108435154, -0.0041984329000115395, 0.03558637946844101, 0.00033212665584869683, 0.0012721989769488573, 0.015920963138341904, 0.02431079000234604, 0.026999477297067642, -0.017933959141373634, 0.004691123962402344, 0.0028030609246343374, 0.009910130873322487, -0.015597195364534855, -0.0006224625394679606, 0.021481335163116455, -0.0024458598345518112, 0.008805095218122005, 0.0017983227735385299, 0.015470502898097038, 0.0022804562468081713, -0.014428813010454178, 0.006774503737688065, 0.006521119736135006, 0.04220251739025116, -0.0014261649921536446, -0.01936980150640011, -0.024198174476623535, 0.00379372201859951, -0.0426248237490654, -0.0048530083149671555, 0.019397955387830734, 0.01573796384036541, -0.013978352770209312, 0.010072015225887299, 0.010761783458292484, 0.0074818674474954605, -0.027182476595044136, 0.016132116317749023, -0.013520853593945503, 0.008889556862413883, -7.126425771275535e-05, 0.014372505247592926, -0.009938284754753113, 0.03507961332798004, -0.01872226409614086, 0.00532458396628499, -0.013056316412985325, -0.02911101095378399, 0.014639966189861298, 0.002466975012794137, -0.0029930989257991314, 0.017919881269335747, -0.01789172738790512, -0.016385501250624657, 0.025296172127127647, 0.0030494064558297396, 0.013358969241380692, 0.011993510648608208, -0.0108603211119771, 0.0022452641278505325, -0.03257392719388008, 0.003207771573215723, -0.02297348529100418, -0.038964834064245224, 0.029505163431167603, 0.0068167345598340034, 0.0025690326001495123, 0.000995060196146369, 0.0011754203587770462, -0.013373046182096004, -0.01637142337858677, 0.0037409337237477303, 0.006992695387452841, -0.005057123024016619, 0.009769362397491932, 0.015484579838812351, -0.014738504774868488, -0.023311331868171692, -0.0064894468523561954, 0.0257607102394104, -0.0017499334644526243, -0.008157557807862759, 0.009023287333548069, 0.009889015927910805, -0.005841909907758236, 0.038401756435632706, 0.01385869923979044, -0.023494331166148186, -0.010501360520720482, -0.027140244841575623, 0.006911753211170435, -0.01245804876089096, -0.006309966091066599, 0.005226045846939087, 0.007228483445942402, -0.005521660204976797, 0.0035368187818676233, -0.01949649304151535, 0.005050084553658962, -0.018159188330173492, 0.003769087605178356, 0.0247049443423748, 0.006795619148761034, -0.015695733949542046, -0.017919881269335747, 0.007538175210356712, -0.007685982622206211, -0.03857067972421646, 0.012282087467610836, -0.01727234572172165, 0.005676506087183952, -0.00020510470494627953, -0.014822966419160366, -0.017680574208498, 0.011409319937229156, -0.023339485749602318, -0.01607580855488777, -0.0018634283915162086, -0.008030866272747517, 0.005018411669880152, -0.009332978166639805, 0.0055568525567650795, 0.002167841186746955, -0.025817018002271652, 0.015850579366087914, -0.002371956128627062, -0.01385869923979044, -0.008037904277443886, -0.00876990333199501, -0.0020006780978292227, 0.008087174035608768, -0.02136871963739395, 0.01914457231760025, -0.004244182724505663, -0.03153223544359207, 0.0045749894343316555, -0.0027854647487401962, -0.0015035879332572222, -0.03730376064777374, -0.0034611555747687817, 0.04135790467262268, -0.006000274792313576, -0.014034660533070564, 0.0016557943308725953, -0.18345004320144653, 0.0027784262783825397, 0.00036907847970724106, -0.006116408854722977, 0.04130159690976143, 0.0025021673645824194, 0.03099731355905533, -0.0019126975676044822, -0.01566758006811142, -0.002030591480433941, 0.015695733949542046, -0.010360592044889927, -0.018975649029016495, -0.01970764808356762, 0.00259366724640131, -0.012598817236721516, -0.022959409281611443, 0.017497574910521507, 0.03569899499416351, 0.007545213680714369, 0.020425569266080856, -0.005894698202610016, 0.01755388267338276, 0.002604224719107151, 0.018229573965072632, 0.01699080690741539, 0.006633734796196222, 0.01782134361565113, -0.000586830370593816, -0.02409963682293892, -0.009804554283618927, -0.004106932785362005, 0.0051802960224449635, 0.002591907512396574, -0.00887547992169857, -0.006102332379668951, -0.01152193546295166, 0.002387792570516467, -0.027492167428135872, 0.011430435813963413, 0.0041315676644444466, 0.026436401531100273, -0.015188965015113354, -0.004078778903931379, -0.01663888432085514, 0.03626207262277603, 0.017582036554813385, -0.01776503585278988, -0.002623580629006028, -0.011543050408363342, -0.002044668421149254, -0.014527351595461369, 0.002030591480433941, 0.01069139875471592, -0.0013082710793241858, 0.004356797784566879, 0.02062264457345009, 0.002989579690620303, -0.016667038202285767, -0.027604782953858376, 0.014808889478445053, -0.03029346838593483, 0.004149163607507944, -0.0001288915373152122, -0.03364377096295357, -0.01915864832699299, 0.01330266147851944, -0.0005797919584438205, -0.022790485993027687, 0.016695192083716393, 0.002563753630965948, 9.716353815747425e-05, 0.00960747804492712, -0.03133516013622284, 0.018426649272441864, 0.012282087467610836, -0.009494862519204617, 0.0037338952533900738, -0.009973477572202682, 0.02346617728471756, 0.014203582890331745, 0.029477009549736977, -0.026492709293961525, -0.012373587116599083, -0.00395208690315485, 0.005623717792332172, 0.011789396405220032, 9.391925414092839e-05, 0.0084602115675807, -0.036093149334192276, 0.015118580311536789, -0.03572715073823929, -0.01831403560936451, -0.014210620895028114, -0.003339742310345173, 0.01998918503522873, 0.01762426644563675, 0.0069047147408127785, -0.006419062148779631, -0.005226045846939087, -0.01333785429596901, 0.015681656077504158, -0.005915813148021698, 0.012577702291309834, 0.027140244841575623, 0.01962318643927574, 0.01748349890112877, -0.01065620593726635, 0.019482417032122612, -0.01169789582490921, -0.018891187384724617, -0.010508399456739426, 0.01309150829911232, 0.0183844193816185, -0.008924748748540878, 0.029082857072353363, 0.019468339160084724, -0.0034066075459122658, 0.010142399929463863, 0.00940336287021637, 0.06509154289960861, -0.004655931610614061, -0.012584740296006203, -0.0007443156209774315, -0.012697355821728706, -0.01211316417902708, -0.14763842523097992, -0.014105044305324554, -0.014949657954275608, 0.026858707889914513, 0.004779104609042406, 0.01142339687794447, -0.010438014753162861, 0.003313348162919283, -0.006619657855480909, 0.03530484065413475, -0.0037831643130630255, -0.03806391358375549, -0.031081775203347206, -0.0037972412537783384, 0.017384959384799004, -0.028013013303279877, 0.005599083378911018, -0.026309708133339882, -0.025817018002271652, 0.03327776864171028, -0.008059020154178143, -0.011993510648608208, -0.0016909865662455559, -0.03203900530934334, -0.007854904979467392, -0.00218367762863636, -0.0242967139929533, 0.015569041483104229, 0.01392204500734806, 0.010944782756268978, -0.002250542864203453, -0.012507317587733269, 0.011796434409916401, -0.016864115372300148, -0.005222526378929615, -0.018285881727933884, -0.027815936133265495, -0.0018370342440903187, 0.018567418679594994, -0.02987116202712059, 0.00876990333199501, 0.017117498442530632, 0.014977811835706234, -0.016399577260017395, -0.0034717132803052664, -0.0004948906716890633, -0.046200357377529144, 0.03417868912220001, -0.02882947213947773, 0.0017992026405408978, -0.025605864822864532, 0.010459129698574543, -0.008558749221265316, -0.02471902035176754, 0.013936121948063374, -0.006619657855480909, 0.011226320639252663, 0.008643210865557194, -0.0018317553913220763, -0.004571470431983471, 0.010789936408400536, -0.003790202783420682, -0.022804563865065575, 0.028533857315778732, 0.013225238770246506, -0.010811052285134792, 0.0030722813680768013, 0.016047654673457146, 0.011972395703196526, -0.006609100382775068, -0.007742289919406176, 0.015639426186680794, 0.0051662190817296505, 0.03781053051352501, -0.030124546959996223, -0.012725509703159332, -0.00469816243276, -0.02422632835805416, 0.012451009824872017, -0.022804563865065575, -0.008622095920145512, -0.02152356691658497, 0.003807798959314823, -0.007664867211133242, -0.004691123962402344, 0.022874947637319565, 0.0035808091051876545, 0.024902019649744034, -0.0032711175736039877, 0.0003640195936895907, 0.005687064025551081, 0.030800236389040947, 0.006088255438953638, -0.02249487116932869, -0.022508949041366577, 0.009706016629934311, -0.020566336810588837, -0.013591238297522068, 0.002957906574010849, 0.0009924208279699087, -0.025887401774525642, -0.02981485426425934, -0.020580414682626724, 0.022677870467305183, -0.0023561196867376566, 0.0017745679942891002, 0.001324987388215959, 0.004289932548999786, 0.015540887601673603, 0.018116958439350128, -0.007728212978690863, 0.030096393078565598, -0.019243109971284866, 0.04408882185816765, -0.00025536358589306474, 0.0011648627696558833, -0.021354643628001213, -0.014555505476891994, -2.5005727366078645e-05, -0.018891187384724617, -0.012134280055761337, 0.0010170553578063846, -0.01997510902583599, 0.009741208516061306, 0.0036775877233594656, 0.028730934485793114, 0.026267478242516518, 0.007826751098036766, -0.020481877028942108, 0.010768821462988853, -0.001937332097440958, 0.0015071071684360504, 0.023128332570195198, -0.012549548409879208, 0.019721724092960358, 0.013492699712514877, -0.004377913195639849, -0.00545127596706152, 0.0052823531441390514, 0.02742178365588188, 0.006383869796991348, 0.027731474488973618, -0.02812562696635723, -0.001814159331843257, 0.00810828898102045, -0.010712513700127602, -0.00044364199857227504, 0.012809970416128635, -0.02498648129403591, 0.02207256481051445, 0.0010144159896299243, 0.011198166757822037, 0.01644180901348591, 0.009755285456776619, -0.021002721041440964, -0.018426649272441864, -0.026000017300248146, -0.009769362397491932, 0.019820261746644974, -0.005458314437419176, -0.008699518628418446, -0.011472665704786777, 0.015034119598567486, 0.007925289683043957, 0.0036635107826441526, -0.008741749450564384, 0.0158083476126194, -0.007341098506003618, -0.008833249099552631, 0.0006180634954944253, -0.008910671807825565, -0.031504083424806595, -0.016737423837184906, 0.010240938514471054, -0.009347055107355118, 0.041048213839530945, -0.01642773114144802, -0.009825670160353184, -0.015512733720242977, -0.0048494888469576836, -0.007446675095707178, 0.02234002575278282, 0.022114794701337814, 0.004293451551347971, -0.014569582417607307, 0.010219822637736797, 0.021213874220848083, 0.01649811677634716, -0.0034294824581593275, 0.02431079000234604, 0.013471584767103195, 0.017075268551707268, 0.0173286534845829, -0.0006792100029997528, -0.015752041712403297, -0.021762873977422714, 0.009072556160390377, 0.017638344317674637, -0.01277477853000164, 0.027928551658988, 0.0006563350325450301, 0.020650798454880714, -0.00475095072761178, 0.007161618210375309, -0.011930164881050587, -0.004173798020929098, -0.030800236389040947, 0.00760855944827199, -0.005036007612943649, -0.02667570859193802, 0.01173308864235878, 0.02424040623009205, -0.0005212848773226142, -0.0014745543012395501, 0.040625907480716705, 0.008136442862451077, -0.02200218103826046, 0.013218200765550137, 0.010072015225887299, -0.028576089069247246, -0.027928551658988, 0.0006849287310615182, 0.009515978395938873, -0.02556363306939602, 0.02207256481051445, -0.023297253996133804, 0.01782134361565113, -0.002100975951179862, 0.019454263150691986, -0.038120221346616745, -0.002649974776431918, -0.009910130873322487, 0.006820253562182188, 0.0027925032190978527, -0.024676790460944176, -0.006612619385123253, 0.0016857077134773135, -0.028238242492079735, -0.0023526004515588284, 0.022720102220773697, -0.01866595633327961, 0.031504083424806595, 0.002167841186746955, -0.028027089312672615, -0.0068167345598340034, -0.02180510386824608, 0.019552800804376602, 0.0099242078140378, -0.008608018979430199, -0.0079041738063097, -0.005732813850045204, 0.035952378064394, -0.007510021328926086, 0.02652086317539215, -0.016821885481476784, -0.012873317115008831, 0.0028171378653496504, 0.003015973838046193, 0.013380085118114948, 0.009473747573792934, 0.009832708165049553, 0.04003467783331871, -0.013007047586143017, 0.01162751205265522, -0.003015973838046193, 0.0013038719771429896, -0.02173472009599209, 0.016962653025984764, -0.0027098015416413546, -0.03716299310326576, -0.040344368666410446, 0.0013971313601359725, 0.023212792351841927, -0.018074728548526764, -0.004402547609061003, 0.015090427361428738, 0.002630618866533041, -0.0013540209038183093, -0.006637253798544407, 0.021537642925977707, 0.014386582188308239, -0.01517488807439804, 0.015597195364534855, -0.017089344561100006, -0.018834879621863365, -0.01006497722119093, -0.012479163706302643, 0.013239315710961819, -0.005109911318868399, -0.025296172127127647]} +{"id": "test:10000091", "text": "\"Definitely my favorite restaurant in Macau. If you want to experience the authentic Portuguese Macanese dishes, with live one man band (sings with guitar) and cozy ambience. This is it. For extra boost to add on to your one-in-a-lifetime experience, try order Crepe suzette! 3-michelin-stars Chef Antonio himself will invite you to join him cooking at your table and be prepared for a fiery ride!\\nOne of many Signature dish \u2013 Seafood rice!\\nIt\u2019s getting hot in here -Flaming Chouri\u00e7o!\\nCooking crepe suzette! Be careful with the alcohols!\\nTadaaa! Proudly present Crepe suzette by Chef Antonio and Me!\\nAnother popular local traditional Portuguese Macanese restaurant with more mature settings and environment.\\nThe famous african chicken \u2013 this one\u2019s a lil spicy! But superb!\\nSteamed clams! Make sure you dip the bread in the sauce! Yum!\\nEnded with my favorite traditional dessert in Macau! Serradura!\\nA cute romantic Portuguese-Macanese restaurant nearby the famous original Lord Stow\u2019s egg tart bakery!\\nPortugal is also famous for their glazed tiles. So lovely!\\nSauteed shrimp with garlic butter!\\nSoaky wet Portuguese seafood rice! A must!\\nPork ribs!! Love the salty taste!\\nThe french fries sure stole the spotlight from the pork steak!\\nThis cherry whisky is STRONG!! be careful!\\nPineapple, Mango, Peach juices! Loving these rejuvenating refreshments! Just can\u2019t get enough!\\nLovely meal indeed! Bon app\u00e9tit!\\nThe famous Egg tart!! It\u2019s shooo gooood u guyz!!\\nA regular Chinese restaurant, but what impressed me the most is the fact that they have a short tiny table where ladies can put their bag on while eating! Talk about hospitality! A major problem for all ladies who carry handbags, basically everyone! We always need to get extra chair, or hang it on the chair, or let the bag share half of our chair! So Bravo! And obviously the food is Goo-Oood tooo! HEre i picked out my favorite dishes!\\nCrispy pork! Not oily at all!\\nCrispy Spring rolls stick filled with shrimp and mangos!\\nWebsite: http://www.cityofdreamsmacau.com/dining/chinese/jade-dragon Probably the most exclusive and expensive restaurant with millions of dollars on investment, from the interior designs, d\u00e9cor, utensils and obviously the exquisite ingredients for the dishes that are created by Chef Tam Kwok Fung. Be prepared for a fabulous Chinese fine dining.\\nPrepared for a memorable meal!\\nTiny portion, long name, here we go! Crispy yam and vegetable roll serve with sweet and sour sauce, marinated black fungus with garlic and cucumber in aged italian balsamic vinegar, deep fried bean curd with spicy salt, jade dragon dumpling!\\nDon\u2019t let the simplicty appearance fool you. It\u2019s Lychee wood roasted crispy chicken with black truffle! Umm yummz!\\nLooking for Dimsum? Looking for fine dining setting? Looking for traditional Guangdong culinary? Look no further! Look for Lua Azul! Enjoy the numerous dimsum menu they offered, but better yet, the taste will make you smile like a crazy person! Or is that just me?\\nNot a complete dimsum meal without Ha-Gao!\\nHighly recommended \u2013 Spring rolls with smoked salmon and shrimp in mustard sauce. YUMSSSSS !\\nNeed something cool? Or something hot? Not too much, not too little. Just to warm up or cool down. Here I highly recommend you steamed milk pudding with red beans! (Well im a big red beans lover) Everytime I go, the caf\u00e9 is always packed! And I know exactly why! Go try yourself!\\nWhen you see this sign, you know you\u2019re there!\\nThe famous steamed milk pudding! Comes in both hot and cold! Your call!\\n\u201cThe only things more colorful than the d\u00e9cor are the fragrant whiffs of savory goodies.\u201d Their quote didn\u2019t do them justice. Not just the smell, but the taste! The looks! It\u2019s a brillian work of art. You\u2019ll feel very girly at this Pastry Bar being undecisive of which one to eat. A Truly Desserts heaven!\\nCan you feel dat? Dat magic power luring you in!\\nHot cocoa unlike any other place!!! Look at it!\\nPulled pork sandwich worth the try!\\nDon\u2019t be fooled by the looks! Yummiest soup ever!\\nExquisite experience! Feeling pampered at this gorgeous 3 michelin stars restaurant, situated at the dome of Grand Lisboa hotel that overlooks the Macau peninsula. Too bad the day I went was a little foggy though L When you enter this restaurant, you will feel like a royalty, from the excellent service, marvelous food; a truly mixture of art and culinary, spectacular interior design; the chandelier, the plates, the d\u00e9cor, the grand piano\u2026 urgh everything is a dream!\\nYes! Desserts come in carts! So hard to choooose!\\nOther yum yum in the tum tums. Below are some photos from cafes/restaurants worth mentioning!\\nLove the mango froyo with mango bubbles!\\nOther restaurants i\u2019ve been to but didn\u2019t get a chance to take photos, sorry! But undeniably worth mentioning are A lorcha, Fernando\u2019s, and Miramar. Well hope you piglets have fun eating and enjoying all the food in Macau. Let me know what you think! Or recommend any that you think I should try! But for now\u2026. Oink Oink!\\nOMG You just made me sooo hungry\u2026.\"", "vector_field": [-0.017968513071537018, -0.0012417741818353534, 0.011740410700440407, -0.008489219471812248, -0.001801199745386839, 0.0036191237159073353, -0.014128630049526691, -0.016189055517315865, -0.019600799307227135, -0.03283300995826721, 0.007766732946038246, -0.0010193418711423874, -0.013038210570812225, -0.020457079634070396, -0.010529575869441032, 0.019359970465302467, 0.03269921615719795, -9.06244240468368e-05, 0.001518560224212706, -0.047255989164114, -0.01958741992712021, 0.012001308612525463, 0.011292200535535812, -0.013038210570812225, -0.018115686252713203, 0.01032219547778368, -0.006897072773426771, -0.012697036378085613, -0.010268677957355976, 0.03366253152489662, 0.026451043784618378, 0.03184293583035469, -0.00917825847864151, 0.000612524978350848, -0.004157643765211105, 0.003150845179334283, -0.010797163471579552, -0.0006459734286181629, 0.01403497438877821, -0.009894055314362049, 0.009579639881849289, 0.010469368658959866, 0.0015277585480362177, 0.00432488601654768, -0.003465260611847043, 0.009024394676089287, 0.012396000325679779, 0.004809889011085033, -0.01410187128931284, -0.004264678806066513, 0.032458387315273285, 0.012549863196909428, 0.003085620701313019, -0.011713651940226555, -0.01244282815605402, -0.021085910499095917, -0.019533902406692505, 0.024310342967510223, 0.030665550380945206, -0.008221631869673729, 0.002077149460092187, 0.026089800521731377, -0.03178941830992699, 0.0027728774584829807, 0.007311834022402763, 0.007606180384755135, -0.0012401017593219876, -0.0025236865039914846, 0.012436138466000557, 0.011559788137674332, 0.03141479566693306, 0.020925357937812805, -0.0011899290839210153, -0.025260278955101967, 0.02674539014697075, -0.018316378816962242, -9.020631841849536e-05, -0.006164551712572575, 0.022892126813530922, 0.0022126156836748123, 0.024403998628258705, -0.01981486938893795, 0.013379384763538837, 0.03352873772382736, 0.010288747027516365, 0.024537792429327965, -0.023842064663767815, 0.032886527478694916, -0.006335138808935881, -0.01693830080330372, 0.0035990546457469463, 0.016082020476460457, 0.008689910173416138, -1.733832323225215e-05, -0.018383275717496872, 0.010108125396072865, -0.012288965284824371, 0.007739974185824394, -0.0005916196969337761, 0.005017269402742386, 0.01431594230234623, 0.01088412944227457, 0.008937428705394268, -0.025675039738416672, -0.035214539617300034, -0.016416504979133606, -0.0009716778295114636, -0.013359315693378448, 0.026651734486222267, 0.013299108482897282, -0.010241919197142124, 0.013078348711133003, 0.0024601344484835863, -0.023146335035562515, -0.03486667573451996, -0.024698344990611076, 0.02733408287167549, 0.017072094604372978, 0.0012426103930920362, -0.019841628149151802, -0.002448427490890026, 0.03443853557109833, 0.010181711986660957, -0.0003131612320430577, 0.04072684794664383, 0.019761351868510246, -0.022316813468933105, -0.023908961564302444, -0.002102235797792673, -0.00800087209790945, 0.01965431682765484, 0.00885046273469925, 0.01750023476779461, 0.00613110326230526, -0.004488783422857523, 0.0142356650903821, -0.022290054708719254, -0.0004091165028512478, 0.0033716049510985613, -0.002169132698327303, 0.019172659143805504, 0.01653691940009594, -0.016282711178064346, -0.020671149715781212, 0.018945209681987762, 0.04409845173358917, 0.005127649288624525, 0.014757461845874786, 0.010248608887195587, 0.012610070407390594, 0.047470059245824814, -0.042332373559474945, 0.0012752226321026683, 0.014302561990916729, 0.022771712392568588, 0.00024375566863454878, 0.0009934193221852183, 0.01487787626683712, -0.01012150477617979, -0.02865864336490631, 0.007853698916733265, 0.023708270862698555, -0.01139923557639122, 0.004900199826806784, 0.015252498909831047, 0.019533902406692505, 0.014249044470489025, 0.00935888011008501, -0.004518887028098106, 0.005813342519104481, 0.0035488817375153303, 0.021166186779737473, -0.0194803848862648, 0.01822272315621376, -0.00017539536929689348, 0.012964623980224133, 0.003686020616441965, 0.0007981639355421066, -0.016122158616781235, -0.015533465892076492, -0.0016289401100948453, 0.0036358479410409927, 0.02598276548087597, 0.029247336089611053, -0.01961417868733406, -0.010596472769975662, 0.012349172495305538, 0.02093873731791973, -0.012978003360331059, -0.038425594568252563, -0.000352254108292982, 0.015814432874321938, 0.005786583758890629, -0.021085910499095917, -0.6225160360336304, -0.009880675934255123, -0.013713869266211987, -0.010021159425377846, 0.03318087384104729, 0.016590436920523643, 0.015399672091007233, 0.003936883993446827, 0.017914995551109314, -0.02418992854654789, -0.02991630509495735, 0.006144482642412186, 0.015118705108761787, 0.01487787626683712, 0.008536047302186489, -0.02044370025396347, -0.004157643765211105, 0.007786802016198635, -0.02865864336490631, 0.01376738678663969, -0.027253806591033936, 0.03186969459056854, -0.009853917174041271, 0.004114160779863596, 0.00917825847864151, -0.0055189961567521095, 0.0009608070831745863, -0.013874421827495098, -0.0016807852080091834, 0.02001556009054184, -0.03376956656575203, 0.020002180710434914, 0.0006940555758774281, -0.017353061586618423, 0.058066532015800476, -0.017687546089291573, -0.01431594230234623, 0.03655248135328293, 0.014543391764163971, 0.04115498811006546, 0.009184948168694973, 0.0173931997269392, 0.02551448717713356, -0.013580075465142727, 0.010449299588799477, -0.010201781056821346, 0.0375693142414093, 0.02167460322380066, -0.0021390290930867195, -0.006773313507437706, 0.011773859150707722, -0.0035288126673549414, -0.02717353031039238, 0.014891255646944046, -0.0001750817900756374, -0.03585674986243248, 0.029167059808969498, -0.028123466297984123, 0.0010469368426129222, 0.036793310195207596, 0.003595709800720215, 0.02326674945652485, 0.0019333211239427328, -0.03671303391456604, 0.005579203367233276, 0.02084508165717125, -0.01325228065252304, -0.0055189961567521095, -0.0015001635765656829, -5.963233707007021e-05, -0.01146613247692585, 0.010743645951151848, -0.04557018354535103, -0.005706307478249073, -0.007204798981547356, 0.01822272315621376, 0.014262423850595951, 0.009459225460886955, 0.00782025046646595, 0.0022293399088084698, 0.0029501544777303934, -0.00768645666539669, -0.015453189611434937, -0.0051510632038116455, 0.011800617910921574, 0.01431594230234623, -0.030103616416454315, -0.016590436920523643, 0.007733284495770931, 0.011660133488476276, 0.01802203059196472, 0.016844645142555237, 0.017647407948970795, -0.012115033343434334, -0.004013815429061651, 0.031762659549713135, -0.019560661166906357, -0.013994836248457432, -0.002103908220306039, -0.0569961778819561, -0.0118139972910285, -0.018035409972071648, -0.00013891563867218792, -0.004237920045852661, 0.01656367816030979, 0.0024584620259702206, -0.02140701562166214, 0.00047371385153383017, 0.043161895126104355, -0.042171820998191833, -0.0033699325285851955, -0.012222068384289742, -0.0013789128279313445, -0.04476742073893547, 0.016015123575925827, -0.02571517787873745, -0.003373277373611927, 0.010997854173183441, 0.020751425996422768, -0.04987834766507149, -0.0016214142087846994, -0.005134338978677988, 0.005582548212260008, 0.007010797504335642, -0.007224868051707745, 0.01586795039474964, 0.018048789352178574, -0.009733502753078938, -0.007693146355450153, -0.01431594230234623, 0.029300853610038757, 0.01905224472284317, 0.021032392978668213, -0.016376366838812828, 0.010623231530189514, 0.007566042244434357, 0.01157985720783472, 0.0009909106884151697, 0.0191191416233778, -0.03235135227441788, -0.005187856499105692, -0.013613523915410042, 0.0006380294216796756, -0.016817886382341385, -0.004562370479106903, -0.024899035692214966, 0.004388438072055578, -0.012389310635626316, -0.02024300955235958, -0.015747535973787308, -0.0011991274077445269, -0.013366005383431911, 0.0009984365897253156, 0.022785091772675514, 0.0037361932918429375, 0.007800181396305561, -0.013887801207602024, 0.003963642753660679, -0.0451955609023571, -0.01746009662747383, 0.00493699312210083, 0.030531756579875946, -0.024096272885799408, -0.008362115360796452, -0.00017194599786307663, -0.0023731684777885675, 0.004659370984882116, 0.0038097798824310303, -0.016456643119454384, -0.03029092773795128, -0.003478639991953969, -0.0018915105611085892, 0.010803853161633015, 0.010308816097676754, -0.023079438135027885, 0.018048789352178574, -0.027481256052851677, 0.0007806034991517663, -0.006281621288508177, -0.006308380048722029, 0.007713215425610542, -0.001317033194936812, -0.011619995348155499, -0.009787020273506641, 0.024711724370718002, -0.0023882202804088593, 0.013754007406532764, -0.00396698759868741, -0.030959896743297577, 0.02163446508347988, -0.002933430252596736, -0.022838609293103218, -0.01580105349421501, -0.006703071761876345, 0.005268132779747248, 0.005324995145201683, 0.0029886201955378056, 0.002829739823937416, 0.0013922922080382705, 0.019788110628724098, -0.0109510263428092, 0.0030337756033986807, 0.011252062395215034, -0.007044245954602957, 0.030638791620731354, -0.01885155402123928, 0.026451043784618378, -0.013305798172950745, 0.03283300995826721, 0.0006167060346342623, 0.007338592782616615, -0.024203307926654816, -0.008636392652988434, -0.002274495316669345, 0.023641373962163925, 0.031628865748643875, 0.024270204827189445, -0.011606615968048573, -0.004050608724355698, -0.009733502753078938, 0.023213231936097145, 0.012864278629422188, -0.014530012384057045, 0.03029092773795128, -0.013292418792843819, 0.013713869266211987, 0.021781638264656067, 0.01687140390276909, -0.0095595708116889, 0.0030287583358585835, -0.026531320065259933, 0.01905224472284317, 0.012516414746642113, -0.0016531902365386486, 0.0177009254693985, 0.00026277947472408414, 0.013640282675623894, -0.02306605875492096, 0.010181711986660957, -0.003983711823821068, 0.029728993773460388, 0.024310342967510223, 0.025929247960448265, -0.0017125612357631326, 0.020002180710434914, -0.00743893813341856, -0.012456207536160946, -0.0057029626332223415, -0.014275803230702877, 0.0037696417421102524, -0.004722923040390015, 0.018249481916427612, -0.04003112018108368, 0.010643300600349903, 0.0018781311810016632, 0.016229193657636642, -0.0037696417421102524, 0.0010394109413027763, 0.023801926523447037, 0.018811415880918503, -0.008997635915875435, -0.002993637463077903, 0.010743645951151848, 0.006167896557599306, -0.000949936336837709, 0.0020537355449050665, -0.003779676277190447, -0.030959896743297577, -0.025206761434674263, -0.0027344117406755686, 0.006124413572251797, -0.01521236076951027, 0.013506488874554634, -0.004873441066592932, 0.006542519200593233, -0.022785091772675514, -0.02587573044002056, -0.009231775999069214, -0.00858287513256073, 0.00788714736700058, -0.011171786114573479, -0.03767634928226471, 0.03138803690671921, 0.02120632492005825, -0.011686892248690128, -0.029381129890680313, -0.021982328966259956, 0.0042814030312001705, -0.009700054302811623, 0.01952052302658558, 0.014543391764163971, -0.009753571823239326, 0.01604188233613968, -0.0022109432611614466, -0.02587573044002056, -0.007987492717802525, 0.025688419118523598, -0.02124646306037903, 0.018637483939528465, 0.005642755422741175, 0.009526122361421585, -0.015225740149617195, -0.010763715021312237, -0.009064532816410065, 0.003411743091419339, 0.011834066361188889, -0.022918885573744774, 0.006077585741877556, 0.00800087209790945, 0.014583529904484749, 0.0002830575976986438, -0.011907652951776981, -0.012041446752846241, -0.002535393461585045, 0.02703973650932312, -0.006793382577598095, -0.04821930453181267, -0.011827376671135426, 0.024176549166440964, -0.0016874748980626464, -0.02459130994975567, -0.017968513071537018, -0.012884347699582577, 0.006412070244550705, 0.10885467380285263, 0.013533247634768486, -0.013954698108136654, 0.0057364110834896564, 0.02120632492005825, 0.00976026151329279, -0.03687358647584915, -0.014208906330168247, 0.008429012261331081, 0.0012693691533058882, -0.007566042244434357, -0.002376513322815299, 0.004395127762109041, 0.007719905115664005, 0.04618563503026962, -0.006646209396421909, -0.004508852493017912, -0.026089800521731377, 0.017647407948970795, 0.010609852150082588, -0.0037830211222171783, 0.023775167763233185, 0.023534338921308517, 0.036124337464571, 0.010917577892541885, 0.02259778045117855, 0.011760479770600796, 0.014530012384057045, 0.02378854714334011, -0.0032796217128634453, -0.02392234094440937, 0.006044137291610241, -0.010268677957355976, -0.008730048313736916, -0.03087962046265602, -0.013573385775089264, 0.02176825888454914, 0.0005297400057315826, 0.003602399490773678, 0.013198763132095337, 0.00022096889733802527, 0.02906002476811409, 0.006639519706368446, -0.0177009254693985, -0.0023213233798742294, -0.0225710216909647, -0.02724042721092701, 0.017540372908115387, -0.02452441304922104, -0.010282057337462902, 0.010536265559494495, 0.016215814277529716, 0.03237811103463173, -0.005569168832153082, -0.000517614942509681, 0.0019567350391298532, 0.018998727202415466, -0.0024969277437776327, -0.0005230503156781197, -0.007405489683151245, -0.011031302623450756, -0.02793615497648716, -0.005455444101244211, -0.009325431659817696, -0.019774731248617172, -0.031200725585222244, 0.0008964188164100051, -0.006923831533640623, -0.030799344182014465, 0.012382620945572853, -0.023239990696310997, 0.001776113291271031, -0.012081584893167019, 0.00896418746560812, 0.02836429513990879, 0.009653226472437382, 0.006187965627759695, 0.022450607270002365, 0.015065187588334084, 0.03695386275649071, 0.007786802016198635, 0.006589347030967474, -0.01647002249956131, -0.0160017441958189, 0.01081723254173994, 0.014864496886730194, 0.05298236384987831, -0.007719905115664005, -0.021420395001769066, -0.0044653695076704025, 0.029996581375598907, -0.002836429513990879, 0.007405489683151245, 0.0016958370106294751, 0.0036692963913083076, -0.014650426805019379, -0.017607269808650017, -0.005910343490540981, 0.009526122361421585, -0.019333211705088615, 0.009412397630512714, -0.003334811655804515, -0.02591586858034134, 0.010917577892541885, 0.029943063855171204, 0.0009833847871050239, 0.009318741969764233, -0.021393636241555214, -0.0023865478578954935, -0.004087402019649744, -0.0018580621108412743, -0.005753135308623314, -0.014489874243736267, 0.010997854173183441, -0.011492891237139702, 0.03390336036682129, -0.007231557741761208, 0.04573073610663414, -0.033154115080833435, 0.007713215425610542, 0.0076463185250759125, -0.011780548840761185, 0.029247336089611053, 0.04516880214214325, -0.013105107471346855, 0.01772768422961235, -0.012181930243968964, -0.005207925569266081, -0.005953826475888491, -0.010736956261098385, 0.004826613236218691, 0.04672081023454666, -0.016724230721592903, -0.028872713446617126, -0.006997418124228716, -0.018771277740597725, -0.012429448775947094, -0.015346154570579529, 0.00606420636177063, -0.004903544671833515, -0.030237410217523575, 0.006050826981663704, 0.005368478130549192, 0.02004231885075569, 0.007097763475030661, -0.030504997819662094, 0.0020788218826055527, -0.012429448775947094, -0.009619778022170067, 0.004304816946387291, 0.0024718414060771465, -0.012964623980224133, -0.032886527478694916, 0.024939173832535744, -0.004652681294828653, -0.02654469944536686, -0.034277983009815216, -0.027427738532423973, -0.014984911307692528, 0.022209778428077698, 0.047068677842617035, 0.011425994336605072, 0.02071128785610199, -0.009138120338320732, -0.01898534782230854, -0.00628831097856164, -0.0008173967944458127, -0.016657333821058273, -0.020363423973321915, 0.017874857410788536, 0.01467718556523323, 0.018704380840063095, -0.022945644333958626, 0.012188619934022427, -0.013740628026425838, 0.0170319564640522, -0.011004543863236904, -0.027695326134562492, -0.0029752408154308796, -0.028043190017342567, 0.005993964616209269, -0.005254753399640322, -0.01669747196137905, -0.000995091744698584, 0.020965496078133583, -0.0031976730097085238, 0.027748843654990196, 0.0010067987022921443, 0.02753477357327938, 0.005117614753544331, 0.030612032860517502, -0.04562370106577873, 0.0022711504716426134, -0.03240486979484558, 0.014289182610809803, -0.0010561351664364338, -0.01000778004527092, -0.020791564136743546, -0.015894709154963493, -0.0022895473521202803, -0.00545209925621748, 0.00862970296293497, 0.01958741992712021, 0.0037696417421102524, 0.01164675410836935, 0.008275149390101433, -0.012449517846107483, -0.011258752085268497, -0.025434210896492004, -0.030130375176668167, -0.03170914202928543, -0.021795017644762993, -0.008228321559727192, -0.020189492031931877, -0.0024668241385370493, -0.0083353566005826, -0.03430474177002907, 0.00838218443095684, -0.005221304949373007, -0.019038865342736244, -0.0020386837422847748, 0.0076530082151293755, 0.02378854714334011, 0.005201235879212618, 0.0036626067012548447, 0.021019013598561287, -0.006950590293854475, -0.022785091772675514, 0.010302126407623291, 0.012730484828352928, -0.01925293542444706, 0.010342264547944069, 0.025996144860982895, 0.0037228139117360115, -0.014797599986195564, -0.0005247227381914854, 0.020751425996422768, -0.03130776062607765, -0.017513614147901535, 0.00484668230637908, -0.013426212593913078, 0.0015428103506565094, -0.024858897551894188, 0.012054826132953167, -6.70537119731307e-05, 0.020269768312573433, -0.04367031157016754, 0.002654135460034013, -0.0028213777113705873, -0.006569277960807085, -0.025795454159379005, 0.03168238326907158, -0.013807524926960468, -0.006050826981663704, 0.007586111314594746, 0.007472386583685875, -0.001985166221857071, -0.023012541234493256, -0.00889729056507349, 0.022584401071071625, 0.00628831097856164, 0.02820374257862568, -0.030692309141159058, 0.020095836371183395, -0.0044787488877773285, 0.006024068221449852, -0.006097654812037945, -0.0057364110834896564, 0.026531320065259933, 0.03125424310564995, -0.02764180861413479, 0.007064315024763346, 0.007780112326145172, -0.010175022296607494, 0.009766951203346252, -0.012563242577016354, 0.008188183419406414, 0.01954728178679943, 0.011051371693611145, -0.014864496886730194, -0.011325648985803127, -0.01050281710922718, -0.03336818516254425, -0.012088274583220482, 0.0038398834876716137, -0.008342046290636063, 0.004003780893981457, -0.008007561787962914, 0.01363359298557043, -0.0016414832789450884, -0.005749790463596582, -0.011606615968048573, -0.025835592299699783, 0.00896418746560812, 0.013018141500651836, -0.015051808208227158, -0.03098665550351143, 0.012723795138299465, -0.01410187128931284, 0.034545570611953735, -0.005896964110434055, 0.011640064418315887, -0.0065860021859407425, 0.0030354480259120464, 0.0032127248123288155, 0.007867078296840191, 0.012101653963327408, -0.015346154570579529, -0.033983636647462845, -0.030531756579875946, 0.01172703132033348, -0.008415632881224155, 0.013151935301721096, 0.01514546386897564, 0.041369058191776276, -0.019038865342736244, -0.015894709154963493, -0.026531320065259933, -0.010375712998211384, -0.009894055314362049, -0.02203584648668766, -0.0004498818307183683, 0.013553316704928875, 0.005425340496003628, -0.007057625334709883, 0.015279257670044899, -0.02806994877755642, -0.01474408246576786, -0.03652572259306908, 0.008643082343041897, -0.013332556933164597, 0.026196835562586784, -0.0072716958820819855, -0.004656026139855385, -0.014543391764163971, -0.019172659143805504, -0.0250194501131773, 0.016189055517315865, -0.02322661131620407, -0.009967641904950142, 0.022945644333958626, 0.02836429513990879, -0.00994088314473629, -0.008482529781758785, 0.014757461845874786, 0.006930521223694086, -0.007512524724006653, -0.0014332665596157312, -0.02091197855770588, -0.024912415072321892, -0.03374280780553818, 0.020323285833001137, 0.023775167763233185, 0.015961606055498123, -0.019707834348082542, 0.010275367647409439, -0.03224431723356247, -0.012703726068139076, -0.02959519997239113, 0.02740097977221012, 0.01626933179795742, -0.0292205773293972, 0.007900526747107506, -0.002797963796183467, -0.023775167763233185, 0.011820686981081963, -0.01693830080330372, -0.022062605246901512, 0.006903762463480234, -0.04781792312860489, 0.029167059808969498, 0.008435701951384544, -0.05049379914999008, -0.01842341385781765, 0.030665550380945206, 0.01808892749249935, -0.006616105791181326, 0.008837083354592323, -0.024015996605157852, -0.0049737864173948765, -0.010094746015965939, -0.012188619934022427, 0.013165314681828022, -0.008930739015340805, 0.022892126813530922, -0.03711441531777382, 0.006141137797385454, 0.010101435706019402, -0.012275585904717445, -0.014931393787264824, -0.01822272315621376, -0.00015219050692394376, 0.008168114349246025, 0.02021625079214573, 0.00032215050305239856, -0.006652899086475372, 0.024698344990611076, 0.010857370682060719, 0.00994088314473629, -0.0072716958820819855, -0.021192945539951324, 0.016978438943624496, 0.02372165024280548, -0.022022467106580734, 0.004615887999534607, -0.006147827487438917, -0.003241155995056033, -0.0067699686624109745, -0.0026223594322800636, -0.01647002249956131, -0.004866751376539469, -0.011586546897888184, 0.007806871086359024, 0.008536047302186489, -0.009900745004415512, -0.02017611265182495, -0.0015202326467260718, -0.032592181116342545, 0.007311834022402763, 0.03323439136147499, -0.01210834365338087, 0.01869100145995617, -0.020457079634070396, -0.012001308612525463, -0.012777312658727169, 0.002945137210190296, 0.018276240676641464, -0.04131554067134857, -0.009392328560352325, 0.013560006394982338, -0.018971968442201614, -0.017179129645228386, -0.012917796149849892, 0.01097109541296959, 0.008061079308390617, -0.007184729911386967, 0.2209203988313675, 0.02551448717713356, -0.018396655097603798, 0.045650459825992584, 0.01905224472284317, 0.013044900260865688, 0.004809889011085033, 0.02558138407766819, 0.009706743992865086, 0.023507580161094666, -0.011305579915642738, 1.3353253052628133e-05, -0.011981239542365074, -0.0026507906150072813, -0.007579421624541283, -0.03031768649816513, -0.045382872223854065, -0.0319499708712101, 0.0045991637744009495, -0.026732010766863823, 0.007224868051707745, -0.020095836371183395, -0.00507078692317009, -0.009780330583453178, 0.015265878289937973, 0.009700054302811623, -0.00838218443095684, -0.015600362792611122, 0.025072967633605003, 0.02008245699107647, -0.018878312781453133, 0.010268677957355976, -0.015613742172718048, 0.01802203059196472, -0.013553316704928875, 0.0211795661598444, -0.005626031197607517, -0.009599708952009678, 0.00876349676400423, 0.004251299425959587, -0.00917825847864151, -0.0026357388123869896, 0.006539174355566502, -0.006077585741877556, 0.015239119529724121, -0.0037763314321637154, -0.023079438135027885, -0.005743100773543119, 0.009880675934255123, 0.022276675328612328, -0.036927103996276855, -0.018905071541666985, 0.007485765963792801, 0.02959519997239113, 0.019012106582522392, -0.012288965284824371, 0.02349420078098774, 0.011987929232418537, -0.019226176664233208, 0.00889729056507349, -0.02535393461585045, 0.030103616416454315, -0.02048383839428425, 0.013667041435837746, -0.014476494863629341, 0.016148917376995087, -0.013754007406532764, 0.0097401924431324, 0.02176825888454914, -0.004247954580932856, -0.007030866574496031, -0.049851588904857635, 0.00806776899844408, -0.022116122767329216, -0.014891255646944046, -0.02104577235877514, 0.02618345618247986, -0.02697283960878849, 0.04559694230556488, 0.038184765726327896, 0.009258534759283066, -0.017286164686083794, -0.025862351059913635, 0.01282414048910141, -5.665333446813747e-05, -0.033287908881902695, 0.042466167360544205, 0.010195091366767883, 0.00793397519737482, -0.028952989727258682, 0.0038867113180458546, -0.020256388932466507, -0.010797163471579552, -0.013165314681828022, 0.017807960510253906, -0.00586351566016674, 0.012041446752846241, 0.00250194501131773, 0.0044787488877773285, 0.0010318850399926305, -0.011211924254894257, -0.0028665331192314625, -0.009198327548801899, 0.021420395001769066, -0.022142881527543068, -0.004037229344248772, -0.018677622079849243, 0.025889109820127487, 0.009352190420031548, 0.021059151738882065, -0.013178694061934948, -0.01005460787564516, -0.016684092581272125, -0.0019617523066699505, -0.007592801004648209, 0.010302126407623291, 0.011686892248690128, -0.016148917376995087, 0.027079874649643898, -0.01050281710922718, 0.02263791859149933, -0.02001556009054184, -0.013426212593913078, 0.0010293764062225819, -0.03430474177002907, -0.01898534782230854, -0.02091197855770588, -0.010623231530189514, 0.014463115483522415, -0.052741535007953644, 0.029541682451963425, -0.01097109541296959, 0.01938672922551632, -0.011225303635001183, -0.030799344182014465, 0.015747535973787308, 0.01467718556523323, 0.005492237396538258, 0.0011063078418374062, 0.010783784091472626, 0.002234357176348567, 0.017272785305976868, 0.013820904307067394, 0.03098665550351143, -0.0008500090334564447, -0.007010797504335642, -0.005141028668731451, 0.013566696085035801, -0.012168550863862038, -0.0076396288350224495, 0.006549208890646696, -0.001617233152501285, -0.0003215233446098864, -0.0005159425199963152, 0.030076857656240463, -0.0058568259701132774, -0.04337596520781517, 0.008121286518871784, -0.0012417741818353534, -0.013044900260865688, -0.03502722829580307, -0.0038867113180458546, 0.027748843654990196, 0.010656679980456829, -0.011098199523985386, 0.0009683329844847322, -0.16911542415618896, 0.03882697597146034, 0.015680639073252678, -0.010629921220242977, 0.028952989727258682, 0.0021791672334074974, 0.013606834225356579, 0.004304816946387291, -0.00597389554604888, 0.003896745853126049, 0.0033231046982109547, 0.00454899063333869, -0.038318559527397156, -0.03157534822821617, 0.017406579107046127, 0.002371496055275202, -0.038425594568252563, 0.017125612124800682, 0.05089518055319786, -0.009499363601207733, 0.0347863994538784, -0.006796727422624826, -0.015386292710900307, -0.002791274106130004, 0.004759716335684061, -0.0015469914069399238, 0.024644827470183372, -0.010081366635859013, 0.011820686981081963, -0.004850027151405811, 0.012596691027283669, -0.022356951609253883, 0.01146613247692585, 0.005080821458250284, 0.02766856737434864, 0.005980585236102343, 0.0160017441958189, 0.022490745410323143, -0.009954262524843216, 0.0018848208710551262, 0.0021791672334074974, 0.02302592061460018, 0.003142483066767454, 0.014155388809740543, -0.0022694780491292477, 0.006542519200593233, 0.0018496999982744455, 0.0014750772388651967, 0.009345500729978085, 0.014396218582987785, 0.002856498584151268, -0.017072094604372978, 0.02243722788989544, 0.01363359298557043, 0.01097109541296959, 0.010348954237997532, -0.0016347935888916254, 0.03074582666158676, 0.007579421624541283, 0.015854571014642715, -0.013064969331026077, -0.025701798498630524, 0.002043701009824872, 0.004037229344248772, 0.017125612124800682, -0.019172659143805504, 0.006903762463480234, -0.008435701951384544, 0.004361679311841726, -0.00022849481320008636, -0.02448427490890026, -0.0035355023574084044, 0.011138337664306164, -0.019333211705088615, 0.010489437729120255, 0.016082020476460457, -0.022423848509788513, -0.006472277455031872, 0.012817450799047947, -0.026397526264190674, 0.007057625334709883, 0.010496127419173717, -0.034813158214092255, 0.00251365196891129, -0.006746554747223854, 0.019092382863163948, 0.006619450636208057, 0.008375494740903378, -0.012429448775947094, -0.011820686981081963, 0.01474408246576786, -0.013820904307067394, -0.013446281664073467, -0.02876567840576172, 0.00795404426753521, 0.007726594805717468, -0.010128194466233253, 0.029541682451963425, -0.014650426805019379, -0.015225740149617195, 0.003023741068318486, -0.02584897167980671, 0.0027494635432958603, 0.015172222629189491, 0.0173931997269392, 0.02336040511727333, -0.003990401513874531, 0.030103616416454315, 0.03208376467227936, -0.010944336652755737, -0.0160017441958189, 0.033582255244255066, 0.03754255548119545, 0.025554625317454338, -0.016429884359240532, 0.026852425187826157, 0.02073804661631584, -0.027293944731354713, 0.04254644364118576, 0.02587573044002056, 0.028845954686403275, -0.021795017644762993, 0.022490745410323143, 0.01121861394494772, -0.023106196895241737, -0.008161424659192562, -0.10093408077955246, 0.00314415548928082, 0.034679364413022995, 0.04142257571220398, -0.04712219536304474, 0.0006029085488989949, -0.0028330846689641476, 0.024711724370718002, -0.015399672091007233, 0.007592801004648209, -0.009793709963560104, -0.02113942801952362, -0.0028514813166111708, 0.006060861516743898, 0.023012541234493256, -0.02071128785610199, 0.03615109622478485, 0.0020620976574718952, -0.02823050133883953, 0.029943063855171204, -0.0025303761940449476, 0.0016465005464851856, 0.004906889516860247, -0.006100999657064676, -0.016389746218919754, -0.004074022639542818, 0.007004107814282179, 0.01550670713186264, 0.007833629846572876, 0.010783784091472626, 0.004261333961039782, 0.00685024494305253, 0.027253806591033936, -0.014396218582987785, 0.0030354480259120464, -0.03417094796895981, -0.003083948278799653, 0.0049470276571810246, 0.026584837585687637, -0.020590873435139656, -0.006361897569149733, -0.013606834225356579, 0.015346154570579529, -0.0250194501131773, 0.011706962250173092, 0.0032578802201896906, -0.04126202315092087, 0.03304708003997803, 0.01656367816030979, -0.027695326134562492, -0.0034752951469272375, 0.0005585892940871418, -0.016978438943624496, -0.009646536782383919, 0.03465260565280914, -0.005927067715674639, 0.00782025046646595, -0.015560224652290344, -0.025233520194888115, 0.01709885336458683, 0.0042747133411467075, 0.0014583528973162174, -0.006365242414176464, -0.00432488601654768, 0.01889169216156006, 0.015613742172718048, -0.003609089180827141, -0.004107471089810133, -0.009051153436303139, -0.021286601200699806, -0.005054062698036432, -0.010991164483129978, -0.01052288617938757, -0.020952116698026657, -0.035508885979652405, -0.01617567613720894, -0.008937428705394268, -0.011867514811456203, 0.008228321559727192, -0.018610725179314613, -0.0026608251500874758, -0.002784584416076541, -0.007365351542830467, -0.03154858946800232, 0.013305798172950745, 0.00425798911601305, 0.019801490008831024, -0.004495473112910986, 0.007412179373204708, -0.04631942883133888, 0.0028263949789106846, 0.006509070750325918, 0.006438829004764557, -0.005562479142099619, -0.012476276606321335, 0.0008040174143388867, 0.01815582439303398, 0.008241700939834118, 0.005120959598571062, 0.0037495726719498634, -0.02528703771531582, -0.018838174641132355, -0.07128535956144333, -8.75408932188293e-06, 0.011526339687407017, -0.02273157425224781, 0.00935888011008501, -0.005589237902313471, -0.006405380554497242, 0.009506053291261196, 0.00441519683226943, -0.005067442078143358, -0.024042755365371704, 0.004462024662643671, 0.004736302420496941, -0.00662614032626152, -0.00590699864551425, -0.008837083354592323, 0.014075112529098988, -0.008997635915875435, 0.01669747196137905, 0.017379820346832275, -0.020002180710434914, 0.0194803848862648, 0.01474408246576786, -0.002593928249552846, -0.0063953460194170475, 0.011874204501509666, -0.02392234094440937, 0.04284079000353813, 0.0005155244143679738, 0.0012660243082791567, -0.002744446275755763, -0.018236102536320686, -0.017741063609719276, 0.027053115889430046, -0.033555496484041214, -0.03269921615719795, -0.004923613741993904, 0.027427738532423973, 0.007037556264549494, 0.0038532628677785397, -0.017152370885014534, -0.022276675328612328, 0.0007107798010110855, -0.017085473984479904, -0.011459442786872387, 0.010864060372114182, -0.01891845092177391, 0.0012493000831454992, 0.029274094849824905, 0.007445627823472023, 0.013539937324821949, 0.01438283920288086, -0.002654135460034013, -0.016496781259775162, -0.005629376042634249, -0.0012685329420492053, 0.00608762027695775, 0.006392001174390316, -0.021754879504442215, -0.013051589950919151, 0.04334920644760132, -0.010355643928050995, 0.03181617707014084, -0.025434210896492004, 0.021366877481341362, -0.02670525200664997, -0.03660599887371063, 0.005442064721137285, 0.01331917755305767, -0.024912415072321892, -0.024243446066975594, 0.009385638870298862, 0.02345406264066696, 0.02170136198401451, 0.010021159425377846, -0.017687546089291573, -0.008074458688497543, -0.010509506799280643, -0.007786802016198635, 0.015332775190472603, -0.02035004459321499, -0.010088056325912476, -0.023280128836631775, 0.00419109221547842, -0.007552662864327431, -0.005599272437393665, -0.008462460711598396, 0.007706525735557079, -0.00010797581489896402, 0.0055457549169659615, 0.0003990819677710533, 0.005599272437393665, 0.0018580621108412743, -0.01126544177532196, -0.01633622869849205, -0.0023129612673074007, -0.010462678968906403, -0.026624975726008415, 0.016858024522662163, 0.019426867365837097, 0.032030247151851654, 0.020564114674925804, -0.016791127622127533, -0.0008098708931356668, -0.011619995348155499, -0.013513178564608097, -0.044981490820646286, -0.018396655097603798, 0.02522014081478119, -0.006010688841342926, 0.013780766166746616, 0.009198327548801899, 0.011519649997353554, 0.016349608078598976, -0.007760043255984783, -0.000299572799121961, -0.004669405519962311, 0.001978476531803608, -0.006100999657064676, 0.014637047424912453, 0.0017443372635170817, 0.00878356583416462, 0.01826286129653454, -0.0008997636614367366, -0.005559134297072887, 0.029728993773460388, 0.00298694777302444, -0.024738483130931854, 0.018905071541666985, -0.01075033564120531, 0.013553316704928875, 0.0024668241385370493, -0.02176825888454914, -0.0006016542320139706, -0.004860061686486006, -0.006856934633105993, 0.008375494740903378, 0.02932761237025261, -0.02580883353948593, 0.06454215198755264, 0.024470895528793335, -0.006666278466582298, 0.009619778022170067, 0.009907434694468975, 0.044125210493803024, -0.0018095618579536676, -0.0047630611807107925, -0.003110707039013505, 0.005796618293970823, 0.013613523915410042, -0.011987929232418537, -0.011446063406765461, -0.011961170472204685, -0.04923613741993904, 0.013506488874554634, -0.01981486938893795, 0.0029066712595522404, -0.03462584689259529, 0.005037338472902775, 0.011004543863236904, 0.003970332443714142, 0.008214942179620266, 0.013847663067281246, -0.022089364007115364, 0.011278821155428886, 0.017941754311323166, 0.013145245611667633, -0.01925293542444706, -0.029381129890680313, -0.00022703144350089133, -0.0036258134059607983, -0.05410623177886009, -0.014436356723308563, 0.006361897569149733, -0.03251190483570099, 0.013754007406532764, 0.009084601886570454, 0.017406579107046127, 0.006301690358668566, -0.011894273571670055, 0.01383428368717432, -0.013780766166746616, -0.01014157384634018, 0.029381129890680313, -0.007311834022402763, -0.0017476821085438132, -0.015158843249082565, -0.0177009254693985]} +{"id": "test:10000092", "text": "\"Menara Taming Sari (Melaka Tower) is located at Bandar Hilir, beside Dataran Pahlawan Megamall. It is the first and only gyro tower in Malaysia so far. The tower revolves 360\u00b0 to provide a panoramic view of Malacca town. From a height of 80 meters, the ride offers you a spectacular and panaromic view of Historical Melaka City and a far with a host of interesting sights such as St Paul\u2019s Hill, Independence Memorial Building, Samudera Museum and the ship, Flor De La Mar, Melaka River, Dataran Pahlawan, Pulau Selat Mosque, Pulau Besar and the Straits of Malacca.\\nThat apart, you will also see the fast and rapid development taking place in the state.\\nThe air-conditioned circular observation deck can take 66 people at any one time. You'll stay up there for about seven minutes. Once it reaches the top, a breathtaking, sprawling view of Malacca as far as the eye can see, up to the Straits of Malacca, is clearly visible, especially in good weather, with the Melaka River running through the grand panorama below.\\nIn addition to the revolving tower for visitors to enjoy bird view of Melaka city, the ticketing building will also be fitted with 5 commercial outlets, which includes a cafe, restaurant, and souvenir shops.\\nThe admission charges for Menara Taming Sari are RM20 for an adult and RM10 for children below 12 years old. There is a huge car park next to the tower. For further information, please contact Melaka Taming Sari Berhad at +60-6-2881100 or +60-6-2813366.\\nMenara Taming Sari opens at 10:00 am and continues till 10:00 pm, daily.\"", "vector_field": [-0.0057781715877354145, -0.02553376369178295, -0.014156023971736431, -0.046966247260570526, -0.005162979941815138, 0.006677806843072176, -0.020704839378595352, -0.02520301565527916, -0.015267337672412395, -0.024395989254117012, 0.012184764258563519, 0.00814963597804308, -0.004534558393061161, -0.004987683147192001, 0.015703925862908363, -0.002085367450490594, 0.019342156127095222, 0.0025996810290962458, -0.016193432733416557, -0.03331296145915985, -0.03037591651082039, -0.0031669142190366983, -0.0018306914716959, 0.0071970815770328045, -0.01850867085158825, 0.0015321911778301, 0.004597400315105915, -0.026896445080637932, 0.007580749224871397, 0.03291606158018112, 0.0057781715877354145, -0.008804517798125744, 0.0016297618858516216, -0.021895533427596092, -0.023112686350941658, 0.006360288243740797, 0.009697537869215012, -0.024104930460453033, 0.008897127583622932, 0.0012361714616417885, 0.01866742968559265, -0.0015727077843621373, -0.001587591483257711, -0.010802237316966057, -0.02054608054459095, 0.01873357966542244, 0.01788686402142048, 0.007058167364448309, -0.024369528517127037, 0.013157164677977562, -0.015756845474243164, 0.00033219525357708335, -0.009327099658548832, -0.0011774636805057526, -0.005933623295277357, -0.014182483777403831, 0.0073227658867836, 0.00896989181637764, 0.030058398842811584, -0.028338508680462837, 0.0003075958520639688, -0.014235403388738632, -0.013150549493730068, -0.008480384945869446, 0.00027824193239212036, -0.004346031695604324, -0.0016843352932482958, -0.007851962931454182, -0.01348129753023386, -0.01121236477047205, 0.03161953017115593, 0.0449552983045578, 0.011166060343384743, -0.017119528725743294, 0.00789165310561657, -0.03484763205051422, 0.015386407263576984, -0.0008963276632130146, 0.00407481798902154, 0.018548360094428062, 0.01856159046292305, -0.011629107408225536, -0.021670624613761902, 0.02144571579992771, 0.017900094389915466, 0.0045213280245661736, -0.002364849904552102, 0.02219982072710991, 0.018879108130931854, -0.005936930421739817, 0.004547788295894861, 0.03281022235751152, 0.0005755019374191761, 0.005057140253484249, -0.006019617896527052, -0.008883897215127945, -0.019963962957262993, 0.014698451384902, -0.007653513923287392, 0.0036250005941838026, 0.01578330621123314, -0.003734147408977151, -0.005993157625198364, -0.01325638871639967, -0.03693796321749687, -0.006423130631446838, 0.0041442750953137875, -0.02627463825047016, 0.015571626834571362, 0.003312443383038044, -0.017833944410085678, 0.02485903725028038, -0.004382413811981678, -0.016643250361084938, -0.007673359010368586, -0.0036216930020600557, 0.027041975408792496, 0.009624773636460304, 0.02208075113594532, -0.01311085931956768, 0.0315401516854763, 0.019170166924595833, 0.02003011293709278, 0.022451190277934074, 0.03974270820617676, 0.0075212144292891026, -0.0037209175061434507, -0.01782071404159069, -0.00532173877581954, 0.009856296703219414, -0.011298359371721745, 0.011020530946552753, 0.0019629907328635454, 0.005063755437731743, 0.006598426960408688, 0.00693909777328372, -0.004994298331439495, -0.006694343872368336, -0.015703925862908363, -0.033974457532167435, -0.0003392236540094018, 0.001991104334592819, -0.005999772809445858, -0.01367974653840065, -0.003352133324369788, 0.029423361644148827, 0.013130704872310162, 0.0204402394592762, -0.01370620634406805, 0.016391882672905922, 0.02690967544913292, -0.035562049597501755, -0.010550868697464466, 0.024554748088121414, 0.013917885720729828, 0.006770416162908077, -0.020228561013936996, 0.008731753565371037, -0.02111496590077877, 0.003591925837099552, 0.004339416511356831, 0.033683400601148605, 0.008480384945869446, -0.0002360715443501249, 0.01730474643409252, 0.017357666045427322, -0.008480384945869446, -0.0026112571358680725, -0.02765055187046528, 0.007025092374533415, 0.0017480043461546302, 0.030455296859145164, -0.012409673072397709, 0.004805771633982658, -0.024951646104454994, 0.03077281452715397, -0.007514599710702896, -0.009499088861048222, -0.00959169864654541, -0.003651460399851203, -0.04426734149456024, -0.009922446683049202, 0.02055930905044079, 0.042203474789857864, -0.02009626291692257, -0.021842611953616142, 0.011443888768553734, 0.020347630605101585, 0.002053946489468217, -0.031116792932152748, 0.02144571579992771, 0.01906432770192623, 0.0027849001344293356, 0.006013002712279558, -0.6269928216934204, -0.014261863194406033, -0.0017744642682373524, -0.004164120182394981, -0.017622265964746475, -0.001189866685308516, 0.0073095359839499, 0.012502282857894897, -0.0005064582219347358, -0.01121236477047205, -0.021644163876771927, -0.0022953927982598543, -0.018230842426419258, -0.005725251976400614, -0.02611587941646576, -0.013005020096898079, 0.006452898029237986, -0.013296078890562057, -0.004246807191520929, -0.011483578011393547, -0.03855201229453087, 0.038287416100502014, 0.0059600831009447575, 0.01578330621123314, 0.007302920799702406, -0.02259671874344349, 0.004508098121732473, -0.01616697385907173, -0.012442748062312603, 0.03349817916750908, -0.004852076526731253, 0.03212226927280426, -0.008209170773625374, -0.0011013916227966547, 0.03037591651082039, -0.01656387187540531, -0.016590330749750137, 0.02667153626680374, 0.011020530946552753, 0.04625183343887329, -0.02026825211942196, 0.002799783833324909, 0.006945712957531214, -0.03146076947450638, 0.01168864220380783, -0.007355840411037207, 0.026830296963453293, -0.021895533427596092, -0.007230156101286411, -0.028576646000146866, -0.011569572612643242, -0.015399637632071972, -0.0194744560867548, 0.01600821502506733, 0.0195802953094244, 0.004587478004395962, 0.042203474789857864, -0.004200502298772335, 0.0016396843129768968, -0.007686588913202286, 0.005946853198111057, 0.020532850176095963, 0.02116788737475872, 0.002725365338847041, -0.011867246590554714, 0.026036500930786133, -0.02100912667810917, 0.010729472152888775, -0.012237684801220894, -0.03831387311220169, 0.009750457480549812, 0.021207576617598534, -0.022332120686769485, -0.006985402666032314, -0.0035224684979766607, 0.03183120861649513, 0.027002284303307533, -0.00632721371948719, 0.01178125198930502, 0.01918339729309082, 0.0332600399851799, 0.013157164677977562, 0.0012965330388396978, -0.009942291304469109, 0.006958942860364914, -0.02118111588060856, 0.010755931958556175, 0.008890512399375439, 0.023760952055454254, 0.015346718020737171, 0.02054608054459095, -0.006429745350033045, 0.010001826100051403, -0.013163778930902481, 0.009472629055380821, 0.0018918798305094242, -0.007792428135871887, -0.0029684652108699083, 0.01771487481892109, -0.03939872980117798, -0.013540832325816154, -0.007084627170115709, 0.023020075634121895, -0.015439326874911785, 0.014592612162232399, -0.038684312254190445, -0.055195264518260956, -0.005758326500654221, 0.05072354897856712, -0.01432801317423582, 0.014486772008240223, -0.010259809903800488, -0.009479244239628315, 0.01838960126042366, 0.019514145329594612, -0.019143708050251007, 0.040906939655542374, 0.012422903440892696, 0.02611587941646576, -0.006485972553491592, 0.011139600537717342, -0.021207576617598534, 0.03373631834983826, 0.004699932411313057, -0.002067176392301917, 0.022107211872935295, 0.008295165374875069, 0.011906935833394527, -0.011920166201889515, 0.005738481879234314, 0.008705293759703636, 0.012991790659725666, 0.015994984656572342, -0.02530885487794876, 0.027200734242796898, 0.02553376369178295, 0.014539691619575024, -0.027571171522140503, 0.011192520149052143, -0.018892338499426842, -0.04051004350185394, -0.02865602634847164, -0.00899635162204504, -0.01044502854347229, -0.001343664713203907, -0.027782851830124855, -0.0146190719678998, -0.0012419596314430237, -0.03683212399482727, -0.02254379913210869, -0.03180474787950516, 0.006376825738698244, -0.007388915400952101, -0.00509352283552289, -0.007283075712621212, 0.0015569973038509488, 0.016841700300574303, -0.010431799106299877, -0.034768253564834595, -0.004961223341524601, 0.016034673899412155, 0.005500343162566423, -0.044082123786211014, -0.007613824214786291, 0.008295165374875069, -0.017106298357248306, -0.006806798279285431, 0.003968978766351938, 0.0021746696438640356, -0.02979380078613758, 0.0001506971602793783, -0.006820028647780418, -0.007375685498118401, 0.040959861129522324, -0.00842746440321207, 0.0025566837284713984, 0.010464874096214771, 0.011496808379888535, -0.0174899660050869, -0.02133987471461296, -0.002118442440405488, 0.021948453038930893, -0.032545626163482666, 0.0018902261508628726, 0.018865879625082016, 0.01838960126042366, 0.010855156928300858, -0.006958942860364914, -0.00506706302985549, 0.01885264925658703, -0.027994530275464058, 0.031593069434165955, -0.009869527071714401, 0.0015024237800389528, 0.010550868697464466, -0.03151369094848633, 0.02287454716861248, -0.0017132757930085063, -0.017900094389915466, 0.0146190719678998, 0.009955521672964096, 0.02021533064544201, 0.03466241434216499, -0.001696738414466381, -0.003638230497017503, -0.005123290233314037, 0.013666517101228237, -0.02825912833213806, 0.03098449297249317, -0.0062379115261137486, 0.020625459030270576, -0.04548449441790581, -0.00899635162204504, 0.009413094259798527, 0.013044710271060467, 0.014089873991906643, 0.0049215336330235004, 0.013408533297479153, -0.0245150588452816, 0.0007966897683218122, 0.006188299506902695, -0.012508898042142391, 0.024263689294457436, -0.005331661552190781, -0.01069639716297388, -0.008301780559122562, 0.03270438313484192, 0.027968069538474083, -0.011708486825227737, -0.03405383601784706, -0.033974457532167435, 0.0097901476547122, -0.012581662274897099, 0.010914691723883152, 0.028867704793810844, 0.03468887507915497, 0.0013420109171420336, -0.00510675273835659, 0.030719894915819168, 0.011926781386137009, 0.017926553264260292, 0.00012589104881044477, 0.038684312254190445, -0.03365693986415863, 0.013210084289312363, 0.009214645251631737, 0.033974457532167435, 0.004220347385853529, -0.004518020898103714, 0.013917885720729828, -0.016484491527080536, -0.00822901539504528, -0.013302694074809551, 0.0014048530720174313, -0.010835312306880951, 0.0069986325688660145, 0.01623312383890152, -0.024382758885622025, 0.0049215336330235004, -0.001971259480342269, 0.005133212544023991, 0.002197821857407689, 0.03161953017115593, -0.023906482383608818, 0.00820255558937788, 0.005573107395321131, 0.05114690586924553, 0.0036183856427669525, -0.011873860843479633, -0.00043700108653865755, 0.00326283136382699, 0.0075212144292891026, 0.039822086691856384, -0.029873179271817207, 0.006009695120155811, -0.02810036949813366, -0.016140513122081757, 0.001447023474611342, 0.015637775883078575, 0.008248860947787762, -0.01365328673273325, -0.043843984603881836, 0.01992427371442318, -0.0003361228737048805, 0.008321625180542469, -0.028920624405145645, -0.034927014261484146, 0.0003313683846499771, -0.007633668836206198, 0.05119982361793518, 0.004551095422357321, -0.002695597941055894, -0.019911043345928192, -0.00919480063021183, -0.03458303585648537, -0.020122721791267395, 0.01628604345023632, 0.0006631502183154225, 0.03450365364551544, -0.010021671652793884, -0.015875915065407753, 0.0006916772108525038, 0.00696555757895112, -0.021829383447766304, 0.006390055641531944, -0.003479471430182457, -0.025732211768627167, -0.0013180316891521215, 0.0010749317007139325, -0.013216699473559856, -0.018005933612585068, -0.006853103172034025, -0.009452784433960915, -0.005368043668568134, 0.005500343162566423, 0.0009376712259836495, -0.004809079226106405, -0.018627740442752838, -0.0015181343769654632, 0.0004948820569552481, -0.006664576940238476, 0.0012179802870377898, 0.00023090360627975315, 0.006747263949364424, 0.10361680388450623, -0.003965671174228191, -0.0012981868349015713, 0.004289804492145777, -0.021075276657938957, -0.003370324382558465, -0.018323451280593872, -0.026989055797457695, 0.012760266661643982, 0.018363142386078835, -0.012819801457226276, -0.016828469932079315, 0.0014544653240591288, 0.00885082222521305, 0.015280568040907383, -0.0015859376871958375, -0.006694343872368336, -0.0042798821814358234, 0.02900000475347042, 0.028364967554807663, 0.0010509524727240205, 0.002808052347972989, 0.019686134532094002, 0.04863321781158447, 0.043791066855192184, 0.015862684696912766, 0.0022953927982598543, 0.010597173124551773, 0.004346031695604324, -0.04974453151226044, -0.005821168888360262, 0.022967156022787094, -0.003386861877515912, 0.0035786957014352083, -0.014963049441576004, 0.0012725538108497858, 0.004934763535857201, -0.0016281080897897482, -0.004633782431483269, -0.01118590496480465, 0.011794481426477432, 0.02434306964278221, 0.001827383995987475, -0.003370324382558465, 0.007091241888701916, -0.011159445159137249, 0.006724111270159483, 0.014552921988070011, -0.00811656191945076, 0.0005453211488202214, 0.04588139429688454, -0.004124430473893881, -0.006224681623280048, -0.010980840772390366, -0.010597173124551773, -0.0052820490673184395, 0.005513573065400124, -0.012971945106983185, -0.0027815925423055887, 0.005394503474235535, -0.014208943583071232, -0.026129109784960747, 0.024448908865451813, 0.0002811359881889075, -0.007461680099368095, -0.008844207972288132, -0.008017336949706078, -0.004918226040899754, -0.03206934779882431, -0.008553149178624153, -0.0005597913987003267, -0.009307255037128925, -0.017675185576081276, 0.004402258899062872, 0.025044254958629608, -0.004852076526731253, 0.01719890721142292, -0.01487044058740139, 0.005510265473276377, 0.04971807450056076, 0.010861772112548351, 0.0012361714616417885, -0.015307027846574783, -0.02644662745296955, -0.02032117173075676, 0.01484398078173399, 0.03344526141881943, -0.027174273505806923, -0.02905292436480522, 0.0016057826578617096, 0.0013056285679340363, 0.023575734347105026, 0.02645985782146454, -0.019963962957262993, -0.0023152376525104046, -0.0003038749273400754, 0.0009318831143900752, 0.01430155336856842, 0.00956523884087801, -0.009571854025125504, 0.0026625231839716434, -0.024409219622612, -0.007084627170115709, 0.009823222644627094, 0.036752741783857346, 0.0008582916343584657, 0.026975825428962708, 0.028470806777477264, 0.009466013871133327, 0.0060758450999855995, -0.02145894430577755, 0.0037143025547266006, -0.007746123243123293, -0.003853216767311096, -0.024250460788607597, -0.008262090384960175, 0.015055659227073193, 0.04262683168053627, -0.019209856167435646, -0.018204381689429283, -0.016616791486740112, -0.026975825428962708, 0.02673768624663353, -0.0021614397410303354, 0.002945312997326255, -0.002430999418720603, -0.014685221016407013, 0.017833944410085678, -0.00896989181637764, -0.009975366294384003, 0.0018769961316138506, -0.006082459818571806, -0.008513459004461765, -0.025017796084284782, -0.021366335451602936, -0.015042428858578205, 0.0009294025367125869, -0.013024864718317986, -0.008090101182460785, 0.007858578115701675, -0.031196173280477524, 0.024369528517127037, 0.0022805090993642807, 0.00692586787045002, -0.03281022235751152, -0.01567746512591839, -0.012469207867980003, -0.004676779732108116, -0.03378923982381821, 0.021300185471773148, -0.015584856271743774, 0.003006501356139779, -0.019143708050251007, 0.008295165374875069, 0.010365649126470089, -0.03082573413848877, -0.0274653322994709, -0.03415967524051666, 0.03865785151720047, 0.020519619807600975, 0.02708166465163231, -0.0012742074904963374, 0.02473996765911579, 0.0052985865622758865, 0.004676779732108116, -0.0037870672531425953, -0.013183624483644962, 0.014195714145898819, -0.021829383447766304, 0.01370620634406805, 0.03183120861649513, 0.022729018703103065, -0.001282476237975061, -0.003312443383038044, -0.013468068093061447, 0.024144619703292847, 0.01512180920690298, -0.01370620634406805, -0.027491793036460876, -0.0409863218665123, -0.005854243412613869, 0.000622220104560256, -0.0098166074603796, -0.002025832887738943, -0.0068795629777014256, -0.006472742650657892, 0.03013777919113636, 0.0014222173485904932, 0.021273726597428322, -0.01405018474906683, 0.002217666944488883, -0.03392153978347778, 0.04053650423884392, -0.0073359957896173, 0.020175641402602196, 0.008533304557204247, -0.019262775778770447, -0.022041061893105507, -0.014354472979903221, 0.01041195448487997, -0.013263003900647163, -0.012356753461062908, 0.02100912667810917, -0.023020075634121895, -0.005123290233314037, 0.01427509356290102, -0.01633896306157112, -0.004293112084269524, -0.017185678705573082, -0.007448450196534395, -0.01243613287806511, -0.040166065096855164, -0.024541517719626427, -0.03450365364551544, 0.004336109384894371, 0.008698678575456142, -0.0176487248390913, 0.014195714145898819, -0.015227648429572582, -0.02190876193344593, 0.01365328673273325, 0.0031421082094311714, 0.03082573413848877, 0.03648814558982849, 0.026155570521950722, 0.00879790261387825, -0.017953014001250267, -0.003836679505184293, 0.025983581319451332, 0.016590330749750137, -0.015042428858578205, 0.021604474633932114, 0.028867704793810844, -0.033683400601148605, -0.013745896518230438, 0.008976507000625134, -0.003664690302684903, -0.016378652304410934, -0.019156936556100845, 0.030349457636475563, -0.011754792183637619, -0.010107665322721004, -0.020704839378595352, 0.010557482950389385, -0.029264602810144424, -0.00550695788115263, 0.0018439213745296001, -0.0036779202055186033, -0.0008806171244941652, -0.0009203069494105875, -0.016709400340914726, 0.021697083488106728, -0.056624095886945724, -0.00510675273835659, 0.018693890422582626, -0.039875004440546036, 0.006985402666032314, 0.0037076876033097506, -0.024488598108291626, 0.019104016944766045, 0.007058167364448309, 0.018177922815084457, -0.002156478352844715, 0.0034893937408924103, 0.014566151425242424, -0.02104881778359413, -0.010431799106299877, -0.024620898067951202, -0.01809854246675968, 0.025070715695619583, 0.004170734900981188, 0.005814553704112768, 0.014169254340231419, 0.010173815302550793, 0.01180771179497242, 0.001151830656453967, 0.0010600481182336807, -0.01804562285542488, -0.009115421213209629, -0.025520533323287964, 0.017529655247926712, 0.018945258110761642, -0.018191153183579445, -0.014764600433409214, -0.016378652304410934, -0.017556115984916687, -0.012303833849728107, 0.003558850847184658, 0.002935390453785658, -0.01940830610692501, -0.025798361748456955, -0.02423723042011261, -0.0350857712328434, 0.0008037181687541306, 0.003849909408017993, 0.002632755786180496, 0.0014974626246839762, 0.03042883612215519, -0.01066993735730648, 0.008308395743370056, 0.03638230636715889, 0.04614599421620369, -0.011973085813224316, 0.0011493500787764788, 0.01393111515790224, 0.003117301966995001, 0.005447423085570335, -0.0076402840204536915, -0.002642678329721093, -0.020069802179932594, 0.01690784841775894, 0.007064782083034515, -0.003003193996846676, -0.01305132545530796, -0.0007069743587635458, -0.007931342348456383, 0.009485859423875809, 0.003122263355180621, -0.027571171522140503, 0.003945826552808285, -0.005721944384276867, -0.009009581990540028, 0.0146455317735672, -0.018892338499426842, 0.0058939335867762566, 0.008897127583622932, -0.011695257388055325, -0.002903969492763281, -0.022517338395118713, 0.02214690111577511, 0.00468339491635561, 0.006621579639613628, -0.009254335425794125, -0.007646899204701185, -0.04148905724287033, -0.020241791382431984, -0.01203262060880661, 0.025864511728286743, 0.005278741475194693, -0.009843067266047001, 0.004167427774518728, 0.03513869270682335, 0.01038549467921257, 0.027624091133475304, -0.01844252087175846, -0.0016479530604556203, -0.003246293868869543, -0.01001505646854639, -0.033286500722169876, -0.035509128123521805, -0.0057947090826928616, 0.04159489646553993, 0.003123916918411851, -0.005156364757567644, -0.013540832325816154, -0.00056557945208624, -0.05805292725563049, -0.01385173574090004, 0.00817609578371048, 0.012793341651558876, 0.028523726388812065, 0.012747036293148994, -0.002903969492763281, 0.006350365933030844, -0.003982208669185638, -0.009472629055380821, 0.0013767394702881575, 0.011920166201889515, 0.012065695598721504, -0.027518251910805702, 0.010398724116384983, -0.0098166074603796, -0.04807756096124649, -0.021260496228933334, 0.028470806777477264, -0.01918339729309082, -0.006813413463532925, 0.031063873320817947, -0.001795962918549776, 0.014433852396905422, -0.00961815845221281, 0.012614737264811993, 0.012522127479314804, -0.0013023210922256112, 0.03003193810582161, -0.019447995349764824, -0.005070370156317949, -0.018151462078094482, 0.011278514750301838, -0.014592612162232399, -0.0015140000032261014, -0.02218659035861492, -0.0122839892283082, 0.010941151529550552, 0.012952100485563278, -0.012952100485563278, 0.006307368632405996, -0.029873179271817207, 0.02395940199494362, 0.011132985353469849, -0.005123290233314037, 0.0008814440225251019, 0.013891425915062428, -0.007865192368626595, 0.0024773043114691973, 0.0067009590566158295, -0.02332436479628086, -0.0122575294226408, 0.021617703139781952, -0.03275730460882187, 0.020400550216436386, -0.014433852396905422, 0.014513231813907623, -0.0006056826678104699, 0.004405566491186619, 0.002287124050781131, -0.03339233994483948, -0.0408804789185524, 0.030746355652809143, 0.018720349296927452, -0.010273040272295475, -0.01248243823647499, -0.0015479016583412886, -0.02668476663529873, 0.009254335425794125, 0.004140967968851328, 0.02418431080877781, -0.012297218665480614, -0.00035452074371278286, -0.00307099730707705, -0.03217518702149391, 0.011152829974889755, 0.017635496333241463, 0.017503196373581886, -0.004005360882729292, -0.014380932785570621, 0.20924456417560577, 0.021432485431432724, -0.03116971254348755, 0.039583947509527206, 0.00020413367019500583, 0.011285129003226757, 0.005268819164484739, 0.01771487481892109, -0.004283189307898283, 0.018416061997413635, -0.037440698593854904, -0.011252054944634438, 0.009055886417627335, 0.0026277946308255196, 0.00676710857078433, -0.0030776122584939003, -0.014063414186239243, -0.0274124126881361, -0.005123290233314037, -0.013415148481726646, 0.012568432837724686, -0.015426097437739372, -0.03376277908682823, -0.006251141428947449, -0.0017992703942582011, 0.00902281142771244, -0.013692976906895638, -0.013613596558570862, 0.019672904163599014, 0.01373266614973545, -0.016828469932079315, -0.034424275159835815, 0.005248974543064833, -0.006562044844031334, 0.00368122779764235, 0.0025418000295758247, 0.0006519874441437423, -0.012389828450977802, 0.011060221120715141, 0.025229474529623985, -0.023231755942106247, 0.007461680099368095, -0.009829836897552013, 0.01248243823647499, -0.011787867173552513, -0.010537638328969479, -0.00552018778398633, 0.003555543487891555, -0.007693203631788492, 0.0007189639727585018, -0.025613142177462578, -0.026499547064304352, 0.02458120882511139, 0.009201415814459324, 0.004266652278602123, -0.002459113020449877, 0.02394617162644863, -0.010603788308799267, 0.0015421136049553752, 0.029211683198809624, -0.02765055187046528, 0.021088507026433945, -0.03103741444647312, -0.001795962918549776, -0.03659398481249809, 0.009426324628293514, -0.008315009996294975, 0.02468704804778099, 0.01737089641392231, 0.011622492223978043, 5.850109300808981e-05, -0.02786223031580448, -0.01365328673273325, 0.0032413327135145664, -0.0098166074603796, -0.01935538649559021, 0.047707125544548035, -0.01935538649559021, 0.03087865374982357, 0.014963049441576004, -0.014883670024573803, -0.012217839248478413, 0.010001826100051403, 0.010081205517053604, 0.024898726493120193, -0.02037409134209156, 0.011331434361636639, -0.004124430473893881, -0.013891425915062428, -0.019117247313261032, -0.013382073491811752, -0.010835312306880951, -0.0024524980690330267, 0.011080065742135048, 0.016722630709409714, 0.025216244161128998, 0.018826188519597054, 0.021128196269273758, -0.027280114591121674, -0.0015189612749963999, 0.011046990752220154, -0.026949364691972733, 0.0146719915792346, -0.0014800983481109142, -0.017357666045427322, -0.0036183856427669525, 0.012985175475478172, 0.011007300578057766, -0.006492587737739086, -0.029423361644148827, -0.012078925035893917, -0.04013960435986519, 0.0174899660050869, -0.0020142567809671164, 0.0025351850781589746, 0.00874498300254345, 0.00857299380004406, -0.009426324628293514, -0.005735174287110567, -0.00034687219886109233, 0.01350775733590126, -0.023708032444119453, 0.001693430938757956, -0.006380133330821991, -0.014394163154065609, -0.018151462078094482, -0.0273594930768013, -0.019818434491753578, 0.001655394909903407, -0.039583947509527206, -0.007871807552874088, 0.005268819164484739, 0.031116792932152748, -0.0020870212465524673, -0.005811246111989021, -0.012766881845891476, 0.005831091199070215, -0.007316150702536106, -0.0030114625114947557, -0.0037308400496840477, 0.009406480006873608, 0.008023952133953571, 0.0203873198479414, 0.015862684696912766, -0.0009294025367125869, -0.019963962957262993, 0.0003911097883246839, 0.018984947353601456, -0.017516426742076874, -0.013957574963569641, -0.030058398842811584, -0.03265146538615227, 0.008692063391208649, -0.004412181209772825, 0.03741423785686493, -0.004230269696563482, -0.01816469244658947, -0.017278287559747696, 0.014936589635908604, -0.024395989254117012, -0.014579381793737411, -0.008950047194957733, 0.03410675749182701, 0.0065190475434064865, -0.01873357966542244, 0.0033504795283079147, -0.16701462864875793, 0.02332436479628086, 0.03558851033449173, -0.019950732588768005, 0.015439326874911785, 0.0014371010474860668, 0.03103741444647312, -0.0035092385951429605, -0.024501828476786613, -0.0014652146492153406, 0.01868066005408764, -0.0014900207752361894, -0.01405018474906683, -0.020519619807600975, -0.0036845351569354534, 0.005222514271736145, -0.005864166188985109, 0.004054973367601633, 0.028682485222816467, 0.0036051557399332523, 0.011152829974889755, -0.007203696295619011, 0.010028285905718803, 0.0025103790685534477, -0.0073227658867836, 0.01016058586537838, -0.02197491191327572, 0.005073677748441696, 0.0017182370647788048, -0.014367703348398209, -0.016788780689239502, 0.015359947457909584, 0.03177829086780548, -6.821682291047182e-06, 0.012892565689980984, -1.3772562851954717e-05, 0.006188299506902695, -0.004508098121732473, -0.017860403284430504, 0.008341469801962376, 0.019778743386268616, -0.01061701774597168, 0.030217157676815987, -0.0004465100937522948, -0.006634809542447329, 0.03598540648818016, 0.003955748863518238, -0.005430886056274176, 0.028364967554807663, -0.0122310696169734, 0.008956662379205227, -0.015399637632071972, 0.02661861665546894, -0.001183251733891666, 0.002601334825158119, 0.008665603585541248, 0.014698451384902, 0.0034497040323913097, 0.03127555176615715, -0.002756786532700062, -0.03490055352449417, -0.012105384841561317, -0.003628307953476906, -0.02082390896975994, -0.028126828372478485, -0.03704380244016647, -0.01203262060880661, 0.03003193810582161, -0.01588914543390274, 0.028312047943472862, -0.010325959883630276, -0.004795849323272705, 0.019223086535930634, 0.012317064218223095, 0.007058167364448309, 0.01242951862514019, -0.004544480703771114, -0.013864965178072453, 0.013375458307564259, -0.025957120582461357, 0.0022292430512607098, 0.05704745277762413, -0.015598086640238762, 0.002143248450011015, 0.02639370784163475, 0.00013508998381439596, -0.015399637632071972, 0.007382300216704607, -0.0015404598088935018, -0.0011973085347563028, 0.0031040720641613007, -0.0274653322994709, 0.0038135270588099957, -0.017463507130742073, 0.015452557243406773, 0.014936589635908604, -0.0005378792993724346, 0.010431799106299877, 6.966384535189718e-05, -0.03146076947450638, -0.01001505646854639, 0.0013668170431628823, -0.01240305881947279, 0.018416061997413635, 0.022054292261600494, 0.040721721947193146, -0.011926781386137009, 0.029608581215143204, 0.01370620634406805, -0.0004998432705178857, -0.003839986864477396, -0.008678833022713661, 0.019659673795104027, 0.009525548666715622, -0.006380133330821991, 0.02894708514213562, 0.00020403030794113874, -0.024700278416275978, -0.011132985353469849, 0.02815328910946846, 0.026076190173625946, -0.007005247287452221, -0.012535357847809792, -0.005910470616072416, -0.016497721895575523, -0.012310449033975601, -0.08938140422105789, -0.017384126782417297, 0.018085312098264694, 0.010464874096214771, -0.029714420437812805, 0.018720349296927452, -0.007554289419203997, 0.01370620634406805, -0.029661500826478004, 0.021101737394928932, -0.026936136186122894, -0.020903287455439568, -0.003013116307556629, -0.004468408413231373, 0.03151369094848633, -0.019514145329594612, 0.03659398481249809, -0.0069986325688660145, -0.018601279705762863, 0.00550695788115263, -0.011245439760386944, 0.00904927123337984, 0.00953877903521061, -0.0022093981970101595, -0.015598086640238762, 0.008864052593708038, -0.035112231969833374, 0.00814963597804308, 0.010775777511298656, 0.03267792612314224, 0.007117701694369316, -0.0038069121073931456, -0.005536725278943777, 0.017450276762247086, -0.008930201642215252, -0.0056954845786094666, -0.02196168154478073, -0.009055886417627335, 0.0015321911778301, -0.01628604345023632, -0.009353559464216232, 0.0029535817448049784, 0.02366834320127964, -0.02413139119744301, -0.018707118928432465, -0.0013254735385999084, -0.0331542007625103, 0.009829836897552013, 0.006267678923904896, -0.02145894430577755, -0.0122575294226408, -0.013772356323897839, -0.01168864220380783, 0.00634375074878335, 0.026949364691972733, -0.005530110560357571, -0.015703925862908363, -0.00877144280821085, 0.010848541744053364, -0.0037771447096019983, -0.0014577727997675538, 0.014976279810070992, 0.023588962852954865, -0.006651347037404776, 0.020188871771097183, 0.012568432837724686, 0.010795622132718563, 0.010802237316966057, 0.023813871666789055, 0.003024692414328456, -0.003750684903934598, 0.027915149927139282, -0.002548414980992675, -0.005331661552190781, -0.01913047768175602, -0.009346945211291313, -0.03651460260152817, -0.00842746440321207, -0.0005424270639196038, -0.01405018474906683, -0.01680200919508934, -0.010769162327051163, -0.004137660376727581, 0.0039028290193527937, 0.013302694074809551, -0.0008053719066083431, 0.024607667699456215, -0.0175958052277565, 0.04966515302658081, -0.02413139119744301, 0.028576646000146866, 0.01833668164908886, -0.0012427864130586386, -0.0008814440225251019, -0.0006916772108525038, 0.020347630605101585, -0.0013345690676942468, -0.013322538696229458, 0.007223541382700205, 0.019276006147265434, -0.041462596505880356, -0.01573038473725319, -0.0746697187423706, 0.006763801444321871, 0.029211683198809624, -0.007388915400952101, 0.005057140253484249, 0.02264963835477829, 0.01385173574090004, 0.004405566491186619, 0.007845347747206688, -0.0016504336381331086, -0.020360860973596573, 0.004534558393061161, -0.0042600370943546295, -0.008242245763540268, -0.010577328503131866, -0.007355840411037207, 0.011053605936467648, 0.017172448337078094, 0.01393111515790224, 0.01109329517930746, -0.03135493025183678, 0.012753651477396488, 0.002063869033008814, 0.020466700196266174, -0.03151369094848633, 0.027968069538474083, -0.008043796755373478, 0.01856159046292305, 0.01799270324409008, -0.002472342923283577, 0.0048190015368163586, -0.000711522123310715, -0.02997901849448681, 0.020863598212599754, -0.018191153183579445, -0.013355613686144352, -0.0015073850518092513, 0.023760952055454254, 0.0025831435341387987, 0.012350139208137989, -0.03122263215482235, -0.018998177722096443, 0.004336109384894371, -0.00410127779468894, -0.014552921988070011, -0.0014792714500799775, 0.006737341172993183, 0.009260950610041618, 0.02684352546930313, -0.0010501255746930838, 0.015545167028903961, 0.018892338499426842, -0.012422903440892696, -0.0003534871793817729, -0.008387775160372257, -0.01702691800892353, 0.001817461452446878, -0.009922446683049202, 0.004766081925481558, -0.03810219466686249, 0.0245679784566164, 0.011106525547802448, 0.004392336588352919, -0.0036117706913501024, 0.023522814735770226, -0.02140602469444275, -0.03392153978347778, 0.02974088117480278, 0.011840786784887314, -0.029926098883152008, -0.01651095226407051, -0.0009467668132856488, 0.024819346144795418, 0.010332574136555195, 0.012343524023890495, -0.02775639109313488, -0.02884124591946602, -0.006585197057574987, -0.002111827488988638, 0.02997901849448681, -0.008136406540870667, -0.02214690111577511, -0.02905292436480522, 0.007084627170115709, -0.011351278983056545, 0.005503650289028883, 0.0008231496321968734, 0.0008657334838062525, 0.014976279810070992, 0.029396902769804, -0.004534558393061161, 0.014936589635908604, -0.011152829974889755, -0.0009236144251190126, 0.0012990136165171862, 0.010266425088047981, -0.0012469207867980003, 0.0024541518650949, 0.03865785151720047, 0.0467810295522213, 0.02384033240377903, 0.007051552180200815, -0.023139145225286484, -0.013216699473559856, 0.009234490804374218, -0.011648952960968018, -0.015227648429572582, -0.012641197070479393, 0.002430999418720603, -0.011371123604476452, 0.006208144128322601, 0.004246807191520929, 0.04289143159985542, 0.01059055794030428, 0.0015892452793195844, -0.006585197057574987, 0.02865602634847164, 0.009075731039047241, -0.010478103533387184, 0.00630075391381979, -0.017900094389915466, -0.004828924313187599, 0.020056571811437607, -0.0003295079222880304, 0.011476963758468628, 0.05477190762758255, 0.004789234139025211, 0.0006933309487067163, -0.001086507923901081, -0.02270255796611309, -0.00016020616749301553, 0.005864166188985109, -0.015055659227073193, -0.021763233467936516, -0.03638230636715889, 0.018230842426419258, -0.006786953657865524, 0.005103445146232843, -0.00034005052293650806, 0.06911315023899078, 0.018257301300764084, -0.00032661386649124324, 0.021948453038930893, -0.00632721371948719, 0.019937502220273018, -0.0019811817910522223, 0.008295165374875069, -0.04966515302658081, -0.023297905921936035, 0.009459399618208408, 0.0051993620581924915, 0.009598313830792904, 0.009770303033292294, -0.022093981504440308, -0.010683167725801468, -0.017132757231593132, 0.025639602914452553, -0.022265970706939697, 0.0011956548551097512, 0.018627740442752838, -0.028179747983813286, -0.011556343175470829, -0.00530850887298584, -3.648566416813992e-05, 0.0008343124063685536, 0.03183120861649513, 0.02865602634847164, -0.0002695597941055894, -0.02178969234228134, -0.004944685846567154, 0.029714420437812805, -0.02264963835477829, -0.025851281359791756, 0.006333828438073397, -0.01197970099747181, -0.00511005986481905, 0.02905292436480522, 0.0273065734654665, 0.023999091237783432, 0.0073426105082035065, 0.028364967554807663, 0.003373631974682212, -0.01719890721142292, 0.001367643941193819, 0.002259010449051857, 0.01668293960392475, -0.009088961407542229, -0.036408763378858566]} +{"id": "test:10000093", "text": "\"Are going to be able to understand how you might be able to take these things go that much more further into the life that you with actually to be able live? This is the case and we can be able help you do all these things and more as we hope that people going be able understand were going be able help them due through Drones Dallas. People are going to be able have them in more greater ways to be able to know and understand the many different opportunities that they are going be able have them as we can be able help them see how we can be able help them through their our company making sure that they are going be able have that much more better go to utilize these things for yourself.\\nLooking at this, is can be them in more greater ways to be able to see how action might be able have a greater way to be able to understand the many properties of going be able have to the many different services that we do have through Drones Dallas. Some these things going to would magazine quality photography and even HD video tours are going to be able help you do all that you need to more. This is can be something is can be sure that much amazing we hope that people are going to be able to fully recognize different opportunities to be able to make sure that people are going to be able have all of the need and more.\\nAnd making this happen, is can be them in more greater ways that we hope that people going be able to know all that they need and more as a can be able to make sure that you\u2019re going to go to get the much more clearly as understand what were can actually be able help you in doing. This is because we generally care about the quality services you\u2019re actually going to be able to get. This is also one of the reasons that we hope that people will be able to understand that the great quality that were going to be able offer is no because can be able to match as they do not find important of making sure that your needs will be met clearly.\\nThe reason for this is because we also people to be able to understand what we can be able help them due through Drones Dallas. So is things are going to be able include the many different opportunities that people going be able actually be able to get for themselves that they can be able to know how we can be able to make this and more happen for them. This is going to be what we are going to be able hope that people going to be able to do as we can be able to take these things that matter clearly through great customer service as well.\\nSo looking at this, you can also be able to see what is can be the best possible thing for us to be able to do for ourselves as you take these things to the next possible level. That is going to be by going to our website which is can [email protected] We also going that you at 972-885-8823 as we can be able to give you all that you need and more.\\nGoing to be able to understand how we might be able have a better time to all that you need to more? If you are able to do this, you can have them in more greater opportunities to be able to know what you can actually be able help yourself in doing. This is can be something is going to be the much amazing as we hope that people going be able to see what you can actually whenever someday as we can be able to make this and more happen for yourself. This is can be some is can be the much were substantial to also that people going be able to see what is can be the best possible different that we can be able to get you and more. This is can be one of the great services that we can be able to give to you through Drones Dallas as we hope that people can be able to see what is going to become of it.\\nThere\u2019s them in more greater ways to be able to see what is can be the best possible thing for us actually be able to comprehend. These things and more going to be so much more fantastic as we hope that people are going to be able to realize what we can be able help them in doing as we also hope that many people can be able to see what is can be best possible thing for them as they search be able to consider things such as quality services making sure that people going be able before taken care the many different opportunities that we can be able have a making sure that you\u2019re going be able have your property shown and seen as can be the much awesome through Drones Dallas.\\nThis is going to be one of the greatest opportunities to be able to understand what you\u2019re going to be able help yourself in doing as we hope that people going to be able to fully become the much more aware of all the many properties of actually be able to get to people. This is going to go services of ejection be able to do for people accession whenever we consider will going to make sure that you\u2019re going to be able have the great customer service. Great customer service and making sure that you have all these necessary needs them place to be able help you do all that you need and more whenever you\u2019re going to be able to give people them in more greater understandings of what you can actually be able happen do for yourself.\\nDo want to be able to help in all the many different is okay actually be able have you? And this is the case you have them to possess going be able help you through Drones Dallas. This is can be the greatest opportunities to be able to know what you can actually be able help yourself and as we hope that people going to be able to fully realize that were going be able to make sure that you\u2019re going be able have to the many different services that we do have including things such as aerial photography and videography and even 3-D matter port walk-throughs today.\\nSo whenever you\u2019re ready to going to go to our phone give us a call you do for going to do so at 972-885-8823. You also going go to our phone give us cause we would love for be able to be able utilize these things and more today through fullpackagemedia.com.\"", "vector_field": [-0.013796365819871426, 0.0028273386415094137, 0.007585383486002684, -0.03398042172193527, -0.015890691429376602, 0.004502798896282911, -0.020223325118422508, -0.01105410885065794, -0.024909377098083496, -0.028718432411551476, -0.018102820962667465, 0.020328041166067123, -0.011335534043610096, -0.0029647788032889366, 0.02610052563250065, 0.014922065660357475, 0.04222682863473892, -0.005412521306425333, 0.0038286878261715174, -0.04028957709670067, -0.014045067131519318, -0.009097224101424217, 0.007984613999724388, -0.008089330047369003, -0.0006254341569729149, 0.011590779758989811, 0.003458908526226878, -0.006826190743595362, 0.0036781583912670612, 0.009293567389249802, 0.014660274609923363, 0.002372477436438203, -0.031179264187812805, -0.027016792446374893, -0.018914373591542244, -0.014123604632914066, 0.018351523205637932, -0.014254499226808548, -0.00552705442532897, -0.026807360351085663, 0.01115882582962513, -0.026506301015615463, 0.01449011079967022, -0.013351322151720524, -0.01916307397186756, 0.022723427042365074, 0.004610787611454725, -0.016361914575099945, 0.0320955291390419, -0.012775382958352566, 0.023050663992762566, 0.030498607084155083, -0.03934713080525398, -0.036519791930913925, -0.00479076849296689, 0.004270459525287151, -0.01767086796462536, 0.028875505551695824, 0.02180715836584568, -0.007762092165648937, 0.0009195395978167653, 0.00254264147952199, 0.013096076436340809, 0.026859717443585396, -0.02979177236557007, 0.023587334901094437, -0.03513230010867119, 0.0036487069446593523, 0.003025318030267954, 0.0013244969304651022, 0.03316887095570564, 0.025184258818626404, 0.03565588220953941, -0.002313574543222785, 0.017945747822523117, -0.003517811419442296, -0.003380371490493417, -0.006672388408333063, -0.007035622838884592, 0.0372004471719265, 0.011898383498191833, 0.0027488016057759523, -0.016191750764846802, 0.028718432411551476, 0.02591727115213871, 0.013652381487190723, -0.019215431064367294, 0.04175560548901558, -0.0003546444931998849, 0.005932830274105072, -0.01952958106994629, 0.006826190743595362, -0.01084467675536871, 0.01932014897465706, -0.010746505111455917, -0.008023882284760475, -0.02087780274450779, 0.01943795382976532, -0.0038941355887800455, 0.007506845984607935, -9.009279165184125e-05, 0.012448144145309925, -0.005366707686334848, -0.01823371648788452, -0.008023882284760475, -0.012997904792428017, 0.013056807219982147, -0.015799064189195633, 0.00634515006095171, -0.022932859137654305, 0.0006516132270917296, 0.025275884196162224, 0.0030498607084155083, -0.015969227999448776, 0.014018887653946877, -0.006898182909935713, -0.00905795581638813, 0.0018014467786997557, 0.005245629698038101, 0.005674311891198158, 0.010471625253558159, 0.024634497240185738, 0.03497522696852684, -0.0006810646736994386, -0.011086832731962204, 0.022356919944286346, -0.008135143667459488, 0.00470241392031312, 0.029582340270280838, -0.01827298477292061, 0.001612466643564403, 0.011800212785601616, 0.019149985164403915, 0.006708384957164526, -0.03167666494846344, 0.022422367706894875, -0.009372103959321976, 0.01531475130468607, -0.015681259334087372, 0.004555156920105219, 0.01303062867373228, 0.003671613521873951, -0.016872406005859375, 0.0023119384422898293, -0.008678358979523182, 0.005304532591253519, 0.02714768797159195, -0.004113384988158941, -0.003825415624305606, 0.00986950658261776, 0.02057674340903759, -0.02255326323211193, -0.006813101004809141, 0.030996009707450867, 0.010942848399281502, 0.016034675762057304, -0.012461233884096146, 0.007087981328368187, -0.018364612013101578, 0.015275483019649982, 0.01685931719839573, 0.0022497631143778563, 0.008724172599613667, 0.014555558562278748, 0.04175560548901558, 0.028561357408761978, 0.0005370798171497881, 0.00599173316732049, -0.01024255808442831, 0.0077751814387738705, -0.00426718732342124, 0.03764549270272255, -0.02264488860964775, 0.022513993084430695, 0.0024526508059352636, 0.015367109328508377, -0.015903780236840248, 0.008495105430483818, -0.027252404019236565, 0.00709452573210001, 0.018220627680420876, -0.009051410481333733, 0.013148434460163116, 0.028325745835900307, -0.025485318154096603, -0.005137640982866287, -0.00940482784062624, 0.00627643009647727, 0.006459683645516634, -0.011721675284206867, 0.016191750764846802, 0.019071446731686592, 0.011289720423519611, -0.018377702683210373, -0.6345804929733276, 0.0004699959827121347, 0.03761931136250496, -0.005932830274105072, 0.0037730573676526546, 0.016309555619955063, -0.0008745443192310631, 0.01187874935567379, -0.01874420791864395, 0.010635243728756905, -0.02006625197827816, 0.005729942116886377, -0.009594626724720001, -0.01420214120298624, -0.01967356540262699, 0.007526480592787266, 0.02456904947757721, -0.013744007796049118, -0.0034196400083601475, -0.005913195665925741, -0.02435961738228798, 0.009457186795771122, -0.020681459456682205, -0.0016280104173347354, 0.01709492690861225, 0.015668168663978577, 0.008711082860827446, -0.0038614117074757814, -0.0029762322083115578, 0.04445204883813858, -0.03411131724715233, 0.017108017578721046, 0.014660274609923363, -0.00919539574533701, 0.03458254039287567, -0.0072319661267101765, -0.010039670392870903, 0.03976599499583244, 0.0018390791956335306, 0.038352325558662415, -0.020000804215669632, -0.015916870906949043, 0.026833539828658104, 0.005474696401506662, -0.012349972501397133, -0.009640439413487911, 0.01892746239900589, -0.010733415372669697, -0.03227878361940384, -0.003517811419442296, -0.01075304951518774, -0.0012516864808276296, -0.0017130924388766289, -0.04183414205908775, 0.014712633565068245, 0.010327639989554882, 0.024071648716926575, -0.019293969497084618, 0.0054157935082912445, -0.041258201003074646, 0.012696845456957817, -0.012880099005997181, -0.025144988670945168, -0.017330538481473923, -0.02090398222208023, 0.049871113151311874, -0.028482820838689804, -0.007670465391129255, 0.0007076527690514922, 0.01510531920939684, 0.009961132891476154, 0.03172902390360832, -0.019987713545560837, -0.002688262378796935, -0.00333292200230062, 0.035891495645046234, 0.02981795184314251, -0.014647185802459717, -0.007794816046953201, 0.021964233368635178, 0.005006745923310518, -0.025249706581234932, -0.02531515434384346, -0.025275884196162224, 0.014882797375321388, -0.005448517389595509, 0.0026408128906041384, 0.0035014497116208076, 0.014359216205775738, -0.00524235749617219, -0.00659057917073369, 0.011774033308029175, -0.02312920242547989, -0.04076080024242401, -0.004496254026889801, -0.00844274740666151, -0.033456843346357346, -0.002526279538869858, -0.00039104974712245166, 0.0077686370350420475, -0.02819485031068325, -0.008305307477712631, 0.027095329016447067, -0.002063237363472581, 0.003730516415089369, 0.007703189272433519, 0.006512041669338942, -0.003208571346476674, 0.006580761633813381, -0.015066050924360752, 0.008305307477712631, -0.003300198120996356, -0.01451629027724266, -0.004489709157496691, 0.01769704557955265, -0.01946413330733776, 0.010641789063811302, -0.0032282057218253613, 0.02129666693508625, -0.008645635098218918, -0.0073955850675702095, -0.0071076154708862305, 0.021846428513526917, -3.86038918804843e-05, -0.0039039526600390673, 0.024228721857070923, -0.0018227172549813986, -0.02327318675816059, 0.0037239715456962585, -0.007696644403040409, -0.003576714312657714, -0.018992910161614418, 0.004309728275984526, -0.02496173605322838, 0.006911272648721933, 0.012899733148515224, 0.028875505551695824, -0.01941177435219288, 0.002262852620333433, -0.03214788809418678, -0.016505898907780647, -0.011865660548210144, 0.03429457172751427, -0.0049871113151311874, -0.017134197056293488, -0.007631196640431881, -0.03434693068265915, -0.02357424609363079, -0.03165048733353615, -0.00822022557258606, -0.009234664030373096, -0.006309153977781534, 0.011767488904297352, 0.011302810162305832, 0.01248086802661419, 0.004286821465939283, 0.010131297633051872, -0.02399311028420925, -0.0007068346603773534, -0.015995407477021217, 0.008364210836589336, -0.003350920043885708, -0.029058760032057762, -0.008704538457095623, -0.006024456582963467, -0.02561621181666851, 0.03026299551129341, 0.009228119626641273, 0.0011592416558414698, -0.019215431064367294, -0.0017032752512022853, 0.012284524738788605, 0.007565748877823353, 0.03128397837281227, 0.0030531331431120634, 0.03576060011982918, -0.00793225597590208, 0.020864712074398994, 0.007009443826973438, -0.01009202841669321, 0.015471826307475567, 0.010851221159100533, -0.030158279463648796, -0.0032363866921514273, -0.00418537762016058, -0.00039820809615775943, 0.003195481840521097, 0.019071446731686592, -0.016990210860967636, 0.02352188713848591, -0.01913689449429512, 0.015615810640156269, -0.010399632155895233, 0.03662450984120369, 0.0028158854693174362, 0.021911874413490295, -0.022003501653671265, 0.007951890118420124, -0.005883744452148676, 0.009522633627057076, 0.03968745842576027, -0.012709935195744038, 0.006911272648721933, -0.013429858721792698, 1.7240728311662679e-06, -0.021440653130412102, 0.017225822433829308, -0.013547664508223534, 0.018011195585131645, -0.012088182382285595, 2.1973539332975633e-05, -0.026179062202572823, -0.00011391982843633741, 0.01976519264280796, 0.009038321673870087, 0.016087034717202187, -0.02273651584982872, 0.00994149874895811, -0.007879897952079773, 0.021257398650050163, 0.011904928833246231, -0.025864914059638977, 0.03890208527445793, -0.01913689449429512, -0.04055136814713478, -0.0004225464363116771, 0.005327439401298761, 0.013573843985795975, 0.004761316813528538, -0.03002738393843174, -0.02222602441906929, 0.03230496495962143, 0.019477222114801407, 0.03371863439679146, 0.038587938994169235, 0.024372708052396774, 0.007336682174354792, -0.010661423206329346, 0.02777598612010479, -0.02756655216217041, 0.025720929726958275, 0.0077686370350420475, 0.033849526196718216, -0.013744007796049118, 0.003622527699917555, 0.0062077101320028305, 0.006322243716567755, 0.0037697849329560995, -0.0046402388252317905, -0.0012639578199014068, -0.007159973494708538, 0.029687056317925453, 0.011937652714550495, 0.00560886412858963, 0.003599621122702956, -0.031126905232667923, -0.014948245137929916, 0.0026948072481900454, 0.03398042172193527, -0.0034556363243609667, 0.001659098081290722, 0.00741521967574954, 0.025472227483987808, -0.03212171047925949, -0.010661423206329346, 0.00020636463887058198, 0.014058156870305538, -0.022631799802184105, -0.019398685544729233, -0.004404627252370119, 7.664533768547699e-05, 0.0005305350641719997, 0.027854522690176964, -0.005824841558933258, -0.006976719945669174, 0.0009637167677283287, -0.002038694452494383, -0.002559003420174122, 0.00017865165136754513, 0.03824761137366295, -0.026165973395109177, -0.02069454826414585, 0.02301139570772648, 0.00295823416672647, 0.006476045586168766, -0.01068105734884739, -0.007938800379633904, 0.012101272121071815, -0.005530327092856169, 0.010497803799808025, -0.0042115566320717335, 0.0005587593768723309, -0.00665929913520813, 0.0039039526600390673, -0.002559003420174122, 0.016060855239629745, 0.02078617550432682, -0.021493010222911835, 0.014110514894127846, -0.028849326074123383, 0.0029533253982663155, -0.00295823416672647, -0.011132646352052689, 0.003167666494846344, 0.03976599499583244, 0.016178660094738007, -0.009823692962527275, -0.01988299749791622, -0.00844274740666151, -0.0476982519030571, -0.007997703738510609, -0.011283176019787788, -0.02579946629703045, 0.017422165721654892, 0.019189253449440002, -0.012310704216361046, -0.00935901515185833, -0.011047564446926117, 0.010249103419482708, 0.0012304158881306648, 0.006328788585960865, -0.039006803184747696, -0.0013932170113548636, 0.005769210867583752, 0.09382575750350952, 0.011348623782396317, -0.009876050986349583, -0.005968826357275248, -0.012899733148515224, -0.009542268700897694, -0.01653207838535309, -0.017749404534697533, 0.03212171047925949, 0.020720727741718292, -0.00755920447409153, 0.008462381549179554, 0.03151959180831909, -0.02106105536222458, 0.029975026845932007, 0.005762665998190641, 0.003612710628658533, 0.018757298588752747, 0.0014513017376884818, -0.009201940149068832, -0.008776530623435974, 0.02672882378101349, 0.007820994593203068, 0.022095128893852234, 0.04159852862358093, 0.01377018727362156, 0.023495709523558617, 0.03309033438563347, -0.0061520799063146114, -0.006001550238579512, -0.019032178446650505, 0.02195114456117153, 0.02654556930065155, 0.009398283436894417, -0.03633653745055199, -0.005311077460646629, -0.004568246193230152, -0.0037566954270005226, 0.031179264187812805, -0.001558472285978496, -0.01932014897465706, 0.027514195069670677, 0.021218130365014076, -0.034634899348020554, -0.005356890615075827, -0.03460872173309326, -0.019241610541939735, 0.008508195169270039, -0.008999052457511425, -0.005163819994777441, 0.028508998453617096, -0.026270689442753792, -0.01367856003344059, -0.011446795426309109, 0.007225421257317066, -0.028247209265828133, 0.0027569825761020184, -0.018469328060746193, -0.016060855239629745, -0.0016550075961276889, -0.01204236876219511, -0.004332634620368481, 0.013508396223187447, -0.00725814513862133, -0.013207336887717247, -0.031755201518535614, -0.03617946431040764, 0.016518987715244293, -0.033430662006139755, -0.0014946608571335673, 0.017081838101148605, 0.009300111792981625, -0.013429858721792698, -0.019385596737265587, 0.021545369178056717, 0.020524384453892708, 0.005311077460646629, 0.0038679565768688917, 0.004254097584635019, -0.002395384246483445, 0.01871803030371666, -0.028692252933979034, 0.02606125734746456, -0.01114573609083891, -0.003720699343830347, -0.01920234225690365, -0.022592531517148018, -0.020773086696863174, -0.014058156870305538, 0.01700330153107643, -0.0021728621795773506, 0.008351121097803116, 0.01904526725411415, -0.004398082382977009, 0.006374601740390062, 0.01811591163277626, -0.0019405230414122343, 0.004453713074326515, 0.0341898538172245, -0.02120504155755043, 0.00905795581638813, -0.007749002426862717, -0.01234342809766531, -0.005419066175818443, 0.02603507786989212, 0.02120504155755043, -0.009182306006550789, -0.009561902843415737, -0.013626202009618282, -0.006878548767417669, 0.015170766972005367, -0.02213439717888832, 0.009646984748542309, -0.008959784172475338, -0.0052521745674312115, 0.04052519053220749, 0.004774406552314758, 0.0009432643419131637, 0.004142836667597294, -0.010419267229735851, -4.167174847680144e-05, -0.05366707965731621, 0.012971725314855576, 0.01367856003344059, 0.0009203576482832432, 0.00034769068588502705, 0.0004421807243488729, 0.005854292772710323, -0.011106466874480247, 0.018940551206469536, 0.01400579884648323, 0.0006057998980395496, -0.017841031774878502, -0.007343227043747902, -0.018312254920601845, -0.012107816524803638, -0.014895886182785034, 0.024110917001962662, -0.03191227838397026, -0.00858018733561039, -0.010792318731546402, -0.0008859976660460234, 0.010321095585823059, -0.03023681789636612, 0.02178098075091839, -0.03290707990527153, -0.0167938694357872, 0.020380400121212006, 0.011165370233356953, 0.030132101848721504, -0.005363435484468937, -0.023587334901094437, -0.022906679660081863, -0.036703046411275864, -0.01623101904988289, -0.04251479730010033, 0.006377873942255974, -0.01091666892170906, 0.02352188713848591, 0.022055860608816147, 0.02348261885344982, -0.002472285181283951, 0.012114360928535461, 0.020092429593205452, -0.00873071700334549, -0.005477968603372574, 0.008462381549179554, -0.008547463454306126, -0.023089932277798653, 0.03188609704375267, 0.02327318675816059, -0.0044766198843717575, -0.006387691479176283, 0.0034229124430567026, 0.0009465367184020579, 0.030550966039299965, -0.02171553298830986, -0.016492810100317, -0.01890128292143345, -0.0464940145611763, -0.01823371648788452, -0.016178660094738007, 0.005366707686334848, 0.023233918473124504, -0.054400090128183365, -0.008122053928673267, 0.02963469922542572, -0.007389040198177099, 0.023443350568413734, -0.01153187733143568, 0.019896088168025017, -0.02918965555727482, 0.013285874389111996, 0.0056808567605912685, 0.0012885007308796048, -0.004564973991364241, 0.00037571045686490834, -0.030315354466438293, 0.016898585483431816, 0.013122254982590675, 0.01997462473809719, -0.004659872967749834, -0.005844475701451302, 0.009620805270969868, -0.020170968025922775, -0.014738812111318111, -0.0008197319111786783, -0.02426799014210701, -0.0017539971740916371, -0.033456843346357346, -0.029922667890787125, -0.019149985164403915, -0.016846226528286934, -0.006839280016720295, 0.0030907655600458384, 0.0033607371151447296, 0.00485294358804822, 0.011610413901507854, -0.0006303427508100867, -0.01839079149067402, -0.002933691255748272, -0.0016222838312387466, 0.01820753887295723, 0.015445646829903126, 0.006675661075860262, 0.03348302096128464, 0.001737635233439505, -0.017657777294516563, -0.016087034717202187, 0.017081838101148605, -0.008783075027167797, 0.012755747884511948, 0.025328243151307106, -0.037750206887722015, -0.01685931719839573, 0.017225822433829308, -0.02303757518529892, -0.039975427091121674, -0.02243545651435852, 0.01450320053845644, 0.010811952874064445, 0.01003312598913908, -0.0040708440355956554, -0.005131096113473177, -0.004620604682713747, 0.023024486377835274, -0.00336728198453784, 0.020773086696863174, -0.018063552677631378, -0.01532784104347229, -0.010864310897886753, 0.01119154877960682, -0.030969830229878426, -0.00016453949501737952, 0.010098573751747608, 0.018469328060746193, -0.017409076914191246, -0.0011142463190481067, -0.02405855804681778, -0.012055458500981331, -0.01625719852745533, 0.009431007318198681, -0.02354806661605835, -0.011760943569242954, 0.017225822433829308, 0.020131699740886688, -0.011643137782812119, -0.0010839768219739199, 0.026807360351085663, 0.02378367818892002, -0.03290707990527153, -0.007277779281139374, -0.004378448240458965, -0.007637741509824991, -0.002779889153316617, 0.0014881161041557789, -0.007650831248611212, 0.013757097534835339, 0.003596348688006401, -0.015799064189195633, 0.008377299644052982, 0.01105410885065794, -0.01788030005991459, -0.015354020521044731, 0.0003182392392773181, -0.04191267862915993, 0.013174613006412983, 0.004489709157496691, 0.03104836866259575, -0.008141688071191311, -0.006210982799530029, 0.008174411952495575, -0.012474323622882366, 0.02306375466287136, -0.00347527046687901, 0.008704538457095623, -0.006701840087771416, 0.033221229910850525, -0.0059361024759709835, -0.00941791757941246, -0.007133794482797384, 0.017736315727233887, -0.04251479730010033, -0.001038163434714079, 0.0017130924388766289, -0.006286247167736292, -0.002521371003240347, -0.007814450189471245, -0.022723427042365074, -0.02168935351073742, 0.02238309755921364, -0.009712432511150837, 0.003593076253309846, 0.013456038199365139, 0.01955576054751873, 0.012094726786017418, -0.005873927380889654, -0.009967678226530552, -0.06377219408750534, -0.0011821483494713902, 0.016427362337708473, -0.00013150888844393194, 0.022278381511569023, 0.00673456396907568, -0.013744007796049118, -0.0044438960030674934, 0.0043915375135838985, -0.000575121259316802, 0.016518987715244293, -0.018430059775710106, -0.016479719430208206, -0.010739960707724094, -8.411046292167157e-05, -0.0006368875037878752, -0.002380658406764269, 0.00321675231680274, -0.026912076398730278, 0.02900640107691288, 0.0038417773321270943, 0.013207336887717247, 0.03864029794931412, 0.016898585483431816, 0.02045893669128418, 0.009077589958906174, -0.018731119111180305, -0.023888394236564636, -0.0068916380405426025, -0.017147285863757133, -0.04154617339372635, -0.005805206950753927, -0.03107454814016819, 0.03730516508221626, 0.013547664508223534, -0.026074346154928207, -0.010039670392870903, -0.010864310897886753, -0.02531515434384346, -0.008946694433689117, -0.004247552715241909, -0.012847375124692917, -0.0014447569847106934, 0.024843931198120117, -0.007146884221583605, 0.027671268209815025, 0.0007342409226112068, 0.004915119148790836, -0.01018365565687418, -0.030917473137378693, -0.002465740544721484, -0.011558055877685547, -0.0026653558015823364, -0.002537732943892479, -0.02069454826414585, -0.033875707536935806, 0.025694750249385834, -0.00411011278629303, 0.01105410885065794, 0.019987713545560837, -0.004751499742269516, -0.024647587910294533, -0.011675861664116383, -0.011165370233356953, 0.0065873065032064915, 0.004610787611454725, -0.013861813582479954, -0.011106466874480247, 0.00246737664565444, 0.0032232971861958504, -0.010726870968937874, -0.013639291748404503, 0.014058156870305538, 0.007192697376012802, -0.003926859237253666, 0.009215029887855053, -0.001408760785125196, -0.002046875422820449, 0.011400981806218624, 0.020537475124001503, 0.03185991942882538, 0.011060654185712337, -0.0019863364286720753, 0.009149582125246525, 0.033640094101428986, -0.006368056870996952, -0.005546689033508301, 0.01916307397186756, -0.0031987542752176523, 0.012323793955147266, 0.007218876387923956, -0.03473961725831032, 0.008586732670664787, -0.01946413330733776, 0.0027078967541456223, -0.03141487389802933, -0.0056644948199391365, -0.002858426421880722, -0.0030727675184607506, -0.028535177931189537, 0.010910124517977238, 0.024843931198120117, -0.022932859137654305, -0.029948847368359566, -0.008449292741715908, -0.026885896921157837, -0.02066837064921856, -0.019660476595163345, 0.026283778250217438, 0.008462381549179554, -0.000537488900590688, 0.011198094114661217, -0.012389241717755795, -0.015484915114939213, -0.0031349428463727236, 0.011270086281001568, -0.007997703738510609, -0.0062077101320028305, 0.2192758321762085, 0.001408760785125196, 0.001050434890203178, 0.04089169576764107, 0.020616011694073677, 0.02345643937587738, 0.019896088168025017, 0.006063725333660841, -0.011492608115077019, -0.02243545651435852, 0.002015787875279784, -0.004705686587840319, -0.013063352555036545, -0.014136693440377712, 0.01255286019295454, -0.007297413889318705, -0.009902230463922024, -0.02327318675816059, 0.01390108186751604, 0.021885696798563004, -0.0021352297626435757, -0.01502678170800209, 0.005949191749095917, -0.025603123009204865, 0.012997904792428017, 0.024857020005583763, 0.0022039497271180153, -0.0006254341569729149, 0.014791170135140419, -0.003008956089615822, -0.014895886182785034, -0.014346126466989517, 0.03892826661467552, 0.008154777809977531, -0.004038120619952679, -0.002158136572688818, 0.035053763538599014, -0.02939908765256405, 0.031100725755095482, 0.00911685824394226, 0.02229147218167782, 0.010805408470332623, 0.031545769423246384, 0.008841978386044502, 0.027200045064091682, 0.0035276287235319614, -0.0039726728573441505, -0.010890490375459194, 0.018757298588752747, -0.006021184381097555, -0.02480466105043888, -0.009699342772364616, 0.018338432535529137, 0.017291270196437836, 0.0023266642820090055, -0.00643350463360548, -0.002313574543222785, -0.02057674340903759, -0.004237735643982887, 0.03654596954584122, -0.010975572280585766, 0.033666275441646576, -0.0053994315676391125, 0.028351925313472748, -0.014175962656736374, 0.014058156870305538, 0.020223325118422508, 0.025851823389530182, 0.019149985164403915, 0.006721474230289459, 0.005589229986071587, 0.0017392714507877827, -0.02333863452076912, 0.005055831279605627, 0.011610413901507854, -0.02273651584982872, 0.009640439413487911, 0.005422338377684355, 0.036519791930913925, 0.02306375466287136, 0.012101272121071815, 0.0037272439803928137, 0.0025851824320852757, -0.031781382858753204, -0.00741521967574954, -0.01946413330733776, 0.03704337403178215, -0.028535177931189537, -0.013521485961973667, -0.008482016623020172, -0.01105410885065794, 0.008933604694902897, -0.011806757189333439, -0.02030186355113983, 0.005965553689748049, 0.006309153977781534, 0.026768092066049576, 0.009175761602818966, -0.004470075014978647, 0.009921864606440067, -0.013796365819871426, 0.037776388227939606, -0.000933447212446481, 0.01607394404709339, -0.006525131408125162, -0.003609438193961978, 0.017566150054335594, 0.03131015971302986, -0.008750352077186108, -0.0019405230414122343, 0.01450320053845644, -0.03680776059627533, 0.022684156894683838, 0.0007910985732451081, 0.016911674290895462, 0.020197147503495216, 0.01366547029465437, -0.015484915114939213, 0.003234750358387828, 0.006786921992897987, -0.006322243716567755, -0.021636994555592537, -0.006031001452356577, 0.008338031359016895, -0.007192697376012802, 0.0002716077724471688, -0.02918965555727482, 0.003265838138759136, 0.00325111229903996, -0.022147485986351967, 0.0016852770932018757, 0.003138215048238635, 0.03374481201171875, 0.00317584746517241, -0.0005076283705420792, -0.008717628195881844, 0.0064564114436507225, -0.01920234225690365, -0.010772684589028358, 0.023089932277798653, 0.0013678560499101877, 0.009817148558795452, -0.015236214734613895, 0.0007150156307034194, 0.004283549264073372, -0.01399270910769701, 0.02285432070493698, -0.02129666693508625, 0.011139190755784512, -0.008645635098218918, -0.01105410885065794, 0.018011195585131645, 0.010340729728341103, -0.003192209405824542, 0.03625800088047981, 0.024333437904715538, -0.03275000676512718, -0.002658810932189226, 0.02238309755921364, 0.008619456551969051, -0.03002738393843174, 0.005520510021597147, 0.011918018572032452, 9.602398495189846e-05, -0.016414271667599678, -0.008868157863616943, -0.16482338309288025, 0.020380400121212006, 0.023836037144064903, -0.02693825587630272, 0.03547263145446777, -0.014110514894127846, 0.024320349097251892, 0.005225995555520058, -0.015995407477021217, -0.021650085225701332, -0.0026080890092998743, 0.012199442833662033, -0.02222602441906929, -0.02600889839231968, 0.02127048745751381, -0.009463731199502945, -0.01007893867790699, 0.039006803184747696, 0.017356717959046364, 0.009110313840210438, 0.026257600635290146, -0.009679708629846573, 0.015877600759267807, -0.015170766972005367, -0.0038875907193869352, -0.004528977908194065, -0.007729368284344673, -0.009175761602818966, -0.02774980664253235, -0.033456843346357346, -0.009921864606440067, -0.0006008913042023778, 0.033456843346357346, 0.012775382958352566, 0.030158279463648796, -0.00966007448732853, 0.016178660094738007, 0.011459884233772755, -0.013325142674148083, 0.03848322108387947, 0.009228119626641273, 0.030393891036510468, 0.00478422362357378, 0.0029238739516586065, 0.008639090694487095, 0.011793667450547218, 0.009103769436478615, 0.003609438193961978, 0.00462714908644557, -0.013495306484401226, 0.019189253449440002, -0.033823348581790924, -0.0056874011643230915, 0.018312254920601845, 0.002526279538869858, -0.0005923013086430728, -0.017618509009480476, 0.010072394274175167, -0.00822022557258606, -0.027069151401519775, -0.004404627252370119, 0.003341102972626686, -0.023652782663702965, -0.010052760131657124, -0.004489709157496691, -0.013429858721792698, -0.018992910161614418, -0.005579412914812565, -0.024228721857070923, 0.0035570801701396704, -0.02066837064921856, -0.003946493845432997, -0.016545167192816734, 0.01151878759264946, 0.003202026477083564, 0.0019568849820643663, -0.01053052768111229, 0.0037828744389116764, -9.571719419909641e-05, 0.007303958293050528, -0.00325765716843307, 0.022723427042365074, -0.015079139731824398, -0.004293366335332394, -0.005743031855672598, -0.013940351083874702, 0.010569796897470951, 0.018351523205637932, -0.007762092165648937, -0.0017474524211138487, 0.030393891036510468, -0.01823371648788452, -1.3255707017378882e-05, -0.02459522895514965, -0.008187501691281796, 0.02129666693508625, 0.019686654210090637, -0.013757097534835339, 0.002068145899102092, -0.031545769423246384, -0.022959038615226746, -0.00016024449723772705, -0.0017621781444177032, 0.00793225597590208, 0.001450483687222004, 0.011787123046815395, 0.021767890080809593, 0.01901908963918686, 0.0027160777244716883, -0.019686654210090637, -0.012094726786017418, -0.007474122568964958, 0.02939908765256405, 0.010373453609645367, -0.0008467290317639709, 0.005563050974160433, 0.007585383486002684, 0.012277980335056782, 0.032802365720272064, 0.01366547029465437, 0.016728421673178673, 0.0032691103406250477, -0.03167666494846344, 0.011034474708139896, -0.005857565440237522, 0.015432557091116905, -0.09497763961553574, -0.018678762018680573, 0.015406378544867039, 0.010196744464337826, -0.02090398222208023, 0.004231190774589777, -0.0017016390338540077, 0.00858018733561039, -0.021165771409869194, 0.031388696283102036, -0.01697712205350399, -0.02375749871134758, -0.013809455558657646, -0.005801934748888016, -0.003917042165994644, -0.024857020005583763, 0.009758245199918747, -0.00888124667108059, -0.008272583596408367, 0.040001608431339264, 0.016872406005859375, 0.007022533565759659, 0.008252949453890324, -0.012448144145309925, -0.024307260289788246, -0.003192209405824542, -0.03047242946922779, -0.008148233406245708, 0.001721273409202695, 0.004479892086237669, -0.0024771937169134617, -0.012304159812629223, 0.019333237782120705, -0.03193845599889755, -0.007794816046953201, 0.012088182382285595, -0.0031267618760466576, -0.03782874718308449, 0.0057561215944588184, -0.013796365819871426, -0.01860022358596325, 0.013063352555036545, -0.015825243666768074, -0.050525590777397156, 0.005448517389595509, 0.0046402388252317905, -0.03293326124548912, 0.005553233437240124, -0.017356717959046364, -0.020406579598784447, -0.03521084040403366, 0.005641588009893894, -0.016833137720823288, -0.011099922470748425, 0.001996153499931097, -0.028090134263038635, -0.0007407856755889952, -0.014123604632914066, -0.013298964127898216, 0.01225180085748434, -0.02712150849401951, 0.010288371704518795, -0.03408513963222504, 0.013056807219982147, 0.02507954277098179, -0.04450440779328346, -0.025118811056017876, -0.014647185802459717, -0.0013138616923242807, -0.001004621502943337, -0.022095128893852234, 0.035917673259973526, -0.007173063233494759, -0.0023184833116829395, -0.04178178310394287, -0.0074872118420898914, 0.011034474708139896, 0.009771334938704967, 0.02735712006688118, -0.0002943099243566394, -0.00935901515185833, -0.010321095585823059, 0.028037775307893753, -0.016322646290063858, 0.018220627680420876, 0.001842351513914764, 0.01952958106994629, -0.016191750764846802, 0.010674512945115566, -0.02498791553080082, -0.008789620362222195, 0.017238913103938103, 0.003579986747354269, 0.016636794432997704, -0.013174613006412983, -0.007820994593203068, -0.001729454263113439, -0.0016558257630094886, 0.014778080396354198, 0.004025030881166458, -0.036729224026203156, -0.01904526725411415, -0.021964233368635178, 0.014869707636535168, -0.002045239321887493, 0.010962482541799545, 0.0007379223243333399, -0.018377702683210373, 0.004761316813528538, -0.008115509524941444, 0.012605218216776848, -1.3153445252100937e-05, -0.0056874011643230915, 0.019293969497084618, 0.0034687258303165436, 0.003717426909133792, -0.0339018851518631, -0.012991359457373619, 0.01901908963918686, -0.006878548767417669, -0.0028813329990953207, -0.0027864340227097273, -0.014463932253420353, 0.0016558257630094886, 0.024830840528011322, 0.021977322176098824, -0.012579039670526981, 0.03853558003902435, -0.015131497755646706, 0.002806068165227771, 0.004496254026889801, -0.03319505229592323, 0.0013285874156281352, -0.008985963650047779, 0.007984613999724388, 0.0038614117074757814, -0.01541946828365326, -0.020197147503495216, -0.014398484490811825, 0.030158279463648796, 0.004830037243664265, 0.02759273163974285, -0.0035112667828798294, -0.01316806860268116, 0.008266039192676544, -0.024124005809426308, 0.020524384453892708, 0.003687975462526083, -0.019987713545560837, -0.009300111792981625, 0.02183333784341812, 0.01992226578295231, 0.004146108869463205, -0.005000201053917408, -0.029975026845932007, -0.00791916623711586, -0.01841697096824646, -0.03604856878519058, -0.025969630107283592, -0.029739415273070335, 0.012749203480780125, 0.001500387559644878, 0.05261991545557976, -0.0014913884224370122, -0.009666618891060352, -0.02210821770131588, 0.022068949416279793, 0.013442948460578918, -0.0197913721203804, 0.026375405490398407, -0.004617332015186548, -0.02492246776819229, -0.030786577612161636, 0.005733214784413576, 0.02147992141544819, 0.008900880813598633, 0.011800212785601616, -0.002814249135553837, -0.0001692435471341014, -0.0034458190202713013, -0.025393690913915634, 0.02774980664253235, -0.00081277807475999, 0.027042971923947334, -0.007225421257317066, 0.0184431504458189, 0.005743031855672598, 0.0056874011643230915, 0.007526480592787266, 0.020773086696863174, 0.0013187702279537916, 0.006168441381305456, -0.023469530045986176, -0.0026522662956267595, 0.0004822674090974033, -0.012997904792428017, 0.008586732670664787, -0.005900106392800808, 0.012232166714966297, 0.008122053928673267, 0.02441197633743286, 0.02447742410004139, 0.014215230941772461, 0.0022072221618145704, 0.0017687228973954916, -0.026192152872681618, -0.027828343212604523, 0.02168935351073742, -0.0043195453472435474, -0.023836037144064903, 0.02559003420174122, 0.017631597816944122, 0.008927060291171074, -0.0039563109166920185, 0.03576060011982918, 0.0006724746781401336, -0.011603869497776031, 0.03848322108387947, 0.014333036728203297, -0.029713235795497894, -0.029372908174991608, 0.03400660306215286, 0.013089531101286411, 0.014607916586101055, 0.015576542355120182, -0.02898022159934044, 0.02294594794511795, 0.022304560989141464, 0.0037566954270005226, -0.01952958106994629, 0.010491259396076202, 0.0039955792017281055, 0.03183374181389809, 0.009097224101424217, -0.02127048745751381, 0.006197893060743809, -0.020498206838965416, -0.026113614439964294, 0.0034916324075311422, 0.016937853768467903, -0.01850859634578228, 0.0723065733909607, 0.020629100501537323, -0.017526881769299507, -0.005563050974160433, -0.02552458643913269, 0.025576943531632423, 0.010065849870443344, 0.003933404106646776, 0.006466228514909744, -0.001166604459285736, 0.016937853768467903, 0.014058156870305538, 0.012893188744783401, -0.01531475130468607, -0.02837810292840004, -0.01709492690861225, -0.024817751720547676, 0.02795923873782158, -0.01892746239900589, -0.008023882284760475, 0.02231765165925026, -0.02261870913207531, -0.0057561215944588184, -0.013397134840488434, -0.04159852862358093, -0.0037436059210449457, 0.03369245305657387, -0.006845824886113405, -0.010942848399281502, -0.03225260600447655, 0.0030531331431120634, 0.016244107857346535, -0.04133674129843712, -0.020825443789362907, -0.0038614117074757814, 0.02120504155755043, 0.009967678226530552, -0.03248821571469307, -0.01655825786292553, 0.007618107367306948, 0.0063156988471746445, 0.019058357924222946, -0.017945747822523117, -0.02603507786989212, -0.006129173096269369, -0.0005550779169425368, 0.0012443235609680414, -0.02712150849401951, -0.008285673335194588]} +{"id": "test:10000094", "text": "\"\\\"Extra Eyes for DWIs\\\" on Halloween Here in Fairfax County | McGlone Law Firm, P.C.\\n\\\"Extra Eyes for DWIs\\\" is an effort that will involve seven local police agencies conducting saturation patrols in all seven jurisdictions on Halloween night (this Thursday, October 31). The police officers will be specifally looking for drunk drivers, in an effort to keep them off the streets.\"", "vector_field": [-0.0057866922579705715, 0.009461208246648312, 0.0011055398499593139, -0.04286041483283043, -0.01662919670343399, 0.022529879584908485, -0.00057707674568519, -0.019512485712766647, -0.007751351688057184, -0.0283500999212265, 0.008569400757551193, 0.015194258652627468, -0.024461012333631516, 0.008710213005542755, 0.0020803259685635567, 0.018238473683595657, 0.01583796925842762, -0.02311994880437851, -0.0006638268241658807, -0.01680353470146656, -0.019110165536403656, 0.02891334518790245, 0.004703782964497805, -0.014818760566413403, 0.003996371757239103, 0.0020820023491978645, -0.001153315301053226, -0.012411549687385559, -0.011553268879652023, -0.0016059244517236948, 0.014926045201718807, -0.00420088367536664, -0.020022090524435043, -0.011633732356131077, -0.00457302900031209, 0.0032369939144700766, -0.007771467790007591, -0.011486215516924858, 0.04387962445616722, 0.008985131047666073, 0.03215872123837471, 0.021912990137934685, -0.006980239413678646, -0.0016570525476709008, -0.016910821199417114, 0.02259693294763565, 0.0072216312400996685, -0.012196979485452175, -0.004452333319932222, 0.021242458373308182, 0.008918077684938908, -0.006671794690191746, 0.0005263677448965609, -0.028859702870249748, 0.0020568573381751776, 0.013357000425457954, 0.027089498937129974, 0.005810161121189594, 0.0003639732312876731, -0.02501084841787815, -0.012773637659847736, -0.010031160898506641, -0.006048199720680714, 0.017299728468060493, -0.0024222973734140396, -0.007228336296975613, 0.001386325224302709, 0.01846645586192608, -0.002883288310840726, -0.02553386427462101, 0.031139513477683067, 0.014483494684100151, 0.01195558812469244, -0.039561398327350616, 0.024849921464920044, -0.024072103202342987, -0.024072103202342987, -0.016642607748508453, 0.016213467344641685, -0.010701692663133144, 0.023844122886657715, -0.020303713157773018, -0.04489883407950401, 0.05163097381591797, -0.014563958160579205, 0.0031347377225756645, -0.031246798112988472, 0.024501245468854904, -0.01162702776491642, -0.0003773838689085096, -0.010306078940629959, 0.0037918593734502792, -0.0007392616826109588, 0.007060703355818987, -0.008455410599708557, 0.04685678705573082, -0.008924783207476139, 0.05954325571656227, 0.007389264181256294, 0.00403660349547863, 0.012619415298104286, 0.004868063144385815, -0.015998896211385727, -0.005726344417780638, 0.02719678357243538, -0.0065108672715723515, 0.009856822900474072, 0.011734312400221825, -0.00583362951874733, 0.01344416942447424, -0.006772374734282494, -0.012894333340227604, 0.03714748099446297, -0.021953223273158073, -0.015408828854560852, 0.009239932522177696, -0.009897054173052311, -0.0003432286321185529, -0.035779595375061035, -0.0378984771668911, 0.01720585487782955, 0.003433124627918005, -0.015167437493801117, -0.0023468625731766224, 0.016682839021086693, 0.018305527046322823, 0.006396876648068428, 0.00026171706849709153, 0.01221709605306387, -0.0301471259444952, 0.031997792422771454, -0.013256420381367207, 0.038408081978559494, -0.01076874602586031, -0.024595119059085846, 0.00897172000259161, -0.0012765255523845553, 0.01672307215631008, -0.014470083639025688, -0.0016721395077183843, 0.0051329233683645725, 0.01987457275390625, -0.01943202130496502, -0.005360904615372419, 0.014376209117472172, 0.03344614431262016, 0.008958309888839722, 0.012579183094203472, 0.00493511650711298, -0.00869009643793106, 0.018533509224653244, -0.030281230807304382, 0.032614681869745255, 0.01184830255806446, 0.01925768330693245, 0.015542935580015182, -0.007945805788040161, 0.009964107535779476, -0.021389976143836975, -0.0035035305190831423, -0.034679923206567764, -0.01838599145412445, 0.02851102687418461, 0.00771111948415637, 0.01834576018154621, 0.0006219185888767242, -0.013759319670498371, 0.01243837084621191, -0.005847040098160505, -0.010037866421043873, 0.01917721889913082, 0.03242693468928337, -0.035296812653541565, 0.005572122056037188, 0.005719639360904694, 0.0325610414147377, 0.016200056299567223, -0.008388357236981392, -0.03237329050898552, -0.021752063184976578, 0.010547470301389694, 0.009501440450549126, 0.0033509843051433563, -0.00793910026550293, -0.013893426395952702, -0.02132292278110981, 0.021135173738002777, -0.0053676096722483635, 0.002651954535394907, -0.01493945624679327, -0.0005888110608793795, 0.02290537767112255, 0.010829093866050243, 0.005592238157987595, -0.6505771279335022, -0.015609988011419773, -0.00272068427875638, -0.021215636283159256, -0.001653699902817607, 0.021242458373308182, 0.01597207598388195, -0.006571215111762285, -0.03476038575172424, 0.010359721258282661, 0.01000433973968029, 0.014470083639025688, 0.0007736264378763735, -0.031488191336393356, 0.007288684602826834, -0.01166725903749466, -0.014295745640993118, 0.01676330342888832, 0.020679211243987083, 0.012880922295153141, -0.037013374269008636, 0.019753877073526382, 0.010896147228777409, 0.0008197255665436387, 0.020343944430351257, 0.01790320873260498, 0.01688399910926819, -0.022838324308395386, -0.02821599319577217, -0.002933578100055456, -0.0031598827335983515, -0.00035789652611128986, -0.0053307306952774525, -0.00015244129463098943, 0.04843924194574356, 6.446668976423098e-06, -0.00598785188049078, -0.012351201847195625, -0.0026620125863701105, 0.023790480569005013, -0.026620125398039818, 0.0005439692176878452, 0.016065949574112892, 0.004170709755271673, -0.018922416493296623, 0.002232871949672699, 0.008046385832130909, -0.015918433666229248, -0.010098214261233807, 0.0022429300006479025, -0.0011122451396659017, 0.006396876648068428, -0.013665445148944855, -0.01865420490503311, -0.0007916470058262348, 0.014510315842926502, 0.0063331760466098785, 0.01658896543085575, 0.0022412536200135946, -0.007167988456785679, -0.010996727272868156, 0.01522107981145382, -0.0022747803013771772, -0.0045294445008039474, -0.008830908685922623, 0.027102909982204437, -0.008897961117327213, 0.014107996597886086, 0.02219461463391781, 0.0046769618056714535, 0.015408828854560852, 0.03476038575172424, -0.025775255635380745, -0.011063780635595322, 0.016776714473962784, 0.01061452366411686, 0.01904311217367649, 0.00809332262724638, -0.003295665606856346, 0.003174969693645835, -0.009521556086838245, -0.017822744324803352, -0.0481710284948349, 0.017661817371845245, 0.013745908625423908, -0.01728631928563118, -0.005655938759446144, -0.013725792989134789, 0.0006973534473218024, -0.00188922428060323, 0.04111703112721443, 0.0005280440673232079, 0.005910740699619055, -0.02653966285288334, -0.014684653840959072, 0.014201871119439602, -0.0061688958667218685, 0.02912791632115841, 0.01449690479785204, -0.015154026448726654, -0.01767522655427456, -0.02014278620481491, 0.004070130176842213, -0.03022758848965168, 0.027867315337061882, 0.03154183179140091, -0.007288684602826834, -0.006614799611270428, 0.031139513477683067, -0.02430008538067341, 0.012170158326625824, -0.008857729844748974, -0.01046700682491064, 0.02097424492239952, -0.02737112157046795, -0.024943795055150986, -0.0062560648657381535, -0.01636098511517048, -0.00162855489179492, -0.011559974402189255, 0.02737112157046795, -0.007194810081273317, 0.002132292138412595, -0.01390683650970459, 0.009615430608391762, -0.006779079791158438, -0.004174062516540289, -0.02688833884894848, -0.027183372527360916, -0.025413168594241142, 0.009400860406458378, 0.0011658878065645695, 0.025104723870754242, 0.004234410356730223, 0.010084803216159344, -0.022208023816347122, 0.011747723445296288, -0.0017090188339352608, -0.007885457947850227, -0.009085710160434246, -0.02136315405368805, -0.008012859150767326, 0.010145151056349277, -0.02321382239460945, -0.009219816885888577, 0.01134540420025587, -0.007322210818529129, -0.009836706332862377, -0.014738296158611774, -0.0005137952393852174, -0.0028128824196755886, -0.014067764393985271, -0.009756242856383324, -0.006779079791158438, -0.032185543328523636, -0.012196979485452175, 0.007623950485140085, -0.010741924867033958, -0.011043664067983627, -0.02027689293026924, -0.004241115879267454, 0.018506687134504318, -0.012203685007989407, -0.022047096863389015, -0.0016411273973062634, 0.021912990137934685, 0.008824203163385391, -0.00822742935270071, -0.019794108346104622, -0.032534219324588776, -0.0031598827335983515, -0.024340316653251648, 0.014590779319405556, 0.021872758865356445, -0.0010871002450585365, 0.019981857389211655, 0.002804500749334693, 0.0074563175439834595, -0.002413915703073144, -0.017259497195482254, 0.004720546305179596, 0.009494734928011894, -0.03524316847324371, 0.028242813423275948, 0.014014122076332569, 0.006577920168638229, 0.006618152372539043, 0.016924232244491577, -0.0011172741651535034, -0.003490119706839323, -0.006973534356802702, -0.013061965815722942, 0.0073289163410663605, 0.006618152372539043, 0.00041761581087484956, 0.012223800644278526, -0.014295745640993118, 0.014242102392017841, 0.008126849308609962, 0.05211375653743744, 0.05184554681181908, -0.008844318799674511, 0.02435372769832611, 0.0017668522195890546, 0.006182306446135044, -0.02856466919183731, -0.00028518569888547063, -0.05087997764348984, 0.014724886044859886, 0.011948882602155209, -0.0004163585545029491, -0.00634993938729167, 0.0005758194602094591, 0.022409183904528618, 0.001602571806870401, 0.02286514639854431, -0.023441804572939873, 0.004710488487035036, 0.002522877184674144, 0.004797657486051321, 0.013973889872431755, -0.022959019988775253, 0.036128271371126175, 0.002935254480689764, 0.010373132303357124, -0.024447601288557053, 0.03202461451292038, 0.021001067012548447, 0.010306078940629959, -0.027867315337061882, -0.015449061058461666, 0.02333451807498932, 0.008307892829179764, 0.042055774480104446, 0.011989114806056023, -0.00664832629263401, -0.008998541161417961, -0.0055520059540867805, 0.041653458029031754, -0.015918433666229248, 0.006202422548085451, 0.03843490406870842, 0.03044215962290764, -0.0029184911400079727, -0.002102118218317628, 0.0007501578656956553, 0.009300281293690205, -0.006571215111762285, -0.007778172846883535, 0.007650771643966436, 0.005454779136925936, -0.001964659197255969, 0.003721453482285142, 0.018587151542305946, -0.003040863201022148, 0.019807519391179085, 0.006685205269604921, 0.0006453871610574424, -0.0033426026348024607, 0.020987655967473984, 0.024420781061053276, 0.010165267623960972, 0.0016721395077183843, -0.024769457057118416, 0.0004614099452737719, 0.00398296071216464, 0.014818760566413403, 0.0029989550821483135, 0.007697708904743195, -0.02899380959570408, -0.002777679357677698, -0.011311877518892288, 0.015918433666229248, -0.005719639360904694, 0.036476947367191315, 0.011050369590520859, -0.012840691022574902, -0.000734232715331018, 0.009803179651498795, -0.00045009469613432884, -0.012652941979467869, -0.011834892444312572, -0.0020115964580327272, 0.01794344000518322, -0.01522107981145382, -0.009990928694605827, -0.008167081512510777, 0.008629748597741127, -0.003989666234701872, -0.013403937220573425, -0.00016721394786145538, 9.466028132010251e-05, -0.030200768262147903, -0.0010376485297456384, -0.000525529554579407, -0.011546563357114792, 0.02088037133216858, -0.012927860021591187, -0.011378930881619453, 0.0015506056370213628, -0.004613261204212904, 0.02926202304661274, -0.009347218088805676, -0.03234647214412689, 0.03237329050898552, -0.0073088002391159534, -0.02193981222808361, -0.005796750541776419, 0.00791898462921381, -0.012250622734427452, -0.020384177565574646, 0.010936379432678223, -0.018707847222685814, 0.004807715304195881, -0.0036141681484878063, 0.002970457309857011, -0.008140260353684425, -0.016133002936840057, 0.03749615699052811, 0.01119118183851242, 0.021376565098762512, -0.03285607695579529, -0.011291760951280594, -0.016079360619187355, 0.11683352291584015, 0.01221709605306387, 0.0026083700358867645, 0.02246282622218132, -0.007516665384173393, -0.013558159582316875, -0.026217807084321976, -0.006242654286324978, 0.006380113307386637, -0.012431666254997253, 0.007999448105692863, -0.012230506166815758, -0.019807519391179085, -0.013538043946027756, 0.026512840762734413, 0.013108903542160988, -0.0014743325300514698, -0.010527354665100574, -0.0010166943538933992, 0.0018154657445847988, -0.03387528285384178, -0.030817657709121704, 0.006544393952935934, -0.013605097308754921, -0.009521556086838245, -0.014255513437092304, 0.012860806658864021, 0.04860017076134682, -0.016696250066161156, 0.006826017051935196, -0.023924587294459343, 0.011399046517908573, 0.008066501468420029, -0.014483494684100151, -0.014443262480199337, -0.012860806658864021, 0.005049107130616903, -0.02443419210612774, 0.03623555600643158, -0.01044018566608429, 0.012934564612805843, 0.013484401628375053, -0.0019344852771610022, -0.019190629944205284, 0.03457263857126236, -0.02302607335150242, 0.002492703264579177, 0.019686823710799217, 0.0021943163592368364, -0.030549444258213043, 0.01079556718468666, -0.006913186516612768, 0.020504873245954514, -0.010239025577902794, -9.356018563266844e-05, 0.006926597096025944, -0.006571215111762285, 0.0040231929160654545, -0.01746065728366375, -0.01670966111123562, -0.0027592398691922426, -0.002933578100055456, 0.01908334530889988, -0.000655445153824985, -0.003969550132751465, -0.0035471150185912848, 0.009910465218126774, -0.02255670167505741, -0.010520649142563343, -0.027572281658649445, 0.022006865590810776, -0.007181399501860142, -0.01331676822155714, -0.010239025577902794, -0.004824478644877672, 0.004904942587018013, 0.020290302112698555, -0.0022848383523523808, 0.010574291460216045, 0.011445983313024044, -0.008153670467436314, -0.013920247554779053, -0.01212992612272501, -0.0047138407826423645, -0.021644778549671173, 0.005129571072757244, 0.00015223174705170095, -0.009789769537746906, -0.011915355920791626, -0.0003327515732962638, -0.0034968252293765545, 0.025614328682422638, 0.022824915125966072, -0.029503414407372475, -0.011459394358098507, -0.01164714340120554, 0.013658739626407623, 0.019807519391179085, -0.008301188237965107, -0.007697708904743195, 0.03269514814019203, -0.008871139958500862, -0.007530075963586569, -0.01239143405109644, 0.020464641973376274, -0.0007296227850019932, 0.004888179246336222, -0.0024759399238973856, 0.004884826485067606, 0.01645485870540142, 0.002491026883944869, -0.020075732842087746, 0.01331676822155714, 0.0034297718666493893, 0.015690451487898827, 0.008314598351716995, -0.002755887107923627, -0.016521912068128586, 0.0029151386115700006, -0.0057296971790492535, 0.01048712246119976, -0.04390644654631615, 0.03044215962290764, 0.009548378176987171, 0.01821165345609188, 0.011700785718858242, 0.021725241094827652, -0.026070289313793182, 0.0048412419855594635, -0.00035265801125206053, -0.02281150408089161, -0.0017199149588122964, 0.0003090734244324267, -0.004241115879267454, -0.026338502764701843, -0.009400860406458378, -0.004465743899345398, 0.01987457275390625, -0.027384532615542412, -4.8404039262095466e-05, 0.01449690479785204, -0.011526447720825672, 0.009132647886872292, -0.027652746066451073, 0.02211415022611618, -0.019861161708831787, -0.0005678568850271404, -0.002115529030561447, -0.01956612803041935, 0.009481324814260006, -0.006708674132823944, -0.04492565244436264, -0.0147651182487607, 0.0026569836772978306, -0.023803891614079475, -0.03312428668141365, -0.02636532485485077, -0.004723899066448212, 0.0072417473420500755, 0.00628623878583312, 0.03698655217885971, 0.0016302312724292278, 0.009622136130928993, 0.03478720784187317, -0.006232596468180418, 0.01570386253297329, -0.017795922234654427, -0.006172248162329197, -0.01190865132957697, -0.006638268008828163, -0.00717469397932291, 0.011901945807039738, -0.007235041819512844, -0.006222538184374571, 0.015100384131073952, 0.02202027477324009, -0.01434938795864582, 0.009427681565284729, -0.012686467729508877, -0.02338816039264202, -0.0070070610381662846, 0.0016319076530635357, 0.007798288948833942, 0.041787564754486084, -0.025305883958935738, -0.007771467790007591, 0.03382164239883423, 0.04511340335011482, 0.0015531200915575027, -0.008374946191906929, 0.021443618461489677, -0.01641462743282318, 0.0026670414954423904, 0.012994913384318352, -0.0031095927115529776, -0.0037281587719917297, -0.02725042589008808, -0.013544749468564987, -0.009990928694605827, 0.014081175439059734, 0.040392857044935226, 0.022529879584908485, -0.004727251827716827, 0.014403030276298523, -0.018453044816851616, 0.011425867676734924, 0.003932671155780554, -0.012954681180417538, -0.010225615464150906, -0.029637521132826805, -0.007463022600859404, -0.008254250511527061, -0.039159078150987625, -0.028108706697821617, -0.01059440802782774, 0.002256340580061078, -0.03422395884990692, 0.02320041134953499, 0.009353923611342907, -0.029852090403437614, -0.022784681990742683, -0.015583166852593422, 0.020384177565574646, 0.008649864234030247, 0.027920957654714584, 0.043101806193590164, 0.01254565641283989, -0.005270382855087519, 0.023012662306427956, 0.011855008080601692, -0.005873861722648144, -0.01493945624679327, 0.00796592142432928, -0.033848464488983154, -0.0014961248962208629, -0.007000355515629053, 0.014671243727207184, -0.007798288948833942, -0.03272197023034096, 0.01834576018154621, 0.014081175439059734, 0.007463022600859404, 0.0018054078100249171, -0.02821599319577217, -0.009327102452516556, 0.021886169910430908, 0.007590423803776503, 0.005119512788951397, -0.001072851475328207, -0.019968446344137192, -0.010520649142563343, 0.008602927438914776, -0.039159078150987625, -0.0013494458980858326, 0.01794344000518322, 0.006225890945643187, 0.0031330613419413567, -0.0016754921525716782, 0.013343589380383492, 0.0017618231941014528, -0.009648957289755344, 0.010031160898506641, -0.016025718301534653, -0.006520925089716911, -0.0020652390085160732, 0.005196623969823122, -0.0036845740396529436, -0.006423697806894779, -0.005588885396718979, 0.028618311509490013, -0.015569756738841534, 0.01931132562458515, 0.001284907222725451, 0.01653532311320305, 0.011901945807039738, 0.0071344622410833836, -0.007060703355818987, -0.026217807084321976, -0.0007744646281935275, 0.0008540903218090534, 0.015730684623122215, 0.023347929120063782, -0.013853194192051888, -0.0033442790154367685, -0.014134817756712437, 0.0008876169449649751, -0.023656373843550682, -0.02408551424741745, 0.030522624030709267, -0.008006153628230095, -0.013826373033225536, 0.01689741015434265, -0.0006877145497128367, 0.00923322793096304, 0.01877490058541298, 0.0029805153608322144, 0.011533153243362904, 0.04004418104887009, -0.01705833710730076, 0.010782157070934772, -0.02062556892633438, 0.02224825695157051, -0.038890864700078964, 0.001781939179636538, -0.0066550313495099545, -0.01736678183078766, 0.012378023006021976, -0.0018439634004607797, -0.01767522655427456, -0.02918155863881111, 0.0024239737540483475, 0.005746460519731045, 0.0011399046052247286, -0.00789886899292469, -0.005360904615372419, -0.004405396059155464, 0.011137538589537144, 0.015288133174180984, -0.04910977557301521, 0.02346862480044365, -0.043718695640563965, -0.005464836955070496, 0.030871300026774406, -0.009132647886872292, 0.01539541780948639, -0.009266754612326622, 0.007677593268454075, -0.004113714676350355, -0.01434938795864582, -0.019941626116633415, -0.05514456331729889, 0.03596734255552292, -0.010775451548397541, -0.005860451143234968, -0.009830000810325146, 0.0015874848468229175, -0.00853587407618761, 0.023240644484758377, -0.013383821584284306, 0.011412457562983036, 0.018359169363975525, 0.0013167575234547257, -0.014684653840959072, 0.028001422062516212, -0.014872402884066105, -0.005961030721664429, -0.008039680309593678, 0.0014659508597105742, -0.024031871929764748, -0.0036510475911200047, -0.019753877073526382, 0.026030058041214943, 0.012766932137310505, -0.03167593851685524, -0.004224352538585663, -0.02018301747739315, -0.01522107981145382, 0.005357551854103804, -0.014657832682132721, -0.001892576925456524, 0.01846645586192608, -0.012994913384318352, 0.01618664525449276, -0.0020099200773984194, 0.02110835164785385, 0.017930028960108757, -0.028806060552597046, 0.0005188242648728192, -0.006199069786816835, -0.012015935964882374, 0.0032353175338357687, 0.012029347009956837, -0.021550903096795082, 0.015086973085999489, 0.02711631916463375, 0.006899775937199593, -0.0036007575690746307, 0.023280875757336617, -0.008763855323195457, -0.03382164239883423, -0.005971088539808989, -0.010399953462183475, -0.006815959233790636, -0.025614328682422638, 0.0018942533060908318, -0.043316375464200974, -0.019767288118600845, -0.020102553069591522, 0.004086893517524004, -0.021658187732100487, 0.024984028190374374, 0.03146136924624443, -0.024152567610144615, 0.03435806557536125, -0.02224825695157051, -0.021215636283159256, 0.017272908240556717, -0.018453044816851616, -0.010192088782787323, 0.01417504996061325, 0.00037654570769518614, 0.006172248162329197, 0.03513588383793831, 0.01033290009945631, 0.0050960443913936615, 0.012277443893253803, -0.012337791733443737, -0.008757149800658226, -0.003671163460239768, -0.03111269138753414, 0.02987891249358654, -0.02228848822414875, 0.0023971523623913527, -0.008770560845732689, 0.023844122886657715, -0.012304265052080154, 0.012270738370716572, -0.028940167278051376, 0.029986197128891945, -0.002432355424389243, -0.044416047632694244, -0.011533153243362904, -0.019230861216783524, -0.018010493367910385, -0.011043664067983627, -0.002935254480689764, -0.01287421677261591, -0.0040433090180158615, -0.0012538951123133302, 0.010373132303357124, 0.005699523258954287, 0.0015262988163158298, 0.0010544117540121078, 0.014054353348910809, -0.011050369590520859, 0.02215438149869442, 0.1974046677350998, 0.012579183094203472, -0.0017467362340539694, 0.03746933490037918, -0.02045123092830181, 0.025547275319695473, 0.044416047632694244, -0.009353923611342907, 0.01624028943479061, 0.008931487798690796, -0.008804086595773697, -0.022476237267255783, -0.015033330768346786, 0.0025916066952049732, 0.01852009817957878, -0.01653532311320305, -0.031649116426706314, -0.024729225784540176, -0.021510671824216843, -0.03374117612838745, -0.0033409264869987965, -0.0060414946638047695, -0.03408985584974289, -0.0313272625207901, 0.006054905243217945, -0.01935155689716339, 0.0075367814861238, 0.022717628628015518, 0.0005850393208675086, 0.010379837825894356, -0.014456672593951225, -0.024139156565070152, 0.016991283744573593, 0.002911785850301385, -0.014027532190084457, 0.002288190880790353, -0.013149135746061802, -0.014818760566413403, -0.007811699528247118, 0.021685009822249413, 0.016374394297599792, 0.0038723230827599764, -0.007972626946866512, -0.019069934263825417, -0.018788309767842293, 0.002232871949672699, 0.005344141274690628, -0.018479865044355392, -0.011331993155181408, -0.018533509224653244, -0.04382598027586937, -0.016039129346609116, 0.021483849734067917, 0.03172958269715309, 0.012136631645262241, 0.006544393952935934, 0.0033090761862695217, 0.0062761809676885605, -0.007322210818529129, -0.009045478887856007, -0.018278706818819046, 0.021443618461489677, -0.020826729014515877, 0.026740821078419685, 0.0006386818713508546, 0.029208380728960037, 0.01059440802782774, 0.008281071670353413, 0.004546207841485739, -0.03320475295186043, 0.01780933327972889, -0.016079360619187355, -0.002407210413366556, -0.0016696250531822443, -0.019499074667692184, -0.034948136657476425, 0.005887272302061319, 0.016293931752443314, 0.0026435728650540113, 0.043986909091472626, -0.008750444278120995, -0.024206209927797318, 0.013041850179433823, 0.008079912513494492, -0.017031516879796982, -0.03154183179140091, 0.035162705928087234, -0.010829093866050243, 0.0007157930522225797, 0.013001617975533009, 0.0045495606027543545, -0.007047292776405811, -0.017085159197449684, -0.0037415693514049053, 0.02356250025331974, 0.00649410393089056, -0.009099121205508709, 0.018010493367910385, -0.009897054173052311, 0.000367744971299544, -0.014054353348910809, 0.021819116547703743, 0.018399402499198914, -0.01597207598388195, -0.00941427145153284, 0.014644421637058258, 0.008730328641831875, 0.00835483055561781, 0.0008566048345528543, -0.019458843395113945, 0.003563878359273076, -0.02452806569635868, -0.0027944426983594894, 0.004941822029650211, 0.011861713603138924, -0.004365164320915937, 0.020478051155805588, -0.01736678183078766, -0.011533153243362904, -0.008864435367286205, 0.0061286636628210545, -0.003433124627918005, 0.007463022600859404, -0.0016008955426514149, -0.0002237901062471792, 0.004586440045386553, -0.04886838421225548, -0.013772730715572834, 0.02531929314136505, -0.01715221256017685, -0.0035974050406366587, -0.02808188647031784, 0.019096756353974342, -0.015891611576080322, 0.014899224042892456, -0.02303948439657688, 0.029717985540628433, -0.013330179266631603, -0.004177415277808905, 0.015609988011419773, 0.003785153850913048, -0.00614207424223423, 0.00017873873002827168, 1.0025240953837056e-05, 0.005562064237892628, -0.002963752020150423, 0.011097307316958904, -0.019646592438220978, -0.008616338483989239, 0.0051161604933440685, -0.018882185220718384, -0.011070486158132553, 0.00014563120203092694, 0.010205498896539211, 0.021135173738002777, -0.018801720812916756, -0.012954681180417538, -0.011868419125676155, -0.007053998298943043, 0.017567941918969154, -0.03682562708854675, -0.006953418254852295, 0.034063033759593964, -0.012324380688369274, -0.04229716584086418, 0.008502347394824028, -0.1712270826101303, 0.01432256679981947, 0.007255157921463251, -0.013088787905871868, 0.007161283399909735, -0.0026435728650540113, 0.016092771664261818, 0.008871139958500862, -0.031568653881549835, -0.0034767091274261475, 0.013544749468564987, -0.0010954819153994322, -0.015770915895700455, 0.0036812215112149715, 0.010708398185670376, 0.005719639360904694, -0.011264939792454243, 0.02022324874997139, -0.005699523258954287, -0.014054353348910809, 0.05332071706652641, 0.004311521537601948, 0.016387805342674255, -0.017259497195482254, 0.018359169363975525, -0.004542855080217123, -0.005776634439826012, -0.00020377890905365348, -0.025721613317728043, -0.00836153607815504, -0.00039142314926721156, 0.01272669993340969, 0.022798093035817146, 0.016267109662294388, -0.0015439002308994532, -0.029503414407372475, -0.01252554077655077, -0.014282334595918655, -0.019016291946172714, 0.036959730088710785, 0.010936379432678223, 0.023964818567037582, 0.02543998882174492, 0.01966000162065029, -0.025587506592273712, 0.003245375584810972, 0.005404489114880562, -0.0068025486543774605, -0.0028464088682085276, -0.006420345511287451, -0.006101842503994703, -0.03500177711248398, 0.012082989327609539, -0.008462116122245789, 0.004358458798378706, 0.022033685818314552, 0.004898237530142069, -0.002749181818217039, 0.0069869449362158775, -0.013973889872431755, -0.020960835739970207, -0.031166333705186844, 0.016857178881764412, -0.009273459203541279, -0.006393523886799812, -0.025413168594241142, -0.011533153243362904, 0.007831815630197525, -0.010218909941613674, 0.005944267380982637, -0.011197886429727077, -0.022047096863389015, 0.01852009817957878, -0.011150949634611607, 0.012404845096170902, -0.007851931266486645, -0.015086973085999489, 0.00941427145153284, -0.0011130833299830556, -0.012619415298104286, -0.009219816885888577, 0.040687888860702515, -0.018627382814884186, 0.0006210803985595703, -0.02057192660868168, 0.007999448105692863, -0.007617244962602854, 0.02018301747739315, 0.02904745191335678, -0.0070070610381662846, 0.033848464488983154, -0.004650140646845102, 0.01271999441087246, -0.005572122056037188, -0.018050724640488625, 0.027840495109558105, 0.03358025103807449, -0.013243010267615318, 0.02250305935740471, -0.010165267623960972, -0.0051161604933440685, -0.003354337066411972, -0.020022090524435043, 0.029235200956463814, -0.010587702505290508, 0.007322210818529129, 0.01732655055820942, 0.03411667421460152, 0.002152408240363002, -0.0033928926568478346, -0.01701810583472252, -0.008160375989973545, 0.021309511736035347, -0.02883288264274597, 0.002541316905990243, 0.012713289819657803, 0.00665838411077857, -0.0017232676036655903, 0.0008993512601591647, 0.0038387966342270374, 0.05332071706652641, -0.0025379641447216272, -0.004837889224290848, -0.006416992750018835, -0.012485308572649956, 0.020303713157773018, -0.09543013572692871, -0.02821599319577217, -0.009501440450549126, -5.998119740979746e-05, -0.008757149800658226, 0.02899380959570408, -0.006913186516612768, 0.018627382814884186, -0.038059405982494354, 0.011010137386620045, 0.01917721889913082, 0.008549285121262074, 0.006856190972030163, -0.00031326423049904406, 0.0343044251203537, -0.010688282549381256, 0.010862620547413826, -0.004693725146353245, -0.02864513359963894, 0.027384532615542412, 0.01860056258738041, 0.01970023475587368, -0.004318227060139179, -0.02912791632115841, -0.0015933520626276731, -0.017608173191547394, -0.022342130541801453, 0.030764015391469002, 0.01511379424482584, 0.02110835164785385, -0.017567941918969154, 0.011197886429727077, 0.0070942300371825695, -0.008757149800658226, -0.00822742935270071, -0.0020048911683261395, -0.009575199335813522, -0.022476237267255783, -0.009206406772136688, -0.03411667421460152, 0.010319489985704422, -0.003946081735193729, 0.014577369205653667, -0.0012455134419724345, -0.006091784685850143, 0.00544136855751276, -0.012069578282535076, 0.031005406752228737, 0.007167988456785679, -0.014872402884066105, -0.026646947488188744, -0.00022001836623530835, -0.009360629133880138, -0.013457580469548702, 0.014644421637058258, 0.007845225743949413, 0.002569814445450902, 0.0009152763523161411, -0.024675583466887474, 0.03154183179140091, -0.009642252698540688, 0.01719244383275509, 0.014228692278265953, 0.00969589501619339, 0.008301188237965107, -0.012860806658864021, -0.023414982482790947, -0.013638623990118504, -0.025775255635380745, -0.024031871929764748, -0.015811147168278694, 0.006353292148560286, 0.012820574454963207, -0.005994557403028011, -0.01618664525449276, 0.007677593268454075, 0.022436005994677544, 0.011171065270900726, -0.014563958160579205, -0.014577369205653667, -0.04401373118162155, -0.012310970574617386, 0.008790676482021809, -0.024112336337566376, 0.00039435672806575894, -0.005146334413439035, 0.018278706818819046, 0.0015019919956102967, 0.019499074667692184, -0.017742279917001724, 0.01641462743282318, 0.029020631685853004, 0.021658187732100487, -0.008153670467436314, -0.03234647214412689, 0.018667614087462425, 0.018064135685563087, -0.005042401608079672, 0.00940756592899561, -0.0007375853601843119, -0.000812182086519897, -0.01628052070736885, -0.06807242333889008, 0.005847040098160505, -0.002410563174635172, 0.007603834383189678, -0.03497495502233505, 0.013061965815722942, 0.007751351688057184, 0.006554451771080494, 0.008395062759518623, 0.011479509994387627, -0.01257247757166624, 0.002485997974872589, -0.015328364446759224, 0.021872758865356445, -0.03741569444537163, -0.005910740699619055, -0.015154026448726654, -0.018748078495264053, 0.011204591952264309, 0.007194810081273317, 0.015328364446759224, 0.040258750319480896, 0.008214019238948822, 0.009809885174036026, 0.009488029405474663, 0.0008410987793467939, 0.0068226647563278675, 0.0016411273973062634, -0.00914605800062418, -0.035162705928087234, 0.006051552481949329, -0.0037952119018882513, -0.028269635513424873, 0.013893426395952702, -0.02320041134953499, -0.02601664699614048, 0.013350294902920723, 0.008294482715427876, -0.002618428086861968, -0.011392340995371342, -0.03293653950095177, -0.00014060220564715564, 0.015314954333007336, -0.0041505941189825535, -0.021121762692928314, -0.012813868932425976, -0.008388357236981392, -0.011419162154197693, 0.0010024455841630697, -0.0013745909091085196, 0.020169606432318687, 0.018748078495264053, -0.01987457275390625, 0.009897054173052311, -0.02443419210612774, -0.00755689712241292, -0.029154738411307335, -0.03272197023034096, 0.002519524423405528, -0.0070070610381662846, 0.05479588732123375, 0.018882185220718384, 0.011010137386620045, 0.012894333340227604, 0.048546526581048965, 0.028537848964333534, -0.001964659197255969, -0.006054905243217945, 0.023683195933699608, -0.005213387310504913, -0.029342487454414368, -0.0029922497924417257, 0.017755690962076187, 0.01917721889913082, 0.019861161708831787, 0.008307892829179764, 0.026137342676520348, 0.02267739735543728, -0.016039129346609116, 0.02430008538067341, 0.020196428522467613, -0.00388238113373518, -0.01208969485014677, 0.017970262095332146, 0.015194258652627468, -0.005763223860412836, -0.020303713157773018, 0.01542223896831274, 0.00809332262724638, 0.010715103708207607, -0.019807519391179085, 0.017702048644423485, -0.004586440045386553, -0.006973534356802702, 0.008965014480054379, 0.00969589501619339, -0.019190629944205284, 0.005853745620697737, 0.005159744992852211, 0.01886877417564392, 0.006249359808862209, 0.02431349642574787, 0.001911016646772623, 0.003384510986506939, -0.01432256679981947, 0.02058533765375614, 0.0001342111936537549, -0.01736678183078766, -0.009595314972102642, 0.0014341006753966212, 0.035645488649606705, 0.0015531200915575027, 0.006973534356802702, 0.010011045262217522, -0.017272908240556717, 0.014255513437092304, 0.01952589675784111, -0.023535678163170815, -0.0013410643441602588, 0.009300281293690205, 0.01328994706273079, 0.0042947581969201565, -0.008985131047666073, -0.01983434148132801, 0.03561866655945778, -0.01622687838971615, 0.008636454120278358, -0.042967699468135834, 0.009367333725094795, 0.007302095182240009, 0.0005008036969229579, -0.03567231073975563, -0.00272571318782866, -0.006530982907861471, -0.014255513437092304, 0.00635999720543623, 0.004505976103246212, 0.010212204419076443, 0.006068315822631121, 0.038676295429468155, 0.023441804572939873, -0.016334163025021553, 0.011680670082569122, -0.026123931631445885, 0.005334082990884781, -0.0020602100994437933, 0.02193981222808361, -0.027545461431145668, 0.0025128191336989403, 0.015341775491833687, -0.006051552481949329, 0.032614681869745255, -0.03395574912428856, -0.024863332509994507, 0.00042096845572814345, -0.013960478827357292, 0.018976058810949326, 0.016200056299567223, 0.004066777415573597, 0.0386226512491703, 0.0008775589521974325, 0.023991640657186508, 0.013216188177466393, -0.00020629340724553913, -0.021617956459522247, 0.0343044251203537, -0.0004429703112691641, -0.019016291946172714, -0.016213467344641685, 0.007013766095042229, 0.021215636283159256, 0.0032973417546600103, -0.022932199761271477, 0.004536150023341179, -5.269963730825111e-05, -0.005870508961379528, -0.006098489742726088, 0.02294561080634594, 0.02202027477324009, 0.002605017274618149, 0.008374946191906929, 0.012471897527575493, 0.006728790234774351, 0.008716917596757412, -0.022275077179074287, -0.024340316653251648, -0.015301543287932873, -0.004251173697412014]} +{"id": "test:10000095", "text": "\"Educating women, in particular, has unmatched transformative power. It not only empowers them but also has a positive effect on their family and surroundings. Women with more education tend to have fewer children, which benefits them, their families and the society at large. Education is also one of the most powerful elements of improving people\u2019s health. It saves the lives of millions of mothers and children, helps prevent diseases, and is an essential component of efforts to reduce malnutrition. Educated mothers are better informed about diseases; therefore, they take preventative measures, recognize signs of illness early and tend to use health care services more often.\\nTo create awareness regarding the importance of girl child eductaion, SOC Films in collaboration with Chime of Change, created a series of short documentaries on young girls who want to achieve their dreams and aspirations to seek excellence in various professions in future whether it's sports, literature, science, math, activism, or so on and so forth. Chime for Change, a global movement is aimed at strengthening voices speaking out to support the cause of promoting justice, health and education for girls and women the world over. These documentaries were screened at an event in New York on 11th October this year, which is also celebrated as the International Day of the Girl Child. The event was also featured in Huffington Post.\\nAreeba Fatima, a student of Dawood Public School, one of the biggest girls\u2019 school in Karachi, is also featured in one of the documentaries where she expresses her desire to become a diplomat to be able to represent Pakistan at the UN and paint a positive picture of the country. The documentary can be viewed here: http://vimeo.com/107582646 . Dawood Public School is among the very few schools that aim to empower women by giving them the confidence of facing the world through participation in many extracurricular activities including sports, debates, performing arts etc.\"", "vector_field": [-0.017089538276195526, -0.001430576085112989, -0.01733175292611122, -0.03154163807630539, -0.0024154111742973328, -0.009103206917643547, -0.010711238719522953, -0.010913083329796791, -0.016605110839009285, -0.044325150549411774, 0.006203367840498686, 0.022418245673179626, 0.023023780435323715, 0.020238319411873817, -0.021476302295923233, 0.02294304221868515, 0.03393686190247536, -0.009688557125627995, -0.012628765776753426, -0.022283682599663734, -0.03818906471133232, -0.00025924461078830063, 0.02301032468676567, 0.011027461849153042, -0.014761594124138355, -0.0046188849955797195, 0.022081837058067322, -0.03808141499757767, 0.00402007857337594, -0.018367890268564224, 0.009338692761957645, 0.02088422328233719, -0.019081074744462967, -0.009782752022147179, -0.030814995989203453, 0.008107438683509827, -0.009749110788106918, -0.02586306631565094, 0.026455143466591835, -0.00047812022967264056, 0.024234849959611893, -0.006314382888376713, -0.0024641903582960367, 0.026576250791549683, -0.012453833594918251, 0.025755414739251137, -0.007004019804298878, 0.004053719341754913, -0.009103206917643547, 0.02839285507798195, 0.010549762286245823, 0.020870767533779144, -0.02319871261715889, 0.0011505162110552192, -0.03251049295067787, -0.007205864414572716, -0.008490944281220436, 0.022673916071653366, 0.014317534863948822, -0.0022858940064907074, 0.011000549420714378, 0.0005988066550344229, -0.03867349028587341, 0.005920785013586283, -0.021489758044481277, 0.014990351162850857, -0.02650897018611431, 0.007818127982318401, 0.015098001807928085, 0.036143701523542404, 0.026482056826353073, 0.030303655192255974, 0.0022926221136003733, 0.009950956329703331, 0.013368863612413406, -0.010953452438116074, -0.020722748711705208, -0.0004760176525451243, 0.018785037100315094, -0.0024927849881350994, 0.0019696701783686876, -0.013456329703330994, -0.015138370916247368, -0.006815630942583084, -0.0038619665428996086, -0.0010563218966126442, 0.0012590078404173255, 0.03356008604168892, -0.00916376058012247, 0.002171515254303813, -0.009035925380885601, -0.00964146014302969, 0.011458064429461956, 0.008975371718406677, 0.0059342412278056145, 0.029792314395308495, -0.0002836342027876526, 0.01302572712302208, 0.022054923698306084, -0.0018788400338962674, -0.018408259376883507, 0.017210645601153374, -0.0020823669619858265, -0.008847536519169807, -0.016161052510142326, 0.0025129695422947407, -0.00787868071347475, 0.0038619665428996086, 0.020265232771635056, -0.043652333319187164, -0.03905026987195015, 0.007306787185370922, 0.0007922414224594831, -0.045966822654008865, 0.04018060117959976, -0.022081837058067322, -0.021005330607295036, 0.0001861809432739392, 0.002220294438302517, 0.005409444682300091, 0.0185428224503994, 0.005298429634422064, 0.035443972796201706, -0.0030512227676808834, -0.0034851895179599524, -0.013658174313604832, -0.01707608252763748, -0.0031302787829190493, -0.0029334798455238342, -0.014707768335938454, -0.006768533959984779, 0.001524770399555564, 0.012339454144239426, 0.0017308203969150782, -0.014169515110552311, -0.006189911626279354, -0.016470547765493393, 0.012077055871486664, -0.036951079964637756, -0.01870429888367653, 0.0077575743198394775, 0.03283344581723213, -0.012568212114274502, -0.01193576492369175, -0.023763878270983696, 0.02018449455499649, 0.006714708637446165, -0.0015163602074608207, 0.009136848151683807, 0.029415536671876907, 0.016739673912525177, 0.007441350258886814, 0.0054868184961378574, 0.0066339704208076, 0.009870218113064766, 0.015568973496556282, 0.02855433151125908, 0.0028830186929553747, -0.00848421547561884, 0.0002306499081896618, 0.0028678802773356438, -0.012245260179042816, -0.006943466141819954, 0.0017291384283453226, 0.014761594124138355, 0.014761594124138355, -0.00037803876330144703, 0.0068391794338822365, -0.01052284985780716, -0.008961915969848633, -0.005762673448771238, 0.04244126379489899, -0.019430940970778465, 0.0015298165380954742, -0.006284106057137251, 0.015622799284756184, 0.017654703930020332, 0.02576887235045433, 0.016510916873812675, -0.024746190756559372, -0.02727598138153553, 0.014707768335938454, 0.022068381309509277, 0.010643957182765007, -0.04066503047943115, -0.0013321766164153814, 0.002930115908384323, -0.01005860697478056, 0.0031841041054576635, -0.012944988906383514, 0.006465766578912735, 0.004655890166759491, 0.009890402667224407, -0.015811188146471977, -0.654946506023407, -0.014613574370741844, 0.02262009121477604, 0.0067786262370646, 0.0054868184961378574, 0.00863223522901535, 0.00763646699488163, 0.0023464474361389875, -0.006166363134980202, 0.021637778729200363, -0.010657412931323051, 0.018314065411686897, -0.025015316903591156, 0.00332875968888402, -0.006284106057137251, -0.019175270572304726, 0.01879849284887314, -0.046155210584402084, 0.02679155208170414, -0.024880753830075264, 0.0034851895179599524, 0.0044035837054252625, -0.008046885021030903, 0.0011698596645146608, -0.006997291464358568, 0.00925795454531908, -0.010872714221477509, -0.014815418981015682, -0.005412808619439602, 0.03183767572045326, -0.00626392150297761, 0.02239133231341839, -0.005961154121905565, 0.03574001044034958, 0.04459427669644356, -0.017910374328494072, -0.015636255964636803, 0.009204129688441753, 0.0238715298473835, 0.022969955578446388, -0.03837745264172554, -0.01819295808672905, 0.004992298316210508, -0.00939924642443657, 0.01992882415652275, 0.02009030058979988, 0.018973425030708313, -0.01970006711781025, 0.019242551177740097, 0.008511128835380077, -0.013362135738134384, -0.020776573568582535, 0.003143734997138381, -0.01761433482170105, 0.021557040512561798, 0.0016786771593615413, 0.015219109132885933, -0.01886577345430851, 0.016322528943419456, 0.0033523081801831722, 0.021906904876232147, -0.0006231962470337749, -0.012480746023356915, 0.013792738318443298, -0.015165283344686031, 0.03921174630522728, -0.014115690253674984, 0.01566316746175289, -0.00020142443827353418, -0.004837550688534975, -0.005977974738925695, 0.030734257772564888, -0.02009030058979988, -0.023575490340590477, 0.014990351162850857, 0.02611873671412468, 0.01195594947785139, -0.01501726359128952, -0.02377733401954174, 0.011881939135491848, 0.011114928871393204, -0.01838134601712227, 0.014479011297225952, -0.0147750498726964, -0.0018855682574212551, -0.024234849959611893, 0.015098001807928085, 0.006862728390842676, -0.02646860107779503, -0.006519591901451349, 0.01879849284887314, 0.005752581171691418, 0.0080536138266325, -0.012191434390842915, 0.025042230263352394, 0.01092654000967741, -0.008914818055927753, 0.0017812816658988595, 0.0295500997453928, -0.009950956329703331, -0.02595726028084755, -0.0132679408416152, 0.00991731509566307, 0.015730449929833412, 0.0071654957719147205, 0.002842649584636092, -0.007804671302437782, -0.0031302787829190493, 0.02971157617866993, -0.024880753830075264, -0.00400325795635581, -0.0013473150320351124, -0.010004781186580658, -0.010805432684719563, -0.004198374692350626, -0.015568973496556282, 0.015932293608784676, 0.00801997259259224, 0.02364277094602585, -0.006297562271356583, -0.008968643844127655, -0.008928274735808372, -0.015057632699608803, -0.015098001807928085, 0.015595885924994946, 0.002038633916527033, -0.0017711893888190389, 0.022216400131583214, 0.0014978577382862568, -0.02146284654736519, -0.010220082476735115, 0.001952849910594523, 0.007569185458123684, -0.0006963650812394917, 0.02695302851498127, 0.013806194067001343, -0.0018939784495159984, -0.01585155725479126, 0.017210645601153374, -0.0011109882034361362, 0.007804671302437782, -0.00527824554592371, 0.02442323789000511, -0.031460899859666824, -0.002649214817211032, -0.022808479145169258, -0.030976470559835434, -0.02255280874669552, -0.022149119526147842, -0.013900388963520527, -0.011377327144145966, -0.01768161728978157, -0.008174720220267773, 0.0380006767809391, 0.011700279079377651, 0.0019141628872603178, -0.0033035289961844683, 0.012198163196444511, 0.0059140571393072605, -0.020224863663315773, 0.003592840163037181, 0.028742721304297447, -0.006812267005443573, -0.003569291438907385, -0.012870979495346546, -0.005392624065279961, -0.0077643021941185, 0.008968643844127655, -0.008087254129350185, -0.02403300441801548, 0.002676127478480339, -0.018919600173830986, -0.001767825335264206, 0.02512296847999096, 0.03272579237818718, 0.029792314395308495, -0.01102073397487402, 0.03086882084608078, -0.0043060253374278545, 0.003338851733133197, -0.0021849714685231447, -0.043302468955516815, -0.042306702584028244, -0.012830610387027264, 0.008497672155499458, 0.010206625796854496, -0.002839285647496581, -0.010933267883956432, -0.01953859068453312, -0.010643957182765007, -0.01649746112525463, 0.017721986398100853, -0.005409444682300091, -0.00813435111194849, 0.008349652402102947, -0.0048308223485946655, 0.019592415541410446, 0.002563430927693844, 0.03608987480401993, 0.020978417247533798, 0.03350626304745674, -0.009244498796761036, 0.022028012201189995, 0.0011496752267703414, 0.0178430937230587, -0.03334478661417961, -0.003502009902149439, -0.016147594898939133, 0.027666214853525162, 0.00791232194751501, 0.0007148674922063947, -0.01652437262237072, 0.0003435569233261049, 0.0027232246939092875, -0.015165283344686031, 0.01501726359128952, -0.0070847575552761555, -0.00032989034662023187, -0.025432463735342026, 0.00011406341945985332, -0.0026105279102921486, -0.025876522064208984, 0.022324051707983017, 0.007111670449376106, 0.006795446388423443, 0.032671969383955, 0.006284106057137251, -0.012198163196444511, 0.022350963205099106, -0.022350963205099106, -0.012985358014702797, 0.01900033839046955, 0.00864569190889597, 0.030734257772564888, -0.008046885021030903, 0.018946511670947075, 0.015555516816675663, -0.009762567467987537, 0.026926115155220032, -0.011707006953656673, 0.008901362307369709, 0.012211618945002556, 0.029415536671876907, -0.002311124699190259, 0.001053798827342689, -0.03159546107053757, 0.016093770042061806, -0.006018343381583691, -0.011128384619951248, 0.019107988104224205, -0.007347156293690205, 0.01451937947422266, -0.008114166557788849, -0.0030814995989203453, -0.0027938704006373882, -0.02646860107779503, 0.01114856917411089, 0.011410967446863651, 0.04074576497077942, 0.028958022594451904, 0.024948036298155785, 0.00904938206076622, 0.0076095545664429665, -0.020615097135305405, -0.00848421547561884, 0.021637778729200363, 0.017143364995718002, -0.015232564881443977, -0.04211831092834473, -0.03923865780234337, -0.020130669698119164, -0.009634732268750668, 0.015246021561324596, -0.008470759727060795, 0.009977868758141994, 0.00450787041336298, -0.0027989165391772985, 0.016820412129163742, 0.019874999299645424, 0.02903875894844532, -0.01267586275935173, 0.003290072549134493, -0.006923281587660313, 0.025822697207331657, -0.004501142539083958, -0.009150303900241852, -0.025499744340777397, -0.011242763139307499, 0.00488464767113328, 0.009964412078261375, -0.008457303047180176, 0.0012707822024822235, -0.004739992320537567, -0.0050091189332306385, -0.005675206892192364, -0.018475539982318878, 0.014452097937464714, 0.009735655039548874, 0.023831160739064217, -0.004063811618834734, -0.0016685848822817206, -0.009365605190396309, -0.029415536671876907, 0.013577437028288841, 0.010818889364600182, 0.004265656694769859, -0.0015811187913641334, -0.0026340766344219446, -0.013718727976083755, -0.025836152955889702, 0.009971139952540398, -0.022969955578446388, 0.0022034740541130304, -0.000510079029481858, 0.012070327997207642, 0.0033270774874836206, 0.010408471338450909, 0.007360612507909536, 0.006697888020426035, 0.012009774334728718, -0.013449601829051971, -0.010590131394565105, -0.018731210380792618, 0.0073875249363482, 0.10576675087213516, 0.0058669596910476685, -0.012958445586264133, 0.012083783745765686, -0.007125126663595438, 0.001169018680229783, -0.014142602682113647, -0.015057632699608803, 0.002857788000255823, 0.008349652402102947, 0.007737389765679836, -0.022969955578446388, -0.006405212916433811, -0.013886932283639908, -0.01517874002456665, 0.010213354602456093, 0.006300926208496094, -0.004713079426437616, -0.000540776236448437, -0.005786221940070391, 0.01953859068453312, -0.0001415017177350819, 0.012366367504000664, 0.00490146828815341, 0.023158343508839607, -0.01177428849041462, 0.011034190654754639, 0.03439437970519066, -0.009870218113064766, -0.00930505245923996, 0.0004118898359593004, 0.025419006124138832, 0.02384461648762226, -0.006041891872882843, -0.012574939988553524, 0.02602454274892807, 0.00525469658896327, -0.030761169269680977, 0.01427716575562954, -0.002526425989344716, 0.024974947795271873, 0.03533632308244705, 0.006122630089521408, -0.009136848151683807, 0.012635493651032448, 0.004689530935138464, 0.0011454700725153089, 0.024826928973197937, 0.01879849284887314, 0.0006976266158744693, 0.03401760011911392, 0.006055348552763462, -0.024853840470314026, -0.003848510328680277, 0.021382108330726624, -0.015165283344686031, 0.005547372158616781, 0.0022001098841428757, 0.009500169195234776, 0.01030082069337368, -0.02984613925218582, -0.01315356232225895, 0.011357142589986324, -0.01468085590749979, 0.014156059361994267, -0.023239081725478172, -0.003962889313697815, -0.007845040410757065, -0.012837338261306286, 0.02454434521496296, -0.0027720038779079914, -0.0421721376478672, -0.03423290327191353, -0.0017257742583751678, 0.024786559864878654, 0.010805432684719563, 0.01953859068453312, 0.017695073038339615, -0.0015903699677437544, 0.016860781237483025, -0.006781990174204111, -0.026078367605805397, -0.022229857742786407, -0.015044176951050758, -0.0004810637910850346, 0.0076768361032009125, -0.013913844712078571, -0.016484003514051437, -0.05436357110738754, 0.03417907655239105, 0.011444608680903912, 0.014088776893913746, 0.00750863179564476, -0.012110697105526924, -0.007865224964916706, 0.02755856327712536, 0.016187964007258415, 0.01909453235566616, 0.014115690253674984, -0.03493263199925423, 0.007979603484272957, -0.012756600975990295, -0.02650897018611431, -0.00263071246445179, 0.0002086782333208248, 0.007616282440721989, -0.018852317705750465, 0.019323289394378662, -0.005163866560906172, -0.033102571964263916, 0.014398273080587387, -0.03183767572045326, 0.01624179072678089, 0.004245472140610218, -0.01603994518518448, 0.022660458460450172, 0.02060164138674736, 0.03590148687362671, -0.00626392150297761, -0.014573205262422562, 0.003660121699795127, -0.026751182973384857, 0.021974187344312668, 0.010617043823003769, -0.021974187344312668, -0.009056109935045242, 0.003276616334915161, -0.008396749384701252, 0.0012581668561324477, 0.016806956380605698, 0.00628074211999774, 0.025715045630931854, -0.006748349405825138, -0.01976734772324562, -0.011040918529033661, -0.01844862848520279, -0.01179447304457426, 0.0032547498121857643, -0.03740859776735306, 0.008511128835380077, 0.008531313389539719, -0.00502930348739028, -0.014573205262422562, -0.04873882606625557, 0.03969617187976837, -0.027935341000556946, -0.00988367386162281, -0.004346394445747137, 0.014505923725664616, -0.006714708637446165, 0.00751536013558507, 0.012447104789316654, -0.014721225015819073, 0.0058366828598082066, -0.0034263180568814278, -0.021826166659593582, -0.008026700466871262, -0.010630500502884388, 0.020399795845150948, 0.01819295808672905, 0.03089573234319687, -0.023212168365716934, 0.007320243399590254, 0.0066406987607479095, 0.013328494504094124, -0.006990563124418259, -0.00025588052812963724, -0.0009638096089474857, -0.017560509964823723, 0.031245598569512367, 0.01893305592238903, -0.010132616385817528, -0.0068055386655032635, -0.01800457015633583, 0.014761594124138355, 0.03536323457956314, -0.008141079917550087, -0.04688185453414917, -0.012003046460449696, -0.03385612741112709, -0.023319819942116737, 0.015891926363110542, -0.01206360012292862, 0.011929036118090153, -0.036816518753767014, 0.017062626779079437, 0.016147594898939133, -0.010354645550251007, 0.021731972694396973, -0.018919600173830986, 0.011061103083193302, -0.001515519106760621, 0.006371572148054838, -0.01591883786022663, 0.002186653669923544, -0.014801963232457638, -0.013146834447979927, -0.020399795845150948, 0.013012271374464035, 0.0112965889275074, 0.005173958837985992, 0.004565059673041105, -0.004282476846128702, -0.016214877367019653, -0.0147750498726964, 0.017372122034430504, -0.0023683139588683844, -0.03216062858700752, -0.009574178606271744, -0.027639301493763924, -0.006311018485575914, -0.001764461281709373, -0.0040705399587750435, -0.043302468955516815, 0.017802724614739418, -0.0041041807271540165, -0.0140214953571558, 0.015838099643588066, 0.00477363308891654, -0.01068432629108429, -0.0006000681896694005, -0.009681829251348972, 0.01684732548892498, 0.007771030534058809, 0.02519024908542633, -0.006307654548436403, 0.004914924502372742, 0.0018166045192629099, 0.02903875894844532, 0.00450787041336298, 0.006586873438209295, 0.038162149488925934, 0.007845040410757065, -0.003781228559091687, -0.0004219820839352906, -0.002164787147194147, 0.02213566191494465, -0.006997291464358568, -0.01988845504820347, 0.015003807842731476, 0.0208573117852211, -0.017654703930020332, -0.0157573614269495, 0.014828875660896301, -0.013389048166573048, 0.007663379888981581, 0.006129358429461718, 0.009372333995997906, -0.002679491648450494, -0.012783513404428959, -0.012447104789316654, -0.008908090181648731, -0.010125888511538506, 0.016255246475338936, -0.031945325434207916, -0.0069501944817602634, -0.01317374687641859, -0.024207936599850655, -0.026926115155220032, 0.016551285982131958, -0.007320243399590254, 0.012830610387027264, -0.0014810373540967703, 0.002928433706983924, 0.011740648187696934, -0.014600117690861225, 0.0038720588199794292, 0.0063917567022144794, -0.0052446043118834496, 0.031164860352873802, -0.002227022545412183, -0.00413109315559268, 0.008060341700911522, -0.015165283344686031, -0.008686061017215252, -0.006489315070211887, -0.009345420636236668, -0.022539352998137474, -0.017896918579936028, -0.020561272278428078, -0.01065068505704403, 0.000522273825481534, -0.024275219067931175, -0.025822697207331657, -0.00848421547561884, 0.002529789926484227, -0.008524584583938122, -0.01582464389503002, 0.027908427640795708, 0.0038586026057600975, -0.020063387230038643, 0.004985569976270199, -0.006549868732690811, -0.003192514181137085, -0.004565059673041105, -0.009823121130466461, -0.013375591486692429, 0.024948036298155785, -0.03393686190247536, -0.006435489747673273, 0.01076506357640028, 0.012312541715800762, -0.0029570285696536303, -0.02358894608914852, -0.009210857562720776, -0.00012447104381863028, 0.007252961862832308, -0.003152145305648446, -0.009964412078261375, 0.01893305592238903, -0.0005521300481632352, 0.028069904074072838, -0.0038619665428996086, -0.014627030119299889, 0.007717205211520195, 0.011114928871393204, -0.00018123994232155383, -0.03853892907500267, -0.0412571057677269, 0.013806194067001343, 0.0025684768334031105, -0.01519219670444727, 0.031111033633351326, -0.021287914365530014, 0.021032243967056274, -0.016107227653265, 0.02753164991736412, -0.0010958499042317271, -0.007407709490507841, 0.01302572712302208, -0.01684732548892498, 0.005819862708449364, 5.690029865945689e-06, -0.01469431258738041, 0.004208466969430447, -0.008477487601339817, -0.014734680764377117, 0.020574728026986122, 0.003249703673645854, 0.01258166879415512, 0.010495937429368496, 0.0034851895179599524, 0.001888932310976088, 0.007804671302437782, 0.00600825110450387, -0.005533915478736162, -0.023091061040759087, -0.013664903119206429, -0.03958852216601372, 0.0014919706154614687, 0.006041891872882843, 0.03574001044034958, 0.0208573117852211, -0.010341189801692963, -0.005678571294993162, 0.005752581171691418, -0.044136762619018555, -0.01427716575562954, -0.008174720220267773, 0.010549762286245823, 0.011397511698305607, 0.016699304804205894, 0.0020302238408476114, -0.014492467045783997, -0.015219109132885933, 0.024113742634654045, -0.04873882606625557, -0.006553232669830322, 0.009715470485389233, -0.028984934091567993, -0.0014944936847314239, -0.002891428768634796, -0.027424000203609467, 0.006462402176111937, 0.04424441233277321, -0.012083783745765686, 0.002491103019565344, -0.006374936085194349, 0.0025281079579144716, 0.013409232720732689, -0.008827351965010166, 0.014627030119299889, 0.02092459239065647, 8.357642400369514e-06, 0.01717027649283409, -0.021543584764003754, 0.013261212967336178, -0.004036898724734783, -0.0017240922898054123, -0.014115690253674984, -0.012171249836683273, -0.023064149543642998, -0.004921652842313051, 0.015595885924994946, -0.01268931943923235, 0.016672393307089806, 0.013012271374464035, -0.01067086961120367, 0.019552046433091164, 0.005796314217150211, 0.0008805485558696091, 0.01155225932598114, 0.02625329978764057, 0.0032059706281870604, 0.004161369986832142, 0.004770268686115742, -0.004225287586450577, -0.008235273882746696, 0.0001214223520946689, -0.016981888562440872, -0.018314065411686897, -0.017654703930020332, -0.024275219067931175, -0.003848510328680277, 0.009850033558905125, -0.016322528943419456, -0.01608031429350376, -0.035605449229478836, 0.015124914236366749, -0.019054163247346878, -0.005251332651823759, 0.0011740648187696934, -0.006284106057137251, -0.01758742332458496, -0.015232564881443977, -0.013792738318443298, -0.00864569190889597, -0.01963278464972973, -0.007562457118183374, 0.002317852806299925, -0.0005517094978131354, -0.00490146828815341, -0.0009368969476781785, 0.025688134133815765, -0.010966909117996693, 0.0024877390824258327, 0.20475152134895325, 0.0068862768821418285, -0.006341295316815376, 0.013738912530243397, 0.006849271710962057, 0.008154535666108131, 0.029334798455238342, -0.008363109081983566, -0.00048484839498996735, -0.0039595249108970165, -0.013415960595011711, 0.0018182866042479873, -0.022162575274705887, -0.00475681247189641, -0.024073373526334763, -0.023225625976920128, -0.03143398463726044, -0.012595124542713165, -0.0016290568746626377, -0.014290622435510159, 0.017156820744276047, -0.019081074744462967, -0.02175888605415821, -0.012366367504000664, 0.018206413835287094, 0.017062626779079437, -0.0032934367191046476, 0.008544769138097763, 0.0271145049482584, 0.0017695074202492833, -0.014600117690861225, -0.0147750498726964, -0.003626480931416154, 0.0011656545102596283, -0.025782328099012375, 0.0014473964693024755, 0.010872714221477509, -0.028931109234690666, 0.0026441689115017653, 0.006324475165456533, 0.003791320836171508, 0.03549779951572418, -0.010852529667317867, -0.020803485065698624, 0.01643017865717411, 0.04319481924176216, -0.00814780779182911, 0.012958445586264133, -0.009029197506606579, -0.0064253974705934525, -0.02150321565568447, 0.002650897018611431, 0.0059476979076862335, 0.017883462831377983, -0.02092459239065647, -0.004326209891587496, 0.0030932738445699215, 0.005123497452586889, -0.02971157617866993, 0.009446343407034874, 0.0032968008890748024, 0.020332515239715576, -0.016120683401823044, 0.03164928779006004, -0.016161052510142326, 0.010738151147961617, -0.009580906480550766, -0.011323501355946064, -0.008342924527823925, -0.004144549369812012, -0.00501248287037015, -0.009157032705843449, -0.02146284654736519, 0.007670107763260603, -0.02130137011408806, -0.012958445586264133, 0.03003452718257904, 0.03358699753880501, 0.004608793184161186, 0.045643869787454605, -0.015528604388237, -0.008329467847943306, -0.031111033633351326, -0.0017325024819001555, -0.01492306962609291, -0.019713522866368294, 0.0007060368079692125, -0.00829582754522562, -0.015582430176436901, 0.001801466103643179, -0.01787000522017479, -0.009587635286152363, -0.009331964887678623, -0.0035221942234784365, -0.003091591876000166, -0.0024490519426763058, 0.00863223522901535, 0.004124365281313658, -0.0002109910419676453, 0.018596647307276726, -0.013469786383211613, 0.03867349028587341, 0.015420953743159771, 0.0066743395291268826, 0.019646242260932922, -0.0016374671831727028, 0.0015676624607294798, -0.0012127517256885767, -0.007528816349804401, -0.031164860352873802, 0.0032631598878651857, -0.028231380507349968, 0.00864569190889597, -0.008329467847943306, -0.012716231867671013, 0.028904195874929428, 0.00980293657630682, -0.022512439638376236, 0.023952266201376915, -0.010859258472919464, -0.01105437520891428, -0.020642010495066643, -0.002955346368253231, 0.03205297887325287, -0.01754705421626568, -0.0112965889275074, -0.0380006767809391, 0.0333716981112957, -0.02069583535194397, -0.00250287726521492, 0.010334460996091366, -0.020803485065698624, -0.005688663572072983, -0.013160290196537971, -0.018233327195048332, -0.0016879283357411623, 0.019430940970778465, 0.0029015210457146168, -0.0014692629920318723, 0.021974187344312668, -0.008591866120696068, -0.023212168365716934, 0.010065334849059582, 0.018314065411686897, 0.01320738811045885, -0.043140992522239685, -0.006815630942583084, -0.0019965828396379948, 0.011532074771821499, -0.009661644697189331, -0.012857522815465927, -2.3022939785732888e-05, 0.012776785530149937, -0.03224136680364609, 0.010004781186580658, 0.006475858855992556, -0.02755856327712536, -0.009157032705843449, 0.0012581668561324477, 0.0025079234037548304, -0.008201632648706436, 0.024396326392889023, 0.02201455645263195, 0.013066096231341362, 0.008141079917550087, 0.0034195897169411182, -0.17181041836738586, 0.02971157617866993, 0.011498433537781239, -0.03307566046714783, 0.006687795743346214, -0.02412720024585724, 0.027491282671689987, 0.010146073065698147, -0.023158343508839607, -0.031164860352873802, -0.001366658485494554, 0.009924042969942093, -0.03221445530653, -0.005096585024148226, 0.004282476846128702, 0.017183732241392136, -0.009459800086915493, 0.03805449977517128, 0.02785460278391838, 0.008201632648706436, 0.021355194970965385, -0.018731210380792618, 0.004457409027963877, -0.014088776893913746, -0.008161264471709728, 0.007966146804392338, -0.005755945108830929, 0.012985358014702797, -0.011754103936254978, -0.0329141840338707, -0.0032513856422156096, -0.0009713788167573512, 0.020574728026986122, 0.001840153126977384, 0.004400219768285751, -0.00048190480447374284, -0.027639301493763924, -0.02772003971040249, -0.014357903972268105, 0.014398273080587387, 0.0140214953571558, 0.01214433740824461, -0.015932293608784676, -0.0035793837159872055, 0.02593034692108631, 0.021355194970965385, 0.006617150269448757, -0.011982861906290054, 0.02855433151125908, -0.0015987801598384976, 0.027101047337055206, -0.009042653255164623, 0.004911560565233231, -0.005604561418294907, -0.0014179607387632132, 0.0003456594713497907, -0.01102073397487402, 0.01992882415652275, -0.008275642991065979, -0.016699304804205894, -0.005624745972454548, -0.017439402639865875, 0.009553994052112103, 0.0077575743198394775, -0.001460852799937129, -0.006832451559603214, -0.024086831137537956, 0.021409019827842712, -0.027908427640795708, 0.020870767533779144, -0.02412720024585724, 0.00031937757739797235, -0.0248000156134367, -0.005712212063372135, -0.010852529667317867, -0.00729333097115159, -0.027033766731619835, 0.0025365182664245367, -0.0027198607567697763, -0.0008313488797284663, -0.019363658502697945, 0.017896918579936028, -0.03641282767057419, -0.0037946850061416626, -0.007347156293690205, -0.014075321145355701, -0.02187999151647091, -0.0012606899254024029, -0.009937499649822712, -0.010260451585054398, 0.015138370916247368, -0.02852742001414299, -0.01992882415652275, -0.0016046672826632857, 0.01928292028605938, 0.01113511249423027, 0.009372333995997906, 0.012467289343476295, -0.023117974400520325, -0.012554755434393883, 0.005789585877209902, -0.022162575274705887, 0.000844384718220681, 0.01951167732477188, 0.01976734772324562, 0.0017274563433602452, 0.012447104789316654, -0.016066858544945717, 0.051968347281217575, 0.009405974298715591, -0.011538802646100521, -0.00613608630374074, 0.02461162768304348, 0.00016841437900438905, 0.01180792972445488, 0.03821597620844841, 0.0009495122940279543, 0.005718939937651157, -0.0018536094576120377, 0.0023817704059183598, 0.07029587030410767, -0.002714814618229866, -0.023669684305787086, 0.006879548542201519, -0.004602064844220877, -0.027935341000556946, -0.11852335184812546, 0.002371678128838539, 0.012998814694583416, 0.021234087646007538, -0.0034919176250696182, 0.0059006004594266415, -0.01493652630597353, 0.008215089328587055, -0.009984596632421017, 0.02169160358607769, -0.016658935695886612, -0.032456666231155396, 0.010415199212729931, 0.0008069592877291143, 0.041310932487249374, -0.005772765260189772, -0.012500930577516556, -0.00012320952373556793, -0.026172561571002007, 0.03926556929945946, -0.005062944255769253, -0.018663929775357246, 0.0016862463671714067, -0.006795446388423443, 0.0077104768715798855, 0.002925069769844413, -0.019753891974687576, 0.02442323789000511, -0.002958710538223386, 0.0005660068709403276, -0.020144125446677208, -0.008464031852781773, 0.028312118723988533, -0.024948036298155785, -0.0071318550035357475, -0.006351387593895197, -0.01277005672454834, -0.01649746112525463, 0.0028830186929553747, -0.03969617187976837, 0.0022102021612226963, 0.017533598467707634, 0.014034952037036419, -0.021099524572491646, 0.004121000878512859, -0.03135325014591217, -0.01909453235566616, 0.012325998395681381, 0.01874466799199581, -0.009029197506606579, -0.056355107575654984, -0.0012396643869578838, -0.0382428877055645, 0.003777864621952176, 0.012655678205192089, 0.006620514206588268, 0.014263710007071495, -0.009358877316117287, 0.0023346731904894114, 0.008289098739624023, -0.0034851895179599524, 0.006748349405825138, -0.030922645702958107, 0.030492043122649193, 0.015568973496556282, -0.04793144762516022, -0.0208573117852211, 0.021718516945838928, 0.011659909971058369, -0.03189150243997574, -0.010321005247533321, 0.007064573001116514, -0.007172223646193743, 0.020171038806438446, -0.017816180363297462, 0.04421750083565712, -0.017533598467707634, -0.01649746112525463, 0.019969193264842033, -0.006566688884049654, -0.008928274735808372, -0.016564741730690002, 0.021745428442955017, -0.004978842101991177, -0.012299085035920143, -0.015447866171598434, 0.009890402667224407, 0.0006429602508433163, -0.007394253276288509, -0.013294854201376438, 0.00020510390459094197, 0.023414013907313347, -0.018731210380792618, -0.022660458460450172, -0.002378406235948205, 0.03557853773236275, 0.0023195347748696804, -0.014721225015819073, 0.031945325434207916, 0.010677597485482693, -0.018717754632234573, -0.012884436175227165, -0.03514793515205383, 0.01819295808672905, 0.013792738318443298, 0.007703748997300863, -0.009850033558905125, -0.014452097937464714, 0.005604561418294907, -0.01127640437334776, 0.016578197479248047, 0.003404451534152031, -0.015259478241205215, 0.021731972694396973, 0.004699623212218285, 0.015286390669643879, -0.04123019427061081, -0.032779619097709656, 0.02920023538172245, 0.020911136642098427, 0.019901912659406662, 0.024181025102734566, -0.008706245571374893, -0.00613608630374074, 0.00826218631118536, 0.022741196677088737, 0.004299297463148832, 0.014627030119299889, -0.0027316350024193525, 0.003465004963800311, -0.001603826298378408, -0.0031874680425971746, -0.0012985358480364084, -0.031111033633351326, 0.004141185432672501, -0.012736416421830654, 0.0011564033338800073, -0.005483454558998346, 0.009587635286152363, 0.018717754632234573, 0.016632024198770523, 0.028150642290711403, -0.009473255835473537, -0.015905382111668587, -0.014384816400706768, -0.03207989037036896, 0.0105430344119668, 0.011659909971058369, -0.017789268866181374, -0.007340427953749895, 0.01076506357640028, 0.007441350258886814, 0.05460578575730324, 0.020615097135305405, -0.0053556193597614765, 0.0062941983342170715, 0.021866535767912865, -0.01339577604085207, 0.0014835603069514036, 0.004013350233435631, -0.010125888511538506, 0.0030377665534615517, 0.0012657360639423132, 0.003121868474408984, 0.010051878169178963, 0.00867933314293623, 0.02130137011408806, -0.021731972694396973, -0.01067086961120367, 0.007851768285036087, 0.023763878270983696, -0.03272579237818718, -0.006600329652428627, -0.0033994053956121206, 0.017721986398100853, 0.01842171512544155, -0.0025499744806438684, 0.004329574294388294, 0.005059579852968454, 0.00526142492890358, 0.012225075624883175, 0.046128299087285995, -0.018758123740553856, 0.0248000156134367, -0.042629651725292206, 0.03318331018090248, 0.019040707498788834, 0.009829849004745483, -0.01937711425125599, -0.02101878635585308, 0.005829954985529184, -0.00010775576811283827, 0.00039254638249985874, 0.025405550375580788, 0.004840914625674486, 0.013860019855201244, 0.01201650220900774, 0.012870979495346546, 0.008248730562627316, -0.02002301812171936, 0.011377327144145966, 0.017210645601153374, 0.026320580393075943, -0.010839073918759823, -0.005099948961287737, -0.011047646403312683, -0.01026717945933342, 0.006788718514144421, -0.016605110839009285, -0.017049169167876244, -0.0015441138530150056, 0.019619328901171684, -0.008618779480457306, -0.0023767242673784494, 0.0018216506578028202, -0.0017594151431694627, -0.024557800963521004, 0.01116875372827053, 0.025109510868787766, -0.021583953872323036, -0.026522425934672356, 0.03135325014591217, -0.0073000588454306126, -0.0023212169762700796, 0.020319057628512383, -0.006654154974967241, 0.049976807087659836, 0.0004127308784518391, 0.026495512574911118, -0.027746951207518578, 0.03221445530653, 0.00157270859926939, 0.00750863179564476, 0.0185428224503994, -0.03740859776735306, -0.008302555419504642, 0.008282370865345001, -0.027127960696816444, 0.017277928069233894, 0.010697782039642334, -0.012971902266144753, 0.038646578788757324, 0.020453620702028275, -0.0027349989395588636, 0.019054163247346878, 0.006788718514144421, 0.02547283284366131, 0.015367128886282444, 0.040611203759908676, 0.010442111641168594, 0.003387630917131901, 0.01838134601712227, -0.02968466281890869, 0.0034885534550994635, -0.002247207099571824, -0.009560721926391125, 0.0157573614269495, -0.02739708684384823, 0.026778096333146095, 0.011175481602549553, 0.017816180363297462, 0.02554011344909668, -0.005251332651823759, 0.013886932283639908, 0.0031336427200585604, -0.019982649013400078, 0.004639069549739361, 0.01052284985780716, 0.013106465339660645, -0.01787000522017479, -0.00729333097115159, -0.011101472191512585, 0.0100249657407403, -0.03436746448278427, -0.02463853918015957, 0.01330158207565546, -0.022768110036849976, -0.0014558066613972187, 0.008995556272566319, 0.016806956380605698, 0.023884985595941544, 0.027827689424157143, 0.0008540564449504018, -0.018811948597431183, -0.027235612273216248, -0.0061495425179600716, 0.010771792382001877, -0.009654916822910309, -0.005567556247115135, -0.029630837962031364]} +{"id": "test:10000096", "text": "\"I was satisfied with how they allocated time.\\nBuild Vacation time the day you start. Receive so many hours per pay period.\\nThere is no encouragement not to use vacation.\\nWhat can you tell job seekers about Fulton County (Georgia)'s Vacation & Paid Time Off?\\\"\\nGlassdoor is your resource for information about the Vacation & Paid Time Off benefits at Fulton County (Georgia). Learn about Fulton County (Georgia) Vacation & Paid Time Off, including a description from the employer, and comments and ratings provided anonymously by current and former Fulton County (Georgia) employees.\"", "vector_field": [0.009536589495837688, 0.0183065515011549, 0.016812948510050774, -0.05535583943128586, -0.027862967923283577, 0.02084435522556305, -0.039811793714761734, -0.005439094267785549, -0.023897649720311165, -0.012821194715797901, 0.020024854689836502, 0.0169451255351305, -0.006000848021358252, -0.00119950866792351, -0.009463892318308353, 0.004682379774749279, 0.02069895900785923, -0.02211325615644455, 0.0070516569539904594, -0.010666705667972565, 0.0013275553938001394, 0.021267320960760117, -0.01928466372191906, -0.015094643458724022, -0.007593583781272173, 0.01747383363544941, 0.008142119273543358, -0.0037604430690407753, -0.016522157937288284, 0.028550289571285248, 0.01432801503688097, 0.0019248314201831818, 0.01525325607508421, 0.0007658020476810634, -0.00917971134185791, -0.01216030865907669, -0.01738131046295166, -0.008168554864823818, 0.011248284950852394, -0.01191578060388565, 0.0015506044728681445, 0.005908323917537928, 0.0014828636776655912, 0.004368458408862352, -0.003724094480276108, 0.04509888216853142, -0.0031061656773090363, -0.004712119698524475, -0.010541136376559734, 0.007243314292281866, 0.009477109648287296, 0.01792323775589466, -0.022232215851545334, 0.0008137162658385932, 0.008921965025365353, -0.0051747397519648075, 0.00020446169946808368, 0.015887707471847534, -0.024545317515730858, -0.05403406545519829, 0.021002966910600662, -0.013409383594989777, 0.004788121208548546, 0.02006450854241848, -0.00549196545034647, -0.016773294657468796, -0.024690713733434677, 0.006572514306753874, -0.035079844295978546, 0.0068864356726408005, 0.02479645423591137, 0.020831136032938957, -0.0018851782660931349, 0.0011986825847998261, 0.03740616515278816, -0.011816647835075855, -0.008955009281635284, -0.015226820483803749, 0.0011292895069345832, 0.017566358670592308, -0.01635032705962658, -0.015345780178904533, -0.009391194209456444, 0.025510212406516075, 0.006199113558977842, 0.041371483355760574, 0.0035588727332651615, 0.008928574621677399, -0.027519306167960167, -0.010349479503929615, -0.007871155627071857, 0.02568204328417778, 0.03486836329102516, 0.015808401629328728, 0.004487418103963137, 0.021901773288846016, -0.019007090479135513, 0.014605587348341942, 0.005531618371605873, -0.03875437378883362, -0.011056628078222275, 0.04121287167072296, 0.005211088806390762, -0.0030565992929041386, -0.020394951105117798, -0.017949672415852547, 0.020923661068081856, -0.01801576092839241, 0.028497418388724327, -0.031088093295693398, -0.024241309612989426, 0.016389980912208557, 0.004408111795783043, -0.012807976454496384, -0.04390928894281387, -0.016271021217107773, -0.016046319156885147, -0.009073968976736069, -0.031008785590529442, -0.011241676285862923, 0.0008178467978723347, 0.013376339338719845, 0.015993449836969376, 0.013237552717328072, 0.005429181270301342, -0.00976129062473774, 0.012318921275436878, 0.010785664431750774, 0.009463892318308353, -0.017460616305470467, 0.01810828596353531, 0.037723392248153687, 0.006129720713943243, -0.01567622274160385, -0.03378450870513916, -0.0028814643155783415, -0.006688169669359922, -0.014037225395441055, -0.033810943365097046, 0.018095066770911217, 0.022827014327049255, 0.018240462988615036, 0.007864546962082386, -0.016627900302410126, 0.013045895844697952, 0.021743159741163254, 0.012140481732785702, 0.026911290362477303, -0.003101208945736289, -0.005630751606076956, 0.014658458530902863, -0.02796870842576027, 0.00047418594476766884, -0.0024188440293073654, 0.003978535532951355, -0.01325737964361906, -0.009272235445678234, -0.018280114978551865, 0.02870890125632286, -0.040102582424879074, 0.014235490933060646, 0.013812524266541004, 0.016376763582229614, 0.01806863211095333, -0.00886248517781496, 0.028180193156003952, 0.02807445079088211, -0.010818708688020706, 0.011783602647483349, 0.008769961073994637, -0.00019816262647509575, 0.011354027315974236, -0.026052137836813927, 0.02572169527411461, 0.005630751606076956, 0.019654758274555206, 0.010408959351480007, -0.013838959857821465, 0.01303267851471901, 0.025840654969215393, -0.023897649720311165, 0.010012427344918251, -0.005673708859831095, 0.032779961824417114, -0.005310221575200558, -0.0038364450447261333, 0.037273988127708435, 0.012147090397775173, 0.030109981074929237, -0.0030731214210391045, 0.0010210693581029773, 0.00045890294131822884, 0.021174797788262367, -0.02841811254620552, -0.6344508528709412, -0.02089722454547882, -0.019297881051898003, -0.027995144948363304, -0.001407687901519239, 0.00793063547462225, -0.004114017356187105, 0.0024105829652398825, -0.017857149243354797, -0.006010761018842459, -0.003978535532951355, 0.030929479748010635, 0.019549017772078514, 0.008439518511295319, 0.000738540489692241, -0.02089722454547882, 0.0031656455248594284, 0.013283815234899521, 0.0004593159828800708, 0.017011214047670364, -0.033863816410303116, 0.031907591968774796, -0.01305911410599947, 0.030136415734887123, 0.02046103961765766, 0.021994296461343765, 0.022086821496486664, 0.005306917242705822, -0.023686164990067482, 0.0031639933586120605, -0.028100885450839996, 0.02562917210161686, 0.0019413535483181477, -0.012120654806494713, 0.05014805495738983, 0.007772023323923349, -0.011433333158493042, -0.01777784153819084, 0.0009111970430240035, 0.04427938535809517, -0.020632870495319366, -0.006285028997808695, 0.025113679468631744, -0.006780693773180246, -0.01839907467365265, -0.04121287167072296, 0.004728641826659441, 0.014724547043442726, 0.012781541794538498, -0.006311464589089155, -0.015530828386545181, 0.03114096261560917, -0.003107817843556404, -0.010554354637861252, -0.01806863211095333, -0.012847630307078362, 0.015173950232565403, 0.008756743744015694, -0.013402774930000305, -0.011016975156962872, 0.002192490268498659, 0.017407745122909546, -0.03994397073984146, -0.003390346886590123, -0.029951367527246475, 0.03373163938522339, 0.022879883646965027, -0.014169402420520782, 0.010864971205592155, 0.0054655298590660095, 0.011869518086314201, 0.026435453444719315, -0.01044861227273941, -0.002423800528049469, 0.017420964315533638, 0.009285452775657177, 0.00232466752640903, -0.00035770473186858, -0.007765414193272591, -0.00937136821448803, 0.0028500722255557775, -0.015028554946184158, -0.01995876617729664, 0.020527129992842674, -0.0011953781358897686, -0.014433757402002811, -0.019456492736935616, -0.006070240866392851, 0.021412717178463936, -0.026990598067641258, 0.005035953596234322, 0.01699799671769142, 0.011294547468423843, -0.027598612010478973, -0.02894682064652443, 0.017936455085873604, 0.009259017184376717, -0.006156156305223703, 0.022337958216667175, -0.03222481533885002, -0.012946763075888157, -0.04031406715512276, -1.9878221792168915e-05, 0.01007851678878069, 0.03267421945929527, 0.007758805528283119, 0.00820159912109375, -0.02079148404300213, 0.019932331517338753, -0.014975683763623238, 0.009338323958218098, -0.024584971368312836, 0.007282967213541269, -0.0022321436554193497, -0.012312312610447407, -0.03523845970630646, 0.001238335738889873, -0.01166464388370514, -0.01772497035562992, 0.004576637875288725, -0.005072302650660276, 0.0009756334475241601, 0.0272813867777586, -0.030797302722930908, -0.016046319156885147, 0.021544894203543663, 0.004973169416189194, -0.037273988127708435, 0.0021264017559587955, 0.01723591610789299, -0.03119383379817009, 0.009906685911118984, 0.037723392248153687, -0.0004878167237620801, 0.010534527711570263, -0.01479063555598259, 0.026937726885080338, -0.023937301710247993, 0.00849899835884571, -0.05488000065088272, -0.0018108285730704665, 0.01991911418735981, -0.005696840118616819, -0.011149152182042599, 0.0007806719513610005, -0.0012441185535863042, -0.011215240694582462, 0.00603058747947216, 0.0006042979075573385, 0.009662157855927944, 0.0076134102419018745, -0.002592326607555151, -0.020593218505382538, -0.00817516352981329, -0.014063660986721516, -0.00864439271390438, 0.008360211737453938, -0.03164323791861534, -0.004209845792502165, -0.022589094936847687, 0.003340780269354582, 0.0005316004389896989, -0.011307764798402786, -0.016535375267267227, -0.015953795984387398, 0.005237523932009935, 0.021121926605701447, -0.005776146426796913, 0.0017100434051826596, -0.010673314332962036, 0.011109499260783195, 0.0008244556956924498, 0.007170616649091244, 0.032436300069093704, 0.024135569110512733, 0.0005303612560965121, 0.0010838535381481051, -0.0019281358690932393, -0.015173950232565403, -0.010375915095210075, 0.0046427263878285885, 0.014222273603081703, -0.005277177318930626, 0.006807128898799419, 0.024928631260991096, -0.005062389187514782, -0.001746392110362649, 0.043433450162410736, -0.047372329980134964, 0.009060751646757126, 0.018147937953472137, -0.013601040467619896, -0.015041772276163101, 0.014552717097103596, 0.003193733049556613, 0.016099190339446068, -0.022205781191587448, 0.007058266084641218, 0.015808401629328728, 0.0011730731930583715, 0.012173525989055634, 0.007778631988912821, 0.004543593619018793, 0.018372640013694763, -0.007468015421181917, -0.019588669762015343, 0.024029826745390892, -0.03989109769463539, 0.002640240825712681, 0.009107013233006, 0.004708814900368452, -0.011486204341053963, -0.006741040386259556, 0.0020834440365433693, 0.034392524510622025, 0.008677436970174313, -0.02367294766008854, 0.0007600192911922932, 0.021690288558602333, 0.009655549190938473, 0.03774982690811157, 0.0013060765340924263, 0.024373486638069153, 0.015002119354903698, -0.01645606942474842, 0.014063660986721516, 0.0045204623602330685, -0.00014642762835137546, 0.017698535695672035, 0.006443641614168882, -0.00443124258890748, 0.03320292755961418, -0.0020884007681161165, 0.027202080935239792, 0.04306335374712944, 0.02158454805612564, 0.01249735988676548, -0.01855768822133541, 0.014513063244521618, -0.003413477912545204, 0.001298641669563949, -0.0017050866736099124, 0.02894682064652443, -0.03140531852841377, -0.011776993982493877, 0.006496512331068516, 0.02899969182908535, -0.015041772276163101, -0.03188115730881691, -0.006760866846889257, -0.010038862936198711, 0.007884373888373375, -0.023739036172628403, 0.010858362540602684, 0.004338718485087156, -0.0027410259936004877, -0.01301285158842802, -0.0029971194453537464, 0.029845627024769783, 0.022139692679047585, -0.008617958053946495, 0.017658881843090057, 0.003446522168815136, -0.0367981493473053, -0.010461830534040928, 0.009886858984827995, -0.009800944477319717, -0.01850481703877449, 0.014936030842363834, 0.0004717076080851257, -0.0034498265013098717, -0.017156608402729034, 0.0013267293106764555, 0.02635614573955536, 0.009734855964779854, -0.01981337182223797, -0.032542042434215546, -0.009946338832378387, 0.010289999656379223, -0.00011162782902829349, -0.01191578060388565, -0.028338804841041565, 0.009662157855927944, 0.029184740036725998, 0.012451098300516605, -0.0031557322945445776, -0.00802976917475462, -0.011922389268875122, -0.012484142556786537, 0.009563025087118149, -0.007924026809632778, 0.00524743739515543, -0.02176959626376629, 0.002769113751128316, 0.011109499260783195, 0.011776993982493877, 0.013944701291620731, -0.00341678224503994, -0.01714339107275009, -0.001100375666283071, -0.013971136882901192, 0.011512639932334423, -0.02533838152885437, -0.00126064068172127, 0.017592793330550194, 0.01703764870762825, -0.0021594460122287273, -0.011750558391213417, -0.009992601349949837, -0.005905019119381905, 0.02236439287662506, -0.0017744797514751554, 0.021743159741163254, -1.967169373529032e-05, 0.004794730339199305, 0.0036712235305458307, -0.015491175465285778, -0.004599768668413162, 0.025166550651192665, -0.014037225395441055, -0.00798350665718317, 0.001307728816755116, -0.023170674219727516, -0.003329214872792363, 0.11747915297746658, -0.00961589626967907, 0.008102466352283955, 0.0016290848143398762, 0.009272235445678234, 0.010422177612781525, 0.019839806482195854, -0.03240986540913582, 0.011433333158493042, -0.00773897860199213, 0.011030192486941814, -0.008512215688824654, -0.030347900465130806, -0.009741464629769325, 0.009146667085587978, -0.0038794027641415596, -0.0016051277052611113, -0.032885704189538956, -0.009807553142309189, 0.00500290933996439, -0.011624990031123161, 0.017606010660529137, 0.007454797625541687, -0.00021437500254251063, -0.014460192993283272, 0.02406948059797287, 0.03240986540913582, 0.016376763582229614, -0.005482051987200975, -0.004897167440503836, 0.0002191251260228455, -0.004507244564592838, 0.02026277408003807, 0.01997198536992073, 0.004897167440503836, 0.011697688139975071, -0.007890982553362846, 0.0022486657835543156, 0.027030250057578087, 0.0013349903747439384, 0.012728670611977577, -0.0030565992929041386, -0.016442852094769478, -0.017645664513111115, 0.0429576113820076, 0.0022734489757567644, -0.002440322656184435, 0.011102890595793724, 0.006575819104909897, 0.0015092991525307298, 0.02225865051150322, -0.001283771707676351, 0.010369306430220604, -0.022126473486423492, 0.013627476058900356, -0.013838959857821465, -0.00200083339586854, 0.004593160003423691, -0.0063048554584383965, -0.010838535614311695, -0.02245691791176796, -0.034445393830537796, -0.006800520233809948, -0.004404807463288307, 0.0010483309160917997, 0.005211088806390762, 0.01481707114726305, -0.01167125254869461, -0.008948400616645813, -0.0110103664919734, 0.013561387546360493, -0.008122293278574944, -0.016667552292346954, -0.000884761568158865, 0.0353442020714283, -0.0036910499911755323, -0.0024882371071726084, 0.013244162313640118, -0.019271444529294968, -0.013072331435978413, 0.008710482157766819, -0.014565934427082539, -0.002425452694296837, 0.011486204341053963, -0.02367294766008854, 0.02118801511824131, -0.007719152141362429, 0.004940125159919262, 0.030136415734887123, 0.016773294657468796, 0.003030163701623678, 0.0038959248922765255, 0.015398651361465454, -0.03365233168005943, -0.0006906262133270502, 0.02640901692211628, 0.004946734290570021, -0.0021858816035091877, 0.025007938966155052, -0.010878188535571098, 0.03671884536743164, -0.023263199254870415, 0.002663371851667762, -0.014024008065462112, -0.016720423474907875, 0.007884373888373375, 0.004408111795783043, 0.0037736608646810055, -0.007045048289000988, -0.01669398881494999, -0.01366712898015976, -0.004913689568638802, -0.00042028239113278687, 0.0042329770512878895, -0.0050987377762794495, 0.026858419179916382, -0.026831984519958496, 0.016575029119849205, -0.0037373120430856943, -0.008684046566486359, -0.0023494509514421225, -0.012993024662137032, 0.005567967426031828, 0.013363121077418327, -0.019099615514278412, 0.00886248517781496, 0.031801849603652954, -0.0353442020714283, 0.005240828730165958, -0.00016460199549328536, -0.006767475977540016, 0.022430481389164925, -0.01510786172002554, -0.015226820483803749, -0.033123623579740524, -0.012166917324066162, -0.02484932541847229, 0.00017668382497504354, -0.04393572360277176, -0.010924451053142548, 0.002230491256341338, -0.01727556809782982, -0.005452312063425779, -0.02557630091905594, 0.03367876634001732, -0.024426357820630074, -0.02118801511824131, -0.01567622274160385, 0.01464524120092392, 0.03206620365381241, -0.016429634764790535, 0.010323043912649155, -0.030982350930571556, 0.010461830534040928, 0.008459344506263733, -0.010785664431750774, -0.018425511196255684, 0.0003641070507001132, 0.04446443170309067, 0.009166493080556393, 0.02401660941541195, -0.0049797785468399525, -0.006020674481987953, 0.0022833622060716152, -0.004464287310838699, -0.007223487365990877, -0.014856724068522453, -0.008723699487745762, -0.008082639425992966, 0.014076878316700459, 0.023408593609929085, 0.0066253854893147945, 0.027651483193039894, 0.0003711289609782398, 0.006218940485268831, 0.020712178200483322, -0.0058455397374928, -0.017857149243354797, -0.03169610723853111, -0.019218573346734047, -0.0008657610742375255, -0.002339537488296628, 0.0005708405515179038, 0.012021522037684917, -0.00958946067839861, -0.02124088630080223, 0.03293857350945473, 0.00336886802688241, 0.007626628037542105, 0.003160688793286681, 0.01118219643831253, -0.01678651198744774, -0.01389183010905981, 0.007996724918484688, 0.014367668889462948, 0.0023428420536220074, 0.0010805490892380476, -0.0326213501393795, 0.0036943545565009117, 0.03447183221578598, 0.02533838152885437, 0.007547321729362011, -0.016389980912208557, 0.01271545235067606, -0.020328862592577934, 0.008763352409005165, -0.03835784271359444, -0.009444065392017365, -0.018095066770911217, -0.008267687633633614, -0.003139209933578968, -0.04026119410991669, -0.0020867486018687487, 0.01732843928039074, -0.00942423939704895, 0.022734489291906357, 0.0007067352999001741, 0.004953342955559492, 0.015887707471847534, -0.02544412389397621, 0.0007451493293046951, -0.017050867900252342, 0.011605164036154747, -0.0017050866736099124, 0.018319768831133842, 0.0014498194213956594, 0.012074393220245838, -0.009741464629769325, 0.0005022736149840057, 0.000546057301107794, 0.011036801151931286, 0.01810828596353531, -0.010898015461862087, 0.006093371659517288, -0.029845627024769783, 0.031801849603652954, 0.029554836452007294, 0.0010813752887770534, -0.0010598964290693402, -0.006972350645810366, 0.030453640967607498, -0.0010731141082942486, -0.005541531834751368, 0.008181772194802761, 0.016416415572166443, 0.03465687856078148, 0.01076583843678236, -0.0019826588686555624, -0.009430848062038422, 0.013164855539798737, -0.017050867900252342, 0.022337958216667175, -0.020038073882460594, 0.04345988482236862, 0.017024431377649307, -0.0023808430414646864, -0.016535375267267227, 0.0018670038552954793, -0.02528551034629345, 0.01320450846105814, -0.003238342935219407, 8.632827666588128e-05, -0.018412292003631592, 0.02865603007376194, -0.027122775092720985, 0.008651002310216427, 0.011003756895661354, 0.0027806791476905346, -0.002615457633510232, 0.022483352571725845, -0.013482080772519112, -0.0219282079488039, -0.03436608985066414, 0.021716725081205368, 0.0041834102012217045, -0.008399865590035915, -0.021359845995903015, -0.034736186265945435, -0.0037439209409058094, 0.008756743744015694, 0.019099615514278412, 0.0016604769043624401, -0.008776570670306683, -0.017011214047670364, 0.024770019575953484, -0.007686107885092497, 0.012206570245325565, 0.004669161979109049, 0.004497331567108631, -0.014222273603081703, -0.01596701331436634, -0.005805886350572109, 0.007600192911922932, 0.00917971134185791, -0.019905896857380867, -0.011340809054672718, -0.004579942207783461, 0.023355722427368164, -0.00888892076909542, 0.02494185045361519, -0.02430739812552929, -0.002655110787600279, -0.061488863080739975, -0.0020619654096663, -0.024294180795550346, -0.004510549362748861, 0.02841811254620552, 0.0009681984665803611, -0.009576242417097092, -0.026250405237078667, 0.006562601309269667, 0.013072331435978413, -0.020196685567498207, 0.002268492244184017, 0.005330048035830259, -0.0010706358589231968, 0.0015357346273958683, 0.0075407130643725395, -0.017024431377649307, 0.007996724918484688, -0.005908323917537928, -0.015094643458724022, 0.03431321680545807, -0.004391589667648077, 0.0018934393301606178, -0.02362007647752762, 0.030295029282569885, -0.0071573988534510136, -0.005078911315649748, -0.03925664722919464, 0.016337109729647636, -0.0036910499911755323, -0.014579152688384056, -0.011823256500065327, -0.023567205294966698, 0.004180105868726969, -0.009186320006847382, 0.02773079089820385, -0.03066512569785118, -0.037908438593149185, 0.03129957616329193, 0.024888979271054268, 0.006268506869673729, 0.024664277210831642, -4.069226633873768e-06, -0.006179287098348141, -0.006731127388775349, -0.032806396484375, 0.006741040386259556, -0.012166917324066162, 0.01435445062816143, 0.04057842120528221, 0.005442399065941572, -0.02876177243888378, -0.02323676273226738, -0.030797302722930908, -0.0270038153976202, -0.023553987964987755, 0.02631649374961853, 0.028470981866121292, 0.02968701347708702, 0.003684441326186061, 0.019496146589517593, 0.016138844192028046, 0.0007600192911922932, 0.015200385823845863, -0.007507668808102608, -0.01727556809782982, 0.012345356866717339, -0.006526252254843712, 0.023910867050290108, -0.008333776146173477, 0.004196627996861935, 0.018385857343673706, -0.004176801536232233, 0.0038100096862763166, -0.017500270158052444, -0.015940578654408455, 0.01709051989018917, 0.0009954599663615227, -0.01415618509054184, 0.015781965106725693, 0.004530375823378563, -0.0029029431752860546, 0.011320983059704304, -0.033863816410303116, -0.010369306430220604, 0.027466434985399246, -0.009648940525949001, -0.01486994232982397, -0.009278844110667706, 0.005022736266255379, -0.014711329713463783, 0.01605953834950924, 0.009311888366937637, -0.028867514804005623, 0.009265625849366188, -0.022496569901704788, -0.009556416422128677, 0.04174157977104187, -0.01079227402806282, -0.00814872793853283, 0.03447183221578598, -0.015623352490365505, 0.01530612725764513, -0.0014936031075194478, -0.037326861172914505, 0.008631175383925438, 0.003390346886590123, -0.005204479675740004, 0.009212755598127842, -0.013627476058900356, 0.026052137836813927, -0.013601040467619896, -0.010283390991389751, -0.03309718519449234, 0.003998362459242344, -0.03196046128869057, 0.008188381791114807, 0.004890558775514364, 0.00029554011416621506, 0.00269145960919559, -0.02860316075384617, -0.005482051987200975, -0.025259075686335564, -0.0007253227522596717, 0.001961180241778493, -0.024915413931012154, -0.011611772701144218, 0.007197052240371704, -0.0036877456586807966, 0.00018081437156070024, -0.02245691791176796, 0.031563930213451385, -0.02704346738755703, 0.025351598858833313, 0.20577356219291687, 0.011413507163524628, 0.008988053537905216, 0.017592793330550194, -0.02796870842576027, 0.008095857687294483, 0.007434971164911985, 0.006813738029450178, 0.007910809479653835, 0.008320558816194534, -0.010726184584200382, -0.00500290933996439, 0.02095009572803974, 0.0005324265221133828, -0.01054774597287178, -0.033070750534534454, -0.005230915267020464, -0.0245585348457098, -0.01772497035562992, 0.0076596722938120365, 0.007719152141362429, 0.005772842094302177, -0.007699325680732727, 0.0026038920041173697, 0.0008979793055914342, -0.018610559403896332, -0.013415992259979248, -0.010997148230671883, -0.01022391114383936, 0.030453640967607498, -0.010818708688020706, -0.024862542748451233, 0.01461880560964346, 0.021558111533522606, -0.025034373626112938, 0.008485780097544193, -0.012259441427886486, -0.019549017772078514, 0.0232896339148283, 0.01812150329351425, -4.217667992634233e-06, -0.004074363969266415, -0.027862967923283577, -0.027572177350521088, 0.015689441934227943, 0.02396373823285103, -0.004219759255647659, -0.00444446038454771, -0.011195414699614048, -0.002636936493217945, -0.028338804841041565, -0.02225865051150322, 0.009767900221049786, 0.015491175465285778, -0.005574576091021299, -0.001054113730788231, 0.014037225395441055, -0.023408593609929085, -0.02069895900785923, 0.0164031982421875, 0.025457341223955154, 0.022879883646965027, -0.005217697471380234, 0.01743418164551258, -0.01079227402806282, -0.0010532875312492251, 0.003324258141219616, 0.01279475912451744, 0.03521202132105827, -0.013561387546360493, 0.010534527711570263, 0.012338747270405293, -0.009404412470757961, 0.016416415572166443, -0.004807948134839535, -0.0038760981988161802, 0.02773079089820385, 0.04256108030676842, 0.00513839116320014, 0.010812100023031235, -0.028973257169127464, -0.025351598858833313, 0.001237509655766189, -0.0158612709492445, 0.0030202504713088274, -0.03309718519449234, 0.03891298547387123, -0.0035820037592202425, 0.01435445062816143, -0.04179444909095764, 0.0022734489757567644, 0.019853025674819946, -0.02621075138449669, -0.00639737956225872, 0.0016654335195198655, 0.00683356449007988, -0.004361849743872881, 0.016918689012527466, 0.007454797625541687, 0.020381733775138855, -0.016072755679488182, 0.04433225467801094, -0.004748468287289143, -0.006879826541990042, -0.009794334881007671, 0.007917418144643307, 1.3243542525742669e-05, 0.011638208292424679, 0.0068401736207306385, -0.01859734021127224, 0.010785664431750774, -0.017553141340613365, 0.006767475977540016, -0.01008512545377016, 0.01547795720398426, 3.805259621003643e-05, 0.015438304282724857, -0.034736186265945435, 0.03470974788069725, -0.006691474001854658, 0.01191578060388565, -0.025946397334337234, -0.01122845895588398, -0.01301285158842802, 0.005449007730931044, -0.025801001116633415, -0.03328223526477814, -0.0017331744311377406, -0.002653458621352911, -0.03838427737355232, 0.009437456727027893, 0.01600666716694832, 0.013138419948518276, -0.03928308188915253, -0.032779961824417114, 0.00035481335362419486, 0.0048112524673342705, 0.0021825770381838083, 0.0004721206787507981, -0.024188438430428505, 0.019311098381876945, -0.00286163785494864, 0.019654758274555206, -0.006569209974259138, -0.01777784153819084, -0.01761922985315323, 0.02362007647752762, -0.00942423939704895, -0.03222481533885002, -0.01933753304183483, -0.014116532169282436, -0.012933545745909214, 0.002544412389397621, -0.015438304282724857, 0.027202080935239792, 0.005095433443784714, -0.028814643621444702, -0.02270805463194847, 0.005762928631156683, -0.0040644509717822075, -0.022337958216667175, -0.02973988465964794, 0.037617649883031845, -0.01274188794195652, -0.027307823300361633, 0.00022841883765067905, -0.16590890288352966, 0.027413563802838326, -0.015742311254143715, 0.005670404527336359, 0.008056203834712505, 0.01968119479715824, 0.021558111533522606, 0.009285452775657177, -0.017804278060793877, 0.018874913454055786, 0.015385433100163937, -0.017169825732707977, -0.0009838944533839822, 0.00896161887794733, 0.015385433100163937, 0.008584912866353989, -0.05858096480369568, 0.004751772619783878, 0.02289310283958912, -0.008327167481184006, 0.022390829399228096, -0.005280481651425362, 0.002498150337487459, -0.013224335387349129, 0.004325501155108213, 0.025655606761574745, -0.013111984357237816, 0.0021264017559587955, 0.011902562342584133, -0.023884432390332222, -0.024730365723371506, -0.0025047592353075743, -0.0029987716116011143, -0.011096280999481678, 0.003352345898747444, -0.02796870842576027, -0.003608439350500703, -0.013211117126047611, -0.0036910499911755323, 0.007732369937002659, 0.009781117551028728, 0.0030483382288366556, -0.0011094629298895597, 0.011704296804964542, -0.014975683763623238, 0.004870732314884663, 0.005482051987200975, 0.011849692091345787, -0.00038785766810178757, -0.028735337778925896, 0.00958946067839861, -0.019271444529294968, 0.015729093924164772, 0.032198380678892136, 0.011730732396245003, -0.006516339257359505, -0.0006600602064281702, 0.009496936574578285, -0.004411416128277779, -0.013627476058900356, -0.02416200377047062, -0.01781749539077282, 0.0007149963639676571, -0.004936820827424526, 0.016125626862049103, -0.024730365723371506, -0.02474358305335045, -0.007395317777991295, -0.0013267293106764555, -0.010071907192468643, -0.006529557052999735, -0.01276832353323698, 0.011188805103302002, 0.014724547043442726, 0.007474624086171389, 0.001176377641968429, -0.03357302397489548, -0.0024056262336671352, 0.006146242842078209, 0.009034316055476665, -0.03357302397489548, 0.02825949899852276, -0.026197534054517746, -0.01850481703877449, -0.0021594460122287273, 0.010263564996421337, -0.0003564655489753932, 0.05215714871883392, 0.007527495268732309, -0.004556811414659023, 0.011545684188604355, 0.0010747663909569383, -0.02128054015338421, -0.009331715293228626, 0.015940578654408455, 0.010349479503929615, 0.01645606942474842, -0.012940154410898685, -0.010884798131883144, -0.013587823137640953, 0.014737765304744244, -0.009767900221049786, -0.0389658585190773, 0.01908639632165432, 0.030109981074929237, 0.0031590366270393133, 0.01547795720398426, 0.024373486638069153, 0.028048016130924225, -0.0029805973172187805, -0.007785240653902292, -0.014539498835802078, 0.0006802998832426965, -0.005805886350572109, -0.001761262072250247, 0.035767167806625366, 0.023791907355189323, -0.019734065979719162, 0.015583699569106102, -0.007527495268732309, 0.053716838359832764, -0.001853786176070571, -0.01027678232640028, 0.004464287310838699, -0.028285933658480644, 0.015689441934227943, -0.09590782225131989, 0.0013820785097777843, 0.011558901518583298, 0.01166464388370514, -0.0022552746813744307, 0.02430739812552929, 0.013627476058900356, 0.004563420079648495, 0.009523372165858746, 0.020910443738102913, 0.00841969158500433, 0.008624566718935966, -0.0054655298590660095, 0.0054357899352908134, 0.019311098381876945, -0.008842659182846546, -0.004487418103963137, -0.0028798121493309736, -0.004292456433176994, 0.03856932744383812, 0.013706781901419163, -0.01904674433171749, -0.019059961661696434, -0.017064085230231285, -0.02802157960832119, -0.01461880560964346, -0.016667552292346954, 0.007269749417901039, 0.02260231226682663, 0.0169451255351305, -0.013052504509687424, -0.0041371481493115425, 0.006780693773180246, 0.021108709275722504, -0.02128054015338421, -0.012431271374225616, -0.04129217565059662, -0.014539498835802078, 0.008617958053946495, -0.02773079089820385, -0.006893044337630272, -0.013217726722359657, 0.008234643377363682, -0.016680771484971046, 0.03196046128869057, -0.010997148230671883, -0.030083544552326202, 0.01796288974583149, 0.04094851762056351, -0.0053895278833806515, -0.003568786196410656, -0.018901348114013672, -0.040842775255441666, -0.03817279264330864, 0.0027129382360726595, -0.009021098725497723, 0.008981444872915745, 0.034736186265945435, 0.0029525095596909523, 0.027651483193039894, -0.02714920975267887, -0.0028517243918031454, -0.005719970911741257, -0.0006191678694449365, 0.03867506608366966, 0.011585337109863758, 0.0014531237538903952, -0.004223063588142395, -0.009173101745545864, -0.007454797625541687, -0.02089722454547882, 0.012735279276967049, -0.010289999656379223, 0.0183065515011549, -0.027122775092720985, -0.022086821496486664, -0.03613726422190666, 0.002539455657824874, -0.002602239837870002, -0.013574604876339436, -0.010243738070130348, -0.019694412127137184, -0.009529980830848217, -0.006195809226483107, 0.02923761121928692, 0.0067939115688204765, 0.010422177612781525, 0.005310221575200558, 0.014486628584563732, 0.005571271758526564, -0.0035026974510401487, 0.05022735893726349, 0.01566300541162491, -0.006232157815247774, -0.008895529434084892, 0.016720423474907875, 0.00526065519079566, -0.010567571967840195, 0.012094219215214252, -0.0021346628200262785, -0.05107329413294792, -0.03463044390082359, -0.05815799534320831, 0.0005799277569167316, 0.002532846760004759, -0.003892620326951146, -0.01899387314915657, 0.02020990289747715, 0.019945548847317696, 0.0036249614786356688, -0.012616319581866264, -0.05210427567362785, -0.007626628037542105, -8.070041803875938e-05, -0.02020990289747715, -0.017976107075810432, -0.0027955491095781326, -0.007639845833182335, -0.016707206144928932, 0.00020105400471948087, 0.016826165840029716, 0.025021156296133995, -0.0070053949020802975, 0.0009037620620802045, 0.005415963474661112, 0.0019975288305431604, -0.008915356360375881, 0.020672524347901344, -0.025615952908992767, 0.013905048370361328, -0.008221426047384739, 0.00044114163029007614, 0.007593583781272173, -0.009444065392017365, -0.0005939715774729848, 0.028285933658480644, -0.022589094936847687, 0.012464315630495548, -0.012979807332158089, 0.013415992259979248, 0.01476419996470213, -0.009827379137277603, -0.01615206152200699, -0.010171040892601013, 0.02031564526259899, -0.024545317515730858, -0.03455113619565964, -0.009609286673367023, 0.010137995705008507, 0.005974412430077791, -0.024386705830693245, 0.04089564457535744, 0.006661734078079462, 0.012940154410898685, -0.010455221869051456, -0.012940154410898685, 0.0136406933888793, -0.01977371796965599, -0.004454373847693205, 0.001607605954632163, -0.007507668808102608, 0.00028418112196959555, 0.05390188843011856, 0.018531251698732376, 0.030347900465130806, -0.0059248460456728935, 0.027942273765802383, 0.009093795903027058, -0.019165704026818275, 0.027810096740722656, 0.00029905105475336313, -0.01527969166636467, -0.025986049324274063, -0.012728670611977577, 0.029819190502166748, 0.01420905627310276, -0.013812524266541004, 0.02392408438026905, 0.007877765223383904, -0.009748073294758797, 0.011770385317504406, 0.03095591440796852, 0.0351591520011425, -0.0329914465546608, -0.022523006424307823, 0.017738189548254013, 0.025840654969215393, 0.028629595413804054, -0.010151213966310024, 0.0042627169750630856, 0.0007860416662879288, 0.023461464792490005, -0.010627051815390587, -0.00477159908041358, 0.000831064535304904, -0.03764408454298973, -0.009867032989859581, -0.0018884827150031924, 0.010706358589231968, 0.013878612779080868, 0.009278844110667706, 0.037855569273233414, 0.021161580458283424, 0.007144181057810783, 0.018518034368753433, 0.0041834102012217045, -0.021544894203543663, 0.007177225314080715, -0.010151213966310024, -0.031220270320773125, -0.007593583781272173, 0.05102042481303215, 0.030770866200327873, 0.022232215851545334, 0.004384980536997318, 0.009093795903027058, -0.029766319319605827, 0.008406474255025387, 0.018716299906373024, 0.0032135597430169582, -0.0060438052751123905, 0.010124778375029564, -0.0017496965592727065, 0.0018934393301606178, 0.02162420004606247, -0.011895953677594662, 0.0008128901827149093, -0.02508724480867386, -0.0077059343457221985, -0.02211325615644455, -0.024836108088493347, 0.0006410597125068307, 0.0007781936437822878, -0.002729460597038269, -0.04335414245724678, -0.027942273765802383, 0.008968227542936802, -0.004312283359467983, 0.0019330924842506647, 0.03354658931493759, -0.02026277408003807, 0.037961311638355255, 0.018980655819177628, -0.015292909927666187, -0.007349055726081133, 0.007785240653902292, 0.014777418226003647, 0.01962832361459732, 0.0070053949020802975, -0.0016051277052611113, -0.027202080935239792, 0.028550289571285248, -0.01728878542780876, 0.0010268521727994084, -0.01476419996470213, -0.023937301710247993, 6.552068225573748e-05, 0.01678651198744774, 0.0021164885256439447, 0.0110103664919734, 0.023157456889748573, 0.020619653165340424, -0.01650894060730934, -0.004755076952278614, -0.011975260451436043, -0.02030242793262005, -0.023791907355189323, 0.013111984357237816, 0.00502934493124485, 0.011168979108333588, -0.047768864780664444, 0.004368458408862352, -0.014856724068522453, -0.0010053733130916953, -0.0006299072992987931, 0.011294547468423843, -0.008545259945094585, -0.01738131046295166, -0.028444547206163406, -0.0009012836962938309, 0.00912684015929699, -0.007269749417901039, 0.006367639638483524, -0.012021522037684917, -0.009014489129185677, -0.008941791951656342, 0.01181003823876381, 0.011274720542132854, 0.0006026457413099706, -0.014301580376923084]} +{"id": "test:10000097", "text": "\"Private equity group Penta and the company J 23 a.s. yesterday reached an agreement on the sale of the ADAST a.s., a manufacturer of printing presses based in Adamov, Czech Republic. J 23, a company with a significant interest in machinery manufacturing, is purchasing 100 % of the shares in ADAST. The value of the transaction has not been disclosed. The new owner plans to keep the polygraphic manufacturing activities at ADAST.\\nPenta purchased ADAST out of bankruptcy in 2003. The company was in poor condition and it was having a hard time surviving in the highly competitive international market for printing equipment. In ADAST Penta invested primarily in the areas of production optimization, R&D and innovations. Although ADAST retained its existing production, it increasingly started to focus on cooperative arrangements. The company will continue these trends after its change of ownership.\"", "vector_field": [-0.0032574692741036415, 0.0005289146210998297, 0.0008516924572177231, -0.010383183136582375, -0.027647770941257477, 0.03338905796408653, -0.04826483502984047, -0.00472672376781702, -0.01176082156598568, -0.00603649765253067, 0.01730530522763729, 0.03642936423420906, -0.002575436607003212, -0.0006459798896685243, -0.007804353255778551, 0.006348671857267618, 0.012975585646927357, -0.01769891567528248, -0.0037087644450366497, -0.005164445843547583, -0.006467433646321297, 9.394914377480745e-05, -0.00732930563390255, 0.008829099126160145, -0.008903749287128448, 0.004278821405023336, 0.015337251126766205, -0.008245469070971012, -0.01258197519928217, -0.01566299796104431, 0.021607879549264908, -0.019096912816166878, 0.004451874643564224, 0.008455847389996052, -0.005052470602095127, -0.009792766533792019, -0.019327649846673012, 0.0017127161845564842, 0.022571546956896782, -0.021173549816012383, 0.0007664383738301694, -0.010071009397506714, 0.01992485299706459, -0.024159563705325127, -0.019219068810343742, -0.0026823224034160376, 0.01845899224281311, -0.01790250837802887, 0.006888190284371376, 0.01818753592669964, -0.008964826352894306, 0.0418856255710125, -0.02872001938521862, 0.008089382201433182, 0.0006366486195474863, -0.00600256584584713, 0.015527269802987576, 0.016721675172448158, 0.008863030932843685, -0.0015277869533747435, 0.034882064908742905, -0.01685740239918232, -0.007241082843393087, 0.0055885957553982735, -0.032058924436569214, 0.004149880260229111, -0.014278572984039783, -0.012358023785054684, -0.0029978896491229534, -0.0014845237601548433, 0.045278821140527725, 0.011394355446100235, -0.0058939834125339985, -0.005045684054493904, 0.007607548031955957, -0.003888603998348117, -0.011611520312726498, 0.00589737668633461, -0.00806223601102829, 0.019531242549419403, -0.0018917074194177985, -0.028747165575623512, -0.03338905796408653, 0.005927915219217539, 0.022883720695972443, -0.013023090548813343, -0.007017131429165602, -0.00026997123495675623, 0.012887362390756607, 0.010919308289885521, 0.02732202224433422, 0.01772606186568737, -0.006715137045830488, 0.009412729181349277, -0.021662170067429543, -0.00015895007527433336, -0.0012207026593387127, 0.013274187222123146, -0.014617892913520336, -0.03067450225353241, -0.015323678031563759, 0.019327649846673012, 0.005374824162572622, -0.0025347182527184486, -0.004587602335959673, 0.008863030932843685, 0.005510552320629358, 0.006481006275862455, 0.01636878214776516, -0.017712488770484924, -0.005435901693999767, 0.03797666355967522, 0.012120500206947327, -0.013009517453610897, 0.015106513164937496, -0.0013632168993353844, 0.01093966793268919, 0.010783580131828785, 0.003600182244554162, -0.02072564885020256, 0.0033694447483867407, 0.02376595325767994, 0.002838409272953868, -0.01273806206882, 0.030403045937418938, 0.015961598604917526, -0.01930050551891327, -0.01678953878581524, 0.008985185995697975, -0.0019459985196590424, 0.011041463352739811, -0.010946453548967838, 0.034664902836084366, -0.0122155100107193, -0.020508483052253723, 0.01741388812661171, -0.002487213583663106, -0.0007821319159120321, -0.053341057151556015, -0.042944300919771194, 0.010186377912759781, 0.022001489996910095, -0.003391500562429428, -0.00790614914149046, -0.003583216108381748, 0.013606720604002476, 0.010328891687095165, -0.004757262300699949, 0.008903749287128448, 0.008964826352894306, -0.005751469172537327, -0.016287345439195633, -0.0040582637302577496, 0.0008415128686465323, 0.01317239087074995, -0.011530083604156971, 0.006117934361100197, -0.013654225505888462, -0.01085144467651844, 0.013063808903098106, -0.024132417514920235, 0.0034593644086271524, -0.0049031698144972324, -0.015486551448702812, 0.012636265717446804, 0.04289001226425171, -0.007919722236692905, -0.021852189674973488, -0.012018703855574131, -0.02103782258927822, -0.022557973861694336, 0.013185963965952396, -0.014835057780146599, -0.003749482799321413, 0.011204336769878864, 0.007370023988187313, -0.0058329058811068535, 0.04631035402417183, -0.026358354836702347, -0.009989572688937187, 0.029018621891736984, 0.01608375459909439, -0.0051406933926045895, 0.012242655269801617, -0.004407763015478849, -0.0018662584479898214, -0.008584789000451565, 0.00745146069675684, 0.008415129035711288, 0.008204750716686249, 0.008659439161419868, 0.014821484684944153, -0.014726475812494755, -0.007553256582468748, -0.6506251692771912, -0.021621452644467354, -0.014522884041070938, -0.016355210915207863, -0.0012257924536243081, 0.03982255980372429, 0.016667384654283524, 0.01231730543076992, -0.02072564885020256, -0.005598775111138821, -0.006698171142488718, 0.02143143303692341, -0.006426715292036533, 0.014427874237298965, -0.0020749401301145554, -0.016382355242967606, 0.013002730906009674, -0.02048133686184883, 0.023888107389211655, 0.005259455647319555, -0.005605561658740044, 0.014007117599248886, 0.0016041339840739965, -0.00790614914149046, 0.012269800528883934, 0.005388396792113781, 0.0004585057613439858, -0.02523181401193142, -0.012045850045979023, 0.014278572984039783, -0.007634693291038275, 0.025801870971918106, -0.018581148236989975, -0.007614334113895893, 0.04538740590214729, 0.0327918566763401, -0.02718629501760006, 0.040772657841444016, 0.02090209349989891, 0.03528925031423569, -0.004333112854510546, -0.005327319260686636, 0.014332864433526993, 0.02096995897591114, 0.000904287036973983, 0.017223868519067764, 0.014265000820159912, -0.010376396588981152, 0.001164714922197163, -0.01404783595353365, -0.006301166955381632, -0.024064553901553154, 0.008048663847148418, -0.024933211505413055, 0.011910121887922287, 0.0018136638682335615, 0.02261226624250412, -0.029507242143154144, 0.016423074528574944, -0.020861376076936722, -0.003556070616468787, 0.01737316884100437, -0.020888522267341614, -0.020779939368367195, -0.010261028073728085, 0.006559050176292658, 0.011849043890833855, 0.02736274152994156, 0.0023786311503499746, -0.01821468211710453, -0.01210692711174488, -0.0034322189167141914, 0.003245593048632145, 0.0034084664657711983, 0.01626020111143589, -0.007783994078636169, -2.550199678807985e-05, 0.04679897427558899, 0.00920913740992546, 0.024417446926236153, -0.012846644036471844, -0.010016717948019505, -0.01341670099645853, -0.006430108565837145, 0.030375899747014046, 0.00732930563390255, -0.004696184769272804, -0.00332363648340106, 0.006094181910157204, 0.005571629852056503, -0.00958238821476698, 0.013579574413597584, 0.008462633937597275, 0.005836299154907465, -0.007105354685336351, -0.003394893603399396, -0.018757592886686325, 0.010172804817557335, 0.009148059412837029, -0.0019527849508449435, -0.03822097182273865, -0.01786178909242153, 0.021227840334177017, -0.008449060842394829, 0.028068525716662407, 0.01894761249423027, -0.010335678234696388, 0.00855085626244545, 0.021200696006417274, -0.02523181401193142, 0.010688571259379387, -0.002434619003906846, -0.015418687835335732, -0.00848299264907837, -0.01404783595353365, -0.03154316172003746, 0.01814681850373745, -0.013579574413597584, 0.011699743568897247, -0.012500538490712643, -0.0013784862821921706, -0.013097740709781647, 0.0032320201862603426, 0.02885574847459793, -0.0001850352855399251, 0.010613920167088509, 0.0017950012115761638, -0.03322618454694748, -0.020155590027570724, -0.0160566084086895, 0.018160391598939896, -0.009134486317634583, 0.016843831166625023, -0.010179591365158558, -0.005754862446337938, -0.01832326501607895, -0.009168418124318123, 0.006582802161574364, 0.0031607630662620068, -0.020603492856025696, -0.029534388333559036, -0.007946867495775223, 0.0023090706672519445, 0.020807085558772087, 0.0010688570328056812, -0.023820243775844574, -0.009236282669007778, 0.006949267815798521, 0.012391955591738224, 0.02523181401193142, -0.010037076659500599, -0.015120086260139942, -0.007057849783450365, 0.03431200981140137, 0.0011019407538697124, 0.002556774066761136, 0.007471819873899221, -0.01972126215696335, 0.0025126624386757612, -0.008801952935755253, 0.021417859941720963, 0.01892046630382538, -0.03412199020385742, -0.009982786141335964, -0.00855085626244545, -0.02201506309211254, -0.0028587684500962496, 0.004404369741678238, -0.002872341312468052, -0.014332864433526993, 0.010817511938512325, -0.002424439415335655, -0.019761979579925537, -0.0027909046038985252, -0.032086070626974106, 0.0011944053694605827, -0.0056700324639678, -0.01371530257165432, 0.004926922265440226, 0.0018764380365610123, 0.0025737399701029062, 0.027294877916574478, -0.013613507151603699, -0.0028604650869965553, 0.016938839107751846, 0.007587188854813576, -0.00344579154625535, 0.0032523793634027243, -0.011767607182264328, 0.007566829677671194, -0.009507738053798676, 0.017264587804675102, 0.0106410663574934, 0.02020988240838051, -0.029860135167837143, 0.0027569725643843412, 0.0017364686354994774, 0.017875362187623978, 0.008944467641413212, 0.01121790986508131, 0.011808325536549091, 0.0001551327295601368, 0.021797897294163704, -0.016477365046739578, 0.010071009397506714, -0.018866175785660744, 0.01441430114209652, -0.026860548183321953, 0.03336191549897194, -0.007838285528123379, -0.0012495447881519794, -0.033416204154491425, -0.005388396792113781, -0.0013861210318282247, 0.004638500511646271, 0.026928411796689034, 0.012602333910763264, -0.005507159046828747, -0.00266026658937335, 0.007159645669162273, -0.013342050835490227, -0.005486799869686365, 0.01233766507357359, -0.02236795611679554, -0.006355458404868841, -0.011000744998455048, 0.025557560846209526, 0.004037904553115368, -0.003055573906749487, -0.029887279495596886, -0.005306960083544254, 0.020929239690303802, 0.010993958450853825, 0.017427461221814156, 0.011177191510796547, 0.017536042258143425, 0.03300902247428894, 0.0027790283784270287, 0.027769925072789192, -0.007587188854813576, 0.008116527460515499, 0.008218323811888695, 0.014862203039228916, 0.005242489743977785, -0.015310104936361313, -0.003888603998348117, 0.03447488322854042, 0.0012486965861171484, -0.006803360302001238, 0.013823885470628738, -0.044437307864427567, 0.008177605457603931, 0.012629479169845581, -0.009073409251868725, 0.021377142518758774, -0.011686170473694801, 0.005049077328294516, 0.018757592886686325, 0.009528097696602345, 0.014848630875349045, 0.001974840648472309, 0.0171017125248909, 0.025734007358551025, 0.034529175609350204, 0.027932798489928246, -0.011774393729865551, -0.014482165686786175, 0.00016796325508039445, 0.018961185589432716, -0.020752793177962303, -0.0035255318507552147, -0.007845072075724602, 0.0054664406925439835, 0.00079612887930125, 0.010627493262290955, -0.013864603824913502, -0.015092941001057625, -0.02770206145942211, -0.0008406646084040403, -0.001601589028723538, -0.023820243775844574, -0.04598460718989372, 0.021200696006417274, 0.016341637820005417, -0.004238103050738573, -0.0013946039834991097, 0.019246213138103485, -0.00519837811589241, -0.01936836913228035, 0.016423074528574944, -0.017536042258143425, 0.0063283126801252365, -0.016518084332346916, 0.01985698938369751, 0.0026059753727167845, -0.034773483872413635, 0.01502507645636797, -0.025204667821526527, 0.011787966825067997, -0.007912935689091682, -0.004991393070667982, -0.006837292108684778, -0.008014731109142303, -0.02432243712246418, 0.06514938175678253, 0.02247653715312481, -0.002558470703661442, -0.009507738053798676, -0.01200513169169426, -0.007783994078636169, 0.011367210187017918, 0.008191177621483803, -0.02550327032804489, 0.0014209012733772397, 0.004224530421197414, 0.0003198088379576802, 0.0062604486010968685, -0.009154845960438251, 0.06514938175678253, 0.0025398079305887222, -0.003338905982673168, -0.0009093768312595785, -0.011625093407928944, 0.02421385422348976, 0.09240354597568512, 0.02487892098724842, -0.013898535631597042, -0.01259554736316204, -0.004048084374517202, 0.006956053897738457, -0.024539601057767868, -0.016450218856334686, 0.004489199724048376, 0.016626665368676186, 0.018024662509560585, -0.009399156086146832, 0.02787850797176361, -0.006195977795869112, 0.005873624235391617, 0.01170653011649847, -0.020861376076936722, -0.0007808594964444637, 0.008211537264287472, -0.013884962536394596, 0.01657237485051155, 0.0025296283420175314, 0.03105453960597515, 0.029127204790711403, 0.02439030073583126, 0.003627327736467123, 0.001634672749787569, 0.005035504698753357, 0.01892046630382538, 0.007437888067215681, 0.007098568137735128, 0.002602582098916173, -0.012303732335567474, 0.02847570925951004, -0.030755938962101936, -0.01943623274564743, -0.007390383165329695, -0.009032690897583961, 0.003471240634098649, 0.013864603824913502, 0.0373523123562336, 0.022843003273010254, -0.0011452040635049343, 0.0018238434568047523, -0.007275014650076628, -0.009874203242361546, -0.029235785827040672, 0.019042622298002243, -0.0012274890905246139, -0.027308451011776924, 0.028177108615636826, -0.032058924436569214, -0.015405114740133286, 0.007288587279617786, 0.0023989903274923563, 2.262573252664879e-05, -0.006935694720596075, -0.022829430177807808, -0.01892046630382538, -0.01825539954006672, 0.006087395828217268, -0.008082595653831959, 0.01748175173997879, -0.02045419253408909, 0.003678225679323077, -0.009826699271798134, -0.020712075755000114, -0.007213937118649483, -0.016531655564904213, 0.008768021129071712, -7.364297925960273e-05, -0.03119026869535446, -0.00398700637742877, 0.016667384654283524, 0.015622279606759548, -0.0014930067118257284, 0.025693288072943687, -0.004333112854510546, -0.0010128692956641316, 0.0018713482422754169, -0.01137399673461914, -0.004231316968798637, 0.00629777368158102, -0.0352349579334259, 0.0122155100107193, 0.022096499800682068, -0.027783498167991638, -0.006623520981520414, 0.0006519180024042726, 0.03987685218453407, 0.02299230359494686, -3.3666878152871504e-05, 0.007051063701510429, -0.00912769977003336, -0.00921592302620411, -0.005364644806832075, 0.01881188526749611, 0.025530414655804634, 0.005252669099718332, 0.022354383021593094, -0.011469006538391113, -0.011889762245118618, -0.019951999187469482, -0.052119508385658264, 0.020494909957051277, 0.020752793177962303, -0.00033635066938586533, -0.004431515466421843, -0.0214585792273283, -0.02718629501760006, -0.0027993875555694103, -0.0038512786850333214, -0.017020275816321373, 0.0009212529985234141, 0.0007583795813843608, 0.016219481825828552, -0.017603905871510506, 0.001279235351830721, 0.00591434258967638, -0.007180004846304655, -0.020074153319001198, -0.004482413176447153, 0.01639592833817005, 0.024417446926236153, -0.02585616149008274, 0.02578829787671566, 0.009711329825222492, -0.0052696350030601025, -0.015079367905855179, 0.008754448965191841, -0.01268377061933279, 0.03382338955998421, 0.010552843101322651, -0.0031013821717351675, -0.04826483502984047, -0.014183564111590385, -0.002429529093205929, -0.01841827481985092, -0.0017356203170493245, -0.026073327288031578, 0.0033694447483867407, -0.019246213138103485, -0.015920881181955338, 0.004255068954080343, -0.010091368108987808, -0.022775139659643173, -0.02456674724817276, -0.003888603998348117, 0.017875362187623978, 0.02638550102710724, -0.01999271661043167, -0.0025279319379478693, 0.013375982642173767, 0.003444095142185688, -0.0342034250497818, -0.03189605101943016, 0.006704957690089941, -0.00851692445576191, 0.01916477642953396, 0.015255814418196678, 0.029480095952749252, -0.01176082156598568, -0.007811139803379774, 0.018201109021902084, -0.00591434258967638, -0.0008915625512599945, -0.019585533067584038, 0.01590730808675289, -0.0006065339548513293, 0.0036917985416948795, 0.012921294197440147, 0.028530001640319824, 0.03043019212782383, -0.0018306298879906535, -0.004394190385937691, 0.007872217334806919, -0.011787966825067997, -0.01650451123714447, -0.00018811036716215312, -0.025123231112957, -0.006562442984431982, -0.007132500410079956, -0.011380783282220364, 0.023494496941566467, -0.05532268434762955, 0.000564967340324074, 0.027159148827195168, 0.006569229532033205, -0.0020121659617871046, -0.001722895773127675, 0.01856757514178753, -0.028149962425231934, 0.016558801755309105, 0.023888107389211655, 0.004312753677368164, -0.014726475812494755, -0.009765621274709702, -0.02421385422348976, -0.0024481916334480047, 0.012391955591738224, -0.0014395639300346375, -0.005418935790657997, 0.01818753592669964, -0.004736903123557568, -0.005958454217761755, 0.033307623118162155, -0.0006264690309762955, -0.01174046192318201, 0.019286932423710823, -0.02376595325767994, -0.02041347324848175, -0.014278572984039783, -0.029751552268862724, -0.019273359328508377, 0.03496350347995758, -0.009989572688937187, -0.0174546055495739, 0.005313746631145477, -0.006029711104929447, -0.014197136275470257, -0.007648266386240721, 0.0025957957841455936, 0.039415378123521805, -0.01636878214776516, 0.01601589098572731, 0.03811239078640938, 0.003915749490261078, -0.02219150960445404, -0.03602217882871628, -0.011360423639416695, 0.01674882136285305, 0.0013767897617071867, 0.003776628291234374, -0.03702656552195549, 0.0025618637446314096, -0.010810726322233677, 0.015853017568588257, -0.021648596972227097, -0.006671025417745113, 0.024783911183476448, -0.012765207327902317, 0.0014641645830124617, -0.004502772353589535, -0.0012911114608868957, 0.0011545353336259723, 0.006470826920121908, -0.018201109021902084, 0.016233054921030998, -0.020752793177962303, -0.019381942227482796, -0.017983945086598396, 0.011869403533637524, -0.010037076659500599, 0.031000249087810516, 0.011692957021296024, -0.020684929564595222, 0.011278986930847168, -0.021472150459885597, 0.00708499550819397, 0.01734602451324463, -0.0005450322641991079, 0.0024566748179495335, -0.010342464782297611, 0.01514723151922226, -0.003349085571244359, 0.0027705454267561436, -0.02725415863096714, -0.01106182299554348, -0.017508897930383682, 0.02174360677599907, 0.01818753592669964, 0.0035866093821823597, 0.0038648515474051237, 0.010566416196525097, 0.03043019212782383, 0.00419738469645381, -0.010328891687095165, 0.008808739483356476, -0.013979972340166569, -0.007831498980522156, -0.016952412202954292, 0.0014242944307625294, -0.008292973972856998, -0.007410742342472076, -0.008130100555717945, -0.02711843140423298, -0.0009534884011372924, 0.0034118597395718098, -0.016735248267650604, -0.018852602690458298, -0.02114640362560749, 0.009202350862324238, -0.015961598604917526, -0.006691384594887495, 0.003993792925029993, -0.0024736407212913036, -0.018934039399027824, -0.010104941204190254, 0.002149590291082859, 0.018377555534243584, -0.004838699009269476, -0.004350078757852316, -0.015920881181955338, 0.008523711003363132, 0.004289001226425171, 0.004519738722592592, 0.020332036539912224, -0.01594802550971508, 0.0022496897727251053, 0.0007062091608531773, -0.006012745201587677, -0.003235413460060954, 0.017929652705788612, -0.01362707931548357, 0.00650815200060606, 0.007322519551962614, 2.8630100132431835e-05, 0.008537284098565578, -0.02198791690170765, -0.01329454593360424, -0.0023175536189228296, 0.01821468211710453, 0.004095588810741901, -0.0334976427257061, -0.0009102251497097313, -0.006287594325840473, -0.0015456012915819883, -0.008557642810046673, -0.02749846875667572, -0.021268559619784355, -0.016341637820005417, 0.015282959677278996, -0.009365224279463291, 0.01923264190554619, -0.010926094837486744, -0.022313663735985756, 0.004920135717839003, 0.0040514771826565266, -0.0003490751550998539, 0.009154845960438251, 0.022666556760668755, 0.0160566084086895, -0.0037562691140919924, 0.004482413176447153, -0.01182868517935276, -0.0064504677429795265, 0.01100753154605627, 0.0006561594782397151, -0.001923942705616355, -0.010817511938512325, -0.02369808778166771, 0.03132599592208862, 0.007241082843393087, -0.027335595339536667, -0.0397682711482048, 0.009080195799469948, -0.024838203564286232, 0.002341306069865823, -0.0032150542829185724, 0.01551369670778513, 0.05491550266742706, 0.031950343400239944, -0.01359314750880003, 0.009901349432766438, -0.0007532897288911045, 0.02152644284069538, -0.010199950076639652, -0.011041463352739811, 0.019558388739824295, -0.0038750311359763145, -0.015215096063911915, -0.027647770941257477, -0.05296102166175842, -0.014400728978216648, 0.007648266386240721, -0.008774807676672935, 0.008706944063305855, -0.01097359973937273, -0.006586195435374975, 0.015527269802987576, 0.0003880969306919724, 0.004163452889770269, 0.004261855501681566, 0.007220723666250706, 0.028937185183167458, -0.031461723148822784, 0.002149590291082859, 0.008367624133825302, 0.0006540387403219938, -0.011027890257537365, 0.008578002452850342, 0.020630639046430588, -0.019843416288495064, -0.008238682523369789, -0.0064572542905807495, -0.01188297662883997, -0.0017814284656196833, -0.029507242143154144, 0.01981627196073532, 0.004265248775482178, 0.011238268576562405, 0.02411884441971779, 0.016952412202954292, -0.006514938548207283, 0.0009348258026875556, -0.007926508784294128, -0.014522884041070938, -0.041125547140836716, -0.012174791656434536, -0.026751965284347534, 0.0037359099369496107, -0.03615790978074074, -0.008014731109142303, 0.008326905779540539, -0.002320946892723441, -0.024865347892045975, 0.011190763674676418, 0.0043161469511687756, 0.027417032048106194, 0.011774393729865551, -0.029914425686001778, 0.019585533067584038, 0.0030114625114947557, 0.009487379342317581, -0.024593893438577652, -0.024553174152970314, -0.01322668232023716, -0.017780352383852005, -0.014007117599248886, 0.010953240096569061, -0.023793097585439682, -0.021092113107442856, 0.008340478874742985, 0.026602664962410927, -0.008788380771875381, 0.009141272865235806, 0.19696830213069916, 0.01526938658207655, 0.012066208757460117, 0.004784408025443554, -0.0010400149039924145, 0.015120086260139942, 0.013342050835490227, 0.007390383165329695, 0.003932715393602848, -0.007044277153909206, -0.013443847186863422, 0.024336010217666626, -0.021892907097935677, 0.002078333171084523, 0.02721344120800495, -0.006667632143944502, -0.03496350347995758, -0.00019415449060034007, -0.03056591935455799, 0.03192319720983505, 0.010654638521373272, -0.010647852905094624, 0.020264172926545143, -0.0007634693756699562, -0.005486799869686365, -0.016043035313487053, -0.012765207327902317, 0.01852685585618019, 0.04690755531191826, -0.0037189440336078405, -0.016110900789499283, 0.000526369723957032, 0.018621865659952164, -0.002731523709371686, 0.018119672313332558, -0.018825458362698555, -0.014577174559235573, 0.005021931603550911, 0.012507324106991291, 0.014753621071577072, 0.0004146062710788101, -0.005975420121103525, -0.023535214364528656, -0.013389555737376213, -0.006436895113438368, 0.015337251126766205, 0.00372573034837842, -0.01176082156598568, -0.034637756645679474, -0.016843831166625023, -0.020467765629291534, -0.01678953878581524, 0.017983945086598396, 0.017264587804675102, 0.016721675172448158, 0.013851030729711056, 0.006599768530577421, 0.025082513689994812, -0.003610361833125353, -0.01198477204889059, -0.017020275816321373, 0.01072928961366415, 0.003569643246009946, -0.0035967889707535505, -0.01298237219452858, 0.03056591935455799, 0.019422659650444984, 0.007295373827219009, 0.00855085626244545, -0.026779111474752426, 0.019001903012394905, -0.016219481825828552, -0.013572788797318935, 0.01930050551891327, -0.028014235198497772, -0.020847802981734276, 0.03249325603246689, 0.02376595325767994, 0.012032276950776577, 0.007953654043376446, 0.008557642810046673, 0.023616651073098183, -0.01797037199139595, 0.017821071669459343, -0.031081685796380043, -0.03252040222287178, 0.004224530421197414, 0.002141107339411974, -0.01231730543076992, 0.002193701919168234, 0.012636265717446804, 0.0018458992708474398, -0.005578415933996439, -0.002853678772225976, 0.02243581973016262, 0.015554415993392467, -0.01296201255172491, -0.011720103211700916, 0.008245469070971012, -0.008082595653831959, -0.021553587168455124, 0.058960191905498505, 0.0036307210102677345, 0.02614119090139866, -0.016382355242967606, 0.006796573754400015, -0.017074568197131157, 0.013878175988793373, 0.009073409251868725, -0.013552429154515266, 0.00011430832819314674, 0.006005958653986454, 0.0010103244567289948, -0.019205495715141296, -0.010546056553721428, 0.03971397876739502, 0.004041297826915979, -0.013647438958287239, 0.017427461221814156, -0.014753621071577072, 0.03265612944960594, -0.009867417626082897, -0.001827236614190042, 0.009589174762368202, 0.0066269137896597385, -0.010803939774632454, -0.02132285013794899, 0.009643466211855412, 0.027932798489928246, -0.0064436811953783035, 0.0005331560969352722, 0.008686584420502186, 0.024417446926236153, -0.02833998203277588, -0.015228668227791786, 0.008801952935755253, 0.013830671086907387, -0.0014370189746841788, 0.0031862121541053057, 0.03496350347995758, 0.011577588506042957, -0.004451874643564224, -0.005412149243056774, -0.00012629055709112436, 0.014387155883014202, -0.030403045937418938, 0.021309277042746544, 0.007987585850059986, -0.010953240096569061, -0.011197550222277641, -0.01563585177063942, -0.0050253248773515224, -0.02010129950940609, 0.010240668430924416, 0.004034511279314756, 0.0031726392917335033, -0.0266976747661829, -0.018309691920876503, 0.00033401785185560584, 0.021594306454062462, -0.0474504679441452, 0.01692526787519455, 0.02247653715312481, 0.0019375155679881573, -0.01563585177063942, 0.001896797213703394, -0.17394885420799255, 0.014875776134431362, -0.006725316867232323, 0.019490523263812065, 0.014034262858331203, -0.013878175988793373, 0.01411569956690073, 0.012873790226876736, 0.013145245611667633, 0.007064636331051588, 0.019517669454216957, 0.018133245408535004, -0.03262898325920105, -0.020359182730317116, 0.005144086666405201, 0.001096850959584117, 0.011475792154669762, 0.008109740912914276, 0.0059550609439611435, 0.008937681093811989, 0.01146221999078989, 0.002541504567489028, 0.010722503066062927, 0.002261565765365958, -0.00020963595306966454, 0.03165174275636673, 0.0015159108443185687, -0.005116941407322884, -0.025842588394880295, -0.02030489221215248, -0.0014480468817055225, -0.014455019496381283, 0.03509923070669174, 0.013158818706870079, -0.003391500562429428, -0.006861044559627771, -0.006935694720596075, -0.01392568089067936, -0.016273774206638336, 0.021377142518758774, 0.04481734707951546, 0.021472150459885597, -0.012507324106991291, -0.0039971861988306046, 0.023440206423401833, 0.02738988772034645, 0.01490292139351368, -0.014984358102083206, 0.004686005413532257, 0.00019383638573344797, 0.0305387731641531, -0.011109326966106892, 0.021662170067429543, 0.02201506309211254, 0.04633750021457672, 0.014604320749640465, 0.01233766507357359, 0.015880161896348, 0.025530414655804634, -0.018961185589432716, -0.006606554612517357, 0.0003289280575700104, 0.03841099143028259, 0.007186791393905878, 0.0067626419477164745, -0.038058098405599594, -0.0014997931430116296, 0.01362707931548357, -0.022327236831188202, 0.02896433137357235, -0.017291732132434845, -0.01423785462975502, 0.0009288876899518073, -0.0033796243369579315, -0.010403542779386044, -0.00215976987965405, 0.005289994180202484, 0.02785136178135872, -0.011150045320391655, -0.01721029542386532, -0.02554398775100708, 0.014767194166779518, -0.008795167319476604, 0.004624927882105112, -0.00946023315191269, 0.004251676145941019, 0.02896433137357235, 0.015472978353500366, -0.003211661009117961, 0.007790780626237392, 0.012486965395510197, -0.02980584278702736, 0.013301332481205463, 0.008645866066217422, 0.02707771211862564, 0.010654638521373272, -0.010369610041379929, 0.021485723555088043, -0.004977819975465536, -0.011068608611822128, -0.0004008214164059609, -0.012208723463118076, -0.03371480479836464, 0.012676984071731567, 0.007125713862478733, -0.007071422878652811, 0.006324919406324625, 0.003678225679323077, 0.020874949172139168, -0.023019449785351753, -0.03363336995244026, 0.03333476930856705, 0.009222709573805332, 0.039062485098838806, 0.0028655549976974726, 0.042944300919771194, 0.014332864433526993, -0.003610361833125353, 0.007987585850059986, -0.008496565744280815, 0.05032790079712868, -0.01590730808675289, -0.010797153227031231, 0.02058991976082325, -0.0015371183399111032, -0.027294877916574478, -0.09918993711471558, -0.003871637862175703, 0.026005461812019348, 0.00909376796334982, -0.007858644239604473, 0.003593395696952939, 0.01723744161427021, 0.013789952732622623, -0.011631879955530167, 0.007872217334806919, 0.016477365046739578, -0.01845899224281311, -0.0008605996263213456, -0.003698584856465459, 0.04584887996315956, -0.0026687495410442352, -0.004967640619724989, -0.008815526030957699, -0.03309045732021332, 0.006599768530577421, -0.009202350862324238, -0.014835057780146599, 0.023128030821681023, -0.053123895078897476, -0.008082595653831959, -0.013708516024053097, -0.020576346665620804, 0.009032690897583961, 0.021784326061606407, 0.0030403046403080225, 0.014672184363007545, -0.0007062091608531773, 0.006080609280616045, -0.007180004846304655, 0.003949681296944618, -0.013287760317325592, -0.051875196397304535, 0.014007117599248886, 0.03300902247428894, -0.026507655158638954, 0.017807498574256897, -0.002349789021536708, 0.006416535936295986, -0.01759033463895321, -0.004811553284525871, -0.0037698419764637947, -0.013009517453610897, 0.0225036833435297, -0.003664652816951275, -0.018241828307509422, -0.04622891917824745, -0.008801952935755253, -0.004825126379728317, 0.005646280013024807, 0.0054596541449427605, -0.0373523123562336, 0.014997931197285652, 0.01947695203125477, -0.02610047161579132, -0.019558388739824295, 0.004166846163570881, -0.0030318216886371374, 0.010430688038468361, -0.020142018795013428, 0.0016660598339512944, -0.00909376796334982, -0.02114640362560749, 0.00797401275485754, 0.021662170067429543, -0.021553587168455124, 0.004407763015478849, 0.00708499550819397, -0.018866175785660744, 0.011265414766967297, -0.025842588394880295, -0.0023430027067661285, -0.02048133686184883, -0.016599521040916443, 0.01923264190554619, 0.035750724375247955, -0.017508897930383682, -0.012778780423104763, 0.010661425068974495, 0.001157928491011262, 0.0157037153840065, 0.007295373827219009, -0.0030657534953206778, -0.022951584309339523, 0.01712885871529579, -0.026154763996601105, -0.013477778993546963, 0.023290904238820076, 0.013328478671610355, -0.011774393729865551, -0.018228255212306976, 0.015649424865841866, 0.00830654613673687, -0.01453645620495081, -0.03718943893909454, 0.021757179871201515, -0.025001076981425285, -0.028394272550940514, -0.07019846141338348, 0.0030827196314930916, 0.00409219553694129, -0.008991972543299198, -0.012867003679275513, -0.007071422878652811, 0.020630639046430588, -0.0029978896491229534, -0.0021360176615417004, -0.03165174275636673, -0.03583216294646263, -0.004865844734013081, -0.0025957957841455936, -0.018296118825674057, -0.021119259297847748, -0.011435073800384998, 0.019029049202799797, 0.020671356469392776, 0.001418356434442103, -0.01783464476466179, 0.012609120458364487, -0.01298237219452858, 0.025693288072943687, 0.006789787206798792, -0.007512538228183985, -0.009507738053798676, -0.018784739077091217, 0.004492592997848988, 9.299480734625831e-05, 0.0027332203462719917, 0.022761566564440727, -0.026955557987093925, -0.0052153440192341805, 0.004190598614513874, -0.0006909397779963911, -0.01779392547905445, -0.06846114248037338, 0.014102127403020859, -0.014305719174444675, 0.01429214607924223, -0.011448646895587444, -0.014346437528729439, 0.012201936915516853, -0.033443350344896317, -0.001634672749787569, 0.003600182244554162, 0.005656459368765354, 0.001174046192318201, -0.0017203509341925383, 0.014197136275470257, 0.04392154514789581, 0.012867003679275513, -0.014957212843000889, -0.03604932501912117, -0.014455019496381283, -0.018784739077091217, 0.022286519408226013, -0.015133659355342388, 0.01685740239918232, -0.006219730246812105, 0.04006687179207802, -0.009718116372823715, 0.006535297725349665, -0.006494579371064901, 0.020671356469392776, -0.007159645669162273, -0.021241413429379463, -0.01380352582782507, -0.0017950012115761638, -0.03300902247428894, -0.0020223455503582954, -0.017332451418042183, 0.022951584309339523, 0.008252255618572235, 0.018404701724648476, -0.0200605820864439, 0.0011112720239907503, -0.008048663847148418, -0.014767194166779518, 0.029751552268862724, -0.003949681296944618, -0.020779939368367195, -0.012276587076485157, -0.005846478510648012, 0.022381527349352837, -0.011557229794561863, -0.021404286846518517, -0.003908962942659855, -0.014916494488716125, 0.025910453870892525, 0.0010790367377921939, 0.03056591935455799, -0.017821071669459343, -0.01563585177063942, -0.0349092110991478, -0.020264172926545143, 0.004764048848301172, 0.007600761484354734, -0.022666556760668755, 0.023860961198806763, 0.01856757514178753, 0.017603905871510506, -0.02449888363480568, -0.011733675375580788, -0.023155177012085915, 0.032574690878391266, -0.025109658017754555, -0.04424729198217392, 0.0011774393497034907, 0.013179177418351173, 0.020535629242658615, 0.00026975918444804847, 0.03102739341557026, 0.02132285013794899, -0.01514723151922226, 0.01252768374979496, -0.01216121856123209, -0.006050070282071829, -0.014862203039228916, 0.02658909186720848, 0.00699677225202322, -0.01843184605240822, 0.01859471946954727, 0.011088968254625797, 0.02885574847459793, -0.0005734502919949591, 0.007919722236692905, -0.038655299693346024, 0.0030148555524647236, 0.026887694373726845, -0.026684101670980453, -0.0003425008326303214, -0.021485723555088043, -0.021580733358860016, -0.024906067177653313, 0.005941488314419985, -0.008618720807135105, 0.04093553125858307, 0.0007884941878728569, 0.08534569293260574, 0.03078308328986168, -0.024648183956742287, 0.0018170570256188512, 0.020440619438886642, -0.007152859587222338, 0.021648596972227097, 0.0016516386531293392, 0.0030827196314930916, -0.023888107389211655, -0.004960854072123766, -0.0009331292239949107, 0.01490292139351368, -0.023888107389211655, -0.002923239255324006, -0.011631879955530167, 0.007098568137735128, 0.03520781174302101, -0.041858479380607605, 0.0017610692884773016, 0.026466937735676765, -0.008801952935755253, -0.01905619539320469, 0.0019697509706020355, -0.0327918566763401, 0.00032065712730400264, 0.010403542779386044, 0.006379210390150547, 0.00622991006821394, -0.026643384248018265, 0.004960854072123766, 0.007152859587222338, -0.03615790978074074, -0.008218323811888695, 0.011469006538391113, -0.004434908740222454, -0.017780352383852005, -0.018961185589432716, 0.025109658017754555, 5.129559576744214e-05, -0.004967640619724989, 0.03968683257699013, -0.004557063803076744, 6.038830542820506e-05, 0.0008326057577505708, -0.01176082156598568, -0.008808739483356476, -0.014088554307818413, -0.018825458362698555]} +{"id": "test:10000098", "text": "\"The High brightness is being compared to 350 Watts conventional xenon light source that can be utilized for critical illness like laparoscopy without any problem and with utmost ease.\\nIt has an extended service life, which means it can easily last for 60000 hours at a max, which is 120 times of xenon. This further emphasises that one doest not need to change the bulb for a number of years and can be used for a long time.\\nWe have an extensive range of power from 100 to 240 V / AC; 50 / 60 Hz.\\nThe perfect colour temperature ranges from 5000 K to 6500 K, and the colour rendering is over 70.\\nIt does not gives out light in the Ultra Violet or Infrared Rays.\\nThe customized connector, it accepts fibre light directs with dynamic and on the go areas varying from 3 mm to 10 mm in diameter or span.\\nIt helps saves half the energy i.e. it will be more than 50%.\\nIt is considered environment friendly.\"", "vector_field": [0.041516851633787155, 0.016929201781749725, -0.010486699640750885, -0.03181615099310875, -0.011595159769058228, 0.01257597841322422, -0.01680827885866165, -0.02805410511791706, -0.01443012896925211, -0.005038452800363302, 0.006791834253817797, 0.02091965638101101, -0.014819769188761711, -0.01061434019356966, -0.0178159698843956, 0.017399456351995468, 0.037539832293987274, -0.002146380953490734, 0.0007423320785164833, -0.012461773119866848, -0.006959782913327217, 0.012159465812146664, -0.04648812487721443, 0.026361186057329178, -0.028725899755954742, 0.00364112202078104, 0.020503144711256027, -0.019267043098807335, 0.00911624077707529, -0.0017197919078171253, 0.005962169263511896, 0.008827369660139084, -0.006711218971759081, -0.02159144915640354, -0.00970070157200098, 0.013126849196851254, 0.011030852794647217, -0.005921861622482538, 0.0030936100520193577, -0.01176310796290636, 0.01883709616959095, 0.0018642275827005506, 0.00794060155749321, 0.014913820661604404, -0.014161411672830582, 0.02126898802816868, 0.01608274132013321, -0.008343677967786789, 0.010876339860260487, 0.012710336595773697, -0.011158493347465992, 0.013422437943518162, -0.002986123086884618, -0.007490499876439571, -0.0028164952527731657, -0.012139311991631985, -0.0036982244346290827, 0.0067985523492097855, 0.008652702905237675, -0.01991196535527706, 0.006133476737886667, -0.013623976148664951, -0.00771219190210104, 0.015504999086260796, -0.02474888041615486, -0.015652794390916824, -0.027812259271740913, 0.017587559297680855, 0.005021657794713974, -0.004417043644934893, 0.01593494601547718, 0.03471829742193222, 0.005975604988634586, -0.0010605943389236927, 0.035820040851831436, 0.0031003279145807028, -0.014524180442094803, -0.008874394930899143, 0.0033925583120435476, -0.0126028498634696, 0.012750644236803055, -0.014913820661604404, -0.015558741986751556, -0.004581633023917675, 0.006613809149712324, 0.0015249716816470027, -0.031386204063892365, 0.020382221788167953, -0.01938796602189541, -0.008162293583154678, -0.00832352414727211, 0.008410857059061527, 0.029155846685171127, 0.001704676542431116, 0.010298597626388073, 0.022155756130814552, 0.008316806517541409, 0.034798912703990936, 0.009035625495016575, -0.03487952798604965, -0.01710386760532856, -0.0063350144773721695, -0.02464139275252819, -0.015652794390916824, -0.013026080094277859, -0.016217099502682686, 0.005391144659370184, 0.018850531429052353, 0.031278714537620544, -0.00046899597509764135, -0.01827278919517994, 0.04433838650584221, 0.006170425098389387, -0.0038930445443838835, -0.010930083692073822, -0.015075051225721836, 0.03356281295418739, -0.018823659047484398, -0.0015022986335679889, -0.0211346298456192, -0.008162293583154678, 0.017856277525424957, -0.005522144492715597, 0.006294707302004099, 0.02896774560213089, 0.005720323417335749, -0.015867767855525017, -0.02086591348052025, -0.0029290206730365753, 0.0004790728853549808, -0.0007225981098599732, 0.004951119888573885, 0.028591539710760117, 0.017587559297680855, -0.01112490426748991, 0.022357294335961342, 0.009425265714526176, 0.016620175912976265, -0.002090957947075367, -0.01178326178342104, 0.015599049627780914, 0.019078942015767097, -0.013892694376409054, -0.019159557297825813, -0.028806515038013458, 0.01867586560547352, -0.006543270777910948, 0.026401493698358536, -0.014685411006212234, 0.010023161768913269, 0.009465573355555534, -0.017224790528416634, 0.026186518371105194, 0.014389821328222752, -0.0015552024124190211, -0.020100068300962448, 0.006979936733841896, 0.01980447955429554, -0.03152056038379669, 0.0011244147317484021, -0.03238045796751976, 0.0014577923575416207, -0.011998235248029232, 0.004138249438256025, 0.015142230316996574, 0.004050916060805321, 0.011111468076705933, 0.012629722245037556, 0.001595510053448379, -0.0052433498203754425, 0.013550079427659512, 0.06164379045367241, -0.009942546486854553, 0.03874906152486801, 0.0022118808701634407, 0.026898620650172234, 0.010070187970995903, 0.0003451340307947248, -0.01680827885866165, -0.017842840403318405, -0.013026080094277859, 0.013093259185552597, 0.012300542555749416, 0.01872960850596428, -0.0272345170378685, -0.0025158673524856567, 0.03135932981967926, 0.00995598267763853, 0.028269078582525253, -0.010493418201804161, 0.011131621897220612, 0.04323664307594299, -0.007147884927690029, -0.013879258185625076, -0.648791491985321, -0.027651028707623482, -0.008625831454992294, -0.01868930086493492, 0.0183265320956707, 0.021389910951256752, 0.015652794390916824, -0.005051888525485992, -0.02290816605091095, -0.0157871525734663, 0.0029710077214986086, 0.0018642275827005506, -0.015075051225721836, -0.02448016218841076, -0.002008663257583976, -0.033428456634283066, -0.012172902002930641, 0.0028164952527731657, 0.009304342791438103, -0.008081678301095963, 0.03203112259507179, 0.019361095502972603, -0.04718679189682007, 0.018648993223905563, 0.008955010212957859, 0.012972336262464523, 0.026562724262475967, -0.017641302198171616, 0.015854330733418465, 0.01547812670469284, -0.008612395264208317, 0.02143021859228611, -0.0129790548235178, -0.009532752446830273, 0.04318289831280708, -0.0366530641913414, -0.0029004693496972322, 0.016526125371456146, 0.014309206046164036, 0.03090251050889492, -0.02347247302532196, 0.005061965435743332, 0.020503144711256027, -0.008800497278571129, -0.0402270071208477, 0.012918593361973763, 0.009821624495089054, -0.019643248990178108, 0.01659330539405346, -0.008666139096021652, 0.020691245794296265, -0.014524180442094803, 0.02413083054125309, 0.00850490853190422, -0.015048178844153881, -0.014524180442094803, 0.02708672173321247, -0.01721135526895523, -0.016862021759152412, 0.0061469124630093575, -0.012757362797856331, -0.0022018039599061012, 0.012589414604008198, -0.02993512712419033, -0.0223976019769907, 0.0407106988132, -0.01588120311498642, -0.018299661576747894, 0.00794060155749321, 0.013751617632806301, 0.021000271663069725, 0.019764171913266182, -0.02301565185189247, 0.00910952314734459, 0.013355258852243423, 0.016687355935573578, 0.019992580637335777, 0.004386812914162874, 0.022115448489785194, 0.01115177571773529, 0.008625831454992294, -0.02861841209232807, -0.023633703589439392, 0.0001704676542431116, 0.006177143193781376, 0.025823749601840973, 0.0037821985315531492, -0.02321719005703926, 0.007362858857959509, 0.00974772684276104, 0.02159144915640354, -0.0020170605275779963, -0.016378330066800117, -0.041570596396923065, -0.013180593028664589, 0.04253797605633736, -0.02087934873998165, -0.002384867751970887, 0.018796788528561592, -0.015343768522143364, -0.023190319538116455, -0.022814113646745682, -0.00772562762722373, 0.005948733538389206, 0.00990223977714777, 0.0022924961522221565, -0.02265288308262825, 0.0025561749935150146, 0.01203854288905859, -0.02535349503159523, 0.004527889657765627, 0.002586405724287033, -0.00549863139167428, -0.006499604322016239, 0.017735354602336884, -0.02194078266620636, 0.02402334287762642, 0.006697783246636391, -0.010110495612025261, -0.03367030248045921, 0.045332640409469604, -0.01715761050581932, 0.014631667174398899, -0.004961196333169937, 0.0013225938891991973, 0.004880581516772509, -0.00023260856687556952, -0.016472382470965385, 0.026092467829585075, -0.01934765838086605, 0.003123840782791376, -0.0014242026954889297, 0.0028819949366152287, -0.0050317347049713135, -0.006751526612788439, 0.005327323917299509, 0.013933002017438412, -0.018662428483366966, -0.007752499543130398, -0.01761443167924881, -0.013153720647096634, -0.005518785212188959, 0.00039929739432409406, -0.004638735670596361, -0.022921601310372353, -0.039286497980356216, -0.010983827523887157, 0.0025074700824916363, 0.010392649099230766, -0.020100068300962448, -0.012871567159891129, -0.04063008353114128, -0.027812259271740913, 0.02998887188732624, 0.018796788528561592, -0.010574033483862877, -5.5370495829265565e-05, -0.01991196535527706, -0.01721135526895523, -0.013536643236875534, -0.00273084151558578, 0.02810784801840782, -0.02402334287762642, 0.006748167797923088, -0.005018298979848623, -0.005760631058365107, 0.0062006558291614056, 0.035255733877420425, -0.0005059446557424963, -0.045628227293491364, 0.0101776747033, 0.009599932469427586, 0.008948292583227158, 0.032864149659872055, -0.035766296088695526, -0.0018793429480865598, -0.007611422799527645, -0.008269780315458775, 0.029558923095464706, -0.002673738868907094, 0.029908256605267525, 0.01488694828003645, -0.034288350492715836, -0.01969699189066887, 0.02641492895781994, 0.02148396335542202, 0.002077521989122033, -0.003291789209470153, -0.000990895670838654, 0.0059185028076171875, -0.028027234598994255, -0.0007641653646714985, 0.0005848804139532149, 0.01577371545135975, -0.003068417776376009, -0.019522326067090034, 0.0018407147144898772, -0.008027934469282627, 0.0030264307279139757, -0.016969509422779083, 0.046192534267902374, -0.041409365832805634, 0.015894638374447823, 0.00161986262537539, 0.02300221659243107, -0.011003981344401836, 0.019468581303954124, -0.0024554061237722635, 0.024601085111498833, 0.008249626494944096, -0.019011761993169785, -0.007134449202567339, -0.007853268645703793, -0.0021883680019527674, 0.004370017908513546, 0.021524270996451378, 0.0017416252521798015, 0.0064324247650802135, -0.008464600890874863, -0.03420773521065712, 0.0022857780568301678, 0.005952092353254557, 0.025340057909488678, 0.0198582224547863, 0.003327058395370841, 0.0011412096209824085, 0.01889083907008171, 0.0013024400686845183, -0.012213209643959999, -0.009405111894011497, -0.011030852794647217, 0.0034463016781955957, -0.003374083898961544, 0.0074501922354102135, 0.014121104031801224, 0.022666320204734802, 0.0453057661652565, -0.029639538377523422, 0.004346505273133516, 0.0045715561136603355, 0.03074127994477749, 0.0026972517371177673, 0.03649183362722397, -0.021806424483656883, 0.009311060421168804, 0.013583668507635593, -0.005643067415803671, 0.005488554481416941, -0.008195883594453335, 0.0037385320756584406, -0.02449359931051731, 0.023042524233460426, 0.005236632190644741, -0.0011101390700787306, 0.027866004034876823, 0.005972246173769236, 0.0032884301617741585, 0.017224790528416634, 0.01153469830751419, 0.0030029178597033024, -0.003367366036400199, 0.012058696709573269, 0.035658810287714005, -0.0025276239030063152, -0.010271726176142693, 0.012932028621435165, -0.003442942863330245, -0.025004161521792412, -0.013288079760968685, -0.009935828857123852, 0.0003081853792537004, 0.004141608253121376, 0.04216177389025688, -0.02091965638101101, 0.02081216871738434, 0.006926193367689848, 0.004991427063941956, 0.01181013323366642, 0.00732926931232214, 0.022693190723657608, -0.02805410511791706, -0.04264546558260918, 0.014819769188761711, -0.004433838650584221, -0.00014422571985051036, -0.03630373254418373, -0.04592381790280342, -0.005021657794713974, 0.025232572108507156, 0.038722191005945206, -0.01608274132013321, 0.00870644673705101, -0.0003321180120110512, 0.013449310325086117, 0.020140375941991806, -0.027624158188700676, 0.013919565826654434, -0.015531870536506176, 0.019670119509100914, 0.014188283123075962, 0.006959782913327217, -0.02829595096409321, -0.01405392400920391, -0.017533816397190094, 0.009331214241683483, -0.0022992140147835016, -0.01397330965846777, -0.014362949877977371, 0.023660574108362198, -0.02566251903772354, -0.023284370079636574, -0.015491562895476818, -0.01721135526895523, -0.01899832673370838, 0.011501108296215534, 0.009156548418104649, -0.016230536624789238, 0.0029290206730365753, 0.024601085111498833, -0.017735354602336884, -0.0059151435270905495, -0.011648902669548988, -0.01343587413430214, -0.002614957047626376, 0.03251481428742409, -0.007067269645631313, -0.0008691331022419035, 0.025071341544389725, 0.0021329449955374002, -0.012179619632661343, -0.016579868271946907, -0.011111468076705933, 0.020798733457922935, 0.029720153659582138, 0.01479289773851633, 0.022746935486793518, 0.007013526279479265, 0.00043456655112095177, -0.013718027621507645, 0.004191992804408073, 0.015585614368319511, -0.007456910330802202, 0.0074300384148955345, -0.013046233914792538, -0.01224008109420538, 0.012414747849106789, 0.02040909230709076, 0.006402194034308195, -0.003906480502337217, -0.014322642236948013, 0.039474599063396454, 0.005616195499897003, 0.00951931718736887, -0.0028164952527731657, 0.0015594011638313532, 0.007678602356463671, -0.00855193380266428, 0.007073987741023302, 0.0046857609413564205, -0.007537525612860918, -0.005602759774774313, -0.02351278066635132, 0.03340158239006996, -0.008189165033400059, 0.026132775470614433, 0.0045917099341750145, 0.0190923772752285, 0.000702864199411124, 0.018756480887532234, -0.02907523140311241, -0.0034933274146169424, 0.02345903590321541, -0.02692549116909504, -0.0009127997327595949, 0.016472382470965385, -0.010050034150481224, -0.014846640639007092, 0.005673297680914402, -0.009767880663275719, 0.004027403425425291, -0.012703618966042995, -0.022693190723657608, -0.021551141515374184, -0.0006932071410119534, 0.019482018426060677, -0.016002126038074493, -0.0037788397166877985, 0.014215154573321342, -0.04028075188398361, -0.010278443805873394, -0.003394237719476223, -0.01628427952528, -0.01277751661837101, 0.010325469076633453, -0.004091223701834679, -0.013200746849179268, -0.01863555796444416, 0.020341914147138596, 0.005367631558328867, -0.02896774560213089, 0.0029021487571299076, -0.011373467743396759, -0.006156989373266697, 0.017171047627925873, -0.010479982011020184, 0.004642094485461712, 0.008336960338056087, -0.01239459402859211, 0.0065466295927762985, -0.002762751653790474, 0.013691156171262264, -0.038614701479673386, -0.023754626512527466, 0.0029945203568786383, 0.020167246460914612, -0.031628046184778214, 0.007369576953351498, -0.01816530153155327, -0.01989853009581566, -0.009599932469427586, -0.010721827857196331, 0.02183329500257969, 0.0020926373545080423, -0.014577923342585564, -0.015518434345722198, -0.025984980165958405, -0.013361976481974125, -0.012696901336312294, 0.015276589430868626, 0.007591268979012966, -0.007154603023082018, -0.0029558923561125994, -0.009028907865285873, -0.019052069634199142, 0.0002141342411050573, -0.012280388735234737, 0.006573501508682966, -7.00659875292331e-05, -0.0018894198583438993, 0.015142230316996574, -0.04092567414045334, 0.008740035817027092, 0.005095555447041988, -0.018756480887532234, -0.02051657997071743, -0.028537796810269356, 0.024816060438752174, -0.001921329996548593, -0.0036713527515530586, 0.008793779648840427, 0.03987767547369003, 0.005777426064014435, -0.0005382747040130198, 0.004608504939824343, 0.0072419364005327225, 0.014739153906702995, -0.0034798914566636086, -0.015585614368319511, -0.014860076829791069, -0.022988781332969666, -0.002369752386584878, 0.019723864272236824, -0.023378420621156693, -0.020435964688658714, -0.01142721064388752, 0.019374530762434006, 0.013247772119939327, -0.01338884886354208, 0.010647930204868317, -0.011494390666484833, -0.004846991505473852, -0.014376385137438774, 0.007799524813890457, 0.0009086009813472629, 0.02474888041615486, -0.02367401123046875, -0.04466084763407707, -0.01695607416331768, -0.0029004693496972322, 0.00992911122739315, -0.011931056156754494, -0.010412802919745445, 0.018313096836209297, 0.022639447823166847, 0.02794661931693554, -0.017144175246357918, 0.0006671751034446061, 0.021013706922531128, -0.007255372125655413, 0.014698846265673637, -0.031332459300756454, -0.008760189637541771, -0.03291789069771767, 0.020946528762578964, 0.01344259176403284, 0.01112490426748991, -0.012401311658322811, -0.004165120888501406, -0.00772562762722373, 0.03950146958231926, -0.021564578637480736, -0.016217099502682686, -0.015639357268810272, -0.039286497980356216, -0.008807215839624405, -0.020503144711256027, 0.0034328659530729055, 0.007013526279479265, -0.01633802242577076, -0.01321418210864067, 0.026455236598849297, 0.011669056490063667, 0.0018591891275718808, 0.006580219604074955, 0.03353594243526459, -0.0025225854478776455, 0.028322823345661163, 0.0045917099341750145, -0.013308233581483364, 0.013073105365037918, -0.020476272329688072, -0.011695928871631622, -0.002220278140157461, -0.009223727509379387, -0.005545657128095627, 0.00029558924143202603, -0.004003890324383974, -0.02398303523659706, 0.004769735503941774, -0.009062496945261955, 0.0018927787896245718, -0.0027862645220011473, 0.014107667841017246, -0.017520379275083542, -0.005216478370130062, -0.013657566159963608, -0.008894548751413822, -0.017184482887387276, -0.01588120311498642, -0.0020875988993793726, -0.014470436610281467, 0.03160117566585541, 0.011662338860332966, -0.0013973308959975839, 0.016929201781749725, 0.013838950544595718, 0.026307441294193268, -0.018716173246502876, 0.0493902750313282, 0.015263153240084648, 0.010043315589427948, -0.024399546906352043, -0.019011761993169785, -0.011474236845970154, -0.01664704829454422, 0.0029961999971419573, 0.01700981706380844, -0.01420171931385994, -0.0068926033563911915, 0.03702927008271217, 0.005401221569627523, 0.020449399948120117, -0.04251110553741455, 0.010574033483862877, -0.013308233581483364, 0.0175606869161129, -0.0223976019769907, -0.009398394264280796, -0.016929201781749725, 0.019482018426060677, -0.003217891789972782, 0.020530015230178833, 0.013785206712782383, 0.0029290206730365753, 0.002989482134580612, -0.0017701764591038227, -0.002300893422216177, 0.01938796602189541, -0.0011319724144414067, 0.001595510053448379, -0.005139221902936697, -0.020180683583021164, -0.017627866938710213, -0.009599932469427586, -0.006754885893315077, 0.024453291669487953, 0.008417575620114803, 0.026280570775270462, 0.030365075916051865, 0.008545216172933578, 0.007645012345165014, -0.024399546906352043, 0.0028332900255918503, 0.02754354290664196, -0.015652794390916824, -0.0032279687002301216, 0.018971454352140427, 0.011910902336239815, 0.014161411672830582, -0.04218864440917969, -0.02270662784576416, 0.020435964688658714, -0.001236100448295474, 0.02071811817586422, 0.009882085956633091, 0.009808188304305077, -0.006207373924553394, -0.013711309991776943, 0.0011504467111080885, -0.0129857724532485, -0.01960294134914875, 0.014833205379545689, 0.010298597626388073, -0.019078942015767097, 0.0026367902755737305, -0.017547251656651497, 0.0009992931736633182, 0.0037452499382197857, -0.015531870536506176, -0.015840895473957062, 0.012811105698347092, 0.01848776265978813, -0.028725899755954742, -0.005643067415803671, 0.03702927008271217, 0.03807726874947548, -0.015007871203124523, 0.017332278192043304, 0.0006495405687019229, -0.02042252942919731, 0.02042252942919731, 0.009149830788373947, -0.040146391838788986, -0.00506532471626997, 0.012488645501434803, 0.009075933136045933, 0.006103246007114649, -0.03404650464653969, -0.02026129886507988, 0.01562592200934887, -0.004124813247472048, -0.001147087779827416, -0.019482018426060677, 0.0006957263685762882, -0.0129857724532485, 0.019616376608610153, 0.007026962004601955, -0.0211346298456192, -0.016633613035082817, -0.026912055909633636, -0.016405202448368073, -0.017439763993024826, -0.027019543573260307, -0.022007962688803673, -0.00957306008785963, 0.007933883927762508, 0.00013540842337533832, -0.00029852832085452974, -0.016432074829936028, 0.0022034833673387766, -0.026226826012134552, 0.030042614787817, -0.002809777157381177, 0.027462927624583244, 0.024305496364831924, -0.024963853880763054, -0.0017802533693611622, 0.03732485696673393, -0.032353583723306656, -0.002897110302001238, 0.006143553648144007, -0.012743926607072353, -0.010016444139182568, 0.011615313589572906, -0.001194113283418119, 0.02305595949292183, -0.002566251903772354, -0.0315474309027195, -0.01101741660386324, -0.00970070157200098, -0.02230355143547058, -0.017627866938710213, -0.01203854288905859, 0.011111468076705933, -0.008565369993448257, 0.013576950877904892, 0.005276939831674099, 0.019159557297825813, -0.0002846725983545184, 0.009996290318667889, -0.02051657997071743, 0.010836032219231129, -0.0011378505732864141, 0.00034597376361489296, 0.02464139275252819, 0.007631576620042324, -0.03552445024251938, 0.018662428483366966, 0.019979145377874374, 0.011588441208004951, 0.00315575092099607, 0.012085569091141224, -0.00850490853190422, -0.012730490416288376, 0.02285442128777504, 0.014483872801065445, 0.0063350144773721695, 0.022787243127822876, -0.010883058421313763, -0.03074127994477749, 0.004759658593684435, 0.010896493680775166, 0.011796697974205017, 0.0034395838156342506, -0.014618230983614922, 0.037083011120557785, 0.0024822778068482876, 0.01074198167771101, -0.01542438380420208, -0.007383012678474188, -0.001269690110348165, -0.018407147377729416, -0.0027845848817378283, 0.016230536624789238, 0.0008170690853148699, 0.029800768941640854, 0.028376566246151924, -0.02118837460875511, 0.0037922754418104887, 0.0063014249317348, -0.012999208644032478, -0.007678602356463671, 0.012233363464474678, -0.006348450668156147, 0.028376566246151924, -0.012656593695282936, 0.006076374091207981, -0.028699027374386787, -0.005428093019872904, -0.01277751661837101, -0.019643248990178108, -0.041463106870651245, 0.03324035182595253, 0.017023252323269844, -0.010607622563838959, -0.01719791814684868, 0.003943428862839937, -0.015504999086260796, 0.01033218763768673, -0.0013687796890735626, 0.009492444805800915, -0.026401493698358536, -0.002388226566836238, -0.0033052251674234867, -0.025971544906497, -0.04417715594172478, 0.000426169135607779, 0.014806332997977734, 0.016606740653514862, 0.009055779315531254, 0.19014449417591095, 0.01680827885866165, -0.007550961337983608, 0.052050575613975525, -0.0028416872955858707, 0.0061267586424946785, 0.017144175246357918, 0.006392117124050856, 0.007927165366709232, 0.020691245794296265, -0.010399366728961468, 0.007718909531831741, -0.00957306008785963, -0.016217099502682686, 0.03547070920467377, -0.007161321118474007, -0.03439583629369736, -0.018917711451649666, 0.010224699974060059, -0.0015291704330593348, 0.006633962970227003, -0.0003822926082648337, 0.005908425897359848, -0.006761603523045778, 0.016566433012485504, 0.0035940962843596935, -0.003184302244335413, -0.0027325209230184555, 0.03544383496046066, 0.012045261450111866, -0.010278443805873394, -0.019791042432188988, 0.00018621282652020454, 0.01675453595817089, -0.0072016287595033646, 0.012737208977341652, 0.011091314256191254, -0.024359239265322685, 0.0073158335871994495, 0.015343768522143364, -0.011420493014156818, 0.014161411672830582, -0.000297898513963446, -0.011709364131093025, 0.008511626161634922, 0.029639538377523422, -0.015303460881114006, -0.005787502974271774, -0.014215154573321342, 0.009277471341192722, -0.01420171931385994, -0.01919986493885517, 0.012616286054253578, 0.005646426230669022, -0.000350382411852479, -0.017520379275083542, 0.01479289773851633, -0.013281362131237984, 0.00859895907342434, -0.010023161768913269, -0.005421375390142202, 0.030096357688307762, -0.01303279772400856, 0.027651028707623482, -0.00888783112168312, 0.0336165577173233, 0.003278353251516819, 0.017184482887387276, 0.028027234598994255, 0.004333069548010826, 9.714766929391772e-05, 0.004460710100829601, -0.03442271053791046, -0.0039199162274599075, -0.01893114671111107, -0.01872960850596428, 0.014631667174398899, 0.00728224404156208, 0.005387785378843546, 0.006633962970227003, 0.00689932145178318, -0.02641492895781994, 0.008316806517541409, -0.000449681916506961, 0.0043397871777415276, -0.008726600557565689, -0.0007146205753087997, -0.0005941175622865558, 0.008001063019037247, 0.006533193867653608, -0.013012643903493881, 0.008074960671365261, -0.010600904934108257, -0.019885092973709106, -0.012683465145528316, -0.006388758309185505, 0.017896585166454315, -0.0007377134752459824, -0.008820651099085808, 0.013583668507635593, -0.005028375890105963, 0.047724224627017975, -0.007141167297959328, -0.021510833874344826, -0.022128885611891747, -0.0037922754418104887, -0.0025612134486436844, 0.015572178177535534, 0.01058746874332428, 0.00036633751005865633, 0.014067360199987888, -0.0068758088164031506, -0.005716964602470398, -0.007248654030263424, 0.021766116842627525, 0.017036689445376396, -0.0022924961522221565, -0.04514453932642937, 0.003102007554844022, -0.0027207646053284407, 0.014846640639007092, -0.011360031552612782, 0.029720153659582138, -0.009485727176070213, -0.004400248639285564, -0.023176882416009903, -0.03356281295418739, 0.021618321537971497, -0.021094322204589844, -0.03826536983251572, 0.0005067843594588339, -0.023956162855029106, 0.023176882416009903, 0.007188192568719387, 0.013026080094277859, -0.001968355616554618, -0.007295679766684771, -0.020785298198461533, 0.012065415270626545, -0.00773906335234642, 0.001236100448295474, -0.03557819500565529, -0.00507876044139266, -0.0016971188597381115, 0.0004282684822101146, -0.024520469829440117, 0.015316897071897984, -0.017076997086405754, -0.0020758425816893578, -0.012428184039890766, -0.011306287720799446, -0.03375091776251793, -0.008565369993448257, -0.020785298198461533, 0.027019543573260307, -0.0018172019626945257, -0.018138431012630463, -0.01949545368552208, 0.013375412672758102, 0.021604886278510094, -0.03283727541565895, -0.014282334595918655, 0.027301697060465813, 0.0072016287595033646, -0.003977018874138594, -0.00888783112168312, -0.1710118055343628, 0.012555824592709541, 0.006321578752249479, -0.031439945101737976, 0.007557679433375597, -0.006805270444601774, 0.009237163700163364, 0.008693010546267033, -0.005602759774774313, -0.012683465145528316, 0.0034866093192249537, 0.019764171913266182, -0.039635829627513885, -0.035658810287714005, 0.00023491786851081997, -0.00355714769102633, -0.009640240110456944, 0.01991196535527706, 0.024869803339242935, -0.01648581773042679, -0.0015426062745973468, -0.024668265134096146, 0.022316986694931984, -0.025286315008997917, 0.00011210557568119839, -0.003943428862839937, -0.016512690111994743, 0.022236371412873268, 0.02173924446105957, -0.02128242515027523, 0.004608504939824343, -0.003463096683844924, 0.0017483431147411466, 0.0011118185939267278, 0.025729699060320854, -0.01079572457820177, -0.005693451501429081, 0.00837054941803217, -0.012105722911655903, 0.017735354602336884, 0.005542298313230276, -0.009640240110456944, 0.007846551015973091, 0.008874394930899143, -0.00651639886200428, 0.045681972056627274, 0.034906402230262756, 0.0016425356734544039, 0.010829314589500427, 0.0016752855153754354, 0.03356281295418739, -0.018420584499835968, 0.015437819063663483, 0.0007528288406319916, -0.002798020839691162, 0.010668084025382996, -0.000885927933268249, 0.011360031552612782, 0.012287107296288013, 0.0025544955860823393, -0.006143553648144007, -0.042779821902513504, -0.02993512712419033, 0.0007305756444111466, -0.010681520216166973, -0.011984799988567829, 0.0330791212618351, 0.0009304342675022781, 0.004527889657765627, 0.0025645724963396788, -0.003001238452270627, -0.006385399028658867, -0.013093259185552597, -0.0009186778916046023, 0.015236281789839268, -0.016942637041211128, -0.012260234914720058, -0.002512508537620306, -0.01038593053817749, -0.004000531509518623, -0.0027812260668724775, 0.027651028707623482, -0.04143623635172844, 0.004853709600865841, -0.023082831874489784, 0.01054044347256422, 0.014806332997977734, 0.015545306727290154, -0.021927347406744957, 0.012441619299352169, 0.023754626512527466, -0.015330332331359386, -0.00995598267763853, -0.017023252323269844, -0.006113322917371988, 0.002079201629385352, -0.0025276239030063152, 0.002798020839691162, 0.008001063019037247, -0.006808629259467125, 0.01074198167771101, 0.016176791861653328, -0.029290206730365753, 0.039474599063396454, 0.047724224627017975, 0.02398303523659706, 0.0066641937009990215, 0.0015921511221677065, 0.014215154573321342, -0.0012100684689357877, -0.02438611164689064, -0.0031624690163880587, 0.02672395296394825, 0.014470436610281467, -0.015276589430868626, 0.018595250323414803, -0.011037570424377918, -0.008256345055997372, 0.017036689445376396, 0.007530807517468929, 0.023606831207871437, -0.0361425019800663, -0.024829495698213577, -0.004934324882924557, 0.009324496611952782, -0.013563514687120914, -0.10711079090833664, -0.03864157572388649, 0.020220991224050522, 0.03195050731301308, -0.006748167797923088, 0.024466726928949356, 0.005753912962973118, 0.006586937233805656, -0.012945464812219143, 0.04237674921751022, -0.005448246840387583, -0.03289102017879486, -0.008390703238546848, 0.0032733147963881493, 0.03278353437781334, -0.016982944682240486, 0.015908075496554375, -0.017627866938710213, -0.018917711451649666, 0.03517511859536171, 0.01648581773042679, -0.017144175246357918, -0.014551051892340183, -0.027462927624583244, -0.029236461967229843, -0.024816060438752174, -0.030687537044286728, 0.005844605155289173, 0.0017802533693611622, -0.005770707968622446, 0.009257317520678043, -0.013039516285061836, 0.004423761740326881, -0.009411829523742199, -0.01659330539405346, -0.014712282456457615, -0.0028685592114925385, -0.020476272329688072, 0.009297625161707401, -0.053985342383384705, -0.01262300368398428, 0.016942637041211128, 0.021000271663069725, -0.007208346389234066, 0.013456027954816818, -0.017453201115131378, -0.004793248139321804, -0.002050650306046009, -0.004417043644934893, -0.025178829208016396, -0.017130739986896515, -0.003056661458685994, -0.02413083054125309, -0.0018591891275718808, 0.02259914018213749, -0.024359239265322685, 0.001503978157415986, 0.0068724495358765125, -0.01883709616959095, 0.010876339860260487, -0.005156016908586025, 0.00992911122739315, -0.027758516371250153, 0.02947830781340599, 0.0021161502227187157, -0.00857880525290966, 0.0035739424638450146, -0.008518344722688198, -0.014725717715919018, -0.019683556631207466, -0.015491562895476818, 0.018702736124396324, -0.018057815730571747, 0.014067360199987888, -0.02026129886507988, -0.0019179710652679205, -0.00040244642877951264, -0.010903212241828442, 0.001981791341677308, 0.0036511989310383797, -0.025057906284928322, -0.008155575953423977, -0.009788034483790398, -0.009935828857123852, -0.007517371792346239, 0.017372585833072662, 0.0028685592114925385, -0.028672154992818832, 0.03638434782624245, -0.03552445024251938, 0.003382481401786208, 0.03324035182595253, 0.01719791814684868, 0.012764080427587032, 0.0057068876922130585, 0.018662428483366966, -0.0010799083393067122, 0.006539911963045597, 0.022626012563705444, 0.007550961337983608, -0.03710988536477089, -0.012717055156826973, -0.04871847853064537, 0.031332459300756454, 0.009908957406878471, -0.0012377799721434712, 0.004494300112128258, -0.016109613701701164, 0.011030852794647217, 0.020597195252776146, -0.006177143193781376, -0.000879210012499243, -0.005763989873230457, 0.0005987361655570567, -0.031789276748895645, -0.022585704922676086, -0.027113594114780426, 0.005018298979848623, 0.014725717715919018, 0.00264686718583107, -0.004097941797226667, 0.015599049627780914, -0.013919565826654434, -0.012112440541386604, -0.005626272410154343, 0.027731643989682198, -0.03431522101163864, 0.00506532471626997, -0.005300452467054129, -0.017076997086405754, -0.001245337538421154, -0.013368695043027401, 0.035040758550167084, -0.015491562895476818, -0.025057906284928322, 0.03794290870428085, -0.001995227299630642, -0.018608685582876205, -0.0021363040432333946, 0.00446742819622159, 0.020435964688658714, -0.016512690111994743, -0.027651028707623482, -0.019414838403463364, 0.005451606120914221, -0.0030986485071480274, -0.012266953475773335, 0.015545306727290154, -0.015142230316996574, 0.0126028498634696, -0.00857880525290966, 0.042430490255355835, 0.0122199272736907, 0.00954618863761425, 0.0063350144773721695, -0.022209500893950462, -0.014376385137438774, -0.013355258852243423, 0.016270844265818596, 0.00427596690133214, 0.02942456491291523, -0.014806332997977734, 0.02657615952193737, -0.0020724835339933634, -0.005582605954259634, 0.010090341791510582, 0.01945514604449272, -0.0014032091712579131, -0.0032548406161367893, -0.006959782913327217, -0.006143553648144007, -0.03888342157006264, -0.01173623651266098, 0.011433929204940796, 0.0336165577173233, 0.041060030460357666, 0.006351809483021498, -0.012004953809082508, 0.011010698974132538, 0.010285161435604095, -0.005364272743463516, 0.019052069634199142, -0.0014048885786905885, 0.005831169430166483, -0.030580049380660057, 0.013812079094350338, 0.02352621592581272, 0.019172992557287216, -0.031896766275167465, 0.01619022898375988, -0.001470388495363295, -0.0004475825699046254, 0.0026216749101877213, 0.017372585833072662, -0.003266596933826804, 0.008289934135973454, 0.012448337860405445, 0.004319633357226849, -0.004574915394186974, -0.009176702238619328, -0.020744990557432175, 0.015209409408271313, 0.02011350356042385, -0.006069655995815992, 0.018313096836209297, -0.019576068967580795, -0.018460892140865326, -0.0018591891275718808, -0.02662990242242813, -0.026307441294193268, -0.005622913595288992, 0.025622211396694183, 0.004699197132140398, -0.02071811817586422, 0.027476362884044647, 0.011218954809010029, -0.013738181442022324, 0.02352621592581272, -0.017130739986896515, -0.00728224404156208, 0.00024247553665190935, 0.036572448909282684, 0.009405111894011497, 0.005172811448574066, 0.001963317161425948, 0.02051657997071743, 0.03681429475545883, 0.00751065369695425, 0.014403257519006729, 0.012811105698347092, 0.027866004034876823, 0.009264035150408745, 0.0315474309027195, -0.002586405724287033, -0.05111006647348404, -0.01700981706380844, -0.008343677967786789, -0.03909839317202568, 0.015370639972388744, 0.013892694376409054, 0.00789357628673315, 0.09292250871658325, -0.005491913761943579, -0.02937082201242447, 0.033831533044576645, -0.029182719066739082, 0.019119249656796455, 0.020194118842482567, -0.018716173246502876, -0.004830196965485811, -0.00232944474555552, 0.019871657714247704, 0.0024134188424795866, 0.005542298313230276, -0.01858181320130825, -0.02352621592581272, -0.010439674369990826, 0.012878285720944405, -0.004813401959836483, -0.003842659993097186, -0.008175729773938656, 0.02942456491291523, -0.014698846265673637, 0.0019616377539932728, 0.007456910330802202, -0.017426328733563423, -0.016458945348858833, 0.03882967680692673, 0.014080796390771866, 0.0042692492716014385, -0.02728825993835926, -0.01219305582344532, 0.011447364464402199, -0.03264917433261871, -0.007671884261071682, -0.002661982551217079, -0.02000601589679718, -0.014967563562095165, -0.016418637707829475, 0.004652171395719051, -0.0008191684610210359, 0.018716173246502876, 0.05890287086367607, 0.003926634322851896, -0.010903212241828442, 0.0026502262335270643, -0.005297093652188778, -0.011225673370063305, 0.004776453133672476, -0.003483250504359603]} +{"id": "test:10000099", "text": "\"Hulen R. Loyd, Sept. 27, 1884 - Aug. 30, 1959. Margaret F. Loyd, Aug., 24, 1878 - Mar. 14, 1934. Hulen R. Loyd was a son of James S. Loyd & Nancy or Martha Wilkerson. He married Margaret Finley McIntosh on Sept. 5, 1902 in Callaway Co., Missouri. She was a daughter of William McIntosh and Emma Finley.\"", "vector_field": [-0.025047335773706436, -0.0024433238431811333, -0.01587507128715515, -0.02418498694896698, -0.014071977697312832, 0.02330957166850567, 0.006209570448845625, 0.005314555950462818, -0.004024297930300236, -0.03423266485333443, -0.00045853151823394, 0.040478166192770004, 0.0024433238431811333, 0.004318281076848507, -0.003472263691946864, 0.016580630093812943, 0.014241834171116352, 0.007885272614657879, 0.013340286910533905, -0.028535932302474976, -0.032194383442401886, -0.001843925565481186, -0.005151232238858938, -0.013111633248627186, -0.005899255163967609, -0.006634212099015713, 0.02140194922685623, -0.014620745554566383, 0.017142465338110924, -0.007676218170672655, -0.007042521610856056, 0.01400664821267128, 0.0006263467366807163, -0.008355644531548023, -0.029163096100091934, -0.015234842896461487, 0.005922120530158281, 0.0039851004257798195, 0.02764745056629181, 0.0020252149552106857, 0.012079427018761635, -0.004350945819169283, 0.00819232128560543, -0.013627736829221249, -0.017795760184526443, 0.011922636069357395, -0.005654269363731146, 0.019154613837599754, -0.032455701380968094, 0.03365776687860489, 0.014085043221712112, 0.0016144555993378162, 0.02021295204758644, -0.01655449904501438, -0.001758997212164104, -0.017260057851672173, -0.011550257913768291, 0.027856504544615746, -0.014228768646717072, 0.006644011475145817, -0.007179713808000088, 0.001724699162878096, -0.027751978486776352, 0.007532493211328983, -0.02149341069161892, -0.008597363717854023, -0.00893054436892271, -0.005471346899867058, -0.022016046568751335, 0.02866659127175808, 0.009577306918799877, 0.024537766352295876, 0.0073561035096645355, -0.0007516977493651211, 0.009427049197256565, 0.0022734671365469694, -0.04280389845371246, 0.010511518456041813, -0.010165272280573845, -0.007650086190551519, 0.003988367039710283, -0.007401833776384592, 0.0004977292264811695, 0.024877479299902916, 0.03857054561376572, -0.0031782807782292366, 0.018135473132133484, 0.032534096390008926, -0.005147965624928474, -0.019690316170454025, 0.01630624756217003, 0.04238579049706459, 0.013046303763985634, 0.007160114590078592, -0.0022914328146725893, 0.00350166205316782, -0.022617077454924583, 0.016175588592886925, 0.012935243546962738, -0.03235117718577385, -0.02268240787088871, 0.005954785272479057, 0.0038152437191456556, 0.0031913467682898045, -0.03922383859753609, -0.015574555844068527, 0.006209570448845625, -0.01783495768904686, 0.013679999858140945, -0.009975817054510117, -0.006457822397351265, 0.0015572923002764583, 0.021284356713294983, -0.02772584557533264, 0.005206762347370386, -0.03825696185231209, 0.0028744987212121487, -0.015470028854906559, -0.0013727364130318165, -0.0077676791697740555, -0.0016095559112727642, 0.033161260187625885, 0.0030508884228765965, -0.017338452860713005, 0.012125157751142979, 0.023074384778738022, 0.003926303703337908, -0.004752722103148699, -0.010250200517475605, 0.011648252606391907, 0.019781777635216713, 0.0037433812394738197, -0.017652034759521484, -0.0252041257917881, 0.003599656280130148, 0.021219026297330856, -0.03527793660759926, -0.0012935244012624025, -0.017900286242365837, -0.024459369480609894, 0.012634728103876114, 0.025844356045126915, -0.00860389694571495, -0.0023894270416349173, -0.013111633248627186, 0.018945559859275818, 0.012085960246622562, 0.006029914133250713, 0.0023404299281537533, -0.0009995416039600968, 0.019468195736408234, 0.0019713181536644697, 0.015535358339548111, -0.006859599146991968, 0.029764126986265182, 0.0039197709411382675, -0.004804985597729683, -0.005739198066294193, -0.02280000038444996, 0.0032174785155802965, -0.01172664761543274, 0.023466361686587334, 0.01953352428972721, 0.01596653275191784, -0.0009774928912520409, 0.01273272279649973, 0.016711289063096046, -0.005154498852789402, 0.0014103008434176445, 0.011393466964364052, -0.0007721132133156061, 0.0013155731139704585, -0.023740746080875397, 0.013771461322903633, -0.0021166761871427298, 0.0007198496023193002, -0.001829226384870708, -0.01877570152282715, -0.03180240839719772, -0.036506131291389465, -0.022016046568751335, 0.008571232669055462, 0.004857249557971954, 0.04497283697128296, -0.00668320944532752, -0.00022130373690743, -0.0013204728020355105, -0.004654727876186371, 0.021454213187098503, -0.008042063564062119, -0.0038707738276571035, 0.025765961036086082, 0.022604012861847878, -0.017900286242365837, -0.6409609317779541, -0.03530406951904297, -0.011334670707583427, 0.008878281340003014, -0.016319312155246735, 0.0052002291195094585, -0.0037335818633437157, 0.011746246367692947, -0.023035187274217606, -0.002650744980201125, -0.03632320836186409, 0.044580861926078796, 0.0005250859539955854, 0.0037270488683134317, -0.015104183927178383, -0.004213753622025251, -0.032220516353845596, 0.002665444277226925, -0.010635645128786564, 0.0015907736960798502, -0.03645386919379234, 0.028483668342232704, 0.009544641710817814, -0.0312536396086216, 0.01788722164928913, 0.009969283826649189, 0.02500813826918602, 0.008662693202495575, -0.0060266475193202496, -0.021676333621144295, -0.018305329605937004, 0.027098681777715683, 0.008950143121182919, -1.069898371497402e-05, 0.03739461302757263, -0.003753180615603924, -0.009427049197256565, 0.046174902468919754, 0.02730773761868477, 0.01965111866593361, -0.02171553112566471, -0.02963346801698208, 0.01352320984005928, 0.0039197709411382675, -0.0030231233686208725, -0.007140515837818384, 0.016933409497141838, -0.010152206756174564, -0.03459851071238518, -0.012144756503403187, 0.01433329563587904, 0.013954384252429008, -0.008969741873443127, -0.018213868141174316, -0.03360550105571747, -0.015104183927178383, 0.022068310528993607, 0.001744298031553626, -0.00177042989525944, -0.017077134922146797, -0.03127977252006531, 0.013046303763985634, -0.004174556117504835, 0.03242957219481468, -0.009681833907961845, 0.023636218160390854, -0.023636218160390854, 0.018331462517380714, 0.02432871051132679, -0.0003564541693776846, -0.014320229180157185, 0.043483324348926544, 0.007512893993407488, -0.018017880618572235, 0.020944641903042793, 0.0066048139706254005, 0.020892377942800522, 0.0015646418323740363, 0.001786762266419828, 0.014437822625041008, -0.005709799472242594, -0.007349570281803608, -0.0007149498560465872, 0.011582923121750355, 0.019037021324038506, -0.014999656938016415, -0.027464527636766434, 0.003455931320786476, 0.003436332568526268, -0.01315736398100853, 0.007205845322459936, 0.005409283563494682, 0.014424757100641727, -0.025426246225833893, -0.013823725283145905, 0.014973524957895279, -0.01786108873784542, -0.003958968445658684, 0.0021869055926799774, -0.031828537583351135, 0.013248825445771217, -0.005272091832011938, 0.029581204056739807, 0.003655186388641596, 0.009348653256893158, -0.0003525752108544111, 0.009080802090466022, -0.013294556178152561, 0.015247908420860767, -0.02098383940756321, 0.02047426998615265, -0.002815701998770237, -0.000900730665307492, -0.009381318464875221, -0.04201994463801384, -0.021258223801851273, 0.004834384191781282, 0.01074670534580946, 0.01148492842912674, -0.03130590170621872, 0.030182234942913055, -0.02004309557378292, 0.0012935244012624025, -0.013614670373499393, -0.008793353103101254, 0.012170888483524323, -0.0032207451295107603, -0.015496160835027695, -0.005739198066294193, 0.011145215481519699, 0.00827724952250719, -0.0030492551159113646, 0.03065260872244835, -0.014516218565404415, 0.022081375122070312, -0.007957134395837784, 0.020395874977111816, -0.006245501805096865, 0.02704641968011856, 0.006441490259021521, -0.03843988478183746, 0.02869272232055664, 0.010126074776053429, -0.008532034233212471, 0.005598739255219698, -0.02055266499519348, -0.005242693703621626, -0.007708882912993431, -0.03313513100147247, -0.02381914108991623, 0.004922579042613506, 1.5796473235241137e-05, -0.028535932302474976, 0.010452722199261189, -0.030260631814599037, -0.004040630534291267, -0.006333696190267801, 0.006657077465206385, 0.00806819461286068, -0.017730429768562317, 0.006748538929969072, 0.0026442119851708412, -0.007623954210430384, -0.01771736517548561, 0.03830922767519951, -0.019468195736408234, 0.015953468158841133, 0.01885409839451313, 0.004037363920360804, -0.00874762237071991, 0.0039099715650081635, -0.02149341069161892, -0.0015417765825986862, 0.014058911241590977, 0.031880803406238556, -0.004292149096727371, 0.0010697707766667008, 0.00798326637595892, 0.01877570152282715, 0.0066701434552669525, 0.010798968374729156, -0.03039129078388214, -0.010949227027595043, 0.006121375598013401, 0.008048595860600471, -0.0020644126925617456, -0.002640945604071021, 0.014908195473253727, -0.0099366195499897, 0.01498659048229456, -0.008394842967391014, 0.004439140670001507, 0.018475186079740524, 0.004324813839048147, 0.002018681960180402, 0.02830074541270733, -0.006121375598013401, 0.003753180615603924, 0.015744412317872047, 0.02670670486986637, 0.004360745195299387, 0.009342120960354805, 0.01882796548306942, 0.0010289398487657309, -0.008577764965593815, 0.0015899570425972342, 0.00907426979392767, -0.02495587430894375, 0.03958968445658684, -0.0010771204251796007, 0.008159656077623367, -0.0028908310923725367, -0.02004309557378292, -0.042490314692258835, 0.026811232790350914, 0.01481673400849104, 0.014686075039207935, -0.0031096849124878645, -0.011439197696745396, -0.0019337536068633199, 0.014699140563607216, 0.012719656340777874, 0.024171920493245125, 0.009146131575107574, -0.01300057303160429, -0.006536217872053385, -0.011177879758179188, -0.02246028743684292, 0.001835759379900992, -0.0022767335176467896, -0.022381892427802086, 0.0030378224328160286, 0.005804527550935745, 0.02968573197722435, 0.005484412889927626, 0.00786567386239767, 0.010648710653185844, -0.020591862499713898, 0.03608802333474159, -0.007055587600916624, 0.011543725617229939, 0.008714957162737846, 0.04102693498134613, -0.011733180843293667, -0.015404699370265007, -0.02050040103495121, 0.01928527280688286, 0.02302212081849575, -0.004612263757735491, 0.007212378550320864, 0.0012918910942971706, 0.016763553023338318, 0.00016771310765761882, 0.0010624212445691228, 0.021937651559710503, -0.03676744922995567, 0.004883381072431803, -0.0067354729399085045, 0.010942693799734116, 0.02106223627924919, 0.02699415571987629, -0.0048866476863622665, 0.012425673194229603, -0.027621319517493248, 0.012216619215905666, -0.005350487306714058, -0.006774670444428921, 0.00427581649273634, -0.022329628467559814, -0.010655243881046772, -0.01241914089769125, -0.0016838682349771261, 0.02347942814230919, -0.016423840075731277, -0.0023240975569933653, 0.0023289972450584173, 0.009525042958557606, -0.006565616000443697, 0.012203553691506386, 0.010354728437960148, -0.04523415490984917, -0.021388882771134377, 0.03700263798236847, 0.010785902850329876, -0.009492378681898117, -0.01921994239091873, -0.0015434097731485963, 0.0034265329595655203, -0.00358332390896976, 0.018422923982143402, 0.049519773572683334, -0.015208710916340351, -0.004497936926782131, 0.007160114590078592, 0.0006353295175358653, 0.013758395798504353, 0.004857249557971954, -0.01712939888238907, 0.020056160166859627, -0.003929570317268372, 0.005752263590693474, 0.001778596080839634, -0.019089283421635628, -0.006559083238244057, 0.02704641968011856, -0.003580057295039296, 0.010041146539151669, -0.013745330274105072, 0.035539254546165466, -0.026014212518930435, 0.012680458836257458, -0.02285226434469223, -0.010563782416284084, 0.02202911302447319, -0.0015082951867952943, 0.03240343928337097, -0.0005671418621204793, -0.003287707921117544, -0.0038348424714058638, 0.00038932307506911457, -0.013588539324700832, -0.0040504299104213715, -0.04938911274075508, 0.007728481665253639, 0.1187429279088974, 0.004739656113088131, 0.016593696549534798, 0.010511518456041813, -0.012216619215905666, -0.01916768029332161, 0.004148424137383699, -0.03269089013338089, 0.022277364507317543, 0.019742580130696297, 0.02367541566491127, -0.0015866905450820923, -0.023714613169431686, 0.0009440114954486489, 0.0011857306817546487, -0.019611919298768044, 0.008401375263929367, -0.019546590745449066, -0.01686808094382286, -0.001025673351250589, -0.006395759526640177, -0.026079542934894562, -0.014764470048248768, 0.02270853891968727, -0.021597936749458313, -0.019925501197576523, 0.05069570243358612, 0.0016071060672402382, 0.002833667676895857, -0.005004240665584803, -0.0039426363073289394, -0.004749455489218235, 0.0073887682519853115, 0.02965959906578064, -0.010439656674861908, 0.0253217201679945, 0.013118166476488113, -0.01921994239091873, 0.020722521468997, -0.0006275716586969793, 0.0533088855445385, 0.0060266475193202496, -0.0009685100521892309, -0.011589455418288708, 0.007833008654415607, -0.01177891157567501, -0.003557192161679268, 0.01616252213716507, 0.004818051587790251, -0.024054327979683876, 0.021271290257573128, 0.01328149065375328, 0.020918510854244232, -0.024002064019441605, -0.0046808598563075066, -0.03331805393099785, -0.00366825214587152, -0.02554384060204029, -0.00884561613202095, 0.0513751283288002, -0.02319197729229927, -0.02769971452653408, 0.012314613908529282, -0.022381892427802086, 0.009152664802968502, -0.009322521276772022, -0.02903243713080883, -0.018618911504745483, -0.013405616395175457, -0.009165731258690357, 0.02033054456114769, 0.0176912322640419, 0.004938911180943251, 0.007819943130016327, 0.004475071560591459, -0.0012698423815891147, 0.012157822959125042, 0.004929111804813147, 0.006506819743663073, -0.0033383381087332964, -0.009178796783089638, -0.00020364434749353677, 0.008238052017986774, -0.016763553023338318, 0.00924412626773119, -0.0030541548039764166, 0.00014648101932834834, -0.0006581948837265372, 0.004455472808331251, 0.004592665005475283, 0.0015278940554708242, -0.01633237861096859, -0.0003597206377889961, -0.027542922645807266, 0.01364080235362053, -0.008329513482749462, -0.01026326697319746, 0.013046303763985634, -0.003805444110184908, 0.020030029118061066, 0.014764470048248768, -0.013536275364458561, -0.016436906531453133, -0.01820080354809761, -0.006000516004860401, 0.013144298456609249, 0.00941398274153471, 0.019912436604499817, -0.02001696266233921, -0.004729856736958027, -0.004589398391544819, -0.022786933928728104, 0.02466842532157898, 0.011720115318894386, 0.006666876841336489, 0.023910602554678917, 0.008198854513466358, 0.009590372443199158, 0.021415015682578087, -0.015809742733836174, -0.005020573269575834, -0.00508916936814785, 0.01664596050977707, -0.011870373040437698, -0.01449008658528328, 0.03394521400332451, 0.016005730256438255, -0.0365845263004303, -0.012994040735065937, -0.008714957162737846, -0.0033873352222144604, -0.0012314612977206707, -0.011119083501398563, -0.004690659232437611, 0.000969326647464186, -0.010047679767012596, -0.03444172069430351, -0.011896505020558834, -0.015025787986814976, 0.0009546275250613689, 0.022773869335651398, -0.0074540977366268635, 0.012719656340777874, -0.040844012051820755, -0.0040830946527421474, -0.022760802879929543, -0.0023861604277044535, -0.002699742093682289, 0.01888022944331169, 0.028588196262717247, -0.011929169297218323, -0.006715874187648296, -0.023374900221824646, 0.010981891304254532, -0.0004332163371145725, 0.01307243574410677, -0.0058861891739070415, 0.017926419153809547, 0.009191862307488918, 0.02963346801698208, 0.025818224996328354, -0.006036447361111641, 0.031227508559823036, -0.018605845049023628, 0.009139599278569221, 0.0005736747989431024, 0.0008215186535380781, 0.004501203540712595, -0.028823381289839745, 0.01669822447001934, 0.04745535925030708, 0.025556905195116997, 0.003674785140901804, 0.012556333094835281, 0.017625903710722923, 0.042202867567539215, -0.0003195021708961576, 0.003753180615603924, 6.124642095528543e-05, -0.035513125360012054, -0.005958051886409521, -0.018096275627613068, -0.01185077428817749, 0.0011661318130791187, -0.012183954007923603, -0.018475186079740524, 0.0019174212357029319, 0.011308538727462292, -0.005105501506477594, -0.015535358339548111, 0.0014829799765720963, -0.02333570271730423, 0.018919426947832108, 0.009054671041667461, -0.026915760710835457, -0.01604492962360382, -0.01812240667641163, -0.009361719712615013, -0.00533742131665349, -0.006134441588073969, 0.0010289398487657309, -0.00819232128560543, 0.005448481533676386, -0.011432665400207043, 0.006379426922649145, 0.012543266639113426, -0.024903610348701477, -0.007545558735728264, 0.02001696266233921, -0.02767358161509037, -0.02330957166850567, 0.0015605587977916002, -0.017064068466424942, -0.013575472868978977, -0.007682750932872295, 0.0022979655768722296, -0.008322980254888535, 0.030156103894114494, 0.007911404594779015, -0.020722521468997, -0.010224069468677044, -0.0009856590768322349, 0.008564699441194534, -0.01582280918955803, 0.02645845338702202, 0.020787851884961128, -0.0029496275819838047, -0.03966807946562767, 0.04727243632078171, 0.011948768049478531, -0.0011285673826932907, 0.025909684598445892, -0.004207220859825611, -0.018396791070699692, 0.022656274959445, 0.0002703008649405092, 0.0025462177582085133, -0.011347736231982708, -0.03062647581100464, 0.004246418364346027, 0.019376734271645546, -0.014581548050045967, -0.006284699309617281, -0.011680916883051395, -0.01416343916207552, -0.003965501673519611, 0.004520802292972803, 0.005598739255219698, -0.02696802280843258, -0.00814005732536316, -0.027255473658442497, 0.015430831350386143, 0.011772378347814083, 0.02413272298872471, 0.011347736231982708, -0.010439656674861908, 0.0020954441279172897, -0.023414097726345062, 0.0015932235401123762, 0.004399942699819803, -0.0258051585406065, 0.03632320836186409, -0.028248481452465057, -0.016084127128124237, -0.015862006694078445, -0.009616504423320293, -0.026301663368940353, 0.0009701433009468019, -0.008571232669055462, 0.016711289063096046, 0.0006704441620968282, 0.005249226465821266, -0.007748080417513847, 0.0048017194494605064, 0.025334784761071205, -0.013719198293983936, 0.00932905450463295, -0.009747163392603397, -0.025073466822504997, -0.008584298193454742, 0.019664183259010315, -0.013052836991846561, 0.013706131838262081, 0.010119541548192501, -0.022695472463965416, -0.027203209698200226, -0.007833008654415607, -0.013379484415054321, 0.0016773352399468422, -0.028927909210324287, -0.023636218160390854, -0.0032550429459661245, -0.008303381502628326, -0.04141891375184059, -0.01536550186574459, -0.03062647581100464, -0.0070490543730556965, 0.023205043748021126, -0.03603576123714447, 0.007597822695970535, -0.021676333621144295, 0.007682750932872295, -0.03559152036905289, -0.027412263676524162, 0.004834384191781282, -0.02540011517703533, 0.003606189275160432, -0.0019451862899586558, -0.03342257812619209, -0.04541707783937454, 0.02517799474298954, -0.0031717480160295963, -0.006065845489501953, -0.001643853960558772, 0.0034265329595655203, 0.012543266639113426, 0.014764470048248768, -0.019977765157818794, 0.0002935744996648282, 0.03425879776477814, 0.009629570879042149, 0.01635850965976715, -0.01136733591556549, -0.00016026146477088332, 0.028483668342232704, 0.0032566762529313564, 0.025922751054167747, -0.005419083405286074, -0.005824126303195953, 0.003651919774711132, 0.008153123781085014, 0.00741489976644516, -0.005673868115991354, 0.00044260747381486, -0.026419255882501602, -0.014973524957895279, -0.0019517192849889398, 0.011929169297218323, -0.022133639082312584, -0.01202063076198101, 0.014111175201833248, 0.009557708166539669, 0.01803094521164894, 0.019585788249969482, 0.011471862904727459, -0.019638052210211754, -0.001760630402714014, -0.013327221386134624, -0.015182578936219215, -0.01851438358426094, 0.004785386845469475, 0.047559887170791626, -0.010838166810572147, -4.863986396230757e-05, -0.01482979953289032, 0.01834452711045742, -0.031514957547187805, -0.026523781940340996, -0.0013114899629727006, 0.009093868546187878, 0.0006586031522601843, 0.0016275214729830623, 0.0033873352222144604, -0.002072578761726618, -0.01107335276901722, 0.025765961036086082, -0.012399542145431042, -0.029868654906749725, 0.0196249857544899, -0.027569055557250977, 0.024616161361336708, 0.001001174794510007, -0.00891747884452343, -0.010152206756174564, 0.0020056162029504776, 0.002719341078773141, -0.0012404441367834806, 0.000946461339481175, -0.04032137617468834, -0.02384527213871479, -0.006820401176810265, 0.004573065787553787, 0.002618080237880349, -0.008754154667258263, 0.016031863167881966, 0.004873581696301699, 0.013823725283145905, -0.006787736434489489, -0.019180744886398315, -0.0334487110376358, 0.0030868195462971926, 0.020853180438280106, -0.0038577078375965357, 0.005987450014799833, 7.334258407354355e-05, 0.015247908420860767, 0.007447564508765936, 0.003455931320786476, 0.03410200774669647, -0.005651003215461969, 0.0020219485741108656, 0.009061203338205814, 0.015391633845865726, 0.005912321154028177, 0.03433719277381897, 0.006601547356694937, -0.006376160774379969, -0.013928252272307873, -0.014555416069924831, -0.008630028925836086, 0.019089283421635628, -0.015953468158841133, 0.008146590553224087, -0.023570889607071877, -0.018814900889992714, 0.00497484253719449, 0.005046704784035683, -0.03872733563184738, 0.032873813062906265, 0.017064068466424942, -0.02622326649725437, 0.004455472808331251, -0.019324470311403275, -0.015208710916340351, 0.006225902587175369, -0.02464229241013527, 0.005425616167485714, -0.00181126082316041, -0.018814900889992714, 0.007917936891317368, 3.5242210287833586e-05, -0.013444813899695873, -0.011282406747341156, -0.004102693405002356, 0.003795644734054804, 0.03778659179806709, 0.19860172271728516, 0.0050924355164170265, 0.021937651559710503, 0.0015532091492787004, -0.024655358865857124, 0.0008729656110517681, 0.023949800059199333, 0.0022065043449401855, -0.00802246481180191, 0.010870831087231636, -0.0057359314523637295, 0.006333696190267801, 0.001714899786747992, 0.022773869335651398, -0.006872665137052536, -0.016828883439302444, -0.02086624689400196, -0.03700263798236847, -0.014882063493132591, -0.01661982759833336, 0.010655243881046772, 0.030600344762206078, -0.01851438358426094, -0.023936733603477478, 0.028431404381990433, -0.015770545229315758, -0.007774212397634983, 0.014999656938016415, -0.010126074776053429, -0.021088367328047752, -0.028483668342232704, -0.009792894124984741, 0.009819026105105877, -0.000857449893373996, -0.002752005821093917, -0.013875989243388176, -0.0014993123477324843, -0.0038381090853363276, 0.020591862499713898, 0.014320229180157185, 0.0075194272212684155, -0.028117822483181953, -0.018383724614977837, -0.006389226298779249, 0.010171805508434772, 0.03794338181614876, 0.001825959887355566, -0.018279198557138443, -0.026615243405103683, 0.015888137742877007, -0.03614028915762901, -0.002531518694013357, 0.005020573269575834, 0.03242957219481468, 0.007839541882276535, -0.01050498615950346, 0.010152206756174564, 0.026785101741552353, -0.00399163318797946, 0.006588481366634369, -0.017312321811914444, 0.0118115758523345, 0.004687392618507147, -0.008055129088461399, -0.016319312155246735, 0.034128136932849884, 0.015025787986814976, 0.014973524957895279, -0.007872206158936024, -0.025452379137277603, 0.01075977087020874, 0.002887564478442073, -0.002673610346391797, -0.006389226298779249, -0.010870831087231636, -0.005804527550935745, 0.037185560911893845, 0.011889971792697906, -0.005791461560875177, 0.012543266639113426, -0.0023534956853836775, -0.020970774814486504, -0.005873123183846474, 0.006199771072715521, -0.035173412412405014, -0.023858338594436646, 0.0031194842886179686, 0.023270372301340103, -0.01290257927030325, -0.0006471705273725092, -0.005004240665584803, 0.014646877534687519, 0.0077023496851325035, 0.0024433238431811333, 0.013706131838262081, 0.00802246481180191, 0.04131438583135605, 0.03169788047671318, -0.030757134780287743, 0.01010647602379322, -0.034493982791900635, 0.011511060409247875, 0.008061662316322327, -0.001801461330614984, -0.014307163655757904, -0.01098842453211546, 0.005830659065395594, 0.02529558725655079, 0.019350601360201836, -0.023152779787778854, -0.012295014224946499, -0.031906936317682266, -0.0076043554581701756, 0.002761805197224021, -0.004419541452080011, 0.01695954240858555, -0.007525959983468056, -0.015274040400981903, -0.005386418662965298, -0.0013555873883888125, -0.0066701434552669525, -0.013627736829221249, 0.03428493067622185, 0.01596653275191784, 0.011648252606391907, -0.004001432564109564, -0.030077708885073662, -0.008584298193454742, -0.021075300872325897, -0.04293455556035042, 0.02767358161509037, -0.019807908684015274, 0.00016036354645621032, -0.006748538929969072, -0.004965043161064386, -0.0008256017463281751, -0.0176912322640419, 0.0042562177404761314, -0.02001696266233921, -0.009734097868204117, -0.00338080246001482, -0.022839197888970375, 0.003674785140901804, -0.009479312226176262, 0.01067484263330698, -0.022068310528993607, 0.026157937943935394, 0.007761146407574415, -0.026210201904177666, -0.006996790878474712, -0.010034613311290741, 0.007774212397634983, 0.027908768504858017, 0.018213868141174316, 0.016005730256438255, -0.008630028925836086, -0.01400664821267128, -0.014882063493132591, -0.010655243881046772, 0.021245157346129417, -0.011295473203063011, -0.009734097868204117, 0.03039129078388214, -0.010080344043672085, 0.01283071655780077, 0.01897169090807438, -0.16494396328926086, 0.018945559859275818, 0.011530659161508083, -0.019246075302362442, 0.004452206194400787, 0.04405822604894638, 0.016253983601927757, -0.018592780455946922, -0.013967449776828289, 0.003390601836144924, 0.008120458573102951, 0.01034819521009922, -0.010923095047473907, -0.005213295109570026, -0.0014968625036999583, 0.02523025870323181, -0.03825696185231209, 0.024446304887533188, 0.014646877534687519, 0.013549340888857841, 0.03891025856137276, -0.0039948998019099236, 0.024772951379418373, -0.013111633248627186, 0.034833695739507675, 0.0312536396086216, 0.019115416333079338, -0.01400664821267128, -0.010485387407243252, 0.0007974283653311431, -0.03904091939330101, -0.00179166195448488, 0.0007161747780628502, -0.0151433814316988, 0.009440114721655846, -0.004007965791970491, -0.012726189568638802, -0.008643094450235367, -0.021624069660902023, 0.020591862499713898, 0.01251713465899229, 0.007467163726687431, 0.022865330800414085, 0.007610888220369816, -0.01098842453211546, 0.026131805032491684, -0.009877822361886501, 0.02021295204758644, 0.012569398619234562, -0.016084127128124237, -0.018422923982143402, -0.034154269844293594, 0.0256875641644001, 0.016737421974539757, 0.01972951367497444, -0.0024792549666017294, -0.017194727435708046, 0.016946475952863693, -0.011707048863172531, -0.024799084290862083, -0.017939483746886253, -0.00786567386239767, 0.007650086190551519, -0.03133203461766243, -0.0011685816571116447, -0.02563530206680298, -0.03940676152706146, -0.01661982759833336, -0.026863496750593185, 0.004772320855408907, 0.009616504423320293, -0.025622235611081123, 0.011204011738300323, -0.0233879666775465, 0.006379426922649145, 0.015443896874785423, -0.016789685934782028, 0.0029773926362395287, -0.00700985686853528, 0.01034819521009922, -0.025896620005369186, 0.02866659127175808, -0.017429914325475693, -0.011465329676866531, -0.016097191721200943, -0.02614487148821354, -0.020448138937354088, 0.022120574489235878, 0.02246028743684292, -0.03642773628234863, 0.006448023021221161, -0.05471999943256378, -0.005118567496538162, -0.037525273859500885, 0.019010888412594795, -0.020108424127101898, -0.007349570281803608, -0.014241834171116352, -0.019912436604499817, -0.023623151704669, -0.008388309739530087, -0.018867162987589836, -0.012255816720426083, 0.019010888412594795, 0.03739461302757263, -0.0006835100357420743, -0.008002865128219128, 0.015940401703119278, 0.0329522080719471, -0.003495129058137536, -0.0011824641842395067, 0.009093868546187878, 0.026889627799391747, -0.010838166810572147, -0.015326304361224174, 0.005219828337430954, 0.006176905706524849, -0.022172836586833, 0.016685158014297485, 0.009871290065348148, 0.05848298221826553, -0.0004372994299046695, 0.0010918194893747568, 0.007591289468109608, -0.018932493403553963, -0.005399484187364578, -0.0605735257267952, 0.005232894327491522, 0.012484470382332802, 0.01618865318596363, 0.00177042989525944, 0.03470303863286972, 0.0007574140327051282, -0.01034819521009922, 0.012582464143633842, 0.020343611016869545, -0.004517536144703627, -0.014947392977774143, -0.02554384060204029, -0.004354211967438459, 0.04006005823612213, -0.005386418662965298, 0.01644997112452984, -0.03911931440234184, -0.0399816632270813, 0.024576963856816292, 0.0013792694080621004, -0.009838624857366085, -0.006127908360213041, -0.017599770799279213, 0.013810659758746624, -0.015692150220274925, -0.04196767881512642, 0.027438396587967873, 0.020082293078303337, 0.029189227148890495, 0.008081261068582535, -0.00558567326515913, 0.010086877271533012, -0.007271174807101488, -0.008146590553224087, -0.005151232238858938, -0.0312536396086216, 0.016855014488101006, 0.006281432695686817, -0.020435072481632233, -0.003446131944656372, 0.02619713544845581, 0.03096619062125683, -0.01965111866593361, 0.008884813636541367, 0.005004240665584803, -0.02316584624350071, 0.022290430963039398, 0.012406074441969395, -0.0070882523432374, 0.008316447027027607, -0.008113925345242023, -0.01209249347448349, 0.0060593122616410255, 0.019298339262604713, 0.0024367908481508493, -0.006813868414610624, 0.0012714756885543466, -0.011164814233779907, -0.0011089685140177608, -0.005967851262539625, 0.0007284241146408021, -0.04042590409517288, 0.01984710618853569, 0.01754750683903694, 0.029894785955548286, 0.02166326716542244, -0.02129742130637169, -0.0011726648081094027, -0.0248382817953825, 0.008244584314525127, 0.02151954174041748, -0.0007533309399150312, 0.026837363839149475, -0.006807335186749697, 0.030234498903155327, -0.049206189811229706, -0.009407450444996357, -0.014202636666595936, -0.03462464362382889, 0.007748080417513847, -0.012144756503403187, 0.0022881662007421255, -0.01567908376455307, 0.03943289443850517, 0.008055129088461399, -0.001856991439126432, 0.01386292278766632, 0.0049193124286830425, -0.014124240726232529, 0.022956792265176773, 0.03629707917571068, 0.026941891759634018, -0.01621478609740734, 0.012523667886853218, 0.03310899809002876, -0.014581548050045967, -0.015274040400981903, -0.0065721492283046246, 0.005647736601531506, -0.030809398740530014, 0.0056608025915920734, -0.08221065998077393, 0.023113582283258438, 0.004628595896065235, 0.01002154778689146, 0.0019076218595728278, -0.0015613754512742162, 0.008375243283808231, -0.022695472463965416, -0.0012118625454604626, 0.027124814689159393, -0.009557708166539669, 0.03875346854329109, -0.020800916478037834, 0.002861432731151581, -0.014411690644919872, -0.03363163396716118, 0.0028026362415403128, 0.02767358161509037, 0.01249100361019373, 0.028457537293434143, 0.019180744886398315, -0.005765329580754042, -0.012961375527083874, 0.040844012051820755, 0.0038348424714058638, 0.0436139851808548, 0.0006773854256607592, 0.01217742171138525, -0.012719656340777874, 0.00859083142131567, 0.0047690547071397305, -0.031149111688137054, -0.01979484222829342, 0.01854051649570465, -0.0014372493606060743, -0.042568713426589966, -0.0069837248884141445, 0.0020056162029504776, 0.006039713509380817, 0.014607679098844528, -0.006385960150510073, -0.010975358076393604, 0.004569799639284611, 0.012615129351615906, 0.003478796686977148, -8.548979531042278e-05, 0.0012110458919778466, 0.009681833907961845, 0.010171805508434772, 0.01193570252507925, 0.024420171976089478, 0.016946475952863693, -0.018814900889992714, -0.02185925468802452, 0.006389226298779249, 0.01449008658528328, 0.009355186484754086, -0.025569971650838852, -0.024067392572760582, -0.013307621702551842, 0.03491209074854851, 0.023257307708263397, 0.009394383989274502, -0.00716664781793952, 0.020970774814486504, -0.02702028676867485, 0.016907278448343277, 0.018475186079740524, 0.0023943267296999693, -0.013614670373499393, -0.023923669010400772, -0.008558166213333607, -0.0029937250073999166, 0.02018681913614273, 0.009446647949516773, 0.030940057709813118, 0.00836217775940895, -0.002812435617670417, -0.008786819875240326, -0.015443896874785423, 0.015091117471456528, -0.022277364507317543, -0.017795760184526443, 0.025674499571323395, -0.014764470048248768, 0.0068465331569314, -0.007055587600916624, -0.00693146139383316, 0.02430257946252823, 0.01535243634134531, -0.000986475613899529, 0.007859140634536743, 0.006229169201105833, -0.01695954240858555, 0.002654011594131589, 0.0007067836704663932, 0.004631862509995699, -0.022669341415166855, 0.002774871187284589, 0.014790602028369904, -0.00757822347804904, 0.015430831350386143, 0.002639312297105789, -0.01307243574410677, -0.02733386866748333, 0.030417421832680702, -0.014398625120520592, -0.03501661866903305, 0.0014462320832535625, 0.006441490259021521, 0.0037597136106342077, 0.03904091939330101, -0.01384985726326704, 0.010870831087231636, -0.04107919707894325, -0.01141306571662426, 0.021310487762093544, -0.03143656253814697, -0.03159335255622864, 0.0392499715089798, 0.028744986280798912, 0.015051919966936111, 0.02869272232055664, 0.003635587403550744, 0.01590120419859886, -0.009609971195459366, 0.03509501367807388, -0.02350555919110775, -0.024459369480609894, -0.018279198557138443, -0.0029430945869535208, -0.001979484222829342, -0.033187393099069595, 0.008649627678096294, 0.0032076791394501925, 0.008995873853564262, 0.006630945485085249, 0.02086624689400196, -0.01565295085310936, 0.03995553031563759, -0.0013082235818728805, -0.005196962971240282, 0.006575415842235088, -0.002735673449933529, 0.03501661866903305, 0.0015262607485055923, -0.023753810673952103, -0.024485502392053604, -0.0075520919635891914, -0.006957593373954296, -0.015038854442536831, -0.012210085988044739, -0.023126648738980293, -0.009753696620464325, 0.006820401176810265, -0.01797868311405182, -0.002732406836003065, -0.004677593242377043, -0.004733123350888491, 0.005105501506477594, -0.009681833907961845, -0.02898017317056656, 0.00859083142131567, -0.0025511174462735653, 0.012856848537921906, 0.009629570879042149, -0.02038280852138996, -0.015169513411819935, -0.03700263798236847, 0.0007798710721544921, 0.04275163263082504, 0.011844241060316563, -0.017900286242365837, 0.027098681777715683, -0.012497535906732082, -0.014764470048248768, -0.007525959983468056, 0.0108446991071105, 0.01092962734401226, -0.02285226434469223, 0.0016503868391737342, -0.009028539061546326, -0.013575472868978977, 0.03862280771136284, 0.030887793749570847, -0.00693146139383316, 0.0013098567724227905, -0.0016593696782365441]} diff --git a/tests/test_cases/test-view-new.txt b/tests/test_cases/test-view-new.txt new file mode 100644 index 000000000..a421c5a01 --- /dev/null +++ b/tests/test_cases/test-view-new.txt @@ -0,0 +1,32 @@ +function(doc, meta) { + switch (doc.type) { + case "dealership": + if (doc.batch) { + emit([doc.batch, meta.id]); + } + break; + case "vehicle": + if (doc.batch) { + const newDoc = { + batch: doc.batch, + id: doc.id, + type: doc.type + }; + const batches = doc.id.split('::') + if (parseInt(batches[1]) < 10) { + const key = batches[0].slice(0, 2) + emit(key, newDoc) + } else if (parseInt(batches[1]) >= 10 && parseInt(batches[1]) < 20) { + const key = batches[0].slice(2, 4) + emit(key, newDoc); + } else if (parseInt(batches[1]) >= 20 && parseInt(batches[1]) < 30) { + const key = batches[0].slice(4, 6) + emit(key, newDoc); + } else { + const key = batches[0].slice(-2) + emit(key, newDoc); + } + } + break; + } +} diff --git a/tests/test_cases/test-view.txt b/tests/test_cases/test-view.txt new file mode 100644 index 000000000..5ed4d4fa8 --- /dev/null +++ b/tests/test_cases/test-view.txt @@ -0,0 +1,29 @@ +function(doc, meta) { + switch (doc.type) { + case "landmark": + if (doc.name) { + emit([doc.name, meta.id]); + } + break; + case "hotel": + if (doc.name) { + emit([doc.name, meta.id]); + } + break; + case "airport": + if (doc.city) { + emit([doc.city, meta.id]); + } + break; + case "airline": + if (doc.callsign) { + emit([doc.callsign, meta.id]); + } + break; + case "route": + if (doc.airlineid) { + emit([doc.airlineid, meta.id]); + } + break; + } +} diff --git a/tests/test_config.ini b/tests/test_config.ini new file mode 100644 index 000000000..447a7a611 --- /dev/null +++ b/tests/test_config.ini @@ -0,0 +1,37 @@ +[realserver] +; Set this to true if there is a real cluster available +enabled = False +#Local +host = 127.0.0.1 +port = 8091 +admin_username = Administrator +; The administrative password. This is the password used to +; log into the admin console +admin_password = password +bucket_name = default +; If a SASL bucket is being used (i.e. buckets are set up +; per the script, then this is the *bucket* password +; bucket_password sasl_password +bucket_password = password + +[gocaves] +; Set this to enabled to use the GoCAVES mock +enabled = True +; Set the version of GoCAVES to use +version = v0.0.1-78 +; Local path for the GoCAVES mock, if not supplied, will be dowloaded +; path = gocaves-macos +; Where to download it, if not available +url = https://github.com/couchbaselabs/gocaves/releases/download +; provide a directory to store logs, defaults to <SDK repository>tests/test_logs +; log_dir = test_logs +; provide a log filename for logs, defaults to test_logs.txt +; log_filename = test_logs.txt + +; [legacymockserver] +; Set this to enabled to use the mock +; enabled = False +; Local path for the mock +; path = CouchbaseMock-LATEST.jar +; Where to download it, if not available +; url = http://packages.couchbase.com/clients/c/mock/CouchbaseMock-LATEST.jar diff --git a/tests/test_features.py b/tests/test_features.py new file mode 100644 index 000000000..9c04466a4 --- /dev/null +++ b/tests/test_features.py @@ -0,0 +1,384 @@ +# Copyright 2016-2023. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from enum import Enum +from typing import (List, + Optional, + Union) + +import pytest + +from tests.mock_server import MockServerType + + +class ServerFeatures(Enum): + KeyValue = 'kv' + SSL = 'ssl' + Views = 'views' + SpatialViews = 'spatial_views' + Diagnostics = 'diagnostics' + SynchronousDurability = 'sync_durability' + Query = 'query' + Subdoc = 'subdoc' + Xattr = 'xattr' + Search = 'search' + Analytics = 'analytics' + Collections = 'collections' + Replicas = 'replicas' + UserManagement = 'user_mgmt' + BasicBucketManagement = 'basic_bucket_mgmt' + BucketManagement = 'bucket_mgmt' + BucketMinDurability = 'bucket_min_durability' + BucketStorageBackend = 'bucket_storage_backend' + CustomConflictResolution = 'custom_conflict_resolution' + QueryIndexManagement = 'query_index_mgmt' + QueryUserDefinedFunctions = 'query_user_defined_functions' + SearchIndexManagement = 'search_index_mgmt' + ViewIndexManagement = 'view_index_mgmt' + GetMeta = 'get_meta' + AnalyticsPendingMutations = 'analytics_pending_mutations' + AnalyticsLinkManagement = 'analytics_link_mgmt' + UserGroupManagement = 'user_group_mgmt' + PreserveExpiry = 'preserve_expiry' + SearchDisableScoring = 'search_disable_scoring' + Eventing = 'eventing' + EventingFunctionManagement = 'eventing_function_mgmt' + RateLimiting = 'rate_limiting' + Txns = 'txns' + TxnQueries = 'txn_queries' + KeyValueRangeScan = 'kv_range_scan' + SubdocReplicaRead = 'subdoc_replica_read' + UpdateCollection = 'update_collection' + UpdateCollectionMaxExpiry = 'update_collection_max_expiry' + NegativeCollectionMaxExpiry = 'negative_collection_max_expiry' + NonDedupedHistory = 'non_deduped_history' + QueryWithoutIndex = 'query_without_index' + NotLockedKVStatus = 'kv_not_locked' + ScopeSearch = 'scope_search' + ScopeSearchIndexManagement = 'scope_search_index_mgmt' + ScopeEventingFunctionManagement = 'scope_eventing_function_mgmt' + BinaryTxns = 'binary_txns' + ServerGroups = 'server_groups' + Magma128Buckets = 'magma_128_buckets' + + +class EnvironmentFeatures: + # mock and real server (all versions) should have these features + BASIC_FEATURES = [ServerFeatures.KeyValue, + ServerFeatures.Diagnostics, + ServerFeatures.SSL, + ServerFeatures.SpatialViews, + ServerFeatures.Subdoc, + ServerFeatures.Views, + ServerFeatures.Replicas] + + # mock related feature lists + FEATURES_NOT_IN_MOCK = [ServerFeatures.Analytics, + ServerFeatures.BucketManagement, + ServerFeatures.EventingFunctionManagement, + ServerFeatures.GetMeta, + ServerFeatures.Query, + ServerFeatures.QueryIndexManagement, + ServerFeatures.QueryUserDefinedFunctions, + ServerFeatures.RateLimiting, + ServerFeatures.Search, + ServerFeatures.SearchIndexManagement, + ServerFeatures.TxnQueries, + ServerFeatures.UserGroupManagement, + ServerFeatures.UserManagement, + ServerFeatures.ViewIndexManagement, + ServerFeatures.NonDedupedHistory, + ServerFeatures.UpdateCollection, + ServerFeatures.ScopeEventingFunctionManagement, + ServerFeatures.BinaryTxns, + ServerFeatures.ServerGroups] + + FEATURES_IN_MOCK = [ServerFeatures.Txns] + + # separate features into CBS versions, lets make 5.5 the earliest + AT_LEAST_V5_5_0_FEATURES = [ServerFeatures.BucketManagement, + ServerFeatures.GetMeta, + ServerFeatures.Query, + ServerFeatures.QueryIndexManagement, + ServerFeatures.Search, + ServerFeatures.SearchIndexManagement, + ServerFeatures.ViewIndexManagement] + + AT_LEAST_V6_0_0_FEATURES = [ServerFeatures.Analytics, + ServerFeatures.UserManagement] + + AT_LEAST_V6_5_0_FEATURES = [ServerFeatures.AnalyticsPendingMutations, + ServerFeatures.UserGroupManagement, + ServerFeatures.SynchronousDurability, + ServerFeatures.SearchDisableScoring] + + AT_LEAST_V6_6_0_FEATURES = [ServerFeatures.BucketMinDurability, + ServerFeatures.Txns] + + AT_LEAST_V7_0_0_FEATURES = [ServerFeatures.Collections, + ServerFeatures.AnalyticsLinkManagement, + ServerFeatures.TxnQueries] + + AT_LEAST_V7_1_0_FEATURES = [ServerFeatures.RateLimiting, + ServerFeatures.BucketStorageBackend, + ServerFeatures.CustomConflictResolution, + ServerFeatures.EventingFunctionManagement, + ServerFeatures.PreserveExpiry, + ServerFeatures.QueryUserDefinedFunctions, + ServerFeatures.ScopeEventingFunctionManagement] + + AT_LEAST_V7_2_0_FEATURES = [ServerFeatures.NonDedupedHistory, + ServerFeatures.UpdateCollection] + + AT_LEAST_V7_5_0_FEATURES = [ServerFeatures.KeyValueRangeScan, + ServerFeatures.SubdocReplicaRead, + ServerFeatures.UpdateCollectionMaxExpiry, + ServerFeatures.QueryWithoutIndex] + + AT_LEAST_V7_6_0_FEATURES = [ServerFeatures.NotLockedKVStatus, + ServerFeatures.NegativeCollectionMaxExpiry, + ServerFeatures.ScopeSearch, + ServerFeatures.ScopeSearchIndexManagement] + + AT_LEAST_V7_6_2_FEATURES = [ServerFeatures.BinaryTxns, + ServerFeatures.ServerGroups] + + AT_LEAST_V8_0_0_FEATURES = [ServerFeatures.Magma128Buckets] + + AT_MOST_V7_2_0_FEATURES = [ServerFeatures.RateLimiting] + + @staticmethod + def is_feature_supported(feature, # type: str + server_version, # type: float + mock_server_type=None, # type: Optional[MockServerType] + server_version_patch=None # type: Optional[int] + ) -> bool: + try: + supported = EnvironmentFeatures.supports_feature(feature, + server_version, + mock_server_type, + server_version_patch) + return supported is None + except Exception: + return False + + @staticmethod + def check_if_feature_supported(features, # type: Union[str, List[str]] + server_version, # type: float + mock_server_type=None, # type: Optional[MockServerType] + server_version_patch=None, # type: Optional[int] + ) -> None: + + print(f"Server version = {server_version}") + features_list = [] + if isinstance(features, str): + features_list.append(features) + else: + features_list.extend(features) + + for feature in features_list: + try: + supported = EnvironmentFeatures.supports_feature(feature, + server_version, + mock_server_type, + server_version_patch) + if supported is not None: + pytest.skip(supported) + except TypeError: + pytest.skip("Unable to determine server version") + except Exception: + raise + + @staticmethod + def check_if_feature_not_supported(features, # type: Union[str, List[str]] + server_version, # type: float + mock_server_type=None, # type: Optional[MockServerType] + server_version_patch=None # type: Optional[int] + ) -> None: + + features_list = [] + if isinstance(features, str): + features_list.append(features) + else: + features_list.extend(features) + + for feature in features_list: + try: + supported = EnvironmentFeatures.supports_feature(feature, + server_version, + mock_server_type, + server_version_patch) + if supported is None: + pytest.skip(f'Feature: {feature} is supported.') + except TypeError: + pytest.skip("Unable to determine server version") + except Exception: + raise + + @staticmethod + def supports_feature(feature, # type: str # noqa: C901 + server_version, # type: float + mock_server_type=None, # type: Optional[MockServerType] + server_version_patch=None, # type: Optional[int] + ) -> Optional[str]: + + is_mock_server = mock_server_type is not None + is_real_server = is_mock_server is False + + if feature in map(lambda f: f.value, EnvironmentFeatures.BASIC_FEATURES): + return None + + if is_mock_server and feature in map(lambda f: f.value, EnvironmentFeatures.FEATURES_NOT_IN_MOCK): + return f'Mock server does not support feature: {feature}' + + if is_mock_server and feature in map(lambda f: f.value, EnvironmentFeatures.FEATURES_IN_MOCK): + return None + + if feature in [ServerFeatures.Diagnostics.value, ServerFeatures.BasicBucketManagement.value]: + if is_real_server or mock_server_type == MockServerType.GoCAVES: + return None + + return f'LegacyMockServer does not support feature: {feature}' + + if feature in [ServerFeatures.Diagnostics.value, ServerFeatures.BasicBucketManagement.value]: + if is_real_server or mock_server_type == MockServerType.GoCAVES: + return None + + return f'LegacyMockServer does not support feature: {feature}' + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_MOST_V7_2_0_FEATURES): + if server_version > 7.2: + return (f'Feature: {feature} not supported on server versions > 7.2. ' + f'Using server version: {server_version}.') + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V5_5_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 5.5: + return (f'Feature: {feature} only supported on server versions >= 5.5. ' + f'Using server version: {server_version}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V6_0_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 6.0: + return (f'Feature: {feature} only supported on server versions >= 6.0. ' + f'Using server version: {server_version}.') + # @TODO: couchbase++ looks to choke w/ CAVES + # if feature == ServerFeatures.UserManagement.value: + # return self.mock_server_type == MockServerType.GoCAVES + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V6_5_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 6.5: + return (f'Feature: {feature} only supported on server versions >= 6.5. ' + f'Using server version: {server_version}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V6_6_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 6.6: + return (f'Feature: {feature} only supported on server versions >= 6.6. ' + f'Using server version: {server_version}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V7_0_0_FEATURES): + if is_mock_server: + if mock_server_type == MockServerType.GoCAVES: + return None + return f'Mock server does not support feature: {feature}' + + if server_version < 7.0: + return (f'Feature: {feature} only supported on server versions >= 7.0. ' + f'Using server version: {server_version}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V7_1_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 7.1: + return (f'Feature: {feature} only supported on server versions >= 7.1. ' + f'Using server version: {server_version}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V7_2_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 7.2: + return (f'Feature: {feature} only supported on server versions >= 7.2. ' + f'Using server version: {server_version}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V7_5_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 7.5: + return (f'Feature: {feature} only supported on server versions >= 7.5. ' + f'Using server version: {server_version}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V7_6_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 7.6: + return (f'Feature: {feature} only supported on server versions >= 7.6. ' + f'Using server version: {server_version}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V7_6_2_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 7.6: + return (f'Feature: {feature} only supported on server versions >= 7.6. ' + f'Using server version: {server_version}.') + patch = server_version_patch or -1 + if server_version == 7.6 and patch < 2: + return (f'Feature: {feature} only supported on server versions >= 7.6.2. ' + f'Using server version: {server_version}.{patch}.') + + return None + + if feature in map(lambda f: f.value, EnvironmentFeatures.AT_LEAST_V8_0_0_FEATURES): + if is_mock_server: + return f'Mock server does not support feature: {feature}' + + if server_version < 8: + return (f'Feature: {feature} only supported on server versions >= 8.0. ' + f'Using server version: {server_version}.') + + return None diff --git a/tests/travel_sample_data.json b/tests/travel_sample_data.json new file mode 100644 index 000000000..8c4a5a335 --- /dev/null +++ b/tests/travel_sample_data.json @@ -0,0 +1,7742 @@ +{ + "airports": + { + "query": "SELECT a.* FROM `travel-sample`.`inventory`.airport a WHERE a.id IN [3670, 3682, 3484, 3645, 3830, 3577, 3876, 3576, 3797, 3697]", + "results": + [ + { + "airportname": "Los Angeles Intl", + "city": "Los Angeles", + "country": "United States", + "faa": "LAX", + "geo": + { + "alt": 126, + "lat": 33.942536, + "lon": -118.408075 + }, + "icao": "KLAX", + "id": 3484, + "type": "airport", + "tz": "America/Los_Angeles" + }, + { + "airportname": "Miami Intl", + "city": "Miami", + "country": "United States", + "faa": "MIA", + "geo": + { + "alt": 8, + "lat": 25.79325, + "lon": -80.290556 + }, + "icao": "KMIA", + "id": 3576, + "type": "airport", + "tz": "America/New_York" + }, + { + "airportname": "Seattle Tacoma Intl", + "city": "Seattle", + "country": "United States", + "faa": "SEA", + "geo": + { + "alt": 433, + "lat": 47.449, + "lon": -122.309306 + }, + "icao": "KSEA", + "id": 3577, + "type": "airport", + "tz": "America/Los_Angeles" + }, + { + "airportname": "Detroit Metro Wayne Co", + "city": "Detroit", + "country": "United States", + "faa": "DTW", + "geo": + { + "alt": 645, + "lat": 42.212444, + "lon": -83.353389 + }, + "icao": "KDTW", + "id": 3645, + "type": "airport", + "tz": "America/New_York" + }, + { + "airportname": "Dallas Fort Worth Intl", + "city": "Dallas-Fort Worth", + "country": "United States", + "faa": "DFW", + "geo": + { + "alt": 607, + "lat": 32.896828, + "lon": -97.037997 + }, + "icao": "KDFW", + "id": 3670, + "type": "airport", + "tz": "America/Chicago" + }, + { + "airportname": "Hartsfield Jackson Atlanta Intl", + "city": "Atlanta", + "country": "United States", + "faa": "ATL", + "geo": + { + "alt": 1026, + "lat": 33.636719, + "lon": -84.428067 + }, + "icao": "KATL", + "id": 3682, + "type": "airport", + "tz": "America/New_York" + }, + { + "airportname": "La Guardia", + "city": "New York", + "country": "United States", + "faa": "LGA", + "geo": + { + "alt": 22, + "lat": 40.777245, + "lon": -73.872608 + }, + "icao": "KLGA", + "id": 3697, + "type": "airport", + "tz": "America/New_York" + }, + { + "airportname": "John F Kennedy Intl", + "city": "New York", + "country": "United States", + "faa": "JFK", + "geo": + { + "alt": 13, + "lat": 40.639751, + "lon": -73.778925 + }, + "icao": "KJFK", + "id": 3797, + "type": "airport", + "tz": "America/New_York" + }, + { + "airportname": "Chicago Ohare Intl", + "city": "Chicago", + "country": "United States", + "faa": "ORD", + "geo": + { + "alt": 668, + "lat": 41.978603, + "lon": -87.904842 + }, + "icao": "KORD", + "id": 3830, + "type": "airport", + "tz": "America/Chicago" + }, + { + "airportname": "Charlotte Douglas Intl", + "city": "Charlotte", + "country": "United States", + "faa": "CLT", + "geo": + { + "alt": 748, + "lat": 35.214, + "lon": -80.943139 + }, + "icao": "KCLT", + "id": 3876, + "type": "airport", + "tz": "America/New_York" + } + ] + }, + "airlines": + { + "query": "SELECT a.* FROM `travel-sample`.`inventory`.airline a WHERE a.id IN [1316, 1355, 2009, 24, 3029, 4547, 4687, 5209, 5265, 5331]", + "results": + [ + { + "callsign": "CITRUS", + "country": "United States", + "iata": "FL", + "icao": "TRS", + "id": 1316, + "name": "AirTran Airways", + "type": "airline" + }, + { + "callsign": "SPEEDBIRD", + "country": "United Kingdom", + "iata": "BA", + "icao": "BAW", + "id": 1355, + "name": "British Airways", + "type": "airline" + }, + { + "callsign": "DELTA", + "country": "United States", + "iata": "DL", + "icao": "DAL", + "id": 2009, + "name": "Delta Air Lines", + "type": "airline" + }, + { + "callsign": "AMERICAN", + "country": "United States", + "iata": "AA", + "icao": "AAL", + "id": 24, + "name": "American Airlines", + "type": "airline" + }, + { + "callsign": "JETBLUE", + "country": "United States", + "iata": "B6", + "icao": "JBU", + "id": 3029, + "name": "JetBlue Airways", + "type": "airline" + }, + { + "callsign": "SOUTHWEST", + "country": "United States", + "iata": "WN", + "icao": "SWA", + "id": 4547, + "name": "Southwest Airlines", + "type": "airline" + }, + { + "callsign": "SPIRIT WINGS", + "country": "United States", + "iata": "NK", + "icao": "NKS", + "id": 4687, + "name": "Spirit Airlines", + "type": "airline" + }, + { + "callsign": "UNITED", + "country": "United States", + "iata": "UA", + "icao": "UAL", + "id": 5209, + "name": "United Airlines", + "type": "airline" + }, + { + "callsign": "U S AIR", + "country": "United States", + "iata": "US", + "icao": "USA", + "id": 5265, + "name": "US Airways", + "type": "airline" + }, + { + "callsign": "REDWOOD", + "country": "United States", + "iata": "VX", + "icao": "VRD", + "id": 5331, + "name": "Virgin America", + "type": "airline" + } + ] + }, + "routes": + { + "query": "", + "results": + [ + { + "airline": "B6", + "airlineid": "airline_3029", + "destinationairport": "JFK", + "distance": 870.5264696185337, + "equipment": "E90", + "id": 13971, + "schedule": + [ + { + "day": 0, + "flight": "B6097", + "utc": "23:55:00" + }, + { + "day": 0, + "flight": "B6768", + "utc": "05:54:00" + }, + { + "day": 0, + "flight": "B6467", + "utc": "22:28:00" + }, + { + "day": 1, + "flight": "B6580", + "utc": "09:22:00" + }, + { + "day": 1, + "flight": "B6008", + "utc": "05:28:00" + }, + { + "day": 1, + "flight": "B6901", + "utc": "04:00:00" + }, + { + "day": 2, + "flight": "B6849", + "utc": "13:06:00" + }, + { + "day": 2, + "flight": "B6256", + "utc": "08:36:00" + }, + { + "day": 2, + "flight": "B6514", + "utc": "00:57:00" + }, + { + "day": 2, + "flight": "B6131", + "utc": "18:37:00" + }, + { + "day": 3, + "flight": "B6901", + "utc": "11:45:00" + }, + { + "day": 3, + "flight": "B6229", + "utc": "21:29:00" + }, + { + "day": 3, + "flight": "B6783", + "utc": "12:26:00" + }, + { + "day": 3, + "flight": "B6130", + "utc": "21:28:00" + }, + { + "day": 4, + "flight": "B6880", + "utc": "21:56:00" + }, + { + "day": 4, + "flight": "B6260", + "utc": "21:13:00" + }, + { + "day": 4, + "flight": "B6503", + "utc": "10:51:00" + }, + { + "day": 4, + "flight": "B6912", + "utc": "21:06:00" + }, + { + "day": 5, + "flight": "B6744", + "utc": "20:11:00" + }, + { + "day": 5, + "flight": "B6817", + "utc": "12:07:00" + }, + { + "day": 5, + "flight": "B6795", + "utc": "20:14:00" + }, + { + "day": 5, + "flight": "B6766", + "utc": "08:52:00" + }, + { + "day": 5, + "flight": "B6944", + "utc": "20:53:00" + }, + { + "day": 6, + "flight": "B6098", + "utc": "02:07:00" + }, + { + "day": 6, + "flight": "B6793", + "utc": "07:34:00" + }, + { + "day": 6, + "flight": "B6145", + "utc": "16:54:00" + }, + { + "day": 6, + "flight": "B6677", + "utc": "05:56:00" + }, + { + "day": 6, + "flight": "B6484", + "utc": "14:41:00" + } + ], + "sourceairport": "CLT", + "stops": 0, + "type": "route" + }, + { + "airline": "B6", + "airlineid": "airline_3029", + "destinationairport": "LAX", + "distance": 3974.1999622301605, + "equipment": "320", + "id": 14073, + "schedule": + [ + { + "day": 0, + "flight": "B6437", + "utc": "09:19:00" + }, + { + "day": 1, + "flight": "B6155", + "utc": "21:06:00" + }, + { + "day": 1, + "flight": "B6763", + "utc": "07:25:00" + }, + { + "day": 1, + "flight": "B6890", + "utc": "18:20:00" + }, + { + "day": 2, + "flight": "B6102", + "utc": "15:48:00" + }, + { + "day": 3, + "flight": "B6795", + "utc": "09:07:00" + }, + { + "day": 3, + "flight": "B6354", + "utc": "14:34:00" + }, + { + "day": 4, + "flight": "B6300", + "utc": "08:47:00" + }, + { + "day": 5, + "flight": "B6526", + "utc": "12:25:00" + }, + { + "day": 5, + "flight": "B6153", + "utc": "17:23:00" + }, + { + "day": 5, + "flight": "B6640", + "utc": "08:15:00" + }, + { + "day": 6, + "flight": "B6349", + "utc": "15:55:00" + }, + { + "day": 6, + "flight": "B6745", + "utc": "22:31:00" + }, + { + "day": 6, + "flight": "B6177", + "utc": "10:27:00" + }, + { + "day": 6, + "flight": "B6532", + "utc": "14:11:00" + }, + { + "day": 6, + "flight": "B6888", + "utc": "12:31:00" + } + ], + "sourceairport": "JFK", + "stops": 0, + "type": "route" + }, + { + "airline": "B6", + "airlineid": "airline_3029", + "destinationairport": "JFK", + "distance": 3974.1999622301605, + "equipment": "320", + "id": 14121, + "schedule": + [ + { + "day": 0, + "flight": "B6848", + "utc": "19:48:00" + }, + { + "day": 0, + "flight": "B6183", + "utc": "20:12:00" + }, + { + "day": 0, + "flight": "B6444", + "utc": "00:26:00" + }, + { + "day": 0, + "flight": "B6990", + "utc": "13:27:00" + }, + { + "day": 0, + "flight": "B6130", + "utc": "13:26:00" + }, + { + "day": 1, + "flight": "B6505", + "utc": "11:19:00" + }, + { + "day": 1, + "flight": "B6312", + "utc": "03:59:00" + }, + { + "day": 2, + "flight": "B6989", + "utc": "02:20:00" + }, + { + "day": 2, + "flight": "B6465", + "utc": "07:34:00" + }, + { + "day": 2, + "flight": "B6982", + "utc": "04:37:00" + }, + { + "day": 2, + "flight": "B6020", + "utc": "22:12:00" + }, + { + "day": 3, + "flight": "B6626", + "utc": "16:49:00" + }, + { + "day": 4, + "flight": "B6014", + "utc": "02:24:00" + }, + { + "day": 4, + "flight": "B6484", + "utc": "02:00:00" + }, + { + "day": 4, + "flight": "B6724", + "utc": "09:10:00" + }, + { + "day": 4, + "flight": "B6175", + "utc": "18:03:00" + }, + { + "day": 5, + "flight": "B6486", + "utc": "16:01:00" + }, + { + "day": 5, + "flight": "B6509", + "utc": "18:48:00" + }, + { + "day": 6, + "flight": "B6842", + "utc": "16:47:00" + }, + { + "day": 6, + "flight": "B6602", + "utc": "16:13:00" + }, + { + "day": 6, + "flight": "B6402", + "utc": "19:09:00" + } + ], + "sourceairport": "LAX", + "stops": 0, + "type": "route" + }, + { + "airline": "BA", + "airlineid": "airline_1355", + "destinationairport": "ATL", + "distance": 1174.5341376297392, + "equipment": "M80 M83", + "id": 14484, + "schedule": + [ + { + "day": 0, + "flight": "BA932", + "utc": "05:23:00" + }, + { + "day": 0, + "flight": "BA802", + "utc": "02:48:00" + }, + { + "day": 0, + "flight": "BA259", + "utc": "10:59:00" + }, + { + "day": 0, + "flight": "BA752", + "utc": "19:01:00" + }, + { + "day": 0, + "flight": "BA330", + "utc": "06:09:00" + }, + { + "day": 1, + "flight": "BA861", + "utc": "00:22:00" + }, + { + "day": 1, + "flight": "BA568", + "utc": "23:08:00" + }, + { + "day": 1, + "flight": "BA467", + "utc": "04:36:00" + }, + { + "day": 2, + "flight": "BA506", + "utc": "08:17:00" + }, + { + "day": 2, + "flight": "BA587", + "utc": "04:36:00" + }, + { + "day": 2, + "flight": "BA225", + "utc": "21:25:00" + }, + { + "day": 2, + "flight": "BA637", + "utc": "04:54:00" + }, + { + "day": 3, + "flight": "BA358", + "utc": "07:56:00" + }, + { + "day": 3, + "flight": "BA681", + "utc": "21:37:00" + }, + { + "day": 3, + "flight": "BA728", + "utc": "22:43:00" + }, + { + "day": 3, + "flight": "BA792", + "utc": "20:45:00" + }, + { + "day": 3, + "flight": "BA184", + "utc": "03:18:00" + }, + { + "day": 4, + "flight": "BA861", + "utc": "18:05:00" + }, + { + "day": 4, + "flight": "BA946", + "utc": "22:26:00" + }, + { + "day": 4, + "flight": "BA191", + "utc": "20:32:00" + }, + { + "day": 4, + "flight": "BA579", + "utc": "20:17:00" + }, + { + "day": 4, + "flight": "BA444", + "utc": "02:16:00" + }, + { + "day": 5, + "flight": "BA367", + "utc": "02:35:00" + }, + { + "day": 5, + "flight": "BA569", + "utc": "01:32:00" + }, + { + "day": 6, + "flight": "BA949", + "utc": "18:03:00" + }, + { + "day": 6, + "flight": "BA831", + "utc": "14:07:00" + }, + { + "day": 6, + "flight": "BA313", + "utc": "08:50:00" + }, + { + "day": 6, + "flight": "BA707", + "utc": "06:32:00" + }, + { + "day": 6, + "flight": "BA297", + "utc": "22:08:00" + } + ], + "sourceairport": "DFW", + "stops": 0, + "type": "route" + }, + { + "airline": "BA", + "airlineid": "airline_1355", + "destinationairport": "ATL", + "distance": 959.0618701672437, + "equipment": "ER4 738", + "id": 14817, + "schedule": + [ + { + "day": 0, + "flight": "BA248", + "utc": "20:12:00" + }, + { + "day": 1, + "flight": "BA516", + "utc": "04:53:00" + }, + { + "day": 1, + "flight": "BA267", + "utc": "04:48:00" + }, + { + "day": 1, + "flight": "BA009", + "utc": "20:20:00" + }, + { + "day": 2, + "flight": "BA272", + "utc": "12:39:00" + }, + { + "day": 2, + "flight": "BA904", + "utc": "05:28:00" + }, + { + "day": 2, + "flight": "BA416", + "utc": "10:01:00" + }, + { + "day": 3, + "flight": "BA138", + "utc": "17:54:00" + }, + { + "day": 3, + "flight": "BA460", + "utc": "21:30:00" + }, + { + "day": 3, + "flight": "BA244", + "utc": "12:37:00" + }, + { + "day": 3, + "flight": "BA603", + "utc": "03:40:00" + }, + { + "day": 3, + "flight": "BA860", + "utc": "15:20:00" + }, + { + "day": 4, + "flight": "BA971", + "utc": "17:18:00" + }, + { + "day": 4, + "flight": "BA100", + "utc": "13:46:00" + }, + { + "day": 5, + "flight": "BA268", + "utc": "05:42:00" + }, + { + "day": 5, + "flight": "BA703", + "utc": "16:06:00" + }, + { + "day": 5, + "flight": "BA216", + "utc": "20:45:00" + }, + { + "day": 6, + "flight": "BA640", + "utc": "16:00:00" + } + ], + "sourceairport": "MIA", + "stops": 0, + "type": "route" + }, + { + "airline": "DL", + "airlineid": "airline_2009", + "destinationairport": "JFK", + "distance": 1222.084715099848, + "equipment": "319 752 738 M88 73H ", + "id": 20159, + "schedule": + [ + { + "day": 0, + "flight": "DL378", + "utc": "23:34:00" + }, + { + "day": 0, + "flight": "DL262", + "utc": "04:22:00" + }, + { + "day": 0, + "flight": "DL750", + "utc": "17:56:00" + }, + { + "day": 0, + "flight": "DL953", + "utc": "08:31:00" + }, + { + "day": 0, + "flight": "DL042", + "utc": "09:55:00" + }, + { + "day": 1, + "flight": "DL017", + "utc": "20:52:00" + }, + { + "day": 2, + "flight": "DL324", + "utc": "11:09:00" + }, + { + "day": 3, + "flight": "DL367", + "utc": "06:35:00" + }, + { + "day": 3, + "flight": "DL939", + "utc": "23:54:00" + }, + { + "day": 3, + "flight": "DL970", + "utc": "03:42:00" + }, + { + "day": 3, + "flight": "DL790", + "utc": "23:21:00" + }, + { + "day": 3, + "flight": "DL629", + "utc": "22:32:00" + }, + { + "day": 4, + "flight": "DL279", + "utc": "19:41:00" + }, + { + "day": 4, + "flight": "DL416", + "utc": "04:33:00" + }, + { + "day": 4, + "flight": "DL953", + "utc": "10:18:00" + }, + { + "day": 4, + "flight": "DL265", + "utc": "22:58:00" + }, + { + "day": 5, + "flight": "DL034", + "utc": "05:23:00" + }, + { + "day": 5, + "flight": "DL267", + "utc": "03:44:00" + }, + { + "day": 5, + "flight": "DL301", + "utc": "06:26:00" + }, + { + "day": 6, + "flight": "DL039", + "utc": "14:33:00" + }, + { + "day": 6, + "flight": "DL122", + "utc": "10:11:00" + }, + { + "day": 6, + "flight": "DL074", + "utc": "17:43:00" + }, + { + "day": 6, + "flight": "DL825", + "utc": "05:46:00" + } + ], + "sourceairport": "ATL", + "stops": 0, + "type": "route" + }, + { + "airline": "DL", + "airlineid": "airline_2009", + "destinationairport": "ATL", + "distance": 364.5601258774753, + "equipment": "M90 M88", + "id": 20439, + "schedule": + [ + { + "day": 0, + "flight": "DL753", + "utc": "05:40:00" + }, + { + "day": 1, + "flight": "DL076", + "utc": "00:18:00" + }, + { + "day": 1, + "flight": "DL931", + "utc": "07:37:00" + }, + { + "day": 1, + "flight": "DL770", + "utc": "18:47:00" + }, + { + "day": 1, + "flight": "DL703", + "utc": "10:47:00" + }, + { + "day": 2, + "flight": "DL905", + "utc": "14:16:00" + }, + { + "day": 2, + "flight": "DL632", + "utc": "13:09:00" + }, + { + "day": 2, + "flight": "DL737", + "utc": "18:41:00" + }, + { + "day": 2, + "flight": "DL863", + "utc": "03:17:00" + }, + { + "day": 2, + "flight": "DL814", + "utc": "03:48:00" + }, + { + "day": 3, + "flight": "DL493", + "utc": "16:11:00" + }, + { + "day": 3, + "flight": "DL859", + "utc": "21:47:00" + }, + { + "day": 3, + "flight": "DL721", + "utc": "06:23:00" + }, + { + "day": 3, + "flight": "DL088", + "utc": "04:20:00" + }, + { + "day": 4, + "flight": "DL214", + "utc": "05:15:00" + }, + { + "day": 5, + "flight": "DL711", + "utc": "08:57:00" + }, + { + "day": 5, + "flight": "DL823", + "utc": "20:25:00" + }, + { + "day": 5, + "flight": "DL922", + "utc": "05:39:00" + }, + { + "day": 6, + "flight": "DL950", + "utc": "19:18:00" + }, + { + "day": 6, + "flight": "DL389", + "utc": "23:08:00" + }, + { + "day": 6, + "flight": "DL248", + "utc": "08:24:00" + }, + { + "day": 6, + "flight": "DL304", + "utc": "12:51:00" + }, + { + "day": 6, + "flight": "DL096", + "utc": "08:55:00" + } + ], + "sourceairport": "CLT", + "stops": 0, + "type": "route" + }, + { + "airline": "DL", + "airlineid": "airline_2009", + "destinationairport": "DFW", + "distance": 1586.4201799893763, + "equipment": "319 M88 320", + "id": 20611, + "schedule": + [ + { + "day": 0, + "flight": "DL866", + "utc": "16:59:00" + }, + { + "day": 1, + "flight": "DL493", + "utc": "16:54:00" + }, + { + "day": 1, + "flight": "DL575", + "utc": "05:34:00" + }, + { + "day": 1, + "flight": "DL178", + "utc": "14:19:00" + }, + { + "day": 1, + "flight": "DL582", + "utc": "13:09:00" + }, + { + "day": 1, + "flight": "DL668", + "utc": "08:01:00" + }, + { + "day": 2, + "flight": "DL366", + "utc": "04:41:00" + }, + { + "day": 2, + "flight": "DL413", + "utc": "19:55:00" + }, + { + "day": 2, + "flight": "DL348", + "utc": "05:35:00" + }, + { + "day": 3, + "flight": "DL427", + "utc": "03:15:00" + }, + { + "day": 3, + "flight": "DL177", + "utc": "12:32:00" + }, + { + "day": 3, + "flight": "DL810", + "utc": "08:26:00" + }, + { + "day": 3, + "flight": "DL173", + "utc": "12:45:00" + }, + { + "day": 4, + "flight": "DL601", + "utc": "05:23:00" + }, + { + "day": 4, + "flight": "DL763", + "utc": "10:49:00" + }, + { + "day": 5, + "flight": "DL513", + "utc": "21:04:00" + }, + { + "day": 5, + "flight": "DL684", + "utc": "07:42:00" + }, + { + "day": 5, + "flight": "DL167", + "utc": "19:44:00" + }, + { + "day": 6, + "flight": "DL282", + "utc": "16:07:00" + }, + { + "day": 6, + "flight": "DL255", + "utc": "13:42:00" + } + ], + "sourceairport": "DTW", + "stops": 0, + "type": "route" + }, + { + "airline": "DL", + "airlineid": "airline_2009", + "destinationairport": "DTW", + "distance": 816.7222071725164, + "equipment": "M88 CR9 73H 319", + "id": 20942, + "schedule": + [ + { + "day": 0, + "flight": "DL506", + "utc": "09:32:00" + }, + { + "day": 1, + "flight": "DL668", + "utc": "13:43:00" + }, + { + "day": 1, + "flight": "DL801", + "utc": "06:14:00" + }, + { + "day": 1, + "flight": "DL955", + "utc": "08:22:00" + }, + { + "day": 1, + "flight": "DL901", + "utc": "22:05:00" + }, + { + "day": 1, + "flight": "DL670", + "utc": "19:23:00" + }, + { + "day": 2, + "flight": "DL208", + "utc": "02:07:00" + }, + { + "day": 3, + "flight": "DL431", + "utc": "16:29:00" + }, + { + "day": 3, + "flight": "DL641", + "utc": "22:41:00" + }, + { + "day": 3, + "flight": "DL383", + "utc": "04:20:00" + }, + { + "day": 3, + "flight": "DL844", + "utc": "11:14:00" + }, + { + "day": 3, + "flight": "DL960", + "utc": "03:14:00" + }, + { + "day": 4, + "flight": "DL646", + "utc": "07:00:00" + }, + { + "day": 5, + "flight": "DL394", + "utc": "19:46:00" + }, + { + "day": 5, + "flight": "DL481", + "utc": "16:31:00" + }, + { + "day": 5, + "flight": "DL408", + "utc": "15:25:00" + }, + { + "day": 5, + "flight": "DL365", + "utc": "07:39:00" + }, + { + "day": 5, + "flight": "DL379", + "utc": "07:51:00" + }, + { + "day": 6, + "flight": "DL800", + "utc": "22:15:00" + }, + { + "day": 6, + "flight": "DL332", + "utc": "10:06:00" + }, + { + "day": 6, + "flight": "DL396", + "utc": "23:52:00" + }, + { + "day": 6, + "flight": "DL336", + "utc": "09:31:00" + }, + { + "day": 6, + "flight": "DL846", + "utc": "20:01:00" + } + ], + "sourceairport": "JFK", + "stops": 0, + "type": "route" + }, + { + "airline": "DL", + "airlineid": "airline_2009", + "destinationairport": "ATL", + "distance": 1224.8661560257294, + "equipment": "M88 757 320 738 739 73W", + "id": 21102, + "schedule": + [ + { + "day": 0, + "flight": "DL059", + "utc": "14:35:00" + }, + { + "day": 0, + "flight": "DL031", + "utc": "00:08:00" + }, + { + "day": 0, + "flight": "DL604", + "utc": "13:07:00" + }, + { + "day": 1, + "flight": "DL532", + "utc": "04:01:00" + }, + { + "day": 1, + "flight": "DL443", + "utc": "19:41:00" + }, + { + "day": 1, + "flight": "DL827", + "utc": "10:34:00" + }, + { + "day": 2, + "flight": "DL553", + "utc": "06:03:00" + }, + { + "day": 2, + "flight": "DL747", + "utc": "02:47:00" + }, + { + "day": 2, + "flight": "DL404", + "utc": "20:27:00" + }, + { + "day": 2, + "flight": "DL988", + "utc": "01:20:00" + }, + { + "day": 3, + "flight": "DL589", + "utc": "12:47:00" + }, + { + "day": 4, + "flight": "DL994", + "utc": "19:18:00" + }, + { + "day": 4, + "flight": "DL673", + "utc": "10:25:00" + }, + { + "day": 4, + "flight": "DL470", + "utc": "23:10:00" + }, + { + "day": 4, + "flight": "DL271", + "utc": "11:01:00" + }, + { + "day": 5, + "flight": "DL680", + "utc": "15:11:00" + }, + { + "day": 5, + "flight": "DL398", + "utc": "04:56:00" + }, + { + "day": 5, + "flight": "DL288", + "utc": "10:27:00" + }, + { + "day": 6, + "flight": "DL274", + "utc": "03:13:00" + } + ], + "sourceairport": "LGA", + "stops": 0, + "type": "route" + }, + { + "airline": "DL", + "airlineid": "airline_2009", + "destinationairport": "DTW", + "distance": 1847.0804151368413, + "equipment": "320 M88", + "id": 21298, + "schedule": + [ + { + "day": 0, + "flight": "DL330", + "utc": "15:50:00" + }, + { + "day": 0, + "flight": "DL494", + "utc": "11:27:00" + }, + { + "day": 0, + "flight": "DL324", + "utc": "16:04:00" + }, + { + "day": 1, + "flight": "DL219", + "utc": "21:21:00" + }, + { + "day": 1, + "flight": "DL769", + "utc": "11:37:00" + }, + { + "day": 1, + "flight": "DL601", + "utc": "14:04:00" + }, + { + "day": 1, + "flight": "DL517", + "utc": "14:20:00" + }, + { + "day": 1, + "flight": "DL884", + "utc": "15:23:00" + }, + { + "day": 2, + "flight": "DL861", + "utc": "02:58:00" + }, + { + "day": 2, + "flight": "DL029", + "utc": "00:27:00" + }, + { + "day": 2, + "flight": "DL667", + "utc": "10:53:00" + }, + { + "day": 2, + "flight": "DL258", + "utc": "22:25:00" + }, + { + "day": 2, + "flight": "DL154", + "utc": "20:14:00" + }, + { + "day": 3, + "flight": "DL800", + "utc": "09:40:00" + }, + { + "day": 3, + "flight": "DL643", + "utc": "17:34:00" + }, + { + "day": 3, + "flight": "DL003", + "utc": "16:21:00" + }, + { + "day": 3, + "flight": "DL293", + "utc": "09:30:00" + }, + { + "day": 4, + "flight": "DL984", + "utc": "05:26:00" + }, + { + "day": 5, + "flight": "DL314", + "utc": "16:31:00" + }, + { + "day": 5, + "flight": "DL870", + "utc": "01:48:00" + }, + { + "day": 6, + "flight": "DL313", + "utc": "20:27:00" + }, + { + "day": 6, + "flight": "DL426", + "utc": "22:15:00" + }, + { + "day": 6, + "flight": "DL181", + "utc": "19:55:00" + }, + { + "day": 6, + "flight": "DL702", + "utc": "18:19:00" + } + ], + "sourceairport": "MIA", + "stops": 0, + "type": "route" + }, + { + "airline": "DL", + "airlineid": "airline_2009", + "destinationairport": "ATL", + "distance": 3505.000897033538, + "equipment": "739 76W 757 763", + "id": 21728, + "schedule": + [ + { + "day": 0, + "flight": "DL955", + "utc": "00:13:00" + }, + { + "day": 0, + "flight": "DL536", + "utc": "23:42:00" + }, + { + "day": 1, + "flight": "DL237", + "utc": "14:32:00" + }, + { + "day": 1, + "flight": "DL394", + "utc": "00:31:00" + }, + { + "day": 2, + "flight": "DL332", + "utc": "01:13:00" + }, + { + "day": 2, + "flight": "DL908", + "utc": "09:45:00" + }, + { + "day": 2, + "flight": "DL702", + "utc": "14:06:00" + }, + { + "day": 3, + "flight": "DL913", + "utc": "07:53:00" + }, + { + "day": 3, + "flight": "DL084", + "utc": "12:09:00" + }, + { + "day": 3, + "flight": "DL729", + "utc": "19:10:00" + }, + { + "day": 3, + "flight": "DL406", + "utc": "07:09:00" + }, + { + "day": 4, + "flight": "DL313", + "utc": "17:22:00" + }, + { + "day": 4, + "flight": "DL791", + "utc": "03:57:00" + }, + { + "day": 4, + "flight": "DL027", + "utc": "10:16:00" + }, + { + "day": 5, + "flight": "DL245", + "utc": "12:52:00" + }, + { + "day": 5, + "flight": "DL829", + "utc": "19:39:00" + }, + { + "day": 5, + "flight": "DL895", + "utc": "14:06:00" + }, + { + "day": 5, + "flight": "DL388", + "utc": "11:47:00" + }, + { + "day": 5, + "flight": "DL379", + "utc": "16:33:00" + }, + { + "day": 6, + "flight": "DL355", + "utc": "00:18:00" + } + ], + "sourceairport": "SEA", + "stops": 0, + "type": "route" + }, + { + "airline": "FL", + "airlineid": "airline_1316", + "destinationairport": "ATL", + "distance": 958.2003291112867, + "equipment": "717 73G", + "id": 25068, + "schedule": + [ + { + "day": 0, + "flight": "FL108", + "utc": "10:45:00" + }, + { + "day": 0, + "flight": "FL073", + "utc": "11:18:00" + }, + { + "day": 0, + "flight": "FL234", + "utc": "00:32:00" + }, + { + "day": 1, + "flight": "FL458", + "utc": "17:27:00" + }, + { + "day": 1, + "flight": "FL713", + "utc": "06:57:00" + }, + { + "day": 1, + "flight": "FL514", + "utc": "16:53:00" + }, + { + "day": 2, + "flight": "FL751", + "utc": "10:03:00" + }, + { + "day": 2, + "flight": "FL884", + "utc": "14:34:00" + }, + { + "day": 3, + "flight": "FL852", + "utc": "22:28:00" + }, + { + "day": 3, + "flight": "FL818", + "utc": "20:28:00" + }, + { + "day": 4, + "flight": "FL602", + "utc": "07:35:00" + }, + { + "day": 5, + "flight": "FL891", + "utc": "23:54:00" + }, + { + "day": 5, + "flight": "FL101", + "utc": "19:14:00" + }, + { + "day": 5, + "flight": "FL188", + "utc": "04:52:00" + }, + { + "day": 5, + "flight": "FL524", + "utc": "08:25:00" + }, + { + "day": 5, + "flight": "FL859", + "utc": "00:13:00" + }, + { + "day": 6, + "flight": "FL625", + "utc": "16:11:00" + }, + { + "day": 6, + "flight": "FL036", + "utc": "12:01:00" + } + ], + "sourceairport": "DTW", + "stops": 0, + "type": "route" + }, + { + "airline": "NK", + "airlineid": "airline_4687", + "destinationairport": "LAX", + "distance": 1983.1814544393283, + "equipment": "319", + "id": 43374, + "schedule": + [ + { + "day": 0, + "flight": "NK862", + "utc": "13:17:00" + }, + { + "day": 0, + "flight": "NK384", + "utc": "22:44:00" + }, + { + "day": 1, + "flight": "NK912", + "utc": "20:06:00" + }, + { + "day": 1, + "flight": "NK747", + "utc": "05:37:00" + }, + { + "day": 1, + "flight": "NK551", + "utc": "03:09:00" + }, + { + "day": 2, + "flight": "NK827", + "utc": "19:29:00" + }, + { + "day": 3, + "flight": "NK298", + "utc": "06:13:00" + }, + { + "day": 3, + "flight": "NK242", + "utc": "22:50:00" + }, + { + "day": 3, + "flight": "NK939", + "utc": "15:14:00" + }, + { + "day": 4, + "flight": "NK788", + "utc": "13:32:00" + }, + { + "day": 4, + "flight": "NK748", + "utc": "08:37:00" + }, + { + "day": 4, + "flight": "NK809", + "utc": "09:11:00" + }, + { + "day": 4, + "flight": "NK153", + "utc": "11:57:00" + }, + { + "day": 5, + "flight": "NK096", + "utc": "05:13:00" + }, + { + "day": 5, + "flight": "NK618", + "utc": "09:07:00" + }, + { + "day": 5, + "flight": "NK422", + "utc": "13:29:00" + }, + { + "day": 5, + "flight": "NK428", + "utc": "21:18:00" + }, + { + "day": 6, + "flight": "NK083", + "utc": "10:06:00" + } + ], + "sourceairport": "DFW", + "stops": 0, + "type": "route" + }, + { + "airline": "NK", + "airlineid": "airline_4687", + "destinationairport": "LGA", + "distance": 805.1323615262812, + "equipment": "320", + "id": 43396, + "schedule": + [ + { + "day": 0, + "flight": "NK940", + "utc": "11:13:00" + }, + { + "day": 0, + "flight": "NK712", + "utc": "23:36:00" + }, + { + "day": 0, + "flight": "NK612", + "utc": "21:19:00" + }, + { + "day": 1, + "flight": "NK091", + "utc": "16:08:00" + }, + { + "day": 1, + "flight": "NK165", + "utc": "08:07:00" + }, + { + "day": 1, + "flight": "NK999", + "utc": "00:47:00" + }, + { + "day": 2, + "flight": "NK004", + "utc": "10:27:00" + }, + { + "day": 3, + "flight": "NK232", + "utc": "15:08:00" + }, + { + "day": 3, + "flight": "NK548", + "utc": "23:33:00" + }, + { + "day": 3, + "flight": "NK951", + "utc": "04:13:00" + }, + { + "day": 3, + "flight": "NK014", + "utc": "17:30:00" + }, + { + "day": 3, + "flight": "NK371", + "utc": "01:28:00" + }, + { + "day": 4, + "flight": "NK785", + "utc": "09:08:00" + }, + { + "day": 5, + "flight": "NK819", + "utc": "02:17:00" + }, + { + "day": 5, + "flight": "NK642", + "utc": "18:55:00" + }, + { + "day": 6, + "flight": "NK204", + "utc": "11:27:00" + }, + { + "day": 6, + "flight": "NK599", + "utc": "20:09:00" + }, + { + "day": 6, + "flight": "NK985", + "utc": "16:57:00" + }, + { + "day": 6, + "flight": "NK505", + "utc": "07:58:00" + }, + { + "day": 6, + "flight": "NK942", + "utc": "18:03:00" + } + ], + "sourceairport": "DTW", + "stops": 0, + "type": "route" + }, + { + "airline": "NK", + "airlineid": "airline_4687", + "destinationairport": "DFW", + "distance": 2231.3581987274165, + "equipment": "319", + "id": 43477, + "schedule": + [ + { + "day": 0, + "flight": "NK996", + "utc": "22:50:00" + }, + { + "day": 0, + "flight": "NK493", + "utc": "16:13:00" + }, + { + "day": 0, + "flight": "NK480", + "utc": "17:23:00" + }, + { + "day": 0, + "flight": "NK335", + "utc": "15:27:00" + }, + { + "day": 1, + "flight": "NK398", + "utc": "07:03:00" + }, + { + "day": 1, + "flight": "NK062", + "utc": "05:55:00" + }, + { + "day": 1, + "flight": "NK431", + "utc": "07:19:00" + }, + { + "day": 1, + "flight": "NK080", + "utc": "14:13:00" + }, + { + "day": 1, + "flight": "NK698", + "utc": "02:51:00" + }, + { + "day": 2, + "flight": "NK382", + "utc": "19:47:00" + }, + { + "day": 2, + "flight": "NK884", + "utc": "16:35:00" + }, + { + "day": 2, + "flight": "NK917", + "utc": "16:37:00" + }, + { + "day": 2, + "flight": "NK406", + "utc": "05:16:00" + }, + { + "day": 3, + "flight": "NK815", + "utc": "16:23:00" + }, + { + "day": 3, + "flight": "NK327", + "utc": "06:31:00" + }, + { + "day": 3, + "flight": "NK838", + "utc": "17:44:00" + }, + { + "day": 3, + "flight": "NK334", + "utc": "12:24:00" + }, + { + "day": 3, + "flight": "NK540", + "utc": "17:17:00" + }, + { + "day": 4, + "flight": "NK882", + "utc": "09:16:00" + }, + { + "day": 4, + "flight": "NK260", + "utc": "03:36:00" + }, + { + "day": 4, + "flight": "NK598", + "utc": "20:15:00" + }, + { + "day": 4, + "flight": "NK137", + "utc": "12:52:00" + }, + { + "day": 4, + "flight": "NK768", + "utc": "23:58:00" + }, + { + "day": 5, + "flight": "NK433", + "utc": "00:01:00" + }, + { + "day": 5, + "flight": "NK474", + "utc": "15:17:00" + }, + { + "day": 5, + "flight": "NK695", + "utc": "11:00:00" + }, + { + "day": 6, + "flight": "NK142", + "utc": "00:20:00" + }, + { + "day": 6, + "flight": "NK005", + "utc": "23:04:00" + } + ], + "sourceairport": "LGA", + "stops": 0, + "type": "route" + }, + { + "airline": "NK", + "airlineid": "airline_4687", + "destinationairport": "LGA", + "distance": 1177.0566911346864, + "equipment": "320", + "id": 43526, + "schedule": + [ + { + "day": 0, + "flight": "NK168", + "utc": "04:52:00" + }, + { + "day": 0, + "flight": "NK476", + "utc": "10:40:00" + }, + { + "day": 1, + "flight": "NK540", + "utc": "15:55:00" + }, + { + "day": 1, + "flight": "NK949", + "utc": "13:48:00" + }, + { + "day": 1, + "flight": "NK690", + "utc": "11:22:00" + }, + { + "day": 1, + "flight": "NK143", + "utc": "03:53:00" + }, + { + "day": 1, + "flight": "NK807", + "utc": "07:25:00" + }, + { + "day": 2, + "flight": "NK778", + "utc": "12:47:00" + }, + { + "day": 3, + "flight": "NK155", + "utc": "21:57:00" + }, + { + "day": 3, + "flight": "NK169", + "utc": "00:39:00" + }, + { + "day": 3, + "flight": "NK837", + "utc": "22:56:00" + }, + { + "day": 4, + "flight": "NK276", + "utc": "12:02:00" + }, + { + "day": 4, + "flight": "NK399", + "utc": "17:25:00" + }, + { + "day": 4, + "flight": "NK121", + "utc": "01:50:00" + }, + { + "day": 5, + "flight": "NK705", + "utc": "10:32:00" + }, + { + "day": 5, + "flight": "NK350", + "utc": "20:44:00" + }, + { + "day": 5, + "flight": "NK978", + "utc": "21:29:00" + }, + { + "day": 5, + "flight": "NK377", + "utc": "04:39:00" + }, + { + "day": 5, + "flight": "NK201", + "utc": "10:07:00" + }, + { + "day": 6, + "flight": "NK660", + "utc": "00:02:00" + } + ], + "sourceairport": "ORD", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "CLT", + "distance": 364.5601258774753, + "equipment": "319 320 CR9 321", + "id": 4699, + "schedule": + [ + { + "day": 0, + "flight": "AA447", + "utc": "21:36:00" + }, + { + "day": 0, + "flight": "AA742", + "utc": "01:03:00" + }, + { + "day": 0, + "flight": "AA783", + "utc": "21:23:00" + }, + { + "day": 1, + "flight": "AA493", + "utc": "16:49:00" + }, + { + "day": 1, + "flight": "AA810", + "utc": "03:45:00" + }, + { + "day": 2, + "flight": "AA144", + "utc": "11:39:00" + }, + { + "day": 2, + "flight": "AA979", + "utc": "22:26:00" + }, + { + "day": 2, + "flight": "AA550", + "utc": "06:58:00" + }, + { + "day": 2, + "flight": "AA995", + "utc": "09:30:00" + }, + { + "day": 3, + "flight": "AA875", + "utc": "08:39:00" + }, + { + "day": 4, + "flight": "AA094", + "utc": "05:03:00" + }, + { + "day": 4, + "flight": "AA650", + "utc": "21:05:00" + }, + { + "day": 4, + "flight": "AA755", + "utc": "07:02:00" + }, + { + "day": 4, + "flight": "AA656", + "utc": "10:04:00" + }, + { + "day": 5, + "flight": "AA837", + "utc": "03:12:00" + }, + { + "day": 5, + "flight": "AA175", + "utc": "08:28:00" + }, + { + "day": 5, + "flight": "AA449", + "utc": "21:43:00" + }, + { + "day": 5, + "flight": "AA496", + "utc": "18:32:00" + }, + { + "day": 6, + "flight": "AA693", + "utc": "23:47:00" + } + ], + "sourceairport": "ATL", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "LAX", + "distance": 3412.5368042521163, + "equipment": "321", + "id": 4953, + "schedule": + [ + { + "day": 0, + "flight": "AA859", + "utc": "03:14:00" + }, + { + "day": 0, + "flight": "AA026", + "utc": "19:59:00" + }, + { + "day": 0, + "flight": "AA691", + "utc": "00:31:00" + }, + { + "day": 1, + "flight": "AA027", + "utc": "05:17:00" + }, + { + "day": 1, + "flight": "AA386", + "utc": "06:05:00" + }, + { + "day": 1, + "flight": "AA526", + "utc": "05:23:00" + }, + { + "day": 2, + "flight": "AA059", + "utc": "08:22:00" + }, + { + "day": 2, + "flight": "AA225", + "utc": "10:38:00" + }, + { + "day": 3, + "flight": "AA250", + "utc": "20:00:00" + }, + { + "day": 3, + "flight": "AA394", + "utc": "06:12:00" + }, + { + "day": 3, + "flight": "AA014", + "utc": "03:04:00" + }, + { + "day": 4, + "flight": "AA843", + "utc": "01:41:00" + }, + { + "day": 4, + "flight": "AA807", + "utc": "07:34:00" + }, + { + "day": 4, + "flight": "AA621", + "utc": "21:41:00" + }, + { + "day": 4, + "flight": "AA631", + "utc": "23:17:00" + }, + { + "day": 5, + "flight": "AA888", + "utc": "18:52:00" + }, + { + "day": 6, + "flight": "AA693", + "utc": "00:44:00" + }, + { + "day": 6, + "flight": "AA038", + "utc": "10:31:00" + } + ], + "sourceairport": "CLT", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "SEA", + "distance": 2668.171565254428, + "equipment": "738 757", + "id": 5312, + "schedule": + [ + { + "day": 0, + "flight": "AA887", + "utc": "09:47:00" + }, + { + "day": 0, + "flight": "AA785", + "utc": "13:15:00" + }, + { + "day": 0, + "flight": "AA417", + "utc": "22:08:00" + }, + { + "day": 0, + "flight": "AA542", + "utc": "11:43:00" + }, + { + "day": 0, + "flight": "AA105", + "utc": "03:46:00" + }, + { + "day": 1, + "flight": "AA917", + "utc": "10:00:00" + }, + { + "day": 2, + "flight": "AA520", + "utc": "21:28:00" + }, + { + "day": 2, + "flight": "AA619", + "utc": "01:03:00" + }, + { + "day": 2, + "flight": "AA884", + "utc": "04:46:00" + }, + { + "day": 2, + "flight": "AA882", + "utc": "00:01:00" + }, + { + "day": 2, + "flight": "AA316", + "utc": "22:17:00" + }, + { + "day": 3, + "flight": "AA522", + "utc": "03:25:00" + }, + { + "day": 3, + "flight": "AA491", + "utc": "20:10:00" + }, + { + "day": 3, + "flight": "AA828", + "utc": "20:56:00" + }, + { + "day": 3, + "flight": "AA978", + "utc": "02:19:00" + }, + { + "day": 3, + "flight": "AA898", + "utc": "07:49:00" + }, + { + "day": 4, + "flight": "AA450", + "utc": "19:46:00" + }, + { + "day": 4, + "flight": "AA064", + "utc": "01:44:00" + }, + { + "day": 4, + "flight": "AA452", + "utc": "23:39:00" + }, + { + "day": 4, + "flight": "AA404", + "utc": "22:45:00" + }, + { + "day": 4, + "flight": "AA990", + "utc": "02:04:00" + }, + { + "day": 5, + "flight": "AA030", + "utc": "10:25:00" + }, + { + "day": 6, + "flight": "AA042", + "utc": "18:38:00" + }, + { + "day": 6, + "flight": "AA357", + "utc": "04:09:00" + }, + { + "day": 6, + "flight": "AA775", + "utc": "13:38:00" + } + ], + "sourceairport": "DFW", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "MIA", + "distance": 1847.0804151368413, + "equipment": "738", + "id": 5366, + "schedule": + [ + { + "day": 0, + "flight": "AA956", + "utc": "14:32:00" + }, + { + "day": 0, + "flight": "AA425", + "utc": "23:07:00" + }, + { + "day": 0, + "flight": "AA153", + "utc": "13:50:00" + }, + { + "day": 1, + "flight": "AA314", + "utc": "19:11:00" + }, + { + "day": 1, + "flight": "AA092", + "utc": "10:56:00" + }, + { + "day": 1, + "flight": "AA724", + "utc": "09:53:00" + }, + { + "day": 2, + "flight": "AA819", + "utc": "14:34:00" + }, + { + "day": 2, + "flight": "AA881", + "utc": "11:45:00" + }, + { + "day": 2, + "flight": "AA810", + "utc": "06:07:00" + }, + { + "day": 2, + "flight": "AA336", + "utc": "03:32:00" + }, + { + "day": 2, + "flight": "AA751", + "utc": "11:38:00" + }, + { + "day": 3, + "flight": "AA279", + "utc": "08:05:00" + }, + { + "day": 3, + "flight": "AA530", + "utc": "07:57:00" + }, + { + "day": 3, + "flight": "AA821", + "utc": "19:27:00" + }, + { + "day": 4, + "flight": "AA012", + "utc": "04:35:00" + }, + { + "day": 4, + "flight": "AA084", + "utc": "20:37:00" + }, + { + "day": 4, + "flight": "AA151", + "utc": "03:53:00" + }, + { + "day": 4, + "flight": "AA104", + "utc": "13:22:00" + }, + { + "day": 5, + "flight": "AA803", + "utc": "16:36:00" + }, + { + "day": 6, + "flight": "AA088", + "utc": "11:59:00" + }, + { + "day": 6, + "flight": "AA026", + "utc": "19:10:00" + } + ], + "sourceairport": "DTW", + "stops": 0, + "type": "route" + }, + { + "airline": "UA", + "airlineid": "airline_5209", + "destinationairport": "ORD", + "distance": 976.3282034488873, + "equipment": "CR7 ERJ E70", + "id": 55809, + "schedule": + [ + { + "day": 0, + "flight": "UA473", + "utc": "03:38:00" + }, + { + "day": 0, + "flight": "UA832", + "utc": "13:47:00" + }, + { + "day": 1, + "flight": "UA835", + "utc": "05:04:00" + }, + { + "day": 2, + "flight": "UA371", + "utc": "16:18:00" + }, + { + "day": 2, + "flight": "UA539", + "utc": "07:11:00" + }, + { + "day": 2, + "flight": "UA483", + "utc": "23:54:00" + }, + { + "day": 3, + "flight": "UA713", + "utc": "03:36:00" + }, + { + "day": 3, + "flight": "UA675", + "utc": "23:41:00" + }, + { + "day": 3, + "flight": "UA208", + "utc": "14:13:00" + }, + { + "day": 4, + "flight": "UA443", + "utc": "03:21:00" + }, + { + "day": 4, + "flight": "UA428", + "utc": "03:15:00" + }, + { + "day": 4, + "flight": "UA839", + "utc": "23:29:00" + }, + { + "day": 5, + "flight": "UA031", + "utc": "15:59:00" + }, + { + "day": 6, + "flight": "UA415", + "utc": "18:30:00" + }, + { + "day": 6, + "flight": "UA092", + "utc": "21:56:00" + }, + { + "day": 6, + "flight": "UA884", + "utc": "03:46:00" + }, + { + "day": 6, + "flight": "UA230", + "utc": "12:20:00" + } + ], + "sourceairport": "ATL", + "stops": 0, + "type": "route" + }, + { + "airline": "UA", + "airlineid": "airline_5209", + "destinationairport": "ORD", + "distance": 964.5791472772447, + "equipment": "320 319", + "id": 56020, + "schedule": + [ + { + "day": 0, + "flight": "UA974", + "utc": "14:22:00" + }, + { + "day": 0, + "flight": "UA079", + "utc": "22:09:00" + }, + { + "day": 1, + "flight": "UA341", + "utc": "21:00:00" + }, + { + "day": 2, + "flight": "UA239", + "utc": "05:44:00" + }, + { + "day": 2, + "flight": "UA154", + "utc": "19:46:00" + }, + { + "day": 2, + "flight": "UA324", + "utc": "18:12:00" + }, + { + "day": 2, + "flight": "UA870", + "utc": "10:30:00" + }, + { + "day": 3, + "flight": "UA110", + "utc": "03:45:00" + }, + { + "day": 3, + "flight": "UA725", + "utc": "02:06:00" + }, + { + "day": 3, + "flight": "UA020", + "utc": "07:46:00" + }, + { + "day": 4, + "flight": "UA648", + "utc": "18:16:00" + }, + { + "day": 5, + "flight": "UA294", + "utc": "21:46:00" + }, + { + "day": 5, + "flight": "UA400", + "utc": "05:55:00" + }, + { + "day": 5, + "flight": "UA943", + "utc": "16:25:00" + }, + { + "day": 5, + "flight": "UA655", + "utc": "09:49:00" + }, + { + "day": 6, + "flight": "UA072", + "utc": "10:55:00" + }, + { + "day": 6, + "flight": "UA928", + "utc": "05:10:00" + }, + { + "day": 6, + "flight": "UA050", + "utc": "11:07:00" + }, + { + "day": 6, + "flight": "UA782", + "utc": "00:34:00" + } + ], + "sourceairport": "CLT", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "DFW", + "distance": 2234.8907705863057, + "equipment": "738 757", + "id": 5609, + "schedule": + [ + { + "day": 0, + "flight": "AA165", + "utc": "03:28:00" + }, + { + "day": 0, + "flight": "AA933", + "utc": "13:03:00" + }, + { + "day": 0, + "flight": "AA812", + "utc": "16:01:00" + }, + { + "day": 1, + "flight": "AA801", + "utc": "08:28:00" + }, + { + "day": 1, + "flight": "AA995", + "utc": "05:30:00" + }, + { + "day": 1, + "flight": "AA785", + "utc": "13:07:00" + }, + { + "day": 1, + "flight": "AA709", + "utc": "21:51:00" + }, + { + "day": 1, + "flight": "AA706", + "utc": "16:14:00" + }, + { + "day": 2, + "flight": "AA393", + "utc": "22:35:00" + }, + { + "day": 3, + "flight": "AA085", + "utc": "18:07:00" + }, + { + "day": 3, + "flight": "AA271", + "utc": "05:04:00" + }, + { + "day": 3, + "flight": "AA685", + "utc": "21:54:00" + }, + { + "day": 3, + "flight": "AA558", + "utc": "17:03:00" + }, + { + "day": 3, + "flight": "AA333", + "utc": "04:27:00" + }, + { + "day": 4, + "flight": "AA253", + "utc": "14:34:00" + }, + { + "day": 5, + "flight": "AA896", + "utc": "00:08:00" + }, + { + "day": 5, + "flight": "AA330", + "utc": "16:54:00" + }, + { + "day": 5, + "flight": "AA344", + "utc": "16:31:00" + }, + { + "day": 5, + "flight": "AA874", + "utc": "08:09:00" + }, + { + "day": 5, + "flight": "AA647", + "utc": "21:53:00" + }, + { + "day": 6, + "flight": "AA061", + "utc": "09:58:00" + }, + { + "day": 6, + "flight": "AA625", + "utc": "04:24:00" + }, + { + "day": 6, + "flight": "AA783", + "utc": "19:15:00" + } + ], + "sourceairport": "JFK", + "stops": 0, + "type": "route" + }, + { + "airline": "UA", + "airlineid": "airline_5209", + "destinationairport": "ORD", + "distance": 1290.640549573098, + "equipment": "319", + "id": 56224, + "schedule": + [ + { + "day": 0, + "flight": "UA478", + "utc": "18:44:00" + }, + { + "day": 0, + "flight": "UA048", + "utc": "04:12:00" + }, + { + "day": 0, + "flight": "UA928", + "utc": "07:18:00" + }, + { + "day": 0, + "flight": "UA757", + "utc": "22:55:00" + }, + { + "day": 0, + "flight": "UA755", + "utc": "04:26:00" + }, + { + "day": 1, + "flight": "UA280", + "utc": "05:53:00" + }, + { + "day": 1, + "flight": "UA734", + "utc": "11:54:00" + }, + { + "day": 1, + "flight": "UA676", + "utc": "21:16:00" + }, + { + "day": 2, + "flight": "UA793", + "utc": "08:47:00" + }, + { + "day": 3, + "flight": "UA844", + "utc": "23:13:00" + }, + { + "day": 3, + "flight": "UA087", + "utc": "01:08:00" + }, + { + "day": 3, + "flight": "UA396", + "utc": "16:48:00" + }, + { + "day": 4, + "flight": "UA783", + "utc": "09:36:00" + }, + { + "day": 5, + "flight": "UA870", + "utc": "02:38:00" + }, + { + "day": 5, + "flight": "UA487", + "utc": "21:10:00" + }, + { + "day": 6, + "flight": "UA171", + "utc": "15:44:00" + }, + { + "day": 6, + "flight": "UA795", + "utc": "09:08:00" + } + ], + "sourceairport": "DFW", + "stops": 0, + "type": "route" + }, + { + "airline": "UA", + "airlineid": "airline_5209", + "destinationairport": "LAX", + "distance": 3974.1999622301605, + "equipment": "757", + "id": 56909, + "schedule": + [ + { + "day": 0, + "flight": "UA041", + "utc": "21:34:00" + }, + { + "day": 0, + "flight": "UA958", + "utc": "21:37:00" + }, + { + "day": 1, + "flight": "UA848", + "utc": "19:50:00" + }, + { + "day": 1, + "flight": "UA057", + "utc": "14:37:00" + }, + { + "day": 2, + "flight": "UA941", + "utc": "16:17:00" + }, + { + "day": 3, + "flight": "UA721", + "utc": "18:04:00" + }, + { + "day": 3, + "flight": "UA325", + "utc": "14:41:00" + }, + { + "day": 3, + "flight": "UA323", + "utc": "07:04:00" + }, + { + "day": 4, + "flight": "UA802", + "utc": "16:47:00" + }, + { + "day": 4, + "flight": "UA803", + "utc": "12:11:00" + }, + { + "day": 4, + "flight": "UA040", + "utc": "02:15:00" + }, + { + "day": 4, + "flight": "UA833", + "utc": "20:15:00" + }, + { + "day": 4, + "flight": "UA848", + "utc": "23:27:00" + }, + { + "day": 5, + "flight": "UA177", + "utc": "17:45:00" + }, + { + "day": 5, + "flight": "UA057", + "utc": "12:35:00" + }, + { + "day": 5, + "flight": "UA968", + "utc": "17:45:00" + }, + { + "day": 6, + "flight": "UA083", + "utc": "07:45:00" + }, + { + "day": 6, + "flight": "UA754", + "utc": "00:21:00" + }, + { + "day": 6, + "flight": "UA125", + "utc": "06:26:00" + } + ], + "sourceairport": "JFK", + "stops": 0, + "type": "route" + }, + { + "airline": "UA", + "airlineid": "airline_5209", + "destinationairport": "ORD", + "distance": 2802.1171926467396, + "equipment": "739 738 752 320 319 753", + "id": 56994, + "schedule": + [ + { + "day": 0, + "flight": "UA598", + "utc": "04:44:00" + }, + { + "day": 0, + "flight": "UA581", + "utc": "05:56:00" + }, + { + "day": 0, + "flight": "UA539", + "utc": "15:21:00" + }, + { + "day": 0, + "flight": "UA540", + "utc": "17:04:00" + }, + { + "day": 0, + "flight": "UA035", + "utc": "08:57:00" + }, + { + "day": 1, + "flight": "UA233", + "utc": "21:25:00" + }, + { + "day": 1, + "flight": "UA773", + "utc": "01:37:00" + }, + { + "day": 1, + "flight": "UA192", + "utc": "11:18:00" + }, + { + "day": 1, + "flight": "UA685", + "utc": "08:21:00" + }, + { + "day": 1, + "flight": "UA686", + "utc": "11:45:00" + }, + { + "day": 2, + "flight": "UA869", + "utc": "17:49:00" + }, + { + "day": 3, + "flight": "UA754", + "utc": "11:39:00" + }, + { + "day": 3, + "flight": "UA174", + "utc": "12:09:00" + }, + { + "day": 3, + "flight": "UA833", + "utc": "08:50:00" + }, + { + "day": 3, + "flight": "UA111", + "utc": "14:37:00" + }, + { + "day": 4, + "flight": "UA931", + "utc": "20:01:00" + }, + { + "day": 4, + "flight": "UA721", + "utc": "05:03:00" + }, + { + "day": 4, + "flight": "UA405", + "utc": "07:51:00" + }, + { + "day": 5, + "flight": "UA650", + "utc": "12:48:00" + }, + { + "day": 5, + "flight": "UA295", + "utc": "22:15:00" + }, + { + "day": 5, + "flight": "UA186", + "utc": "16:34:00" + }, + { + "day": 6, + "flight": "UA787", + "utc": "19:09:00" + }, + { + "day": 6, + "flight": "UA006", + "utc": "17:13:00" + }, + { + "day": 6, + "flight": "UA776", + "utc": "23:48:00" + } + ], + "sourceairport": "LAX", + "stops": 0, + "type": "route" + }, + { + "airline": "UA", + "airlineid": "airline_5209", + "destinationairport": "LGA", + "distance": 1177.0566911346864, + "equipment": "738 320 319 752 73G", + "id": 57324, + "schedule": + [ + { + "day": 0, + "flight": "UA129", + "utc": "23:52:00" + }, + { + "day": 0, + "flight": "UA709", + "utc": "23:50:00" + }, + { + "day": 0, + "flight": "UA430", + "utc": "04:09:00" + }, + { + "day": 0, + "flight": "UA110", + "utc": "14:59:00" + }, + { + "day": 1, + "flight": "UA520", + "utc": "02:26:00" + }, + { + "day": 1, + "flight": "UA839", + "utc": "05:56:00" + }, + { + "day": 1, + "flight": "UA820", + "utc": "04:53:00" + }, + { + "day": 1, + "flight": "UA796", + "utc": "01:21:00" + }, + { + "day": 1, + "flight": "UA770", + "utc": "08:22:00" + }, + { + "day": 2, + "flight": "UA296", + "utc": "02:49:00" + }, + { + "day": 3, + "flight": "UA067", + "utc": "04:41:00" + }, + { + "day": 3, + "flight": "UA334", + "utc": "04:14:00" + }, + { + "day": 3, + "flight": "UA314", + "utc": "12:29:00" + }, + { + "day": 3, + "flight": "UA148", + "utc": "23:29:00" + }, + { + "day": 4, + "flight": "UA222", + "utc": "10:30:00" + }, + { + "day": 4, + "flight": "UA044", + "utc": "17:52:00" + }, + { + "day": 4, + "flight": "UA693", + "utc": "22:53:00" + }, + { + "day": 5, + "flight": "UA208", + "utc": "03:13:00" + }, + { + "day": 5, + "flight": "UA895", + "utc": "15:54:00" + }, + { + "day": 5, + "flight": "UA268", + "utc": "22:43:00" + }, + { + "day": 6, + "flight": "UA203", + "utc": "13:59:00" + }, + { + "day": 6, + "flight": "UA534", + "utc": "02:30:00" + }, + { + "day": 6, + "flight": "UA327", + "utc": "03:46:00" + } + ], + "sourceairport": "ORD", + "stops": 0, + "type": "route" + }, + { + "airline": "UA", + "airlineid": "airline_5209", + "destinationairport": "LAX", + "distance": 1536.9098522443824, + "equipment": "739 320", + "id": 57620, + "schedule": + [ + { + "day": 0, + "flight": "UA544", + "utc": "05:01:00" + }, + { + "day": 0, + "flight": "UA170", + "utc": "07:15:00" + }, + { + "day": 0, + "flight": "UA872", + "utc": "21:08:00" + }, + { + "day": 0, + "flight": "UA758", + "utc": "00:38:00" + }, + { + "day": 1, + "flight": "UA115", + "utc": "18:57:00" + }, + { + "day": 1, + "flight": "UA147", + "utc": "18:59:00" + }, + { + "day": 1, + "flight": "UA911", + "utc": "22:33:00" + }, + { + "day": 1, + "flight": "UA057", + "utc": "04:05:00" + }, + { + "day": 2, + "flight": "UA623", + "utc": "18:58:00" + }, + { + "day": 2, + "flight": "UA770", + "utc": "20:08:00" + }, + { + "day": 3, + "flight": "UA130", + "utc": "03:45:00" + }, + { + "day": 3, + "flight": "UA413", + "utc": "08:32:00" + }, + { + "day": 3, + "flight": "UA648", + "utc": "19:37:00" + }, + { + "day": 3, + "flight": "UA555", + "utc": "17:25:00" + }, + { + "day": 4, + "flight": "UA880", + "utc": "15:42:00" + }, + { + "day": 4, + "flight": "UA591", + "utc": "12:47:00" + }, + { + "day": 5, + "flight": "UA543", + "utc": "22:02:00" + }, + { + "day": 5, + "flight": "UA533", + "utc": "22:32:00" + }, + { + "day": 5, + "flight": "UA093", + "utc": "02:07:00" + }, + { + "day": 6, + "flight": "UA040", + "utc": "07:02:00" + }, + { + "day": 6, + "flight": "UA351", + "utc": "18:43:00" + }, + { + "day": 6, + "flight": "UA249", + "utc": "08:42:00" + }, + { + "day": 6, + "flight": "UA915", + "utc": "03:17:00" + } + ], + "sourceairport": "SEA", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "MIA", + "distance": 1768.3749479721741, + "equipment": "738", + "id": 5780, + "schedule": + [ + { + "day": 0, + "flight": "AA139", + "utc": "10:05:00" + }, + { + "day": 0, + "flight": "AA345", + "utc": "00:29:00" + }, + { + "day": 0, + "flight": "AA767", + "utc": "00:18:00" + }, + { + "day": 0, + "flight": "AA296", + "utc": "06:55:00" + }, + { + "day": 1, + "flight": "AA832", + "utc": "16:47:00" + }, + { + "day": 1, + "flight": "AA955", + "utc": "10:24:00" + }, + { + "day": 2, + "flight": "AA969", + "utc": "14:39:00" + }, + { + "day": 2, + "flight": "AA532", + "utc": "11:17:00" + }, + { + "day": 2, + "flight": "AA595", + "utc": "06:12:00" + }, + { + "day": 2, + "flight": "AA852", + "utc": "09:51:00" + }, + { + "day": 2, + "flight": "AA645", + "utc": "16:33:00" + }, + { + "day": 3, + "flight": "AA626", + "utc": "02:42:00" + }, + { + "day": 3, + "flight": "AA344", + "utc": "10:41:00" + }, + { + "day": 3, + "flight": "AA098", + "utc": "04:42:00" + }, + { + "day": 3, + "flight": "AA608", + "utc": "11:51:00" + }, + { + "day": 4, + "flight": "AA756", + "utc": "17:41:00" + }, + { + "day": 4, + "flight": "AA344", + "utc": "17:35:00" + }, + { + "day": 4, + "flight": "AA411", + "utc": "03:12:00" + }, + { + "day": 5, + "flight": "AA220", + "utc": "17:03:00" + }, + { + "day": 5, + "flight": "AA335", + "utc": "12:45:00" + }, + { + "day": 5, + "flight": "AA620", + "utc": "04:47:00" + }, + { + "day": 6, + "flight": "AA625", + "utc": "03:57:00" + } + ], + "sourceairport": "LGA", + "stops": 0, + "type": "route" + }, + { + "airline": "US", + "airlineid": "airline_5265", + "destinationairport": "JFK", + "distance": 3974.1999622301605, + "equipment": "32B 762", + "id": 59507, + "schedule": + [ + { + "day": 0, + "flight": "US233", + "utc": "05:16:00" + }, + { + "day": 1, + "flight": "US396", + "utc": "13:29:00" + }, + { + "day": 1, + "flight": "US652", + "utc": "21:59:00" + }, + { + "day": 1, + "flight": "US723", + "utc": "21:12:00" + }, + { + "day": 1, + "flight": "US771", + "utc": "18:01:00" + }, + { + "day": 2, + "flight": "US834", + "utc": "17:29:00" + }, + { + "day": 2, + "flight": "US664", + "utc": "15:37:00" + }, + { + "day": 3, + "flight": "US942", + "utc": "16:44:00" + }, + { + "day": 3, + "flight": "US010", + "utc": "12:53:00" + }, + { + "day": 4, + "flight": "US227", + "utc": "23:08:00" + }, + { + "day": 4, + "flight": "US689", + "utc": "03:10:00" + }, + { + "day": 4, + "flight": "US032", + "utc": "08:49:00" + }, + { + "day": 5, + "flight": "US146", + "utc": "18:32:00" + }, + { + "day": 5, + "flight": "US589", + "utc": "22:56:00" + }, + { + "day": 6, + "flight": "US437", + "utc": "16:49:00" + }, + { + "day": 6, + "flight": "US382", + "utc": "14:18:00" + }, + { + "day": 6, + "flight": "US286", + "utc": "14:53:00" + }, + { + "day": 6, + "flight": "US496", + "utc": "00:33:00" + }, + { + "day": 6, + "flight": "US087", + "utc": "09:30:00" + } + ], + "sourceairport": "LAX", + "stops": 0, + "type": "route" + }, + { + "airline": "US", + "airlineid": "airline_5265", + "destinationairport": "LAX", + "distance": 3762.637481199252, + "equipment": "777 763 757", + "id": 59729, + "schedule": + [ + { + "day": 0, + "flight": "US099", + "utc": "07:56:00" + }, + { + "day": 1, + "flight": "US892", + "utc": "16:08:00" + }, + { + "day": 1, + "flight": "US619", + "utc": "23:07:00" + }, + { + "day": 1, + "flight": "US632", + "utc": "14:38:00" + }, + { + "day": 2, + "flight": "US482", + "utc": "08:50:00" + }, + { + "day": 2, + "flight": "US993", + "utc": "08:50:00" + }, + { + "day": 2, + "flight": "US689", + "utc": "15:00:00" + }, + { + "day": 3, + "flight": "US737", + "utc": "04:56:00" + }, + { + "day": 3, + "flight": "US916", + "utc": "14:12:00" + }, + { + "day": 3, + "flight": "US231", + "utc": "21:46:00" + }, + { + "day": 3, + "flight": "US113", + "utc": "10:31:00" + }, + { + "day": 4, + "flight": "US375", + "utc": "18:54:00" + }, + { + "day": 4, + "flight": "US370", + "utc": "09:57:00" + }, + { + "day": 5, + "flight": "US806", + "utc": "02:34:00" + }, + { + "day": 5, + "flight": "US452", + "utc": "21:39:00" + }, + { + "day": 5, + "flight": "US916", + "utc": "16:18:00" + }, + { + "day": 5, + "flight": "US820", + "utc": "10:34:00" + }, + { + "day": 6, + "flight": "US998", + "utc": "03:21:00" + }, + { + "day": 6, + "flight": "US708", + "utc": "15:19:00" + }, + { + "day": 6, + "flight": "US957", + "utc": "15:08:00" + }, + { + "day": 6, + "flight": "US356", + "utc": "18:29:00" + } + ], + "sourceairport": "MIA", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "CLT", + "distance": 1049.3950976055385, + "equipment": "319 321 320 737", + "id": 6022, + "schedule": + [ + { + "day": 0, + "flight": "AA657", + "utc": "02:47:00" + }, + { + "day": 0, + "flight": "AA876", + "utc": "16:37:00" + }, + { + "day": 1, + "flight": "AA992", + "utc": "22:31:00" + }, + { + "day": 1, + "flight": "AA304", + "utc": "04:01:00" + }, + { + "day": 2, + "flight": "AA883", + "utc": "01:03:00" + }, + { + "day": 2, + "flight": "AA324", + "utc": "01:38:00" + }, + { + "day": 2, + "flight": "AA560", + "utc": "06:52:00" + }, + { + "day": 3, + "flight": "AA790", + "utc": "04:33:00" + }, + { + "day": 4, + "flight": "AA611", + "utc": "22:47:00" + }, + { + "day": 4, + "flight": "AA058", + "utc": "01:55:00" + }, + { + "day": 4, + "flight": "AA648", + "utc": "19:34:00" + }, + { + "day": 4, + "flight": "AA449", + "utc": "05:35:00" + }, + { + "day": 5, + "flight": "AA751", + "utc": "11:13:00" + }, + { + "day": 5, + "flight": "AA126", + "utc": "11:42:00" + }, + { + "day": 5, + "flight": "AA176", + "utc": "16:06:00" + }, + { + "day": 5, + "flight": "AA726", + "utc": "21:02:00" + }, + { + "day": 5, + "flight": "AA832", + "utc": "03:39:00" + }, + { + "day": 6, + "flight": "AA633", + "utc": "16:15:00" + } + ], + "sourceairport": "MIA", + "stops": 0, + "type": "route" + }, + { + "airline": "US", + "airlineid": "airline_5265", + "destinationairport": "JFK", + "distance": 3886.6616448020814, + "equipment": "738", + "id": 60344, + "schedule": + [ + { + "day": 0, + "flight": "US114", + "utc": "08:20:00" + }, + { + "day": 0, + "flight": "US902", + "utc": "08:37:00" + }, + { + "day": 0, + "flight": "US110", + "utc": "05:36:00" + }, + { + "day": 0, + "flight": "US503", + "utc": "02:43:00" + }, + { + "day": 1, + "flight": "US347", + "utc": "18:45:00" + }, + { + "day": 1, + "flight": "US247", + "utc": "11:54:00" + }, + { + "day": 1, + "flight": "US864", + "utc": "16:23:00" + }, + { + "day": 1, + "flight": "US806", + "utc": "21:59:00" + }, + { + "day": 1, + "flight": "US956", + "utc": "05:26:00" + }, + { + "day": 2, + "flight": "US887", + "utc": "01:54:00" + }, + { + "day": 2, + "flight": "US432", + "utc": "00:30:00" + }, + { + "day": 2, + "flight": "US962", + "utc": "00:09:00" + }, + { + "day": 2, + "flight": "US335", + "utc": "05:31:00" + }, + { + "day": 2, + "flight": "US088", + "utc": "21:32:00" + }, + { + "day": 3, + "flight": "US482", + "utc": "04:43:00" + }, + { + "day": 3, + "flight": "US670", + "utc": "22:24:00" + }, + { + "day": 3, + "flight": "US566", + "utc": "03:01:00" + }, + { + "day": 4, + "flight": "US903", + "utc": "13:58:00" + }, + { + "day": 4, + "flight": "US102", + "utc": "02:46:00" + }, + { + "day": 4, + "flight": "US023", + "utc": "21:14:00" + }, + { + "day": 4, + "flight": "US499", + "utc": "06:33:00" + }, + { + "day": 5, + "flight": "US232", + "utc": "05:37:00" + }, + { + "day": 5, + "flight": "US273", + "utc": "03:48:00" + }, + { + "day": 5, + "flight": "US856", + "utc": "12:19:00" + }, + { + "day": 6, + "flight": "US916", + "utc": "18:58:00" + }, + { + "day": 6, + "flight": "US786", + "utc": "05:17:00" + } + ], + "sourceairport": "SEA", + "stops": 0, + "type": "route" + }, + { + "airline": "VX", + "airlineid": "airline_5331", + "destinationairport": "SEA", + "distance": 1536.9098522443824, + "equipment": "319 320", + "id": 62017, + "schedule": + [ + { + "day": 0, + "flight": "VX516", + "utc": "20:26:00" + }, + { + "day": 0, + "flight": "VX297", + "utc": "19:06:00" + }, + { + "day": 0, + "flight": "VX156", + "utc": "17:16:00" + }, + { + "day": 1, + "flight": "VX307", + "utc": "16:53:00" + }, + { + "day": 1, + "flight": "VX106", + "utc": "04:55:00" + }, + { + "day": 1, + "flight": "VX781", + "utc": "11:44:00" + }, + { + "day": 1, + "flight": "VX822", + "utc": "15:12:00" + }, + { + "day": 1, + "flight": "VX723", + "utc": "18:08:00" + }, + { + "day": 2, + "flight": "VX676", + "utc": "13:39:00" + }, + { + "day": 2, + "flight": "VX263", + "utc": "19:21:00" + }, + { + "day": 2, + "flight": "VX495", + "utc": "05:35:00" + }, + { + "day": 3, + "flight": "VX643", + "utc": "06:21:00" + }, + { + "day": 3, + "flight": "VX680", + "utc": "14:59:00" + }, + { + "day": 3, + "flight": "VX662", + "utc": "11:35:00" + }, + { + "day": 3, + "flight": "VX888", + "utc": "04:36:00" + }, + { + "day": 4, + "flight": "VX772", + "utc": "21:36:00" + }, + { + "day": 4, + "flight": "VX429", + "utc": "21:26:00" + }, + { + "day": 4, + "flight": "VX852", + "utc": "15:13:00" + }, + { + "day": 5, + "flight": "VX908", + "utc": "22:05:00" + }, + { + "day": 6, + "flight": "VX991", + "utc": "14:47:00" + }, + { + "day": 6, + "flight": "VX485", + "utc": "12:03:00" + } + ], + "sourceairport": "LAX", + "stops": 0, + "type": "route" + }, + { + "airline": "VX", + "airlineid": "airline_5331", + "destinationairport": "LAX", + "distance": 2802.1171926467396, + "equipment": "320", + "id": 62022, + "schedule": + [ + { + "day": 0, + "flight": "VX715", + "utc": "10:06:00" + }, + { + "day": 0, + "flight": "VX380", + "utc": "20:49:00" + }, + { + "day": 0, + "flight": "VX896", + "utc": "12:08:00" + }, + { + "day": 0, + "flight": "VX964", + "utc": "17:48:00" + }, + { + "day": 1, + "flight": "VX332", + "utc": "10:24:00" + }, + { + "day": 1, + "flight": "VX195", + "utc": "23:29:00" + }, + { + "day": 2, + "flight": "VX328", + "utc": "08:29:00" + }, + { + "day": 2, + "flight": "VX502", + "utc": "21:42:00" + }, + { + "day": 2, + "flight": "VX855", + "utc": "07:06:00" + }, + { + "day": 2, + "flight": "VX414", + "utc": "03:44:00" + }, + { + "day": 3, + "flight": "VX619", + "utc": "19:20:00" + }, + { + "day": 3, + "flight": "VX340", + "utc": "12:34:00" + }, + { + "day": 3, + "flight": "VX368", + "utc": "03:07:00" + }, + { + "day": 3, + "flight": "VX453", + "utc": "00:39:00" + }, + { + "day": 3, + "flight": "VX988", + "utc": "10:28:00" + }, + { + "day": 4, + "flight": "VX772", + "utc": "15:34:00" + }, + { + "day": 4, + "flight": "VX679", + "utc": "04:35:00" + }, + { + "day": 4, + "flight": "VX513", + "utc": "02:02:00" + }, + { + "day": 4, + "flight": "VX616", + "utc": "19:16:00" + }, + { + "day": 5, + "flight": "VX963", + "utc": "07:23:00" + }, + { + "day": 5, + "flight": "VX355", + "utc": "22:50:00" + }, + { + "day": 5, + "flight": "VX373", + "utc": "12:28:00" + }, + { + "day": 5, + "flight": "VX002", + "utc": "03:55:00" + }, + { + "day": 5, + "flight": "VX607", + "utc": "04:34:00" + }, + { + "day": 6, + "flight": "VX079", + "utc": "20:29:00" + }, + { + "day": 6, + "flight": "VX018", + "utc": "23:47:00" + }, + { + "day": 6, + "flight": "VX344", + "utc": "16:31:00" + }, + { + "day": 6, + "flight": "VX213", + "utc": "15:11:00" + } + ], + "sourceairport": "ORD", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "DFW", + "distance": 1290.640549573098, + "equipment": "M80 M83 738", + "id": 6257, + "schedule": + [ + { + "day": 0, + "flight": "AA692", + "utc": "09:48:00" + }, + { + "day": 1, + "flight": "AA677", + "utc": "09:59:00" + }, + { + "day": 1, + "flight": "AA176", + "utc": "20:07:00" + }, + { + "day": 1, + "flight": "AA784", + "utc": "01:27:00" + }, + { + "day": 1, + "flight": "AA341", + "utc": "18:52:00" + }, + { + "day": 2, + "flight": "AA222", + "utc": "13:41:00" + }, + { + "day": 2, + "flight": "AA986", + "utc": "23:39:00" + }, + { + "day": 2, + "flight": "AA567", + "utc": "13:38:00" + }, + { + "day": 2, + "flight": "AA918", + "utc": "18:59:00" + }, + { + "day": 2, + "flight": "AA602", + "utc": "20:11:00" + }, + { + "day": 3, + "flight": "AA449", + "utc": "16:14:00" + }, + { + "day": 3, + "flight": "AA915", + "utc": "08:55:00" + }, + { + "day": 3, + "flight": "AA968", + "utc": "03:47:00" + }, + { + "day": 3, + "flight": "AA566", + "utc": "00:16:00" + }, + { + "day": 3, + "flight": "AA165", + "utc": "19:11:00" + }, + { + "day": 4, + "flight": "AA514", + "utc": "01:40:00" + }, + { + "day": 4, + "flight": "AA515", + "utc": "23:01:00" + }, + { + "day": 4, + "flight": "AA262", + "utc": "05:51:00" + }, + { + "day": 4, + "flight": "AA682", + "utc": "01:43:00" + }, + { + "day": 5, + "flight": "AA551", + "utc": "22:41:00" + }, + { + "day": 6, + "flight": "AA218", + "utc": "15:58:00" + } + ], + "sourceairport": "ORD", + "stops": 0, + "type": "route" + }, + { + "airline": "WN", + "airlineid": "airline_4547", + "destinationairport": "LGA", + "distance": 1224.8661560257294, + "equipment": "73W ", + "id": 63458, + "schedule": + [ + { + "day": 0, + "flight": "WN657", + "utc": "13:34:00" + }, + { + "day": 0, + "flight": "WN714", + "utc": "13:20:00" + }, + { + "day": 1, + "flight": "WN204", + "utc": "04:20:00" + }, + { + "day": 1, + "flight": "WN544", + "utc": "17:48:00" + }, + { + "day": 1, + "flight": "WN238", + "utc": "08:52:00" + }, + { + "day": 2, + "flight": "WN786", + "utc": "20:18:00" + }, + { + "day": 2, + "flight": "WN710", + "utc": "05:13:00" + }, + { + "day": 2, + "flight": "WN721", + "utc": "04:14:00" + }, + { + "day": 2, + "flight": "WN036", + "utc": "00:34:00" + }, + { + "day": 2, + "flight": "WN369", + "utc": "05:08:00" + }, + { + "day": 3, + "flight": "WN730", + "utc": "20:23:00" + }, + { + "day": 4, + "flight": "WN903", + "utc": "23:02:00" + }, + { + "day": 4, + "flight": "WN955", + "utc": "12:50:00" + }, + { + "day": 4, + "flight": "WN204", + "utc": "01:34:00" + }, + { + "day": 4, + "flight": "WN888", + "utc": "13:49:00" + }, + { + "day": 5, + "flight": "WN718", + "utc": "21:49:00" + }, + { + "day": 5, + "flight": "WN805", + "utc": "08:58:00" + }, + { + "day": 5, + "flight": "WN851", + "utc": "11:16:00" + }, + { + "day": 5, + "flight": "WN077", + "utc": "09:31:00" + }, + { + "day": 5, + "flight": "WN938", + "utc": "00:51:00" + }, + { + "day": 6, + "flight": "WN595", + "utc": "14:38:00" + }, + { + "day": 6, + "flight": "WN486", + "utc": "10:04:00" + }, + { + "day": 6, + "flight": "WN920", + "utc": "21:48:00" + } + ], + "sourceairport": "ATL", + "stops": 0, + "type": "route" + }, + { + "airline": "WN", + "airlineid": "airline_4547", + "destinationairport": "ATL", + "distance": 1224.8661560257294, + "equipment": "73W", + "id": 63995, + "schedule": + [ + { + "day": 0, + "flight": "WN556", + "utc": "01:19:00" + }, + { + "day": 0, + "flight": "WN286", + "utc": "16:45:00" + }, + { + "day": 0, + "flight": "WN385", + "utc": "20:43:00" + }, + { + "day": 1, + "flight": "WN130", + "utc": "22:44:00" + }, + { + "day": 1, + "flight": "WN057", + "utc": "23:02:00" + }, + { + "day": 1, + "flight": "WN427", + "utc": "23:39:00" + }, + { + "day": 2, + "flight": "WN866", + "utc": "04:58:00" + }, + { + "day": 3, + "flight": "WN368", + "utc": "22:20:00" + }, + { + "day": 3, + "flight": "WN160", + "utc": "18:12:00" + }, + { + "day": 4, + "flight": "WN555", + "utc": "17:55:00" + }, + { + "day": 4, + "flight": "WN447", + "utc": "13:39:00" + }, + { + "day": 4, + "flight": "WN973", + "utc": "01:17:00" + }, + { + "day": 5, + "flight": "WN097", + "utc": "19:27:00" + }, + { + "day": 5, + "flight": "WN957", + "utc": "13:42:00" + }, + { + "day": 5, + "flight": "WN228", + "utc": "23:05:00" + }, + { + "day": 5, + "flight": "WN906", + "utc": "07:48:00" + }, + { + "day": 6, + "flight": "WN217", + "utc": "10:24:00" + }, + { + "day": 6, + "flight": "WN219", + "utc": "19:30:00" + }, + { + "day": 6, + "flight": "WN544", + "utc": "09:59:00" + }, + { + "day": 6, + "flight": "WN095", + "utc": "17:09:00" + }, + { + "day": 6, + "flight": "WN802", + "utc": "04:44:00" + } + ], + "sourceairport": "LGA", + "stops": 0, + "type": "route" + }, + { + "airline": "AA", + "airlineid": "airline_24", + "destinationairport": "ORD", + "distance": 2761.7714878234074, + "equipment": "738", + "id": 6758, + "schedule": + [ + { + "day": 0, + "flight": "AA396", + "utc": "01:45:00" + }, + { + "day": 0, + "flight": "AA244", + "utc": "12:45:00" + }, + { + "day": 0, + "flight": "AA446", + "utc": "19:24:00" + }, + { + "day": 1, + "flight": "AA335", + "utc": "04:25:00" + }, + { + "day": 1, + "flight": "AA326", + "utc": "01:48:00" + }, + { + "day": 1, + "flight": "AA812", + "utc": "16:28:00" + }, + { + "day": 1, + "flight": "AA967", + "utc": "07:38:00" + }, + { + "day": 2, + "flight": "AA677", + "utc": "04:55:00" + }, + { + "day": 3, + "flight": "AA035", + "utc": "08:43:00" + }, + { + "day": 3, + "flight": "AA794", + "utc": "13:44:00" + }, + { + "day": 4, + "flight": "AA269", + "utc": "15:37:00" + }, + { + "day": 4, + "flight": "AA923", + "utc": "14:26:00" + }, + { + "day": 4, + "flight": "AA482", + "utc": "19:41:00" + }, + { + "day": 4, + "flight": "AA143", + "utc": "18:01:00" + }, + { + "day": 4, + "flight": "AA809", + "utc": "12:17:00" + }, + { + "day": 5, + "flight": "AA870", + "utc": "22:18:00" + }, + { + "day": 5, + "flight": "AA715", + "utc": "17:13:00" + }, + { + "day": 5, + "flight": "AA523", + "utc": "14:14:00" + }, + { + "day": 6, + "flight": "AA829", + "utc": "06:40:00" + } + ], + "sourceairport": "SEA", + "stops": 0, + "type": "route" + } + ] + }, + "hotels": + { + "query": "SELECT h.* FROM `travel-sample`.`inventory`.hotel h WHERE h.country='United States' AND h.city IN ['Atlanta', 'Detroit','Charlotte', 'Philedelphia', 'Chicago', 'Dallas-Fort Worth', 'Seattle', 'Los Angeles', 'New York', 'Miami' ] LIMIT 20", + "results": + [ + { + "address": "1738 N Whitley Ave", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "This is part of a budget model chain. It offers clean rooms in a convenient location.", + "directions": null, + "email": null, + "fax": "+1 323 464-4645", + "free_breakfast": true, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.102832, + "lon": -118.33313 + }, + "id": 11787, + "name": "Motel 6", + "pets_ok": true, + "phone": "+1 323 464-6006", + "price": "$70 for a double/twin", + "public_likes": + [ + "Dortha Hermiston", + "Harrison Harvey" + ], + "reviews": + [ + { + "author": "Ms. Leland Daniel", + "content": "I stayed at the Ohio House back in the '80's' and it was around $50/night so when I wanted to go 6/03 I was afraid of what the pricing would be. I paid $89/night for a busy Chicago weekend...the hotels DIRECTLY across the street and down the block were asking $125+/night. All I did was watch an hour of TV and crashed...took a shower the next morning and left. I can't see paying for a lap of luxury in the hotel when there are so many places and things to do up there! The rooms are clean and unlike Motel 6 you get your choice of shower or bath! I am going to stay at Ohio House anytime I go for more than a one day stay.", + "date": "2012-06-10 18:55:21 +0300", + "ratings": + { + "Overall": 5 + } + }, + { + "author": "Marie Hamill", + "content": "Family owned Motel takes you back in time to the 1950's or 60's. It's a very well kept place and Clean. You open the door and you can smell the use of bleach and pledge. It's not overwhelming but it's very clean. The paneling has been painted but the look is a back in time experience our bathroom was all yellow the retro look for today. yellow tile, tub, sink with stainless steel legs...Like I said it was CLEAN.. The carpets are indoor/outdoor and very clean. The bed was comfy. (Sorry No Pool) You'll see that it's the older crowd that stay there (people in there 50's and up). Your away from the highway so the noise is minimal. Being in Chicago it's not in a scarey part of town and is a safe place to stay. We were there for the Paul McCartney Concert at Wrigley so we saw alot of the younger crowd (50 years and up) The air cond. was inside and control on the wall but it was cool. Outside the temp was a heat index of 105 and the room was 72 and we could've turned it down if we wanted. I was totally amazed with the cleanliness and the up keep Retro but Very Nice. Wish I had pictures to show you.", + "date": "2013-12-01 06:28:29 +0300", + "ratings": + { + "Cleanliness": 5, + "Overall": 4, + "Service": 5, + "Sleep Quality": 4, + "Value": 4 + } + }, + { + "author": "Madalyn Wintheiser", + "content": "I contacted both AAA and the Manager of the Motel on this issue. I recommend that they do not advertise the Wireless Internet until they have their service together. I attended WorkShops during the day and planned to work back at my motel at night. The service frequently cut off Internet Access and I got messages that the Broadband Link was broken, which I did page capture of the \"Broken Link\" message. I tried to use the Internet early in the morning, with the same result. On my last night's stay, I contacted the front desk and the person on desk suggested that I pull my chair out of the room and sit under the router to get a stronger signal. I did this because I needed to work but of course this did not increase my access. They also suggested that I take my computer to the office. So, I hauled my laptop down to the little lobby (where they said I was only allowed to go) and with the same result--no Internet Access. I was told that there may be Internet Cafes up the street. However, I do not know Chicago well and I did not have a car because we carpooled. However, I was willing to go and find a Cafe but I tried to call out on the hotel phone first to see what Internet Cafes might be open and available and only got a busy signal. My only option was to stay up all night and try to put in work whenever I might per chance get a connection. My hope is that management will take their offer of Free Wireless Internet down until they actually offer a solid service. If you are a business person, who needs to have access to the Internet, please reconsider your motel/hotel options.", + "date": "2013-07-16 05:04:47 +0300", + "ratings": + { + "Location": 5, + "Overall": 1, + "Service": 1, + "Value": 1 + } + } + ], + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "hotel", + "url": "http://www.motel6.com/reservations/motel_detail.asp?MotelId=4044&state=CA&full=California&city=Hollywood", + "vacancy": true + }, + { + "address": "1624 Schrader Blvd", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "The #1 rated hostel in Los Angeles (by hostelworld guests) in 2007 and 2005, USA Hostels is in the heart of Hollywood off Hollywood Blvd on a quiet side street. This 150-bed hostel offers female and mixed 6- and 8-bed dorms and private rooms, free all-you-can-make pancakes, free coffee and tea all day and free wireless internet. The hostel runs many free and discounted activities and tours and a free shuttle three times per week to Venice and Santa Monica beaches.", + "directions": "5 blocks from metro off Hollywood Blvd", + "email": null, + "fax": "+1 323 417-5152", + "free_breakfast": true, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.100585, + "lon": -118.332191 + }, + "id": 11788, + "name": "USA Hostels", + "pets_ok": true, + "phone": "+1 323 462-3777", + "price": "Dorms from $30, private rooms from $90", + "public_likes": + [ + "Mrs. Sven Gusikowski", + "Evangeline Morar", + "Alejandrin Jaskolski", + "Mr. Rafael Weimann", + "Jailyn Huel", + "Jasper Orn", + "Loyce Fadel" + ], + "reviews": + [], + "state": "California", + "title": "Hollywood", + "tollfree": "+1 800 524-6783", + "type": "hotel", + "url": "http://www.usahostels.com/hollywood/", + "vacancy": false + }, + { + "address": "2005 N Highland Ave", + "alias": "Hilton Garden Inn", + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "This boutique-style hotel is relaxed, comfortable, and uniquely personal with 160 rooms fitted with amenities including FACE cosmetics and flat-panel TVs. Starting in March 2012 this hotel will be rebranded as the Hilton Garden Inn.", + "directions": null, + "email": null, + "fax": "+1 323 876-3272", + "free_breakfast": true, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.106661, + "lon": -118.337732 + }, + "id": 11789, + "name": "Hollywood Heights Hotel", + "pets_ok": false, + "phone": "+1 323 876-8600", + "price": "$99+", + "public_likes": + [], + "reviews": + [ + { + "author": "Milford Reichel", + "content": "I used to live in Chicago and always was intrigued by this hotel when I drove by it, mostly due to the big cheesy sign (which I love). I was back in town to do some work and, although I usually stay with friends, I decided to get a hotel for a couple of nights. Since I was using my grant money, I wanted to save on costs so that I can use the money for other things. I read the reviews on here and decided to give the Heart a try. I was not disappointed! I got a large spacious room for less than $90 a night! It is clean, has FREE wireless, and a free breakfast (although the food choices are limited, can't complain about the price!). As a former resident of the city, I can say the location, while not ideal for getting downtown, is very good, especially if you enjoy the neighbourhoods of Chicago, rather than the downtown core. Andersonville is within walking distance, which is a great neighbourhood. Also, the Clark bus (#22) is one block away and this bus will zoom you down to Wrigleyville/Lakeview/Lincoln Park quite easily, or even downtown. Although, if you're heading to Michigan Ave, I would catch the #84 to Bryn Mawr station and take the red line downtown. Essentially, this is a great deal and you will not be disappointed, especially if you like retro things!", + "date": "2014-10-17 18:33:14 +0300", + "ratings": + { + "Business service (e.g., internet access)": 5, + "Check in / front desk": 3, + "Cleanliness": 5, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Value": 5 + } + } + ], + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "hotel", + "url": "http://www.hollywoodheightshotel.com", + "vacancy": false + }, + { + "address": "7000 Hollywood Blvd", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "A boutique hotel right in the center of the action offering spacious, well-appointed rooms and suites with luxury bed and bath linens, bath products and state-of-the-art in-room technologies. It is a Hollywood legend, and was the location of the first Academy Awards ceremony.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": true, + "free_parking": true, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.101375, + "lon": -118.341691 + }, + "id": 11791, + "name": "The Roosevelt Hotel", + "pets_ok": true, + "phone": "+1 323 466-7000", + "price": "$300+", + "public_likes": + [], + "reviews": + [], + "state": "California", + "title": "Hollywood", + "tollfree": "+1 800 950-7667", + "type": "hotel", + "url": "http://www.hollywoodroosevelt.com", + "vacancy": false + }, + { + "address": "929 S Broadway", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "Opened early 2014 in the historic United Artists building that was built in 1927 for what was then a maverick film studio. Includes free Wi-Fi and air-con. Terrace Suites are 633 square foot and include a private kitchenette, private terrace; some have Ace x Rega RP1 turntables and acoustic Martin Guitars.", + "directions": null, + "email": "enquire.dtla@acehotel.com", + "fax": "+1 213 623 6163", + "free_breakfast": false, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.041494, + "lon": -118.256879 + }, + "id": 16630, + "name": "Ace Hotel DTLA", + "pets_ok": false, + "phone": "+1 213 623-3233", + "price": "From $199", + "public_likes": + [ + "Mrs. Mckenna Kassulke", + "Marion Rowe" + ], + "reviews": + [ + { + "author": "Alvah Hodkiewicz", + "content": "We spent 2 nights at the Doubletree before heading over to Rio Grande, PR. It has a central location with easy access from the airport and to Old San Juan. It is also only a 3 block walk to Ocean Park Beach. Perfectly clean -- and I am a clean freak. One of the staff here said it was renovated 2 years ago when purchased by Hilton. All of the staff is super friendly. The pool area is nice with comfy towels and lounge chairs along with a jacuzzi. The hotel is in a heavily populated area so things are pretty compact and there is not much of a view due to the tall buildings all around. They have done a good job of putting palm trees, etc around to make it feel less \"citi-fied.\" There are many restaurants within walking distance so no need to drive around. We enjoyed \"Silk\" --they serve sushi, and chinese cuisine. Parking is a bit much at $20/day valet or $15 self-park, but comes with the terrifory. Oh yeah, and the warm cookies are to die for!", + "date": "2014-01-31 08:14:57 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 4, + "Overall": 5, + "Rooms": 4, + "Service": 5, + "Sleep Quality": 4, + "Value": 5 + } + }, + { + "author": "Octavia Olson", + "content": "Stayed here 1 night pre cruise and everything was what we hoped it would be. Never met a more friendly and helpful hotel staff. Although our upgraded rom wasn't quite ready when we arrived we were given some water and the famous warm cookies and we sat outside for a few minutes in the delightful surroundings of the pool. The room was great, everything you need and very modern. The supermarket across the road was a bonus for buying toiletries etc., to take on the cruise. We walked around the corner to Bebo's Cafe which was full of friendly locals(always a good sign) and we enjoyed an excellant, reasonable meal and a half pitcher of Sangria. We then headed back to the hotel and had a drink under the stars around the pool along with other people who were using the pool or just sitting around. Only thing that could be improved is that the service at breakfast was a bit slow, but we were on holiday so that didn't realy matter. And an added bonus was that we got to meet the charming and very handsome Jose (operations manager) who told us all about his island of Peurto Rico which he obviously loves. Thanks to everyone at the Doubletree.", + "date": "2015-08-11 14:45:24 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 5 + } + }, + { + "author": "Rhea Rowe", + "content": "This is by far the nicest and least expensive hotel in the Cantado area. It is two very short blocks to the beach and the neighborhood is very safe. Our room was spotless and modern, with a design that made great use of the space. You get a microwave, fridge and safe along with great Neutrogena bath products. The place is so modern and beautiful with a fabulous pool. Be sure and ask for Miguel Isaac, an assistant manager. He is so helpful and knows the area well. We left some items there and he helped get them back, a great person and a great hotel!", + "date": "2013-03-10 02:35:13 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 4, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Value": 5 + } + }, + { + "author": "Vivianne Rodriguez DVM", + "content": "After scoring such a great deal on Travelocity, we couldn't have guessed that this hotel would have been as wonderful as it was. The newly renovated rooms are very clean and wonderfully appointed (w/ fridge and microwave), our breakfasts at Michaels and Under the Trees (both on the property) were on point and reasonably priced, and most importantly, the hospitality of the staff (especially Yodil, Peter, and Miguel) was comforting and delightful. It's approx a 3 block walk to the more quiet \"locals\" beach and 4-5 to the main strip of resorts and casinos. For us it was not worth the $300-450 more to stay on the typical resorty main strip (note that most \"resorts\" on the strip seemed older and much less contemporary than this hotel). Pool was small but clean and while families were there, it was certainly not overrun with kids like others. Ask for Yodil Caban, the General Manager, and he will make sure that your stay is outstanding. These folks don't just talk the customer service game, they live it. As they told us, the smaller size of the hotel allows them to be more personally attentive. It's a $0.75 bus ride to Old San Juan or a quick $12 cab ride at night (note that cabs are more expensive at night so it was probably cheaper during the day). If you're in Old San Juan and want a great dinner in a posh, elegant space, check out Marmalade. Expect at least $40-50 per person, but the chef clearly knows and loves great cuisine (note: this is not local Puerto Rican food, but delish nonetheless). Enjoy!", + "date": "2014-03-21 08:19:48 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 4, + "Overall": 5, + "Rooms": 4, + "Service": 5, + "Value": 5 + } + }, + { + "author": "Nellie Kemmer", + "content": "Clean, comfortable room, great bed and shower, friendly and responsive staff. There is a huge 24-hour supermarket across the street and the beach is a short 3 block walk.", + "date": "2014-06-21 20:29:12 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Service": 5, + "Value": 5 + } + }, + { + "author": "Davonte Botsford", + "content": "well ... where to start? all the bad reviews are true! On the positive side, most of the staff is friendly and will go the extra mile but two of the receptionists could do with more training on how to greet/deal with customers and the waiter who served us the first (and only) time we ate at the restaurant could gain on honesty. In term of location, I understand it is a great place for surfers as it is close to many spots but that's it. Although many other interresting places (suggested in their handbook) are from reachable distance if you have your own transportation and are planning to do some sightseeing. On the other side, as we tour the hotel on our first day there, we realised (from the view from the restaurant) that it appeared to be still in construction ... however other bits tended to prove that it was either old or badly maintained. A case in point is \"the whale\" a small slide for kids in the pool area. The paint on it is totally faded and in some area the steel even shows. Another review mentionned the plumbing system, and I thought come on, that was until I realised I could hear my neighboor shower, flush their loo ... and that the pressure of the water in the shower also changed if my neighbour was having a shower at the same time. The fact that I could hear all this was a good indication of the soundproofing as well ... paper thin walls and doors and noisy a/c made my nights there virtually sleepless. Oh nearly forgot ... what they say about the critters in and outside the pool is true too ... in the shower next to the pool we were greeted by a cosy spider (in full swing in its wide web) and a dead cocroach and inside the pool ... an unidentified amphibian. I stayed in Puerto Rico for 10 days and I went to three different hotels, this one was the worst ... and for that price, I will not recommend it to any one I know!!!", + "date": "2013-03-08 21:31:27 +0300", + "ratings": + { + "Cleanliness": 1, + "Location": 5, + "Overall": 2, + "Rooms": 1, + "Service": 2, + "Sleep Quality": 1, + "Value": 1 + } + }, + { + "author": "Tomasa Ryan", + "content": "The view is without a doubt breathtaking. And that's also without a doubt the only good thing about this place. From the disappointing and unwelcoming staff, the lack of maintenance, smelly rooms (and not good flowery smell), torn fridge in the room to the vase of old dirty water and dead flowers at the Front Desk this whole experience was simply HORRIBLE. The \"tongue\" of the whale at the pool is literally black, the stairs leading to the pool are covered with a green moss like $@!#% (slippery) and the pool itself is dirty and full of leaves. The lobby area is very dirty, the floors very dusty. No elevators, and the stairs handrails very rusty and sticky. None of the staff has a uniform, their wear anything they want, any color, any style, and the housekeepers are wearing nurse uniforms. Very unprofessional. And in fact the musty stench in the rooms was so horrible that we could not stay at all. We ended up having to move to another hotel. So please do not waste your time and money. Do not even consider this place. It looks NOTHING like the internet page!!!", + "date": "2012-01-02 21:14:23 +0300", + "ratings": + { + "Business service (e.g., internet access)": 1, + "Check in / front desk": 1, + "Cleanliness": 1, + "Location": 4, + "Overall": 1, + "Rooms": 1, + "Service": 1, + "Value": 1 + } + }, + { + "author": "Mr. Summer Bechtelar", + "content": "service and welcome was great. not from there im from michigan somebody recomendme there and i love it have fun good price. good understood. wao a maissing. i recomend this place 100%. and im a manager in hotel too. radisson hotel in grand rapids michigan. so i know whats a good service and stay is. love it.", + "date": "2013-08-30 07:46:58 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 5 + } + }, + { + "author": "Elaina Morissette", + "content": "Let me just say that this was our first trip to Puerto Rico and we LOVED it here. Loved it so much that we have had a REALLY hard time adjusting to home again and we can't wait to come back soon. As far as the Embassy Suites: Pros:1. The room is a suite. Anyone with small children knows this is important. We could put the kids down for bed or nap and still be in our living room talking and that was great. 2. They serve free breakfast every morning and than we only needed a small lunch or snack before dinner. 3. They have a managers reception at night with snacks, juices, pop, and drinks. We stopped here every night before we left the hotel for dinner. It was a great before dinner appetizer that helped hold all of us over before our later dinner. 4. The beach was beautiful during the week. The Embassy Suites section was nice and it was great to have chairs, towels and umbrellas at the beach. 5. The pool, although crowded, was nice and perfect for the kids to play in for hours while I could read a book. Cons: 1. It's not that clean people. We have traveled to a lot of Embassy Suites and this one is by far the dirtiest one we've stayed at. At first glance the rooms look clean, but then you start looking in the corners, at the carpet, under any furniture, closer at the fixtures in the bathroom and you can tell that these rooms haven't been deep cleaned in a LONG time. Gross 2. One the weekend it was crazy crowded. We had to wait 20 minutes in the breakfast line, and same for managers reception. It was hard to find seats and could barely find anywhere around the pool. That thing that was the worst was the cleanliness at the meals during the busy weekend. For example, the bacon tray had more bacon hanging outside on the counter than in the actual tray. I looked at my husband and said is that supposed to be decoration or something? It was gross and wouldn't pass food codes for sure. I understand that they are busy on the weekends but then that shouldn't surprise them, and they should bring in more staff to handle the bigger crowds. 3. The front desk staff were not great. They were friendly, but always talking to other co-workers and I felt bad interrupting them. I would go down there and ask for towels to be delivered to the room, it never happened. 4. We pre-paid for our trip on the embassy website and it said it covers all taxes and fees. They racked on more than $300 worth of taxes before we checked out. I went to the counter and showed them my receipts and they weren't willing to budge at all. So frustrating when you prepay for things. If they have a \"special tax\" then they should add that in when you prepay. Overall we again LOVED PR, an amazing island with great culture and even better Puerto Rican food. I highly recommend going to see El Morro fort, the El Yunque rain forest, going to Luquillo beach and check out all the food stands, and our favorite going to Old San Juan and spend the day there. As for the Embassy Suites, I wouldn't be that excited to stay there again. It worked for us mainly because we were traveling with small children but I would not be that excited to stay there again. Honestly I would stay somewhere closer to Old San Juan...", + "date": "2014-01-27 07:53:20 +0300", + "ratings": + { + "Cleanliness": 2, + "Location": 4, + "Overall": 3, + "Rooms": 3, + "Service": 3, + "Sleep Quality": 3, + "Value": 3 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": null, + "vacancy": true + }, + { + "address": "550 S Flower St", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "Trendy hotel with designer rooms and a bar and swimming pool on the roof. There is also a West Hollywood location.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.050233, + "lon": -118.256982 + }, + "id": 16631, + "name": "The Standard", + "pets_ok": false, + "phone": "+1 213 892-8080", + "price": "$99+", + "public_likes": + [ + "Kenneth Mosciski", + "Drew Heathcote", + "Norwood Lakin", + "Cynthia Boyle", + "Devonte Treutel", + "Salma Langworth II", + "Miss Ethyl Moore", + "Stephany Bergnaum" + ], + "reviews": + [ + { + "author": "Adonis Friesen MD", + "content": "We stayed here as a layover between flight and cruise. Hotel is in a good location (close to airport and cruise terminal) but there is no shuttle bus to either, meaning $20+ taxi rides are compulsory with many drivers adding extra charges for large parties and baggage. Our room was described as a 2 bedroom suite, but was actually a living area with sofa bed and a bedroom with two small double beds. Having a party of 5, a 2 bedroom suite would have been more than suitable, but the doubles weren't big enough for two people to sleep comfortably meaning 2 of us were made to sleep on the VERY uncomfortable sofa bed (so uncomfortable in fact that we ended up taking the cushions off the sofa's and sleeping on them on the floor for 3 nights, something you don't want after 23 hours of travelling.) There were not enough towels, cups, tea/coffee or pillows and it took three calls and a visit to reception for them to provide us with these. When we asked about the beds we were told that we would not receive help due to booking through HOTELS.COM. The bathroom has no natural light so is very dark due to the lack of artificial lighting also. Our room was 623 opposite the elevators, something that made it noisy at night. Can't complain about breakfast, acceptable especially seen as it was free! There was plenty of loungers at the pool but the pool itself was dirty and cold. Laundry facilities were good. Don't recommend the Outback Steakhouse. Our server was rude and incompetent and lost our debit card! She was never around to assist us and the food was very disappointing. Have stayed at many budget hotels that have been more pleasant that this, a hotel that claims to be part of the Hilton group. Being a Hilton Honours member I am very disappointed with this stay and won't be returning due to the reasons above.", + "date": "2014-06-06 22:24:06 +0300", + "ratings": + { + "Cleanliness": 3, + "Location": 4, + "Overall": 2, + "Rooms": 1, + "Service": 2, + "Sleep Quality": 1, + "Value": 2 + } + }, + { + "author": "Chelsey Lindgren", + "content": "I used my chase visa card only once in puerto rico, to pay for my embassy suites hotel bill in December 2011. 3 months later (last week), an individual in PR began spending $1000/day on the card until visa canceled the card. I do not have proof that it was an employee at that hotel. But I find this very suspicious. If no one else complains of the same thing, call it a fluke. On the other hand, the hotel was actually very nice. The lines at breakfast were long, but the food was good. The pool area was great, the beach area was also a nice touch. Overall I would stay there again, but I would watch my credit card bills closely.", + "date": "2014-07-29 06:37:45 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 4, + "Overall": 2, + "Rooms": 4, + "Service": 4, + "Sleep Quality": 4, + "Value": 4 + } + }, + { + "author": "Stephan Rogahn V", + "content": "The staff very helpful close to the beach and shopping. In and out quickly from the airport. I can't say enough good things about this place. The rates were good and the happy hour up to par.", + "date": "2014-12-11 20:43:28 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Service": 4, + "Sleep Quality": 4, + "Value": 5 + } + }, + { + "author": "Daphnee Shanahan", + "content": "The hotel was in pretty good shape, the lobby was well kept and had some beautiful fish tanks to admire when you walk in. Didn't spend too much time in the casino but the staff was helpful there and throughout the hotel. The manager's reception in the evening was fantastic! The drinks were *strong* and they had popcorn and chips and salsa -- keeping everyone happy. The breakfast in the morning had omelets to order or buffet and we had our fill. The only area that needed some improvement was the room, that had seen some where and tear. Overall, it was a great value!", + "date": "2014-09-06 23:17:33 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 4, + "Overall": 4, + "Rooms": 3, + "Service": 4, + "Sleep Quality": 4, + "Value": 5 + } + }, + { + "author": "Nils Lowe", + "content": "The Embassy Suites is located convenient to the San Juan airport -- it takes three minutes to get there from the airport and ten minutes to get back, thanks to the configuration of the highway system. As another reviewer mentioned, there is no hotel shuttle (there don't actually seem to be such things for any of the properties), and a taxi will cost around $12-15. Parking at the hotel is chaotic--there was never a problem getting a space, but it seemed like we were constantly behind drivers who couldn't work the gate and then insisted on going the wrong way upon entering the garage (it wasn't just us -- we noticed the garage staff seemed a bit bewildered by this behavior, too). Parking, at $16 a day for self park, is less than I've paid at other hotels in the area. I stayed there with my parents between Christmas and New Year's, at the height of the busy season. The hotel was a bit overrun with college swimming teams in town for winter break training, and the fact that the hotel was bursting at the seams showed quite a bit: the line at breakfast and at the manager's reception / happy hour in the afternoon were both incredibly long (forty five minutes for the latter). The first day we were there, the staff seemed to be overwhelmed; by the time we left, systems seemed to be in place to deal with the madness. The hotel is not on the beach - the beach is about a ten minute walk away up a side street. There is a section of Isla Verde beach reserved for guests at the Embassy Suites (look for the flag), where a staff member will provide you with beach chair, towel, and umbrella. Restaurants in the hotel are a bit pricey--we ate at the bar the first night because we arrived late, and the prices were on par with a restaurant in Old San Juan, even if the service and quality was not. There's an Outback Steakhouse in the hotel, which is one of the most expensive places in Isla Verde to eat -- ask at the front desk (not at the concierge stand -- the concierge is a private company) for recommendations for local restaurants within walking distance. There's a Pueblo grocery store next door for basic provisions.", + "date": "2015-06-27 13:25:35 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 4, + "Overall": 3, + "Rooms": 3, + "Service": 3, + "Sleep Quality": 3, + "Value": 3 + } + }, + { + "author": "Otho Mann", + "content": "we loved this place, beautiful grounds, great pool, happy hour, gym, fantastic breakfast. a real surprise as we had been delayed in san juan for a day and were not sure what to expect..", + "date": "2014-04-01 00:44:28 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 5 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": "http://www.standardhotels.com/los-angeles/", + "vacancy": true + }, + { + "address": "120 S Los Angeles St", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "Featuring spacious rooms, modern decor, and a variety of amenities in Little Tokyo.", + "directions": null, + "email": null, + "fax": "+1 213 622-0980", + "free_breakfast": true, + "free_internet": false, + "free_parking": false, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.050739, + "lon": -118.24261 + }, + "id": 16632, + "name": "DoubleTree by Hilton Hotel Los Angeles Downtown", + "pets_ok": true, + "phone": "+1 213 629-1200", + "price": "$110+", + "public_likes": + [ + "Miles Hilpert", + "Jeremie Yost", + "Caleigh Fadel", + "Dahlia Toy", + "Corbin Blick", + "Corene Schaefer", + "Gabrielle Hermann I" + ], + "reviews": + [ + { + "author": "Moshe Beer", + "content": "Yearly trip to PR ends with three days two nights at the Embassy. It is just great hotel, great value for the money and truly representative of the Embassy chain. Had a suite on the 8th floor overlooking the pool area...wow. Check in flawless, complementary two hour cocktail party and breakfast in the morning, flawless as expected. Room was clean, comfortable, the best housekeeping person I have ever met or dealt with bar none. No request was ever too much. The staff at this hotel, everyone I walked by always said hello, good morning, have a good day...from the valet parking attendant to the guy vacuuming the lobby...now thats customer service. All calls to the front desk for special request were handled promptly and courteously....I mean, the only negative on this hotel is that is about two block from the beach...but guess what...they have their own Embassy area at the beach and the excellent customer service extends there too....I honestly cannot say anything wrong about this hotel, this is my third years coming here and they outdo themselves everytime.", + "date": "2015-11-19 15:24:36 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 5 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": "http://www.doubletreeladowntown.com/", + "vacancy": true + }, + { + "address": "333 S Figueroa St", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "Has a small older theater in its basement where you can still see first-run movies for under $8.", + "directions": null, + "email": null, + "fax": "+1 213 621-1505", + "free_breakfast": true, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.054884, + "lon": -118.255501 + }, + "id": 16633, + "name": "The LA Hotel Downtown", + "pets_ok": false, + "phone": "+1 213 617-1133", + "price": null, + "public_likes": + [ + "Olin Pouros", + "Odie Wehner II", + "Gretchen Becker", + "Shyanne Adams", + "Rogelio Rutherford", + "Amely Jacobs", + "Miss Trevor Bauch", + "Bernadette Kiehn", + "Clotilde Walsh" + ], + "reviews": + [ + { + "author": "Tyrel Murphy", + "content": "We returned from a cruise and stayed one night. We were allowed to check into one of our rooms early, 9 a.m. which made using the facilities much easier. The pool is large and the kids really enjoyed it, there were plenty of lounge chairs around deck of the pool. The only negative was the traffic noise on the highway overpass located near the pool but after a few minutes the kids noise overwhelmed the traffic noise anyway. The hotel was clean and attractive with a small casino, about twice the square footage of our cruise ship casino with more table games though. The full breakfast offered to guests was very enjoyable and if you are a Hilton Honors member you can even skip the long lines. The manager's reception in the afternoon was nice with tap beer, soda and mixed drinks but could have offered more than just popcorn and chips, maybe I'm just picky. The beach is a short two block walk away with an Embassy Suites concessionaire providing beach chairs and umbrellas for guests at the hotel, a very nice touch. Overall a nice but short stay, would have liked to stay longer. We would definitely stay here again.", + "date": "2012-09-04 03:20:24 +0300", + "ratings": + { + "Cleanliness": 5, + "Overall": 5, + "Service": 5, + "Sleep Quality": 4, + "Value": 5 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": "http://thelahotel.com", + "vacancy": true + }, + { + "address": "251 S Olive St", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "The OJ jury was sequestered here. Convenient to MOCA, Disney Concert Hall, upscale Noe Restaurant and Bar is onsite.", + "directions": "at California Plaza in Bunker Hill", + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": true, + "free_parking": true, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.052677, + "lon": -118.250337 + }, + "id": 16634, + "name": "Omni", + "pets_ok": true, + "phone": "+1 213 617-3300", + "price": null, + "public_likes": + [ + "Santino Schneider", + "Amber Fritsch", + "Sadie Crona", + "Rafael Mertz", + "Tyree Jast", + "Jordane Rath Sr.", + "Myrna McDermott", + "Maryjane Rohan", + "Megane Boehm" + ], + "reviews": + [ + { + "author": "Mrs. Tyrique Kohler", + "content": "This hotel is a great place to stay with family or with friends. The hotel staff was excellent at all times. The hotel stayed clean the pool area was always clean and well kept. The best part of the hotel is the managers reception every night. I would highly recommend this hotel to anyone with a family or just traveling with a group of friend. The one person who stuck out to me wa Iris at the fron desk. She really made my day with how friendly she was! I am going back next year with my family!", + "date": "2013-06-13 05:40:30 +0300", + "ratings": + { + "Cleanliness": 5, + "Overall": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 4 + } + }, + { + "author": "Kaleigh Kerluke IV", + "content": "Every room is a suite, equipped with 2 plasmas, microwave, refrigerator, coffe maker, desk, free wifi. Extremely clean. They offer complimentary HOT breakfast with multiple choices like omelets, fried eggs, pancakes, fruits, oatmeal, cereals, juices, coffe, etc. Also, every evening, from 5:30 to 7:30 PM they have a Happy Hour with FREE beverages (including alcoholic) for all the guests. The pool is nice and the casino too. The hotel is located near to multiple soteres and restaurants in the middle of a touristic zone. The beach is across the street and the hotel has personnel at the beach, with towels and chairs, at no extra cost.", + "date": "2013-03-29 08:59:38 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 5 + } + }, + { + "author": "Immanuel Bartell", + "content": "Stayed 2/26/11-3/3/11 for my 53rd birthday. Since my sis and I have never been to PR we had no \"real\" idea about location but this was the Best, especially if you are not renting a car. and wanting to do outdoor adventures and seeing Old San Juan. The bus ride was longer than a cab but only 75 cents is right up my alley! I booked through Expedia (air & hotel from Chicago) although we had a late check in of 11:30 pm the front desk staff was so accomodating in getting us checked in quickly and letting us know the lay of the land. We got our bags into the room on the 2nd floor and headed to the bar. It was a bit disappoinng for my sis that the bar keep did not have mint for mojito's or her type of vodka. He was courteous and efficient in setting us up with our 2nd choice cocktails. We did check out the casino but did not gamble. it was loud and yes you can hear it from your living room but as others have said, the bedroom was quiet. I must express that the beds were fantastic! having a bad back and sleeping on a theraputic mattress at home, I was worried without need. The bed and pillows were so comfortable that my sleep was never affected! The room was clean with plenty of towels. I do wish they would put regular blankets in the closet for covering up on the couch though. The complimentary breakfast was superb! The staff friendly and quick. I can not believe how many people do not tip. I know its included but come on, these folks work hard, have a great attitude and take fantastic care of their guests! The happy hour was appreciated after coming back from day long excursions! Again, Tip your bartender! The only negative I would have is the hot tub and parents who believe that is their childrens bath tub or swimming pool. I did read the rules of the tub and it says \"children under 18 must be accompanied by a parent\" Personally I do not want kids in my hot tub but I am not the proprieter. I would highly recommend this location to those who want to go to Fajardo or the rain forest as well as the other direction into OSJ. We will come back to PR and are looking at the Embassey Suites in Dorado. Thanks for a terrific birthday. One more point~ I think its excellent that management responds to negative reviews here. it shows that they do care!", + "date": "2013-06-11 04:46:03 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 5 + } + }, + { + "author": "Joaquin Kutch", + "content": "I stayed here over New Years with my extended family over New Years. It was very close to the airport, so very convenient, but it was quite quiet. No plane noise. The hotel boasts a multistory atrium and is well appointed. There is a daily happy hour with free beer, wine, and chips which is very nice for a quick snack and a full included breakfast everyday with omelettes, eggs, bacon and much more! The rooms were spacious and as the title says are all suites, with both a bedroom, living room with foldout couch, microwave and refridge. The beds were very comfortable and the room updated and clean with good housekeeping. The pool area was quite nice and although very near an overpass was nicely screened by trees and vegitation. The pool was quite cold, and unfortunately the hot tub was broken for 3 of 5 days. I found the staff behind the desk not quite as welcoming as I'm used to. They were unhelpful in assisting with a printer which would not print in the business center, however the business center was free! $19 per day for parking is a bit high as well. The hotel is a block or two from a very nice beach and has included free towels and beach chairs for hotel guests! There are numerous nice restaurants and inexpensive options as well within walking distance. The hotel and airport are about 20 mins from the fort and the old town area, so a car, bus, or taxi will be necessary to visit these areas. Just for new years, we decided not to do the $89 per person buffet and partied in our room, but the open balconies and atrium allowed us to enjoy the confetti, live music and festivities from a commanding view of the 2nd floor open balcony and atrium. And much new years revelrie was occurring on all the hotel balconies, so was the best of both worlds! The rooms were remarkably well insulated, I think helped by the extra space of the living area, because even though we turned in before the festivies were over downstairs, I did not hear any extra noise! There is a casino inside as well which I did not visit.", + "date": "2012-03-08 12:30:39 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 4, + "Overall": 4, + "Rooms": 5, + "Service": 3, + "Sleep Quality": 5, + "Value": 4 + } + }, + { + "author": "Brant Conroy", + "content": "Close the beaches, restaurants, clubs. Nice Hotel, but Puerto Rico was the real selling point. Had to walk about 2 blocks to get to the beach, but its is incredible. The hotel staff are just amazing and the hotel quality is good, clean and in good shape. No balcony and no exceptional view, but you don't need that. The pool was nice, happy hour was good, the breakfast was incredible. A little pricey, though. Great experience, never seen so many good looking women! A that includes the hotel staff.", + "date": "2015-07-31 12:53:28 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Service": 4, + "Sleep Quality": 3, + "Value": 3 + } + }, + { + "author": "Jeremy Baumbach", + "content": "The hotel is conveniently located minutes from the airport. We arrived early in the morning. Melvin at the front desk was wonderful. He directed us to the buffet breakfast with cooks also preparing made to order requests. The breakfast was hot with lots of variety to choose from. The staff was prompt in meeting guests needs while keeping supplies stocked. We then did some gambling in the on site casino and before we knew it our room was ready. When we arrived at the room it was fresh and clean with a big bathroom, coffee maker and mini-refrigerator. There was a nice grocery store and pharmacy within short walking distance. We enjoyed the bartenders at the complimentary happy hour. They were very friendly. The hotel is a brief walk to the beach, however, we enjoyed the pool so much that we never found the need to go to the beech. When I had a problem with my cell phone, Christina at the front desk was very helpful in locating a sprint store nearby. Nice hotel and staff. Would definitely stay here again.", + "date": "2015-12-12 15:03:12 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 5 + } + }, + { + "author": "Willy Price", + "content": "Stay was from May 15th thru the 19th. Arrived in SJU around 1:30; but didnt go directly to the hotel because I knew check in wasnt until 4pm. Rented a car from SJU and we went directly to Plaza Las Americas to check that out. Got to the Embassy Suites around 5pm. Check in was a breeze. I asked for a room overlooking the SJU runway and they quickly accomodated. The suite was very nice. There were 2 beds for the kids and I slept on the pull out couch. Although, when I woke up the next morning I began to itch from what I can only think were some type of bugs. The kids didnt complain of itching so I was okay and decided I could live with it. Anyway, back to check-in on the first day, we went directly to the pool and relaxed. The next morning, Sunday May 16th, my birthday, I woke up to a note under my door that there was a 5.8 tremblor in the zone of Mayaguex which is about 120 miles west of San Juan. It occured about 1am however I didnt feel it. It woke my son up. I didnt notice any items had shifted. But, what a bang to start a birthday. Anyway, back to the hotel. The breakfast buffet is everything other reviewers have said. They cook omlets to order. Plus, there is bacon, sausage, potatoes, corn beef hash, pancakes, french toast, fruits, cereal, juice, coffee, hot chocolate, et al. The atrium area on the weekend had vendors that were showing their wares(sp). There was even a man who was making pottery with a clay machine. It reminded me of Demi Moore in \"Ghost\". The nice part of that was that he would let the kids try their hand at pottery making. The only downside was Carolina experienced some torrential downpours that flooded the streets and consequently affected the roof and subsequently the pipes which caused some problems. Specifically, there was no water coming out of the shower. I called the front desk and they told me it would be rectified in \"10 minutes\" Maintenance came to my room twice. All told it took approximately 90 minutes. The maintenance man was so apologetic that there was no way I could hold onto my disgust. We did experience the opportunity to eat at the Outback Steakhouse on the property and the staff was great. To be honest, the food alright. The kids loved it though. Since it was my birthday, they sang happy birthday to me in both english and spanish. Another night I got pizza for the kids from the Atrium Cafe. They didnt have any complaints. Because there are so many fast food places nearby, like Church's Chicken, Ponderosa, Burger King, Chilis, et al, everyone should be able to dine on what they want and be happy. I came to Puerto Rico for my 50th birthday and a vacation. Lastly, to have a relaxing good time. And I did. Lastly, the Manager Reception was wonderful.", + "date": "2013-03-20 02:54:32 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 4, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 3, + "Value": 5 + } + }, + { + "author": "Danial Schiller", + "content": "Things that I didn't see in other T.A. reviews: (1) There are two pools, an adult pool, and a very shallow kids pool, really more of a toddler pool. (2) The pools aren't heated, and can be quite cold on relatively chilly days. (3) We traveled with two little ones and a nanny, and the suite was adequate size for us. (4) Besides the Outback Steakhouse there is a Noble Romans with (yucky but edible) pizza, hot dogs, and coffee downstairs. (5) Room overlooking the atrium can be noisy, but not all rooms overlook the atrium, so ask for one that doesn't. You might still, however, have a lot of street noise. (6) The free booze at night attracts a rather white-trashy crowd. I'd much prefer if they had less alchohol and some real appetizers, instead of stale chips. (7) Among the restaurants in easy walking distance are Pizza Hut, Taco Bell, KFC, Church's, Subway, Burger King, a Chinese place, a Puerto Rican place (great food, crappy service), Wendy's, and more. The front desk won't tell you about any of these places unless you ask. (8) Don't rent a car to go to Old San Juan, parking is a huge hassle. But if you are going anywhere else, it's likely going to be cheaper to get a car on Hotwire than to take taxis. (9) The area by the kids pool is right by the highway, plus there was a lot of construction noise, not so pleasant. (10) Get to the buffet before 9 if you don't want a huge line. All in all, we had a nice but not fantastic stay. I give the hotel slightly more than 3.5 stars based on value, but I'm sure there are much nicer hotels in San Juan, on the beach, with heated pools, less noisy, and so forth. But traveling with 3 adults, this hotel saved us from having to get two rooms, and we all got a nice free breakfast, so the tradeoff was worth it.", + "date": "2014-06-03 17:33:38 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 4, + "Overall": 4, + "Rooms": 3, + "Service": 3, + "Sleep Quality": 4, + "Value": 5 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": "http://www.omnihotels.com/FindAHotel/LosAngelesCaliforniaPlaza.aspx", + "vacancy": true + }, + { + "address": "939 S Figueroa St", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "For those looking for something unusual, Figueroa Hotel provides Moroccan styled luxury. Mystic and beautiful, this is where Cirque Du Soleil hosted their premiere party of Varekai.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": false, + "free_parking": true, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.045481, + "lon": -118.26387 + }, + "id": 16635, + "name": "Figueroa Hotel", + "pets_ok": true, + "phone": "+1 213 627-8971", + "price": null, + "public_likes": + [ + "Rodolfo Larson", + "Nico Morar IV", + "Gia Jacobson", + "Hilton Christiansen", + "Earlene Larson", + "Felipa Jaskolski", + "Adela Dietrich", + "Jordane Johnson MD", + "Mr. Shawn Gibson" + ], + "reviews": + [ + { + "author": "Rey Cronin", + "content": "The hotel looks like the picture which is a good thing, it's beautiful. Everything from the pool to suites to the dining area was well kept. The location of this hotel is really good. Everything is within walking distance. We enjoyed the conviences of Pueblo's supermarket which is a short walking distance away where we purchased some items to keep in the refrigerator. We ate at a very good restaurant called Don Jose's which is on the main street once you leave the hotel, delicious food. Each night, this hotel offered complimentary drinks, which is a very nice thing. The beach was literally 2 blocks in front of the hotel. If you like beach and R&R, you'll definately want to check out Isla Verde beach. The water was warm and the sand soft. My friend and I rented a beach chair for $4 for the day and stayed there for hours. Our only disappointment was the nightlife for singles...or let me add 35 and older singles:) There didn't seem to be much to offer in this area. Other than that I would recommend this hotel and location to visit San Juan.", + "date": "2012-12-12 03:24:33 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 5, + "Overall": 4, + "Rooms": 4, + "Service": 5, + "Value": 5 + } + }, + { + "author": "Orion Feeney", + "content": "First all before anything....the service I got from the employees was absolutely superb. The front desk clerk was extremely helpful. I had a ton of questions and she did not get annoyed or short with me. She was very helpful. The room was huge and very clean. The amenities in the bathroom was great. I wish they had flat screen tv's in the room, but a tube tv was fine. When I checked out I left my GPS in the room. So right after I checked out i went back upstairs to retrieve my GPS and the housekeeper had it ready for me wrapped up in a plastic bag. I was so impressed that she actually held it. I've stayed in rooms where housekeeping or someone or something has stolen my products. Much Much Kudos to Embassy Suites San Juan! (Also....it's a straight, literally straight shot to the beach. And a FREE very good and big breakfast. And FREE drinks and snacks at happy hour.) Perfecto!", + "date": "2015-12-28 18:24:05 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 4, + "Rooms": 4, + "Service": 5, + "Value": 4 + } + }, + { + "author": "Frida Lebsack", + "content": "Just returned from a trip to San Juan for a cruise. We stayed twice during this time at the Embassy Suites, San Juan. I will begin by saying the staff, front desk personnel,bartenders, waiters, buffet area servers, etc, are all very hard working dedicated people. They are courteous and go out of their way to make your stay to your liking. That being said, I must say that the room quality is becoming an issue. We stayed there last year and at that time I thought the rooms needed some upgrading. They are not lighted properly, very dim in front room area. The sofa sleepers on both of these trips were not worthy to be slept on. We had another couple with us and they said they were just horrible as far as comfort. Also they need to get the latest flat screen TV's. The ones they have are ancient. The rooms just look worn out and need some new decor and brightness.", + "date": "2012-11-01 14:44:02 +0300", + "ratings": + { + "Check in / front desk": 5, + "Cleanliness": 4, + "Location": 4, + "Overall": 3, + "Rooms": 2, + "Service": 5, + "Value": 3 + } + }, + { + "author": "Mr. Irving McDermott", + "content": "We came to Puerto Rico for a friend's wedding. The bride had chosen this hotel for all the guests from the states to stay in. Very smart bride! This hotel was convenient to the airport and lovely Isla Verde beach. Lots of restaurants & a grocery store nearby. The hotel itself has a nice parking garage for your rental car. The desk staff, bar tenders & gift shop girls speak english. The free breakfast is outstanding! The pool is way above average! We never encountered any problems of any kind and would recommend this hotel to any body, especially first-time Puerto Rico visitors.", + "date": "2012-11-11 16:52:34 +0300", + "ratings": + { + "Business service (e.g., internet access)": 3, + "Check in / front desk": 5, + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Value": 5 + } + }, + { + "author": "Mrs. Alanis Will", + "content": "My wife and I stayed here for part of our honeymoon. The rooms are large, clean and perfect for relaxing. The cooked to order breakfast was always good and the free 2 hour happy hour was very nice as well. The hotel is clean and in a great location. Less then a 3 min walk to a perfect beach. The hotel pool is fantastic and the staff go out of their way to help you out. You are also close to Old town San Juan, about 10 min by taxi. Also close by there is a Ritz, El San Juan hotel and and intercontinental. All three are great if you want to get dressed up and go for a great meal, diner and dancing, all with great casinos. This hotel is great value for money. I would not stay anywhere else.", + "date": "2012-02-01 22:54:54 +0300", + "ratings": + { + "Business service (e.g., internet access)": 5, + "Check in / front desk": 5, + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Value": 5 + } + }, + { + "author": "Price Schaden", + "content": "We just came home from a long weekend to Puerto Rico. We picked the Embassy Suites because of the short distance from the airport. The hotel was clean and neat and the extra room in the suite was great! The beach is a 2 minute walk from the hotel which we didn't mind at all. We stayed all day - the water was beautiful. The hotel is also walking distance from nice restaurants so we didn't feel like we needed a car during our short stay.", + "date": "2015-02-26 08:10:40 +0300", + "ratings": + { + "Business service (e.g., internet access)": 3, + "Check in / front desk": 4, + "Cleanliness": 5, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Service": 3, + "Value": 4 + } + }, + { + "author": "August Doyle", + "content": "Hubby, myself and another couple stayed here from Oct 3 - Oct 8. The rooms were very nice - suites- with a separate living room area. Very confortable beds, roomy bathroom and accomodating service. The pool area was enjoyable, free internet access, nice bar / lounge area and the lobby was tropical with an open atrium & lots of foliage. Free full breakfast available, including ready to order omelettes / eggs, bacon, sausage, potatoes, bagels, toast, cereals, oatmeal / farina, coffee, juices, etc, etc. More than enough to start the day. Free evening reception which amounted to free drinks for guests from 5:30pm - 7pm. Two blocks walking distance from the beach and other restaurants & hotels. We enjoy walking and after a full dinner, welcomed the opportunity to stroll along Isla Verde Blvd to visit other hotels. We went to two local restaurants: Barrachina & Mi Casita. Both were within walking distance and very flavorful - check them out! Also, we asked them for a recommendation for a local bakery. They gave us directions to Espana Bakery on Calle Marginal Villamar. This bakery is awesome! Only 5 minute drive from the hotel and they had everything from pastries to sandwiches to quiches to hot meals and even liquor. Delicious! We went there the day we were leaving and picked up sandwiches to eat on the airplane. The other passengers were jealous! Down sides - they do not have an airport shuttle. The front desk said that they are not allowed to have it??? Small casino - okay for a novice, but not sure if real players would be satisfied. Daily parking fee for guests ($10 / day). You would think they wouldn't charge guests for parking. An added hotel fee of 9% (or something like that). They attribute this to providing free internet access, free local phone service and other services. Just be aware of this. Not sure if other hotels have started charging the same \"fee\", but the Embassy does. Regardless, we got a great package deal through Orbitz. Over all, service was great. We enjoyed it, and would stay there again.", + "date": "2015-10-04 02:29:51 +0300", + "ratings": + { + "Business service (e.g., internet access)": 5, + "Check in / front desk": 5, + "Cleanliness": 4, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Service": 4, + "Value": 4 + } + }, + { + "author": "Mr. Cyril Herzog", + "content": "loved the embasyy nick staten island my wife and i stayed ath the embassy after our cruise may 5th-may 7th the hotel is great. the staff is very accomodatng the grounds are beautiful. it is only 2 short blocks to the beach. they have a wonderful free breakfast every morning. at night from 5.30-7.30 pm a managers cocktail party in the lobby. we really enjoyed our stay and will be returning.", + "date": "2012-11-30 05:45:44 +0300", + "ratings": + { + "Business service (e.g., internet access)": 5, + "Check in / front desk": 5, + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Value": 5 + } + }, + { + "author": "Alexandrea Lynch", + "content": "The staff really needs to be reconsidered. While I was staying at the Embassy Suites, I had all sort of trouble and they either kept me ignored or came up with a temporary solution until I left. That is not the way to treat a guest.", + "date": "2014-09-02 19:02:35 +0300", + "ratings": + { + "Business service (e.g., internet access)": 5, + "Check in / front desk": 3, + "Cleanliness": 2, + "Location": 3, + "Overall": 2, + "Rooms": 3, + "Service": 1, + "Value": 2 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": "+1 800 421-9092", + "type": "hotel", + "url": "http://www.figueroahotel.com/", + "vacancy": true + }, + { + "address": "535 S Grand Ave", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "AAA Four diamond, renovated 1920s luxury hotel. Weekend packages are offered.", + "directions": "located adjacent to the Central Library and the Millennium Biltmore in the Financial District", + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": true, + "free_parking": true, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.049557, + "lon": -118.254809 + }, + "id": 16636, + "name": "Hilton Checkers", + "pets_ok": true, + "phone": "+1 213 624-0000", + "price": null, + "public_likes": + [ + "Miss Wellington Larkin", + "Kraig Jewess MD", + "Bertrand Crooks" + ], + "reviews": + [], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": "+1 800 445-8667", + "type": "hotel", + "url": "http://www.hiltoncheckers.com/", + "vacancy": true + }, + { + "address": "506 S Grand Ave", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "The grand-daddy of all downtown hotels, with its gorgeous lobby and fancy restaurants.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": false, + "free_parking": true, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.049757, + "lon": -118.254042 + }, + "id": 16637, + "name": "Millennium Biltmore", + "pets_ok": true, + "phone": "+1 213 624-1011", + "price": null, + "public_likes": + [ + "Ransom Grant", + "Mr. Declan Streich" + ], + "reviews": + [ + { + "author": "Cristal Moore", + "content": "We stayed at this hotel before a cruise and made a reservation for another hotel for our post cruise because Embassy did not have availability. Well when we checked into the \"other\" hotel to simply relax and enjoy our last days of vacation, we were checked into dirty rooms, yes more than one and not just dirty, beds made but the sheets had not been changed from the previous guest we were moved to another room and found the same problem. We called Embassy to check if there were available rooms and thank the good lord above they had a room for us. We checked out of the dirty run down hotel and happily waited for a clean room at the Embassy Suites. We had dinner and had the pleasure of being served by the loviest woman that had such gracious and hospitible skills and made us feel welcomed. Later in our room a knock at the door, room service delivering a beautiful plate of sliced fruit with a large bow and an apology for the inconvenience that we had experienced....At Another Hotel!....This was perfect and such a classy touch on what already measured up to be one great hotel. The front desk staff was top notch and I look forward to our next trip to Puerto Rico and another stay at the Embassy Suites Hotel. Thank you for everything...See you again soon!", + "date": "2015-08-05 03:38:34 +0300", + "ratings": + { + "Business service (e.g., internet access)": 5, + "Check in / front desk": 5, + "Cleanliness": 5, + "Location": 4, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Value": 5 + } + }, + { + "author": "Gilda Nolan", + "content": "We stayed here on a Sat night last June. It is about 1/2 mile and 10 minute taxi ride from the San Juan airport. First off I will mention that the hotel is nice and very clean. It has an attractive lobby and pool (you are near the airport but the noise of landing jets really isn't an issue). However this hotel is not without problems. The hotel does offer free internet access (as mentioned on their website) and there are three free interent kiosks near the check in desk. However, at least one of them was always out of order and the other two were constantly being used. My biggest complaint with the hotel is their handling of crowds. As mentioned we stayed there on a Saturday evening and the hotel was extremely crowded. The receptionist informed us that Saturdays are always busy because the \"pre-cruise\" crowd. (Apparently, a good number of customers stay in this hotel one night prior to their sailing date.) While the managers reception and cooked to order breakfast have always been some of the best things about the Embassy Suites we've stayed at - these are poorly handled at this one. First off the manager's reception. This hotel has a rule that there is a 2 drink maximum per bar visit. This is a problem if you have small children and your spouse has to stay with them while you wait in line for the drinks. The bar tender told me that I could come back for additional drinks, but the line was long and the wait was about 20 minutes. Also, the drinks are served in small palstic cups which don't tend to last very long. I think that, with additional bar tenders and a little more efficiency (larger cups??) the wait time could be drastically reduced. The same thing happened at breakfast. Very long lines (especailly if you wanted cooked to order foods). The wait was again about 20 minutes and finding a place to eat was a challenge. Finally, we asked for late check out on Sunday morning. Being an Hilton Honors member I've never been denied this request. However, this time I was. After having to prod for a reason why this request wasn't honored, the management finally told me that they had another large group checking in and needed to get all of the rooms ready. They did offer to store my luggage but we needed to do laundry so that wasn't much help. We tend to stay at Embassy Suites when we travel. While I might go back to this one, it was the most disappointing of all of the Embassy Suites properties that we've stayed at.", + "date": "2013-01-07 19:02:05 +0300", + "ratings": + { + "Cleanliness": 4, + "Overall": 3, + "Rooms": 4, + "Service": 2, + "Value": 3 + } + }, + { + "author": "Ms. Alec Cassin", + "content": "My family stayed here on Jan 25, 2005. It's funny reading the previous reviews because we had a very different impression of the hotel. I found it to be clean, comfortable and quiet. Yes, there is maintenance being done, but the workers are very careful and warn you if you are approaching a freshly painted area. The beds are fine, the sofa bed is a bit uncomfortable and for one night was acceptable. The breakfasts were fine. Great coffee. The nightly manager party was fun with fruity drinks, and standard junk food of chips, pretzels, party mix and popcorn. The pool was lovely, I can't figure out what people were complaining about. I didn't see any missing tiles and I made a point of searching it out. One traveler said the pool area looked like a hurricane hit it. I can't figure that out either it was simply beautiful. Yes, you can hear the planes either taking off or landing depending on the direction of the wind. After awhile you get use to the sound and you don't notice the aircraft noise. The sounds of the kids screaming and yelling drown the noise out. I don't mind the sound of the kids, I simply put on my headphones and listen to music. It's not as bad as previous reviews write and I consider myself very sensitive to unnecessary noise. The beach is exactly 3 blocks away. One long block and two short. If you have very young kids, this might pose a logistical problem because you need to carry your towels/junk and hold onto kids. I have kids 14, 12,9, so it was no problem. Yes,one of the streets is a highly trafficed street and there is no traffic light. But if you enter the crosswalk the cars did stop for us. No, I wouldn't let the kids do this by themselves. I just don't trust them or the drivers. We walked to the beach and went banana boating which can be found in front of the ESJ hotel. At the same time other kids are on the beach and my kids made instant friends. This worked out well for us. The beach is public and you can rent the chairs for $3-$4 per chair depending where you stop. In front of the ESJ hotel the chairs are $3. This place is good for a short stay and the hotel is designed for business travelers. As we saw mostly flight attendants and business people. If you like the hustle and bustle of San Juan this a decent place. If you are looking for a peaceful surrounding and limited night life you should stay a resort such as the Hyatt in Dorado. The hotel staff is very accomodating and helpful.", + "date": "2012-02-05 08:14:00 +0300", + "ratings": + { + "Cleanliness": 4, + "Overall": 4, + "Rooms": 4, + "Service": 5, + "Value": 4 + } + }, + { + "author": "Newell Ritchie II", + "content": "I stayed at the Embassy suites with my family. I was overall satisfied with our stay there. The staff was very helpful and the room was kept clean. Initially I was put off by the fact that our room had lots of ANTS. The staff was quick to move us to a different suite when we complained about the infestation. So I wasn't a happy camper when we first arrived. I also couldn't believe that a suite which would sleep 4 of us only had 4 drawers and a tiny closet. The drawers were small. But you know what? by the evening of our first day there we forgot all about that other stuff and enjoyed the managers reception-which is free drinks and snacks. The hotel is also conveniently located off the main road which has many souvenir shops,restaurants and a small supermarket. The best place to get puerto rican food inexpensively is La Casita which is in the shopping center near the hotel. I've also stayed at the Intercontinental hotel in the past. These two hotels are 2 that I would recomment. The advantage the Intercontinental Hotel has is that it is right on the beach. The beach is about 2 blocks from the Embassy Suites and easily accessible off the main road . Again overall I was a satisfied guest.", + "date": "2013-06-20 20:27:00 +0300", + "ratings": + { + "Cleanliness": 4, + "Overall": 4, + "Rooms": 3, + "Service": 4, + "Value": 3 + } + }, + { + "author": "Zachariah Hermiston", + "content": "After days, looking for a good place to stay during one night... we chose \"El pedregal\" because it has good reviews on this site and the Marriott Courtyard was fully booked. First of all, the entrance looks like a cheap entrance of a motel, the facilities overall are ok. The pool was clean and refreshing, but the pool bar wasn't open until 6pm and the restaurant too! The restaurant opens just two times during the day: for expensive breakfast (8am) and dinner (6pm). THE ROOM WAS HORRIFYING! Stains everywhere, small tv with poor signal and 8 channels maximum, no decoration at all, uncomfortable mattress, flat pillows, awful bathroom, Hot water! But there's no way to control it so be prepared for a nice burn. DO NOT BELIEVE ANY OFFERS OF THE WEBSITE! they supposedly include breakfast and dinner but all they give you is a $10 coupon and you'll be notified at the restaurant about that scam. Exercise area? haha... not at all!!! At least the service was good. It's better to keep this place as a secret.", + "date": "2012-11-12 01:33:54 +0300", + "ratings": + { + "Cleanliness": 3, + "Location": 2, + "Overall": 2, + "Rooms": 1, + "Service": 5, + "Sleep Quality": 3, + "Value": 2 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": "http://www.millenniumhotels.com", + "vacancy": true + }, + { + "address": "328 E 1st St", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "A downtown Japanese hotel offering classic comfort accommodations, meeting rooms, and a health spa with Shiatsu massage, close to attractions.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": true, + "free_parking": true, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.049693, + "lon": -118.240191 + }, + "id": 16638, + "name": "Miyako Hotel", + "pets_ok": false, + "phone": "+1 213 617-2000", + "price": null, + "public_likes": + [ + "Tamia Koch" + ], + "reviews": + [ + { + "author": "Norwood Strosin", + "content": "I have come back to this property everytime I go to Puerto RICO. Once you learn where this property is at It is easy to get to.The property is always kept clean, if you want to relax one day of your trip this is where you want to be, It is so relaxing beside the pool. If you want to book do it in advance because families from all around the island stay here on weekend getaways. the prices are reasonable. the staff is very helpful especialy one guy by the name of Waldemar he makes your stay there as comfortable as he can. The supervisor is Annie she is also very helpful she speaks English for those that do not speak spanish. She will give you the sightseeing spots in the area and where to dine.", + "date": "2012-07-07 09:09:59 +0300", + "ratings": + { + "Check in / front desk": 4, + "Cleanliness": 5, + "Location": 3, + "Overall": 5, + "Rooms": 3, + "Service": 5, + "Value": 4 + } + }, + { + "author": "Mr. Clifton Kohler", + "content": "Parking! Can you believe guest reserved parking is more than 200 meters away? Hotel employees have bad humor and bad service!", + "date": "2014-02-19 06:45:01 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 3, + "Overall": 2, + "Rooms": 3, + "Service": 1, + "Value": 2 + } + }, + { + "author": "Evelyn Stiedemann", + "content": "This week was my second stay in three months. The rooms were clean and the rooster in the parking lot outside my window was a nice touch. As a city boy, I thought the whole \"cock-a-doodle doo\" thing was a myth -- now I know it's not. The service staff was very nice...the desk staff will provide bottled water at no charge upon request. The restaurant is nothing special but there is an Applebee's about one mile north. And you can't be the $3 blackjack table. Not sure there would be any reason to stay here for vacation but it was fine for business.", + "date": "2014-12-15 18:06:22 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 4, + "Overall": 3, + "Rooms": 3, + "Service": 4, + "Value": 4 + } + }, + { + "author": "Mr. Tiffany Collins", + "content": "The rooms were okay, but not for the price you pay. The pool and restaurants were nice. The view of mountain tops was a plus. The club closes at 2am, and nothing but 40+ dancing salsa. Casino was okay but thought it was too small. Never order a long island ice tea there. Stick with one liquor drinks. It is a relaxing place to stay but want to wait for a special.", + "date": "2012-11-10 11:08:25 +0300", + "ratings": + { + "Overall": 3 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": "http://www.miyakoinn.com/", + "vacancy": true + }, + { + "address": "900 W Olympic Blvd", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "This high-rise hotel is one of the newest in downtown, with stunning views from the upper floors. It's across the street from LA Live, the Convention Center, and the Staples Center.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.045581, + "lon": -118.266793 + }, + "id": 16639, + "name": "Ritz Carlton", + "pets_ok": true, + "phone": "+1 213 743-8800", + "price": "$450+", + "public_likes": + [ + "Dr. Adeline Doyle", + "Korbin Huels III", + "Robb Erdman" + ], + "reviews": + [ + { + "author": "Travis Schimmel", + "content": "My wife and I visit the Ponce area regularly and always stay at this hotel, even though there are plenty of other options in the area with which we are familiar. Our most recent stay was a four-day holiday weekend. It sits on a hill overlooking the Caribbean Sea, a campus-style property with a particularly pretty and ample pool area. Although the hotel's location is convenient for exploring area attractions, a car or some other ground-transportation arrangement is an absolute must for travelers staying at this hotel, which is also true for Puerto Rico as a whole, a car-dependent island. The rooms are very clean, comfortable, and nicely furnished. The front-desk, maintenance and food/beverage staffs are excellent. Rates are on the pricey end for Holiday Inns at large, but one definitely gets what one pays for. The on-site restaurant serves very good food. We will most definitely stay there again.", + "date": "2012-01-02 02:21:38 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 4, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 4, + "Value": 4 + } + }, + { + "author": "Alverta Renner", + "content": "We stayed at the hotel for 10 days and 9 nights. WOW what a veiw and the staff was so kind and helpful. The pool was clean and the water temperture was perfect and the breakfast was just awesome! There staff went out of their way to make us comfortable. I could not believe how close everything was: Walmart, Walgreens, Subway, McDonalds, K-Mart, Sams Club and many more just 5 minutes. Family own bakery 2 minutes away and more...", + "date": "2013-01-04 23:29:12 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 5, + "Overall": 4, + "Rooms": 4, + "Service": 5, + "Sleep Quality": 5, + "Value": 4 + } + }, + { + "author": "Elody Kuphal", + "content": "While I agree that this hotel is nothing extrodinary I would disagree with the rest of the recent reviews. I stayed in this Holiday Inn last weekend and had a blast. The rooms were comfortable, the staff was friendly and polite and the views are breathtaking. The hotel is perched on top of a cliff and you overlook the ocean from the rooms and from the pool. The bar area also has a beatiful view of the ocean. I will definitely stay in this hotel again!", + "date": "2013-12-25 15:01:09 +0300", + "ratings": + { + "Check in / front desk": 5, + "Cleanliness": 4, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Service": 4, + "Value": 3 + } + }, + { + "author": "Queen Hackett", + "content": "We stayed here before and only returned because the web site said or was remolded. Our room and bath were old and our bedding was damp. We were on the 7th floor. During the night a roach walking on us awoke us! They no longer serve a cook to order breakfest and their great long lounge bar has been removed and a hot pizza place has been added. The front desk clerk did not know where the business center was printing off boarding passes. Our group had 3 rooms, one worse then the other. My recommendation is spend a little more money and go down the street!", + "date": "2012-06-03 05:35:56 +0300", + "ratings": + { + "Cleanliness": 2, + "Location": 4, + "Overall": 3, + "Rooms": 2, + "Service": 2, + "Value": 2 + } + }, + { + "author": "Earl Marks", + "content": "Stayed in Howard Johnson for a week with my friends last month. On th arrival it was slight longish, as check in procedure took a really long time. In fact, i think this was the longest hotel check in i ever had. We had a large room that had a big bed, so it was pretty comfy. Tv functioned ok with good choice of programmes, room was cleand & cleaned every day. Bathroom was little bit sloppy, but still ok. It was clean, but it kind of, needs some refreshment.... Breakfast we had was ok only for first day, already second day you don't know what to take because there is literally only few things displayed... A toast, jelly, butter, some cereals, 3 types of fruit, coffee, milk and juice... Not quite good, isn't it? And thing that really bothered me was the fact that 2 girls \"working\" there ate sandwiches with ham and chese during breakfast time, while guests had their empty toasts. Jummy..... Pool area was not too attractive, right next to reception, everyone on the reception can see you plus there are always some locals swimming, which we didnt like too much. Hotel location was good, close to airport (but far enough), bars, shops and beach which we used few times... Parking in the hotel is not free, but as special guest you have a special price, which was fair. Old SJU is some 15 minutes away (by car), which is ok. If you need/ some evening walks, disregard some previous comments about security in hotel's area, cause it is actually very safe. On the other hand, if you act unsafe, there isn't a place on the earth which is safe enough for you.", + "date": "2012-11-03 16:33:18 +0300", + "ratings": + { + "Cleanliness": 3, + "Location": 4, + "Overall": 3, + "Rooms": 3, + "Service": 4, + "Sleep Quality": 4, + "Value": 3 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": "http://www.ritzcarlton.com/en/Properties/LosAngeles/Default.htm", + "vacancy": true + }, + { + "address": "404 S Figueroa St", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "Recognizable from various movies it has appeared in.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.052913, + "lon": -118.255828 + }, + "id": 16640, + "name": "Westin Bonaventure", + "pets_ok": true, + "phone": "+1 213 624-1000", + "price": null, + "public_likes": + [ + "Dayna Crist", + "Lauriane Olson", + "Novella Thiel", + "Ms. Lon Sipes", + "Patrick Moen", + "Myrtle Ryan" + ], + "reviews": + [ + { + "author": "Larue Mitchell", + "content": "I was reading some of the reviews and I can not believe that some people find this place not clean. I had use this hotel at least four times in last two years and every time the rooms are very clean and with some partial view of the Isla Verde beach or the city. Pool area is very nice, as the pizza restaurant in the lobby.", + "date": "2014-02-03 07:48:54 +0300", + "ratings": + { + "Cleanliness": 5, + "Overall": 5, + "Service": 4, + "Sleep Quality": 4, + "Value": 5 + } + }, + { + "author": "Cole Shanahan", + "content": "The hotel is very nice. The furniture is from the 1960s which is all right with me. For us smokers, we have the second floor reserved which is a must and a plus. There is wi-fi in the rooms as advertised. The pizzeria/italian restaurant in the lobby is great and the pizza is the best. There is a parking lot next door - cheap. You will pay the hotel for it when you register. Cristina from the hotel was very accommodating as I asked to be moved to another room and it was done at once. Will definitely go back to this hotel.", + "date": "2014-04-12 06:39:15 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 4, + "Overall": 4, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 4 + } + }, + { + "author": "Max Bins", + "content": "HAMILTON, ONTARIO CANADA FEBRUARY, 2008 We stayed at his hotel as a pre-cruise destination. Arrived at the hotel at 4:00 p.m. It is a very short drive from the Airport. Check in was a breeze. Rooms are clean and include iron, coffee maker, fridge, and microwave. Bathroom is tiny but clean and renovated. Airconditioning worked well noise level was quiet all night. There is a restaurant in the main lobby selling mostly Italian Food. Food is not bad and reaonably priced. We took the bus to Old San Juan. 75 cents per person and about a 15 minute drive. The bus ride was an eye opener as we drove through what looked like a lot of government housing. Wouldn't recommend doing this at night. Took a taxi back from Old San Juan at the end of the night cost was 19.00 dollars. This was a more than acceptable hotel for a very reasonably priced will definately stay here again.", + "date": "2013-06-10 04:01:50 +0300", + "ratings": + { + "Check in / front desk": 5, + "Cleanliness": 5, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Service": 5, + "Value": 4 + } + }, + { + "author": "Toby Wiza", + "content": "My husband and I stayed at Howard Johnson in Isla Verde in March 17-20. We were disappointed as soon as we checked in and saw our room. It was on the lobby floor which may not have been terrible except for the fact that the room was tiny, you could hear everything that was happening outside of the room and worst of all, there were no windows! It was a dark little dungeon. But for the fact that we were only staying 3 nights, I was very close to walking down the street and finding something else. We did ask to change rooms but the staff stated that everything was booked. The staff, by the way, could have been a bit more customer service oriented. They seemed to have attitudes and were not receptive to questions. The service in the hotel restaurant was terrible. The waitstaff were friendly but the service was beyond slow. They don't acknowledge you when you walk in, you have to just guess that you should pick any seat available. The tables were still dirty from prior customers and on one occasion, we waited approx. 30 minutes before anyone even came over to take our order. It was so bad that we complained to the restaurant manager and we weren't the only ones complaining. On the good side, the pool was nice and the beach is conveniently right across the street. The hotel is nicely situated on a relatively busy strip but having visited Old San Juan, I would have rather stayed nearer to there because they seemed to have a much wider and nicer selection of restaurants than where we were and our location seemed better suited for spring breakers. The bus stop to Old San Juan is across the street from the hotel and is approx. 45 mins, it costs about 50 cents. El Yunque is a must see and I would recommend renting a car a driving there as it's a relatively easy drive and was much cheaper than a tour. Our one day car rental was $27 and the visitors center at El Yunque charges about $3.", + "date": "2012-04-18 06:39:37 +0300", + "ratings": + { + "Cleanliness": 2, + "Overall": 2, + "Rooms": 2, + "Service": 1, + "Value": 1 + } + } + ], + "state": "California", + "title": "Los Angeles/Downtown", + "tollfree": null, + "type": "hotel", + "url": "http://www.starwoodhotels.com/westin/search/hotel_detail.html?propertyID=1004", + "vacancy": false + }, + { + "address": "3101 S Figueroa St", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "Close to the campus of USC.", + "directions": null, + "email": null, + "fax": "+1 213 746-9106", + "free_breakfast": true, + "free_internet": false, + "free_parking": false, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.024543, + "lon": -118.279128 + }, + "id": 16648, + "name": "The Vagabond Inn USC Hotel", + "pets_ok": true, + "phone": "+1 213 746-1531", + "price": "$85+", + "public_likes": + [ + "Gladyce Bartell", + "Janice Thiel", + "Tad Ratke" + ], + "reviews": + [ + { + "author": "Chelsey McGlynn", + "content": "Stay Jan 23-27, 2005 For the price of $105 per night this hotel offers a tremendous bargain. It is similar to the Fairfield Inns on the mainland but a bit more upscale. Clean but smallish rooms and bathrooms. Outdoor pool is extremely private but small with no jacuzzi. One of my customers from the area said the Italian restaurant on the 10th floor is his favorite on the Island. The $20-$30 entree fees are typical for this area. There is plenty of fast food nearby for those looking to scrimp. The internet cafe (35 minutes for $5) was less than 5 minutes away. Several nice casinos are no more than 10 minutes walk. If you want a plush pool next to the ocean stay at the Marriot Courtyard but expect to pay $235 per night this time of year.", + "date": "2015-01-10 21:19:55 +0300", + "ratings": + { + "Overall": 4 + } + }, + { + "author": "Jazlyn Ryan", + "content": "I took my sister to Puerto Rico for a girls getaway and we stayed at the Hotel Villa de Sol. We chose the hotel because it was affordable and the pictures of the rooms and bright yellow Spanish style building won us over. We were EXTREMELY disappointed! Let me first say that... IT LOOKS NOTHING LIKE THE PICTURES advertised on the internet. The bar and restaurant were closed the entire time we were there. When we got to our room it was hot and muggy because the a/c had not been turned on as a courtesy and the room reeked of some sort of perfume/disinfectant. The table and chairs on our balcony were covered in mildew. The headboards and frames on our beds were not secured, the pillows were flat, the bed was terribly uncomfortable and we woke up our first morning there with bites on our legs, arms and torso! OMG!... The iron was broken, the ironing board cover was covered in stains and don't even think about asking for extra towels or pillows! They acted as if I were asking for their first born! It was two of us and only one small hand towel. I love my sister but neither of us desired to share the same washcloth. Our room didn't get cleaned until late in the afternoon, and again two people, one washcloth. It was close to the airport so we knew we'd hear airplanes go by... but nothing could have prepared us for what we heard. We could have sworn that the planes literally taking off on the same street! We paid $109 a night, but it was worth every bit of $50 bucks a night. I'm a flight attendant and I'm usually in and out of hotels up to 20 times a month and this was by far THE WORST PLACE EVER!!!", + "date": "2014-04-19 07:53:32 +0300", + "ratings": + { + "Cleanliness": 1, + "Overall": 2, + "Service": 1, + "Sleep Quality": 1, + "Value": 1 + } + }, + { + "author": "Ava Stark", + "content": "My family and I stayed here from July 12-15. We had 3 rooms booked. One that slept 6 and the other 2 that were double rooms. The room we originally were given had water stains on the ceiling and a very musky smell. Upon entering the room I went to the bathroom to wash my daughters face and realized there was very little water pressure. By very little I mean a trickle. There was no way someone could bathe (much less 6 people). I immediately went to the desk and explaiend the problem. They seemed aware of it but the gist of our conversation was there was no way it would be fixed before our stay was over. I requested another room and was offered one on the first floor that only had 2 beds in it with the promise of 2 cots. The room was next to the bar and extremely noisy. Under normal circumstances this would not have bothered me but with all the other issues this just added to my displeasure here. I called the owner and complained. She was not in any way apologetic and when I threatened to leave she said they would charge me for the 3 nights. The other two rooms were musky and generally in poor condition. I travel to Puerto Rico about 4-5 times a year and have stayed at many hotels both budget and higher end resorts. I wanted to try this place as the website made it look very decent. Basic accomodations are never a problem for me as long as the hotel is clean and the service is adequate. Here neither is the case.This was, quite possibly, the worst service I have ever received. The front desk staff was rude and unresponsive, the manager was just as bad, and the owner was indifferent. I was embarrassed as I was travelling with people that had never been to the island and they were left with a bad impression of the service provided in hotels. If this is your budget then stay at the Coral Princess in Condado. Same prices and much better service.", + "date": "2012-01-10 21:37:25 +0300", + "ratings": + { + "Business service (e.g., internet access)": 1, + "Check in / front desk": 1, + "Cleanliness": 3, + "Location": 3, + "Overall": 1, + "Rooms": 1, + "Service": 1, + "Value": 1 + } + }, + { + "author": "Otto Thompson", + "content": "My boyfriend and I loved our stay (5 nights) at Hotel Villa Del Sol. We didn't have a prior reservation, but they managed to accommodate us and give us excellent rates. When I walked in, I asked one of the employees regarding their amenities; she included the employees as one. At the time, I was a bit skeptical due to our interaction with other hotel employees, but she was absolutely right. We were treated wonderfully. Don’t forget to try Annette’s margaritas and Carmen’s Pina Colladas (?). Villa Del Sol is cute, clean and affordable. This place has a lot of character. You will definitely enjoy your stay. FYI, the car rental guy, Carlos (he is not one of their employees), was the only unpleasant person we dealt with at the hotel. You might want to stay away from him.", + "date": "2015-10-10 16:16:50 +0300", + "ratings": + { + "Cleanliness": 5, + "Overall": 5, + "Rooms": 4, + "Service": 4, + "Value": 5 + } + }, + { + "author": "Ollie Lebsack", + "content": "This was a great hotel for the price located in the center of town and convenient to all. The hotel was quaint and charming, the rooms were very comfortable and clean, staff very helpful and courteous, convenient parking and included a very nice continental breakfast. I only stayed 1 night so I couldn't enjoy the pool or relaxing atmosphere, but I would go back and recommend it to friends.", + "date": "2015-11-27 15:17:02 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Sleep Quality": 5, + "Value": 5 + } + }, + { + "author": "Macey Rosenbaum", + "content": "I wish that I could have seen this hotel in its heyday- the lobby is the best-looking part of it today. Still, the bed was comfy (perhaps a little too soft) & the rooftop dining area was nice on a sunny day (even if the view faces away from the square & is somewhat obscured by buildings). We didn't linger & would recommend this for no more than one night. If you're just passing through Ponce like us then why not save a couple of bucks? Downers included mosquitos in our room & the fact that the staff didn't tell us about the parking lot out back until we checked out the next morning. Still, there was only 1 other group besides us at breakfast- more toast or juice anyone?", + "date": "2013-11-12 07:51:01 +0300", + "ratings": + { + "Cleanliness": 3, + "Location": 5, + "Overall": 3, + "Rooms": 3, + "Service": 2, + "Sleep Quality": 3, + "Value": 5 + } + } + ], + "state": "California", + "title": "Los Angeles/South Central", + "tollfree": "+1 800 522-1555", + "type": "hotel", + "url": "http://www.vagabondinn-los-angeles-hotel.com", + "vacancy": true + }, + { + "address": "10740 Wilshire Blvd", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "4 star luxury boutique hotel.", + "directions": null, + "email": null, + "fax": "+1 310 475-5220", + "free_breakfast": true, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.06023, + "lon": -118.43767 + }, + "id": 16686, + "name": "Hotel Palomar", + "pets_ok": false, + "phone": "+1 310 475-8711", + "price": null, + "public_likes": + [ + "Aurelia Langosh", + "Titus Armstrong", + "Antonette Ondricka", + "Jennyfer Weber" + ], + "reviews": + [ + { + "author": "Alexane Lowe", + "content": "We enjoyed our stay at Melia Hotel. The room was spacious and we had great food around the hotel - eg Times Square, Bukit Bintang, Amoy Street, Pavillion.", + "date": "2013-12-02 06:22:20 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 4, + "Overall": 4, + "Rooms": 4, + "Service": 4, + "Value": 4 + } + }, + { + "author": "Stella Wilkinson", + "content": "We stayed at this hotel 1 night and found it to be fairly comfortable. The bed was a little hard and the room a little small but we weren't in Ponce to hang out in the hotel room. The breakfast wasn't great but the rooftop where they serve it was very enjoyable!", + "date": "2013-03-20 13:08:34 +0300", + "ratings": + { + "Business service (e.g., internet access)": 4, + "Check in / front desk": 5, + "Cleanliness": 4, + "Location": 4, + "Overall": 3, + "Rooms": 3, + "Service": 4, + "Value": 4 + } + }, + { + "author": "Hilbert Haag", + "content": "Nice hotel, the oldest in town, big rooms with big beds, private bathrooms, clean, elevator available, breakfast included and served on the roof with a nice view. breakfast has not a big choice, staff is friendly but everytime very busy, don´t ask to much. internet terminals available for free", + "date": "2014-02-14 05:49:57 +0300", + "ratings": + { + "Business service (e.g., internet access)": 4, + "Check in / front desk": 3, + "Cleanliness": 4, + "Location": 4, + "Overall": 3, + "Rooms": 4, + "Service": 3, + "Value": 3 + } + } + ], + "state": "California", + "title": "Los Angeles/West", + "tollfree": null, + "type": "hotel", + "url": "http://www.hotelpalomar-lawestwood.com", + "vacancy": true + }, + { + "address": "701 Stone Canyon Rd", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "This picturesque, five-star hotel is a popular destination for weddings, as well as a playground to the affluent.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": false, + "free_internet": true, + "free_parking": false, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.0863, + "lon": -118.44544 + }, + "id": 16687, + "name": "Hotel Bel-Air", + "pets_ok": false, + "phone": "+1 310-472-1211", + "price": null, + "public_likes": + [ + "Deondre Leffler", + "Zander Bartell", + "Zelda Murazik", + "Marjorie Welch" + ], + "reviews": + [ + { + "author": "Camron Kuhlman", + "content": "We went to Ponce at the last minute and had heard about Hotel Melia as a quaint, historical hotel. We are always looking for interesting places to stay, so we decided to give it a try. The location is convenient since it is located on the Central Square and the \"ice cream\" parlor is definitely a highlight. Twenty-four hour security parking is available, so be sure to ask at check in. The hotel was in fact old and certainly gave us a view of Ponce during its more glamourous years. However, the rooms were disappointing. The furniture is old, which isn't bad since it is an old hotel, but there are stains on the cushions. I asked for my room to be changed when the bathroom floor did not appear to have been clean, and numerous hairs were found on the bed. The room I was changed to was not much better, but since we were only staying one night, we decided to settle in. This is truly unfortunate because the hotel has much potential but seriously needs some TLC and better housekeeping. On a more positive note the continental breakfast was quite good consisting of fresh pastries, juice, and coffee. The staff was also very accommodating and assisted us with many inquiries about the area.", + "date": "2012-10-21 08:26:13 +0300", + "ratings": + { + "Overall": 2 + } + }, + { + "author": "Gino Moore", + "content": "We buy a Gustaso to stay in thi hotel .... This hotel is the worst we ever visit . The hotel is old , dirty and stained . The room smell bad . Our stay it was supose to be 2 nights 3 days ... we left the first night .... we lost our money :(", + "date": "2014-10-01 08:45:26 +0300", + "ratings": + { + "Cleanliness": 1, + "Location": 3, + "Overall": 1, + "Rooms": 1, + "Service": 3, + "Sleep Quality": 1, + "Value": 1 + } + }, + { + "author": "Miss Josh Fritsch", + "content": "I will try to be brief and concise. When we arrived at Perichi’s, the first view of the hotel where hanging mattresses that resembled the hanging Gardens of Babylon. We all thought, “oh well, they are renovatingâ€� so we didn’t put much thought into it. When we got our keys, accompanied with a generic TV remote, I grabbed our baggage and quickly went to our room to try to get a hot shower after a day of sight seeing. Walking towards the room, I saw that the doors seemed old and rustic; I thought it was part of the ambiance. When I opened the door of the room the hurricane sounding air conditioning was on and the moist/molded/cigarette air embalmed my nostrils with pizzazz. The bed covers had a huge unidentified stain right in the middle of the bed. I quickly took the covers and threw them in the corner of the room. When I do a 360 look around the room, I thought I was in the movie Trainspotting. The walls where stained and cracks where noticeable on the ceiling. When I enter the bathroom there was a blue toilet bowl from the 60’s accompanied by a blue sink and blue tub. I have to say that the blue matched perfectly with the blue “flower powerâ€� tiles on the wall. I walked back to the room and noticed that they where triangle shaped patterns on the rug. When I took a closer look, I realized that they where hot iron marks. It seems that a several guests left the iron on the floor. After I laughed and told many times to my girlfriend that I was sorry, we laughed and played along with the hotels crack house theme. I went to the front desk to complain about the room and they just stared at me like I was speaking Cantonese. For the record, I’m from Puerto Rico and my primary language is Spanish. The morning after our arrival we ate breakfast at the hotel. The breakfast was good and I cannot complain about the service. While I was eating, I was looking at the people that where working on the renovation of the hotel. They where spreading a roll of carpet. What was strange is that the carpet seemed old. Then I realized that they where replacing the old carpet with used carpet. Of course, how can you keep a Crack House themed hotel working with new carpet or new appliances. Maybe they just leave the carpet in sea water for days and employ people to intentionally burn the carpets with hot irons just for looks. Overall I have to say that the hotel is a mess, I stayed in the room in the front of the hotel and they seem to be the worst ones in the premises. The staff was nice although the administration keeps the place looking cheap and tasteless. It’s very sad that on this part of the island there is a limited amount of choices when it comes to hotels or bed&breakfast. My advice is, spend 20 dollars more and stay somewhere else. This is definitely not worth the money. Maybe they should change the name from Perichi’s to Perish’ s.", + "date": "2012-07-29 19:24:48 +0300", + "ratings": + { + "Overall": 1 + } + }, + { + "author": "Frieda McKenzie", + "content": "We have just returned from our lovely stay at Posada Porlamar.....where to start..we got a wonderfull room with an incredible view of the Caribeann Sea...we dinned at the hotel restaurant La Pared.... we had a great dinner, ostrich!!! in a white wine reduction...There are no words to describe it....Last but not least, the employees at Posada Porlamar made us feel like family ...We are definitely going back next summer...Two thumps way up!!!", + "date": "2014-12-02 10:56:11 +0300", + "ratings": + { + "Cleanliness": 4, + "Overall": 5, + "Rooms": 5, + "Service": 5, + "Value": 4 + } + }, + { + "author": "Dr. Melyna Dickinson", + "content": "Location is great. Beach is nearby and great restaurants all around. The problem is the rooms. They're too outdated. The lobby is nice and the restaurant is great.", + "date": "2015-12-27 13:18:56 +0300", + "ratings": + { + "Cleanliness": 2, + "Location": 4, + "Overall": 3, + "Rooms": 2, + "Service": 3, + "Value": 2 + } + }, + { + "author": "Adolfo Zboncak", + "content": "My wife and I go to Puerto Rico every year. For 2011/2012 Christmas & New Years holiday we decided to make it special. We wanted to spend it on the island. We had stayed at the San Juan Beach several years ago when they were under renovations. We took that into consideration and put up with the inconveniences, it was a good stay. So we looked at the web site for 2011 and saw what appeared to be new pictures of completed renovations, and booked a 10 day stay. WOW!! What a shock we got when we arrive to find that the owner(s), shifted the emphasis from customer service to greed. The lobby was beautifully redone although simple. A restaurant El Perurrican) had been added, but you must be into strictly seafood (Not much diversity). The free breakfast that was served during our last stay was done away with. We assumed too make way for much better accommodations. WRONG! When we entered the room what a shock we got to find that it smelled musty, was very dirty and with stained carpets. The bathroom was full of mold, mildew, and rust. All the bathroom fixtures had mold, mildew and rust. The window shades were broken so you had to take them down and roll them by hand. The air conditioner (although it worked) sounded like a freight train all night, so we convinced ourselves that it was the sound of the ocean in our room so that we could sleep. We did have an ocean view room (which we payed extra for) We could not get room service to leave us clean towels, and the sheets were not changed. Only the bed was remade again and again for 10 days with the same sheets. I believe we had bedbugs, because we were itching all night an in the morning we both had bites all over our bodies. (we marked the sheets, to make sure). No coffee machine, NO blow dryer although the blow dryer plate was attached to the wall. On the third day we took a ride on the elevator, and were joined by 4 housekeepers. I am Puerto Rican and speak the language my wife is Irish/Italian and only understands it a little bit. Well these 4 housekeepers were discussing in Spanish, how pissed (Encojona) they were that, tourist would not leave the hotel until noon or after, thus causing them to be delayed in their duties. And that they could not find a room on a higher floor where they could watch TV and rest. At check out they attempted to charge us for 10 days of valet parking at $18 a day knowing full well that we only used the valet parking for 7 days, the valets were very good and pleasant . When we did try to launch a complaint the only person we could find did not appear to care about our problems. One employ did tell us that the new management did not appear interested in pleasing the customers only profit. NEVER AGAIN!! Beautiful Condado area, but book at your own risk.", + "date": "2015-03-06 21:55:28 +0300", + "ratings": + { + "Cleanliness": 1, + "Location": 5, + "Overall": 1, + "Rooms": 1, + "Service": 1, + "Sleep Quality": 1, + "Value": 2 + } + } + ], + "state": "California", + "title": "Los Angeles/West", + "tollfree": null, + "type": "hotel", + "url": "http://www.dorchestercollection.com/en/los-angeles/hotel-bel-air", + "vacancy": true + }, + { + "address": "8018 Beverly Blvd", + "alias": null, + "checkin": null, + "checkout": null, + "city": "Los Angeles", + "country": "United States", + "description": "A popular hotel, next door to Swingers.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": false, + "free_parking": false, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.075802, + "lon": -118.364444 + }, + "id": 16708, + "name": "Beverly Laurel Motor Hotel", + "pets_ok": false, + "phone": "+1 323 651-2441", + "price": null, + "public_likes": + [ + "Kip Stehr", + "Rocky Bartell", + "Rebeka Lindgren", + "Felipe Kunze", + "Ova Daugherty" + ], + "reviews": + [ + { + "author": "Reagan Armstrong", + "content": "The staff was rude, had to wait over an hour past the check in time for an available room, no valet parking was available so i had to drive 4 blocks to find a location, was told to check back the second day for parking (but there wasn't any then either so i ended up returning the rental car mid trip) wi-fi doesn't work in the room, was told that someone would fix it and 24 hours later the problem still exists, no phone in the room so i have to go to the lobby to speak to anyone, only one elevator works in the 10 story building, the carpet is rolling and stained (very large spots) in multiple locations, there's no mattress pad under the sheets, the mattresses are decades old and very squishy, the trim is falling off the wall, the windows are stated to open but don't, the windows are filthy so you can't enjoy the view of the ocean, (I can go on but these are the first few items that came to mind...)", + "date": "2012-03-28 09:19:01 +0300", + "ratings": + { + "Cleanliness": 1, + "Overall": 1, + "Service": 1, + "Sleep Quality": 1, + "Value": 1 + } + }, + { + "author": "Cathy Deckow", + "content": "Our mediocre experience started right when checking in. Basically they gave us a room with two single beds instead of the king-size we booked. The only explanation they gave was that the hotel was full and that they run out of king-size bed rooms. In other words, they gave our room to someone else. So we get to our rooms, expecting to get something at least similar to what we saw in the website and so happened that the hotel is undergoing renovations and not all the rooms were fixed. The room we got was worse than what you would get at a cheap motel: Dirty and humid carpets , strong mildew smell, old furniture, no safe box, ornamental phone (didnt even work), the so called \"lagoon view\" was actually a view to a piece of lagoon across the street through a very dirty window. Sleep quality was awful. The sheets were humid to the point of being sticky. The noises coming from the street (honks and loud car stereo) made it impossible to sleep unless you used earplugs. Bottom line: - Poor overall cleanliness. Hallways and room smell to mildew. - Noisy if you get a room with \"lagoon view\" - No breakfast - The hotel has a good location and friendly staff, however, I do not recommend this hotel.", + "date": "2013-02-04 17:17:03 +0300", + "ratings": + { + "Cleanliness": 1, + "Location": 5, + "Overall": 1, + "Rooms": 1, + "Service": 4, + "Sleep Quality": 1, + "Value": 1 + } + }, + { + "author": "Shanelle Hermiston", + "content": "I'm from Puerto Rico and I used to hanout in the Isla Verde Area. When I saw a hotel in Isla Verde by the \"Ventana\" for $89, I knew it wasn't going to be a 5 star hotel, I knew is a beach front property but there is no beach (hotel is sitting in front of the rocky area of the beach) you need to walk a couple minutes to find a good spot is the beach. I picked the hotel because of the location and price. The lobby is real nice it was remodeled and the view is awesome. They have a nice small pool but is a little bit deep for small kids. They have a valet service - $18 dollar daily. If you are a young guy and just want to be in the middle of the night life, walking distance to clubs, restaurants, the real Isla Verde beach and don't mind a motel room. This is the place to stay. If you are looking for a family hotel, with nice rooms and a nice beach go somewhere else.", + "date": "2012-11-15 05:32:38 +0300", + "ratings": + { + "Cleanliness": 2, + "Location": 5, + "Overall": 3, + "Rooms": 2, + "Service": 3, + "Sleep Quality": 3, + "Value": 5 + } + }, + { + "author": "Arjun Koss", + "content": "do not stay here!! it is not worth saving a few dollars, as one of the other reviews read!,... my girls and i stayed here the first night we arrived to puerto rico, we figured we'd stay in pr for one night prior to going on our cruise. the experience at this hotel was a terrible way to start our vacation!... the bathroom was terrible, the bath tub was disgusting, there was no knob to turn the shower on, the hallways smelled like mold, the carpet was stained and looked as if there were layers of dirt. the beds were not well made and the comforter looked as if it had not been washed. there was no bed skirt on the bed, so you were able to see the box-spring and the mattress was disgusting!! you were also able to see the frame of the bed and the wheels were bending, so i was afraid that with any lil' move, the bed would collapse. i had read the previous reviews, but i had already booked the room, .. and i figured what the heck, i'll give it the benefit of the doubt... sure enough my girl-friends and i were totally disgusted!!!... mold everywhere, and according to the receptionist, the hotel is being \"remodeled\". i did some research and its been awhile that this hotel is supposedly being \"remodeled\". seriously this hotel should be shut down!... luckily we only had to stay there one night. we slept all covered up, on top of our towels, on top of the comforter and still, you could feel the dam wires of the mattress poking your back!... we definetly didn't want to un do the bed, cause who knows what the heck would be under the comforters. check out the photos of the bathroom!... SICK!!!", + "date": "2012-02-08 13:27:42 +0300", + "ratings": + { + "Cleanliness": 1, + "Location": 3, + "Overall": 1, + "Rooms": 1, + "Service": 3, + "Sleep Quality": 1, + "Value": 1 + } + }, + { + "author": "Justina Dietrich", + "content": "Booked with Expedia, desk clerk took confirmation info, then hotel double billed after I left. Expedia trying to help, their accountant said they would fix it. It has almost been a month and still not resolved. Next call is to BBB and police. No wash rags, still under construction, loud gay parties at the bar, never warned about that one. Absolute misery, would not have my enemy stay there.", + "date": "2012-04-07 09:55:07 +0300", + "ratings": + { + "Cleanliness": 2, + "Location": 2, + "Overall": 1, + "Rooms": 1, + "Service": 1, + "Value": 1 + } + }, + { + "author": "Justice O'Kon", + "content": "this was my first and will be my last visit to Puerto Rico!!!!...the hotel was unkept overall. the initial room we had was not \"secure\" so we switched, the second was not any better..the shower curtain in the bathroom had mildew and mold (which we requested to be replaced) the hotel floor had a musty smell...my girlfriends clothes remained \"wet\" and clammy from the humidity in the room...I slept on top of the sheets because i was scared i might get bed bug!! The hotel towels were not white at all....I was so relieved to go back home just to get a good night rest and take a nice shower!!! they should have shut down the hotel until it is fully renovated, but i'm not sure they meant the rooms were part of the deal....Unfortunately i will not return due to this experience..I would be ashamed to recommend this place to anyone McKinney, Texas", + "date": "2012-12-12 03:57:37 +0300", + "ratings": + { + "Cleanliness": 1, + "Location": 3, + "Overall": 1, + "Rooms": 1, + "Service": 1, + "Value": 1 + } + }, + { + "author": "Anjali Mante", + "content": "This hotel was a grossly misrepresented on the website, overpriced for the quality, and the epitome of a roach motel. When we first arrived, we entered our rooms, to find the overwhelming stench of mold and mildew slap us in the face. We decided to tough out the first night as it was very late and we were exhausted. As we layed in our beds, the smell of the sheets, pillows, and mattress was nauseating. I could only compare it to leaving your clothes in the washer for a month -wet. Then, I decided to take a shower and took my first step into a slimy, tub with a black bottom and mildew and mold stains. Showering in $@!#% flops would not resolve this issue as the shower would not drain. The second stay - we complained about the room, believing that they would clean the sheets, tub, etc, and we would be able to finish the remainder of our stay at this hotel. Instead, we returned to find the same stench in the beds, and dirty bath. Then, we realized that there was living mold and mildew on the ceiling and walls. See photos attached. We then informed the hotel manger that we would like to check out and would not be staying for the remainder of our visit, but they refused to give us our money back. We decided to check out anyway, as this hotel was a health violation and obviously no cleaning guidelines are enforced. I don't recommend this hotel at all. There isn't really a beach view either. Stay in a Motel 6 before here - Please.", + "date": "2015-07-20 10:49:20 +0300", + "ratings": + { + "Business service (e.g., internet access)": 1, + "Check in / front desk": 1, + "Cleanliness": 1, + "Location": 1, + "Overall": 1, + "Rooms": 1, + "Service": 1, + "Value": 1 + } + }, + { + "author": "Rita Walter", + "content": "Don't check in! This hotel is dirty, smelly and not worth your hard earned money! The beach is there, but you can't use it, the renovations are ongoing causing it to be very loud. You can't charge drinks to your room account, but must pay separately on demand. The beds are most uncomfortable and the bathrooms leave much to be desired. We were to say the very least very, very disappointed in this place. Cindy, Portland, ME", + "date": "2012-12-21 00:07:48 +0300", + "ratings": + { + "Business service (e.g., internet access)": 1, + "Check in / front desk": 2, + "Cleanliness": 1, + "Location": 2, + "Overall": 1, + "Rooms": 1, + "Service": 2, + "Value": 1 + } + }, + { + "author": "Erwin Lakin", + "content": "I would not stay here again, particularly for the price we paid which included a 9% 'Hotel Service Charge' above and beyond the 9% gov't tax and $5pp/night municipal fee. When we asked about the Hotel Service Charge we were told it was another tax. In that situation we figured, what can we do? I took a look at the Holiday Inn Express website and their San Juan hotel right down the street (also in Condado) does not have this service fee. We took a look at the HI Express while we were in town and the pool is smaller than the SJB's, but that seemed to be the only drawback. And the SJB's pool got cloudier by the day, as they seemed to not be caring for it at all. We had to switch rooms after the 1st night because our TV didn't work and you couldn't adjust the temperature on the bathroom faucet. Our new room was only marginally better. And there was one day we came back to no room cleaning after being out of the room the entire day (9-5). At that point we just asked for clean towels for our shower when they offered to clean it for us then. The only food available in the hotel was a continental breakfast for a ridiculous $7+tax. The spread wasn't impressive from the looks of it and I could just grab a better breakfast from a local place for much cheaper. All in all, if I went to San Juan again I would NOT stay at the San Juan Beach. I feel it was way overpriced for the service/amenities.", + "date": "2013-01-23 12:23:14 +0300", + "ratings": + { + "Check in / front desk": 2, + "Cleanliness": 1, + "Location": 4, + "Overall": 2, + "Rooms": 1, + "Service": 3, + "Value": 1 + } + } + ], + "state": "California", + "title": "Los Angeles/Wilshire", + "tollfree": null, + "type": "hotel", + "url": null, + "vacancy": true + }, + { + "address": "4619 Beverly Blvd", + "alias": null, + "checkin": "4PM", + "checkout": "11AM", + "city": "Los Angeles", + "country": "United States", + "description": "A private room hostel with free WiFi internet access, featuring great, comfortable and secure private rooms on a hostel budget. Located on the edge of Korea Town at the intersection of Beverly Blvd and Western Ave. All rooms have 1 full-size memory foam mattress, a work table and a secure wall safe. The hostel features modern bathrooms, a print cafe with WiFi printer and AirPrint for your iphone or iPad, even a loaner laptop and a Keurig coffee maker a large selection, free WiFi in all rooms.", + "directions": null, + "email": null, + "fax": null, + "free_breakfast": true, + "free_internet": false, + "free_parking": false, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.08278, + "lon": -118.31639 + }, + "id": 16709, + "name": "StayON Beverly Hostel", + "pets_ok": true, + "phone": null, + "price": "$50 per night single occupant", + "public_likes": + [], + "reviews": + [ + { + "author": "Ladarius Fadel", + "content": "I loved the ocean view and sound from my room. It did feel a bit damp, however, as I have read in other reviews. The location was wonderful – nice restaurants and bars nearby, and a nice walk to better beaches. The staff was very friendly and helpful but three things were wrong: 1) the website promised a coffee machine and CD player in the room, but they were not there and not provided after I mentioned it, 2) the doors of all rooms are incapable of closing quietly, therefore, there was a lot of slamming noises at all hours and 3) there was an all night party nearby which woke me up throughout the night until 4:00 a.m. I finally had to call the person at the front desk, who handled it promptly and successfully.", + "date": "2014-04-29 15:36:34 +0300", + "ratings": + { + "Cleanliness": 3, + "Overall": 2, + "Rooms": 3, + "Service": 4, + "Value": 3 + } + }, + { + "author": "Concepcion Lind", + "content": "The ONLY 1 good thing about this hotel is the location - it is located in the Condado area of San Juan, and located next door to the Wyndam Condado - which you have access to at the pool, casino, etc. I have read many reviews and prayed nothing would go wrong but my husband is more positive that I am so we stayed at the Regency - the price was right and since we stayed 5 nights and got 1 night free-we tried it. If you just need a place to rest your head, stay here - this MIGHT be the place for you. If you have a car - $5 per day. Positives: Staff is friendly and helpful, hotel is located next door to the Wyndam, balcony in the room with views, price was right for us since we did not want to spend a lot of money per night. Negatives: rooms smell of mildew, our toilet flodded day 2 -but we were switched to another room, whole hotel needs remodeling. Stay here at your own risk.", + "date": "2012-12-03 00:06:50 +0300", + "ratings": + { + "Overall": 2 + } + }, + { + "author": "Mr. Seamus Mertz", + "content": "I am so glad to see Tamarindo getting the reviews it deserves. This is a very special place - you won’t find anything else like it on Culebra. No wonder people return here year after year! This was our first time, but we will be back. When we arrived for a week in February, the place had been undergoing a process of rejuvenation. Everything was newly painted; repairs had been made; things looked fresh and clean. The new resident caretakers welcomed us and made us feel right at home. They were eager to accommodate our needs, and asked for our feedback. The atmosphere was friendly and relaxed. The three cottages on the “estateâ€� are spread out on a hillside facing Luis Pena Cay and the mountains of Puerto Rico beyond. It is a beautiful natural setting: quiet and peaceful. The sunsets and starry skies at night are spectacular. Each cottage has three units with (mostly) covered decks facing the water. The units are simple, but clean and adequate for two people. The covered deck is spacious and comfortable, with table, chairs and a charcoal grill. The bed is a standard double, and there is no hot water except in the shower, (which was not a problem for us, but something to be aware of). There are no cooking supplies provided (except coffee filters), but we found plenty of provisions at the local grocery stores and cooked most of our meals, with occasional forays out for dinner. The small freshwater pool down by the beach is a little oasis – a place to relax, read and cool off after a day in the sun and saltwater. Although the beach at Tamarindo is rough coral, and getting into the water at low tide takes some practice (there is one good spot at the north end by the pool), the snorkeling is incredible. Green turtles feed in the eelgrass just off the beach, and a colorful variety of reef fish make their home in the coral. There are several other good snorkeling spots, notably Carlos Rosario, and the wonderful sandy beach at Flamenco, very near by. Tamarindo is not a resort and Culebra is not a resort island. For those like us, who avoid resorts and instead seek simple, unpretentious places to explore, swim, snorkel, walk, read and relax in beauty and tranquility, it was perfect! Thank you, Dennis and Gail, for your wonderful hospitality. Hope to see you next year!", + "date": "2014-04-07 11:55:37 +0300", + "ratings": + { + "Cleanliness": 4, + "Location": 5, + "Overall": 5, + "Service": 5, + "Sleep Quality": 4, + "Value": 4 + } + }, + { + "author": "Shawn Yost", + "content": "Nice & simple. GREAT snorkeling. Good to be away from town. Stayed on second floor. Good balcony & view. Quiet.", + "date": "2012-08-01 06:48:04 +0300", + "ratings": + { + "Cleanliness": 5, + "Location": 5, + "Overall": 4, + "Rooms": 4, + "Service": 5, + "Sleep Quality": 4, + "Value": 4 + } + } + ], + "state": "California", + "title": "Los Angeles/Wilshire", + "tollfree": null, + "type": "hotel", + "url": "http://www.stayonbeverly.com", + "vacancy": false + } + ] + }, + "landmarks": + { + "query": "SELECT l.* FROM `travel-sample`.`inventory`.landmark l WHERE l.country='United States' AND l.city IN ['Atlanta', 'Detroit','Charlotte', 'Philedelphia', 'Chicago', 'Dallas-Fort Worth', 'Seattle', 'Los Angeles', 'New York', 'Miami' ] LIMIT 20", + "results": + [ + { + "activity": "see", + "address": "6925 Hollywood Blvd", + "alt": null, + "city": "Los Angeles", + "content": "The most famous movie theatre in the world, Grauman's Chinese Theatre opened in 1927 and is home to the cement footprints, handprints, and (in some cases) ''otherprints'' of many of history's most famous movie stars. The theatre is also a former home of the Oscars, and today hosts many movie premieres. The forecourt that showcases the star's prints is free to all visitors. Movies are shown for $10, and half-hour walking tours are available for $5.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.101913, + "lon": -118.340967 + }, + "hours": null, + "id": 11752, + "image": "https://en.wikivoyage.org/wiki/File:8/81/Grauman's Chinese Theatre 01.JPG", + "image_direct_url": "", + "name": "Grauman's Chinese Theater", + "phone": "+1 323 464-8111", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.manntheaters.com/chinese/", + "rating": 1, + "updated": "2022-01-01 13:00:07" + }, + { + "activity": "see", + "address": null, + "alt": null, + "city": "Los Angeles", + "content": "Hollywood's most recognizable landmark is easy to spot high up on Mount Lee in nearby [[Los Angeles/Northwest|Griffith Park]]. You can drive part way up for a closer look, but you can't hike all the way to the sign.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.134031, + "lon": -118.321724 + }, + "hours": null, + "id": 11753, + "image": "https://en.wikivoyage.org/wiki/File:b/b0/Hollywood Sign.JPG", + "image_direct_url": "", + "name": "Hollywood Sign", + "phone": null, + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": null, + "rating": 2, + "updated": "2022-01-01 14:00:07" + }, + { + "activity": "see", + "address": "1750 Vine St", + "alt": "Capitol Studios", + "city": "Los Angeles", + "content": "One of the most iconic buildings in Los Angeles. The circular tower—which contrary to popular belief was not intentionally designed to resemble a stack of records—is home to Capitol Records' west coast operations. Contained inside the building are the renowned Capitol Studios. Unfortunately, tours of the inside are no longer offered to the general public.", + "country": "United States", + "directions": "between Hollywood Blvd and Yucca St", + "email": null, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.103153, + "lon": -118.326365 + }, + "hours": null, + "id": 11754, + "image": "https://en.wikivoyage.org/wiki/File:8/86/USA LosAngeles CapitalRecordBldg.jpg", + "image_direct_url": "", + "name": "Capitol Records Building", + "phone": null, + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": null, + "rating": 3, + "updated": "2022-01-01 15:00:07" + }, + { + "activity": "see", + "address": "6767 Hollywood Blvd", + "alt": null, + "city": "Los Angeles", + "content": "The Hollywood Wax Museum is the longest running wax museum in the United States, with more than 45 years of continuous operation by the same owners since 1965 and featuring over 180 figures of celebrities.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.101757, + "lon": -118.338056 + }, + "hours": "10AM-midnight daily", + "id": 11755, + "image": "https://en.wikivoyage.org/wiki/File:8/80/HollywoodWaxMuseum 01.jpg", + "image_direct_url": "", + "name": "Hollywood Wax Museum", + "phone": "+1 323 462-8860", + "price": "Adults (13+) $15.95, children $8.95, 5 and under free. Discounts online", + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.hollywoodwaxmuseum.com/", + "rating": 4, + "updated": "2022-01-01 16:00:07" + }, + { + "activity": "see", + "address": "1355 N Caheunga Blvd", + "alt": null, + "city": "Los Angeles", + "content": "This museum is in the old Los Angeles City Fire Station 27, opened in 1930. It is fully restored to how it appeared in 1930 and contains historic fire apparatus.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.095878, + "lon": -118.329935 + }, + "hours": "Sa 10AM-4PM", + "id": 11756, + "image": "https://en.wikivoyage.org/wiki/File:e/e9/Engine Co. No. 27 (Los Angeles Fire Department Museum).JPG", + "image_direct_url": "", + "name": "Los Angeles Fire Department Hollywood Museum 27", + "phone": "+1 323 464-2727", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.lafdmuseum.org/museum_hollywood", + "rating": 5, + "updated": "2022-01-01 17:00:07" + }, + { + "activity": "see", + "address": "6780 Hollywood Blvd", + "alt": null, + "city": "Los Angeles", + "content": "Museum that focuses on the odd, the unusual and the unbelievable. Features interactive illusions and a gallery.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.101353, + "lon": -118.338445 + }, + "hours": null, + "id": 11757, + "image": "https://en.wikivoyage.org/wiki/File:0/0e/Ripleys Odditorium Hollywood.jpg", + "image_direct_url": "", + "name": "Ripley's Believe it or Not", + "phone": "+1 323 466-6335", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.ripleys.com", + "rating": 1, + "updated": "2022-02-02 17:00:07" + }, + { + "activity": "see", + "address": "along Hollywood Blvd and also Vine St", + "alt": null, + "city": "Los Angeles", + "content": "The Hollywood Walk of Fame consists of a series of stars embedded in the sidewalk to commemorate famous movie, radio, theatre, and TV personalities. Since 1960, over two thousand stars have been immortalized; the schedule for upcoming star ceremonies is listed on the Walk of Fame's website.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.101641, + "lon": -118.326654 + }, + "hours": null, + "id": 11758, + "image": "https://en.wikivoyage.org/wiki/File:4/45/Hollywood Walk of Fame.jpg", + "image_direct_url": "", + "name": "Walk of Fame", + "phone": null, + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.walkoffame.com/", + "rating": 2, + "updated": "2022-02-02 16:00:07" + }, + { + "activity": "do", + "address": "6801 Hollywood Blvd", + "alt": "formerly the Kodak Theatre", + "city": "Los Angeles", + "content": "Located at the Hollywood & Highland Center (see "Buy" below). Hosts a wide range of live performances, including the annual Academy Awards.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.102076, + "lon": -118.340181 + }, + "hours": null, + "id": 11759, + "image": null, + "name": "Dolby Theatre", + "phone": "+1 323 308-6300", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.dolbytheatre.com", + "rating": 3, + "updated": "2022-02-02 15:00:07" + }, + { + "activity": "do", + "address": "2301 N Highland Ave", + "alt": null, + "city": "Los Angeles", + "content": "America's most famous outdoor theatre hosts the Los Angeles Philharmonic Orchestra as well as numerous other concert events. Traffic and parking can be a nightmare, so the $5 round-trip [http://www.hollywoodbowl.com/visit/getting-to-the-bowl/bowlbus-shuttle public shuttles] are highly recommended.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.112691, + "lon": -118.338907 + }, + "hours": null, + "id": 11760, + "image": null, + "name": "Hollywood Bowl", + "phone": "+1 323 426-2829", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.hollywoodbowl.com/", + "rating": 4, + "updated": "2022-02-02 14:00:07" + }, + { + "activity": "do", + "address": "Hollywood Forever Cemetery, 6000 Santa Monica Blvd", + "alt": null, + "city": "Los Angeles", + "content": "The Cinespia film society screens creepy older movies (recent showings include ''The Shining'', ''Invasion of the Body Snatchers'' and ''Pee Wee's Big Adventure'') every Saturday during the summer on the Fairbanks Lawn in the Hollywood Forever Cemetery. Crowds can be huge, so arrive prior to gates opening if you want a good vantage point. Most people bring a picnic dinner, a drink (wine or beer allowed, no spirits), blanket, pillow (or low chair) and jacket. A DJ plays music prior to the showing to create a fun outdoor atmosphere. Tickets (including parking) often sell out and should therefore be purchased in advance through the Cinespia web site.", + "country": "United States", + "directions": null, + "email": "info@cinespia.org", + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.089097, + "lon": -118.318775 + }, + "hours": "Saturdays at 7PM, May–September", + "id": 11761, + "image": null, + "name": "Cemetery Movie Screenings", + "phone": null, + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://cinespia.org/", + "rating": 5, + "updated": "2022-02-02 13:00:07" + }, + { + "activity": "do", + "address": "Mulholland Drive", + "alt": null, + "city": "Los Angeles", + "content": "If you have a car, it is worth driving up to Mulholland Drive - the home of the stars. Apart from star seeking, the views across Los Angeles and San Fernando Valley are breathtaking.", + "country": "United States", + "directions": "Hollywood Hills", + "email": null, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.11669, + "lon": -118.34355 + }, + "hours": null, + "id": 11762, + "image": null, + "name": "Mulholland Drive", + "phone": null, + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": null, + "rating": 1, + "updated": "2022-03-03 13:00:07" + }, + { + "activity": "buy", + "address": "6801 Hollywood Blvd", + "alt": null, + "city": "Los Angeles", + "content": "Home of the Kodak Theatre (where the Oscars are held) and Grauman's Chinese Theatre, the 387,000-square-foot Hollywood & Highland center is also a major shopping destination.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.102188, + "lon": -118.340044 + }, + "hours": "M-Sa 10AM-10PM, Su 10AM-7PM", + "id": 11763, + "image": null, + "name": "Hollywood and Highland Center", + "phone": "+1 323 467-6412", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.hollywoodandhighland.com/", + "rating": 2, + "updated": "2022-03-03 14:00:07" + }, + { + "activity": "buy", + "address": "6400 W Sunset Blvd", + "alt": null, + "city": "Los Angeles", + "content": "The country's largest independent music store, Amoeba has three locations including Hollywood, [[Berkeley (California)|Berkeley]] and [[San Francisco]]. Prices are slightly higher than at the discount stores, but the selection is enormous and just about any obscure record you could imagine is to be found somewhere on the shelves.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.097753, + "lon": -118.329168 + }, + "hours": "M-Sa 10:30AM-11PM, Su 11AM-9PM", + "id": 11764, + "image": null, + "name": "Amoeba Music", + "phone": "+1 323 245-6400", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://amoeba.com", + "rating": 3, + "updated": "2022-03-03 15:00:07" + }, + { + "activity": "buy", + "address": "6751 Hollywood Blvd", + "alt": null, + "city": "Los Angeles", + "content": "During the golden years of Hollywood, all the superstars were wearing Fredericks, from Greta Garbo to Mae West to Marilyn Monroe. Today, the store is a lot less polished but still a good place to pick up glamorous lingerie.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.101749, + "lon": -118.337507 + }, + "hours": "M-Sa 10AM-9PM, Su 11AM-7PM", + "id": 11765, + "image": null, + "name": "Frederick's of Hollywood", + "phone": "+1 323 957-5953", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.fredericks.com", + "rating": 4, + "updated": "2022-03-03 16:00:07" + }, + { + "activity": "buy", + "address": "6767 W Sunset Blvd Suite 22", + "alt": null, + "city": "Los Angeles", + "content": "If you are tired of walking a long day in Hollywood. Iped offers 1 hour foot massage for as less as $25.", + "country": "United States", + "directions": "at Highland", + "email": null, + "geo": + { + "accuracy": "ROOFTOP", + "lat": 34.098293, + "lon": -118.338196 + }, + "hours": "11AM-10PM daily", + "id": 11766, + "image": null, + "name": "IPED Foot Spa", + "phone": "+1 323 466-1038", + "price": "25+", + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": null, + "rating": 5, + "updated": "2022-03-03 17:00:07" + }, + { + "activity": "eat", + "address": "7009 W Sunset Blvd", + "alt": null, + "city": "Los Angeles", + "content": "This hugely popular Southern California burger chain has a surprisingly basic menu, but serves up some of the most popular burgers around, and does burgers well. In-N-Out has a "secret menu," which was spread by word of mouth before the internet was around. Today it's listed on the company's website, but the secret menu is not listed on menus on-site.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.098222, + "lon": -118.341672 + }, + "hours": "Su-Th 8AM-1AM, F-Sa 8AM-1:30AM", + "id": 11767, + "image": null, + "name": "In-N-Out Burger", + "phone": null, + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.in-n-out.com/", + "rating": 1, + "updated": "2022-04-04 17:00:07" + }, + { + "activity": "eat", + "address": "5900 Hollywood Blvd", + "alt": null, + "city": "Los Angeles", + "content": "Home of the infamous Thai Elvis, who will serenade you through dinner. The decor is authentically cheesy and Elvis sings the hits. While plain dishes such as fried rice or pad Thai are nothing to write home about, the curries (duck and panang), pad prik king, and anything off the "wild things" menu are excellent choices.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.101486, + "lon": -118.31835 + }, + "hours": "Su-Th 11AM-midnight, F-Sa 11AM-2AM", + "id": 11768, + "image": null, + "name": "Palms Thai Restaurant", + "phone": "+1 323 462-5073", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.palmsthai.com", + "rating": 2, + "updated": "2022-04-04 16:00:07" + }, + { + "activity": "eat", + "address": "6145 Franklin Ave", + "alt": null, + "city": "Los Angeles", + "content": "Previously known as the Hollywood Hills Coffee Shop, this place has been popular for years, and the new owners have only improved it. Great selection of sandwiches, burgers, sweet potato French fries, coffee and tea. It's not unheard of to spot celebs here.", + "country": "United States", + "directions": "at Vista Del Mar Ave", + "email": null, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.105411, + "lon": -118.323841 + }, + "hours": "7AM-3AM daily", + "id": 11769, + "image": null, + "name": "101 Coffee Shop", + "phone": "+1 323 467-1175", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "https://www.facebook.com/101coffeeshop", + "rating": 3, + "updated": "2022-04-04 15:00:07" + }, + { + "activity": "eat", + "address": "1155 N Highland Ave", + "alt": null, + "city": "Los Angeles", + "content": "Great for lunch or dinner, excellent and fashionable food.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "APPROXIMATE", + "lat": 34.092347, + "lon": -118.338775 + }, + "hours": "Lunch: M-F 11:30AM-2:30PM; brunch: Sa-Su 10AM-2:30PM; dinner: M-Th 6AM-10PM, F-Sa 5:30PM-11PM, Su 5PM-9PM", + "id": 11770, + "image": null, + "name": "Ammo", + "phone": "+1 323 871-2666", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.ammocafe.com", + "rating": 4, + "updated": "2022-04-04 14:00:07" + }, + { + "activity": "eat", + "address": "7533 W Sunset Blvd", + "alt": null, + "city": "Los Angeles", + "content": "Everyone loves the Cheeb! A play on "cibo" (Italian for food), this place has great and creative food and a fun atmosphere. All-day breakfasts, excellent sandwiches, salads, pizzas by the foot and nice dinners to boot. Eat here for breakfast and you'll be back for lunch.", + "country": "United States", + "directions": null, + "email": null, + "geo": + { + "accuracy": "RANGE_INTERPOLATED", + "lat": 34.098197, + "lon": -118.353777 + }, + "hours": "8AM-11PM daily", + "id": 11771, + "image": null, + "name": "Cheebo", + "phone": "+1 323 850-7070", + "price": null, + "state": "California", + "title": "Hollywood", + "tollfree": null, + "type": "landmark", + "url": "http://www.cheebo.com", + "rating": 5, + "updated": "2022-04-04 13:00:07" + } + ] + } +} diff --git a/txcouchbase/__init__.py b/txcouchbase/__init__.py index b794fd409..77c08a5b1 100644 --- a/txcouchbase/__init__.py +++ b/txcouchbase/__init__.py @@ -1 +1,20 @@ -__version__ = '0.1.0' +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet import asyncioreactor + +from acouchbase import get_event_loop + +asyncioreactor.install(get_event_loop()) diff --git a/txcouchbase/analytics.py b/txcouchbase/analytics.py new file mode 100644 index 000000000..99952b9d4 --- /dev/null +++ b/txcouchbase/analytics.py @@ -0,0 +1,116 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet.defer import Deferred + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.analytics import AnalyticsRequestLogic + + +class AnalyticsRequest(AnalyticsRequestLogic): + def __init__(self, + connection, + loop, + query_params, + row_factory=lambda x: x, + **kwargs + ): + super().__init__(connection, query_params, row_factory=row_factory, **kwargs) + self._query_request_ftr = None + self._query_d = None + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @classmethod + def generate_analytics_request(cls, connection, loop, query_params, row_factory=lambda x: x, **kwargs): + return cls(connection, loop, query_params, row_factory=row_factory, **kwargs) + + def execute_analytics_query(self): + # if self._query_request_ftr is not None and self._query_request_ftr.done(): + if self.done_streaming: + raise AlreadyQueriedException() + + if self._query_request_ftr is None: + self._query_request_ftr = self.loop.create_future() + self._submit_query(callback=self._on_query_complete) + self._query_d = Deferred.fromFuture(self._query_request_ftr) + + return self._query_d + + def _on_query_complete(self, result): + self._loop.call_soon_threadsafe(self._query_request_ftr.set_result, result) + + def _get_metadata(self): + try: + query_response = next(self._streaming_result) + self._set_metadata(query_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + # def _get_metadata(self): + # if self._query_request_ftr.done(): + # if self._query_request_ftr.exception(): + # print('raising exception') + # raise self._query_request_ftr.exception() + # else: + # self._set_metadata() + # else: + # self._loop.run_until_complete(self._query_request_ftr) + # self._set_metadata() + + def __iter__(self): + return self + + def _get_next_row(self): + if self._done_streaming is True: + return + + row = next(self._streaming_result) + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopIteration + + return self.serializer.deserialize(row) + + def __next__(self): + try: + return self._get_next_row() + except StopIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/txcouchbase/binary_collection.py b/txcouchbase/binary_collection.py new file mode 100644 index 000000000..262f161e6 --- /dev/null +++ b/txcouchbase/binary_collection.py @@ -0,0 +1,69 @@ + +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Union) + +from twisted.internet.defer import Deferred + +from couchbase.result import CounterResult, MutationResult + +if TYPE_CHECKING: + from couchbase.options import (AppendOptions, + DecrementOptions, + IncrementOptions, + PrependOptions) + + +class BinaryCollection: + + def __init__(self, collection): + self._collection = collection + + def increment( + self, + key, # type: str + *opts, # type: IncrementOptions + **kwargs, # type: Any + ) -> Deferred[CounterResult]: + return self._collection._increment(key, *opts, **kwargs) + + def decrement( + self, + key, # type: str + *opts, # type: DecrementOptions + **kwargs, # type: Any + ) -> Deferred[CounterResult]: + return self._collection._decrement(key, *opts, **kwargs) + + def append( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: AppendOptions + **kwargs, # type: Any + ) -> Deferred[MutationResult]: + return self._collection._append(key, value, *opts, **kwargs) + + def prepend( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: PrependOptions + **kwargs, # type: Any + ) -> Deferred[MutationResult]: + return self._collection._prepend(key, value, *opts, **kwargs) diff --git a/txcouchbase/bucket.py b/txcouchbase/bucket.py index 7afeb4bf4..8b97740dd 100644 --- a/txcouchbase/bucket.py +++ b/txcouchbase/bucket.py @@ -1,397 +1,161 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. # -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. +# http://www.apache.org/licenses/LICENSE-2.0 # +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. -""" -This file contains the twisted-specific bits for the Couchbase client. -""" +from typing import (TYPE_CHECKING, + Any, + Dict) -from twisted.internet import reactor from twisted.internet.defer import Deferred -from twisted.python.failure import Failure - -from couchbase.async.bucket import AsyncBucket -from couchbase.async.view import AsyncViewBase -from couchbase.async.n1ql import AsyncN1QLRequest -from couchbase.async.events import EventQueue -from couchbase.exceptions import CouchbaseError -from txcouchbase.iops import v0Iops - - -class BatchedRowMixin(object): - def __init__(self, *args, **kwargs): - """ - Iterator/Container object for a single-call row-based results. - - This functions as an iterator over all results of the query, once the - query has been completed. - - Additional metadata may be obtained by examining the object. See - :class:`~couchbase.views.iterator.Views` for more details. - - You will normally not need to construct this object manually. - """ - self._d = Deferred() - self.__rows = None # likely a superlcass might have this? - - def _getDeferred(self): - return self._d - - def start(self): - super(BatchedRowMixin, self).start() - self.raw.rows_per_call = -1 - return self - - def on_rows(self, rowiter): - """ - Reimplemented from :meth:`~AsyncViewBase.on_rows` - """ - self.__rows = rowiter - self._d.callback(self) - self._d = None - - def on_error(self, ex): - """ - Reimplemented from :meth:`~AsyncViewBase.on_error` - """ - if self._d: - self._d.errback() - self._d = None - - def on_done(self): - """ - Reimplemented from :meth:`~AsyncViewBase.on_done` - """ - if self._d: - self._d.callback(self) - self._d = None - - def __iter__(self): - """ - Iterate over the rows in this resultset - """ - return iter(self.__rows) - - -class BatchedView(BatchedRowMixin, AsyncViewBase): - def __init__(self, *args, **kwargs): - AsyncViewBase.__init__(self, *args, **kwargs) - BatchedRowMixin.__init__(self, *args, **kwargs) - - -class BatchedN1QLRequest(BatchedRowMixin, AsyncN1QLRequest): - def __init__(self, *args, **kwargs): - AsyncN1QLRequest.__init__(self, *args, **kwargs) - BatchedRowMixin.__init__(self, *args, **kwargs) +from couchbase.logic.bucket import BucketLogic +from couchbase.logic.views import ViewQuery +from couchbase.result import PingResult, ViewResult +from txcouchbase.collection import Collection +from txcouchbase.logic import TxWrapper +from txcouchbase.management.collections import CollectionManager +from txcouchbase.scope import Scope +from txcouchbase.views import ViewRequest -class TxEventQueue(EventQueue): - """ - Subclass of EventQueue. This implements the relevant firing methods, - treating an 'Event' as a 'Deferred' - """ - def fire_async(self, event): - reactor.callLater(0, event.callback, None) +if TYPE_CHECKING: + from couchbase.options import PingOptions, ViewOptions - def call_single_success(self, event, *args, **kwargs): - event.callback(None) - def call_single_failure(self, event, *args, **kwargs): - event.errback(None) +class Bucket(BucketLogic): -class ConnectionEventQueue(TxEventQueue): - """ - For events fired upon connect - """ - def maybe_raise(self, err, *args, **kwargs): - if not err: - return - raise err + def __init__(self, cluster, bucket_name): + super().__init__(cluster, bucket_name) + self._close_d = None + self._connect_d = self._open_bucket() -class RawBucket(AsyncBucket): - def __init__(self, connstr=None, **kwargs): + @property + def loop(self): """ - Bucket subclass for Twisted. This inherits from the 'AsyncBucket' class, - but also adds some twisted-specific logic for hooking on a connection. + **INTERNAL** """ - if connstr and 'connstr' not in kwargs: - kwargs['connstr'] = connstr - iops = v0Iops(reactor) - super(RawBucket, self).__init__(iops=iops, **kwargs) + return self._cluster.loop - self._evq = { - 'connect': ConnectionEventQueue(), - '_dtor': TxEventQueue() - } + # def _connect_bucket(self): + # """ + # **INTERNAL** + # """ - self._conncb = self._evq['connect'] - self._dtorcb = self._evq['_dtor'] + # def cb(_, self): + # self._connection = self._cluster.connection + # return Deferred.fromFuture(super(Bucket, self)._open_or_close_bucket(open_bucket=True)) - def registerDeferred(self, event, d): - """ - Register a defer to be fired at the firing of a specific event. - - :param string event: Currently supported values are `connect`. Another - value may be `_dtor` which will register an event to fire when this - object has been completely destroyed. + # if not self._cluster.connected: + # d = self._cluster.on_connect() + # d.addCallback(cb, self) + # self._connect_d = d + # else: + # self._connect_d = Deferred.fromFuture(super()._open_or_close_bucket(open_bucket=True)) - :param event: The defered to fire when the event succeeds or failes - :type event: :class:`Deferred` + @TxWrapper.inject_bucket_open_callbacks() + def _open_bucket(self, **kwargs) -> Deferred: + super()._open_or_close_bucket(open_bucket=True, **kwargs) - If this event has already fired, the deferred will be triggered - asynchronously. + @TxWrapper.inject_close_callbacks() + def _close_bucket(self, **kwargs) -> Deferred: + super()._open_or_close_bucket(open_bucket=False, **kwargs) - Example:: + def on_connect(self) -> Deferred: + # only open if the connect deferred doesn't exist and we are not connected + if not self._connect_d and not self.connected: + self._connect_d = self._open_bucket() + self._close_d = None - def on_connect(*args): - print("I'm connected") - def on_connect_err(*args): - print("Connection failed") + return self._connect_d - d = Deferred() - cb.registerDeferred('connect', d) - d.addCallback(on_connect) - d.addErrback(on_connect_err) + def close(self) -> Deferred: + # only close if we are connected + if self.connected and not self._close_d: + self._close_d = self._close_bucket() + self._connect_d = None - :raise: :exc:`ValueError` if the event name is unrecognized - """ - try: - self._evq[event].schedule(d) - except KeyError: - raise ValueError("No such event type", event) - - def connect(self): - """ - Short-hand for the following idiom:: - - d = Deferred() - cb.registerDeferred('connect', d) - return d - - :return: A :class:`Deferred` - """ d = Deferred() - self.registerDeferred('connect', d) - return d - - def defer(self, opres): - """ - Converts a raw :class:`couchbase.results.AsyncResult` object - into a :class:`Deferred`. - - This is shorthand for the following "non-idiom":: - - d = Deferred() - opres = cb.upsert("foo", "bar") - opres.callback = d.callback - - def d_err(res, ex_type, ex_val, ex_tb): - d.errback(opres, ex_type, ex_val, ex_tb) - - opres.errback = d_err - return d - :param opres: The operation to wrap - :type opres: :class:`couchbase.results.AsyncResult` + def _on_okay(_): + super()._destroy_connection() + d.callback(None) - :return: a :class:`Deferred` object. + def _on_err(exc): + d.errback(exc) - Example:: + self._close_d.addCallback(_on_okay) + self._close_d.addErrback(_on_err) - opres = cb.upsert("foo", "bar") - d = cb.defer(opres) - def on_ok(res): - print("Result OK. Cas: {0}".format(res.cas)) - d.addCallback(opres) - - - """ - d = Deferred() - opres.callback = d.callback - - def _on_err(mres, ex_type, ex_val, ex_tb): - try: - raise ex_type(ex_val) - except CouchbaseError: - d.errback() - opres.errback = _on_err return d - def queryEx(self, viewcls, *args, **kwargs): - """ - Query a view, with the ``viewcls`` instance receiving events - of the query as they arrive. - - :param type viewcls: A class (derived from :class:`AsyncViewBase`) - to instantiate - - Other arguments are passed to the standard `query` method. - - This functions exactly like the :meth:`~couchbase.async.AsyncBucket.query` - method, except it automatically schedules operations if the connection - has not yet been negotiated. - """ - - kwargs['itercls'] = viewcls - o = super(AsyncBucket, self).query(*args, **kwargs) - if not self.connected: - self.connect().addCallback(lambda x: o.start()) - else: - o.start() - - return o - - def queryAll(self, *args, **kwargs): - """ - Returns a :class:`Deferred` object which will have its callback invoked - with a :class:`BatchedView` when the results are complete. - - Parameters follow conventions of - :meth:`~couchbase.bucket.Bucket.query`. - - Example:: - - d = cb.queryAll("beer", "brewery_beers") - def on_all_rows(rows): - for row in rows: - print("Got row {0}".format(row)) - - d.addCallback(on_all_rows) - - """ - - if not self.connected: - cb = lambda x: self.queryAll(*args, **kwargs) - return self.connect().addCallback(cb) - - kwargs['itercls'] = BatchedView - o = super(RawBucket, self).query(*args, **kwargs) - o.start() - return o._getDeferred() + def default_scope(self + ) -> Scope: + return self.scope(Scope.default_name()) + + def scope(self, name # type: str + ) -> Scope: + return Scope(self, name) + + def collection(self, collection_name): + scope = self.default_scope() + return scope.collection(collection_name) + + def default_collection(self): + scope = self.default_scope() + return scope.collection(Collection.default_name()) + + @TxWrapper.inject_cluster_callbacks(PingResult) + def ping(self, + *opts, # type: PingOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[PingResult]: + super().ping(*opts, **kwargs) + + def view_query(self, + design_doc, # type: str + view_name, # type: str + *view_options, # type: ViewOptions + **kwargs + ) -> Deferred[ViewResult]: + + query = ViewQuery.create_view_query_object( + self.name, design_doc, view_name, *view_options, **kwargs + ) + request = ViewRequest.generate_view_request(self.connection, + self.loop, + query.as_encodable(), + default_serializer=self.default_serializer) - def n1qlQueryEx(self, cls, *args, **kwargs): - """ - Execute a N1QL statement providing a custom handler for rows. - - This method allows you to define your own subclass (of - :class:`~AsyncN1QLRequest`) which can handle rows as they are - received from the network. - - :param cls: The subclass (not instance) to use - :param args: Positional arguments for the class constructor - :param kwargs: Keyword arguments for the class constructor - - .. seealso:: :meth:`queryEx`, around which this method wraps - """ - kwargs['itercls'] = cls - o = super(AsyncBucket, self).n1ql_query(*args, **kwargs) - if not self.connected: - self.connect().addCallback(lambda x: o.start()) - else: - o.start() - return o - - def n1qlQueryAll(self, *args, **kwargs): - if not self.connected: - cb = lambda x: self.n1qlQueryAll(*args, **kwargs) - return self.connect().addCallback(cb) - - kwargs['itercls'] = BatchedN1QLRequest - o = super(RawBucket, self).n1ql_query(*args, **kwargs) - o.start() - return o._getDeferred() - - -class Bucket(RawBucket): - def __init__(self, *args, **kwargs): - """ - This class inherits from :class:`RawBucket`. - In addition to the connection methods, this class' data access methods - return :class:`Deferreds` instead of :class:`AsyncResult` objects. - - Operations such as :meth:`get` or :meth:`set` will invoke the - :attr:`Deferred.callback` with the result object when the result is - complete, or they will invoke the :attr:`Deferred.errback` with an - exception (or :class:`Failure`) in case of an error. The rules of the - :attr:`~couchbase.connection.Connection.quiet` attribute for raising - exceptions apply to the invocation of the ``errback``. This means that - in the case where the synchronous client would raise an exception, - the Deferred API will have its ``errback`` invoked. Otherwise, the - result's :attr:`~couchbase.result.Result.success` field should be - inspected. - - - Likewise multi operations will be invoked with a - :class:`~couchbase.result.MultiResult` compatible object. - - Some examples: - - Using single items:: - - d_set = cb.upsert("foo", "bar") - d_get = cb.get("foo") - - def on_err_common(*args): - print("Got an error: {0}".format(args)), - def on_set_ok(res): - print("Successfuly set key with CAS {0}".format(res.cas)) - def on_get_ok(res): - print("Successfuly got key with value {0}".format(res.value)) - - d_set.addCallback(on_set_ok).addErrback(on_err_common) - d_get.addCallback(on_get_ok).addErrback(on_get_common) + d = Deferred() - # Note that it is safe to do this as operations performed on the - # same key are *always* performed in the order they were scheduled. + def _on_ok(_): + d.callback(ViewResult(request)) - Using multiple items:: + def _on_err(exc): + d.errback(exc) - d_get = cb.get_multi(("Foo", "bar", "baz)) - def on_mres(mres): - for k, v in mres.items(): - print("Got result for key {0}: {1}".format(k, v.value)) - d.addCallback(mres) + query_d = request.execute_view_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + def collections(self) -> CollectionManager: """ - super(Bucket, self).__init__(*args, **kwargs) - - def _connectSchedule(self, f, meth, *args, **kwargs): - qop = Deferred() - qop.addCallback(lambda x: f(meth, *args, **kwargs)) - self._evq['connect'].schedule(qop) - return qop + Get the CollectionManager. - def _wrap(self, meth, *args, **kwargs): - """ - Calls a given method with the appropriate arguments, or defers such - a call until the instance has been connected + :return: the :class:`.management.collections.CollectionManager` for this bucket. """ - if not self.connected: - return self._connectSchedule(self._wrap, meth, *args, **kwargs) - - opres = meth(self, *args, **kwargs) - return self.defer(opres) - + return CollectionManager(self.connection, self.loop, self.name) - ### Generate the methods - def _meth_factory(meth, name): - def ret(self, *args, **kwargs): - return self._wrap(meth, *args, **kwargs) - return ret - locals().update(RawBucket._gen_memd_wrappers(_meth_factory)) - for x in RawBucket._MEMCACHED_OPERATIONS: - if locals().get(x+'_multi', None): - locals().update({x+"Multi": locals()[x+"_multi"]}) +TxBucket = Bucket diff --git a/txcouchbase/cluster.py b/txcouchbase/cluster.py new file mode 100644 index 000000000..fe20cdc75 --- /dev/null +++ b/txcouchbase/cluster.py @@ -0,0 +1,386 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# used to allow for unquoted (i.e. forward reference, Python >= 3.7, PEP563) +from __future__ import annotations + +from asyncio import AbstractEventLoop +from time import perf_counter +from typing import (TYPE_CHECKING, + Any, + Dict) + +from twisted.internet import task +from twisted.internet.defer import Deferred, inlineCallbacks + +from acouchbase import get_event_loop +from couchbase.diagnostics import ClusterState, ServiceType +from couchbase.exceptions import UnAmbiguousTimeoutException +from couchbase.logic.analytics import AnalyticsQuery +from couchbase.logic.cluster import ClusterLogic +from couchbase.logic.n1ql import N1QLQuery +from couchbase.logic.search import SearchQueryBuilder +from couchbase.options import (DiagnosticsOptions, + PingOptions, + WaitUntilReadyOptions, + forward_args) +from couchbase.result import (AnalyticsResult, + ClusterInfoResult, + DiagnosticsResult, + PingResult, + QueryResult, + SearchResult) +from txcouchbase.analytics import AnalyticsRequest +from txcouchbase.bucket import Bucket +from txcouchbase.logic import TxWrapper +from txcouchbase.management.analytics import AnalyticsIndexManager +from txcouchbase.management.buckets import BucketManager +from txcouchbase.management.queries import QueryIndexManager +from txcouchbase.management.search import SearchIndexManager +from txcouchbase.management.users import UserManager +from txcouchbase.n1ql import N1QLRequest +from txcouchbase.search import FullTextSearchRequest + +if TYPE_CHECKING: + from datetime import timedelta + + from couchbase.options import (AnalyticsOptions, + ClusterOptions, + QueryOptions, + SearchOptions) + from couchbase.search import SearchQuery, SearchRequest + + +class Cluster(ClusterLogic): + + def __init__(self, + connstr, # type: str + *options, # type: ClusterOptions + **kwargs, # type: Dict[str, Any] + ) -> Cluster: + + self._loop = self._get_loop(kwargs.pop("loop", None)) + super().__init__(connstr, *options, **kwargs) + + self._close_d = None + self._twisted_loop = None + self._wur_state = {} + self._connect_d = self._connect() + + @property + def loop(self) -> AbstractEventLoop: + """ + **INTERNAL** + """ + return self._loop + + def _get_loop(self, loop=None) -> AbstractEventLoop: + # no need to check if the loop is running, that will + # be controlled by the reactor + if not loop: + loop = get_event_loop() + + return loop + + @TxWrapper.inject_connection_callbacks() + def _connect(self, **kwargs) -> Deferred: + """ + **INTERNAL** + """ + super()._connect_cluster(**kwargs) + + def on_connect(self) -> Deferred: + if not (self._connect_d or self.connected): + self._connect_d = self._connect() + self._close_d = None + + return self._connect_d + + def close(self) -> Deferred[None]: + if self.connected and not self._close_d: + self._close_d = self._close() + self._connect_d = None + + d = Deferred() + + def _on_okay(_): + super()._destroy_connection() + d.callback(None) + + def _on_err(exc): + d.errback(exc) + + self._close_d.addCallback(_on_okay) + self._close_d.addErrback(_on_err) + + return d + + @TxWrapper.inject_close_callbacks() + def _close(self, **kwargs) -> Deferred: + """ + **INTERNAL** + """ + super()._close_cluster(**kwargs) + + def bucket(self, bucket_name): + return Bucket(self, bucket_name) + + def cluster_info(self) -> Deferred[ClusterInfoResult]: + if not self.connected: + # @TODO(jc): chain?? + raise RuntimeError( + "Cluster is not connected, cannot get cluster info. " + "Use await cluster.on_connect() to connect a cluster.") + + return self._get_cluster_info() + + @TxWrapper.inject_cluster_callbacks(ClusterInfoResult, set_cluster_info=True) + def _get_cluster_info(self, **kwargs) -> Deferred[ClusterInfoResult]: + """**INTERNAL** + + use cluster_info() + + Returns: + Deferred: _description_ + """ + super()._get_cluster_info(**kwargs) + + @TxWrapper.inject_cluster_callbacks(PingResult, chain_connection=True) + def ping(self, + *opts, # type: PingOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[PingResult]: + return super().ping(*opts, **kwargs) + + @TxWrapper.inject_cluster_callbacks(DiagnosticsResult, chain_connection=True) + def diagnostics(self, + *opts, # type: DiagnosticsOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[DiagnosticsResult]: + + return super().diagnostics(*opts, **kwargs) + + @inlineCallbacks + def _wait_until_ready(self, service_types, desired_state): + diag_res = yield self.diagnostics() + endpoint_svc_types = set(map(lambda st: st.value, diag_res.endpoints.keys())) + if not endpoint_svc_types.issuperset(service_types): + yield self.ping(PingOptions(service_types=list(service_types))) + diag_res = yield self.diagnostics() + + if diag_res.state == desired_state: + self._twisted_loop.stop() + + self._wur_state["interval_millis"] += 500 + if self._wur_state["interval_millis"] > 1000: + self._wur_state["interval_millis"] = 1000 + + time_left = self._wur_state["timeout_millis"] - ((perf_counter() - self._wur_state["start"]) * 1000) + if self._wur_state["interval_millis"] > time_left: + self._wur_state["interval_millis"] = time_left + + if time_left <= 0: + raise UnAmbiguousTimeoutException(message="Desired state not found.") + + def wait_until_ready(self, + timeout, # type: timedelta + *opts, # type: WaitUntilReadyOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + final_args = forward_args(kwargs, *opts) + service_types = final_args.get("service_types", None) + if not service_types: + service_types = [ServiceType(st.value) for st in ServiceType] + + desired_state = final_args.get("desired_state", ClusterState.Online) + service_types_set = set(map(lambda st: st.value if isinstance(st, ServiceType) else st, service_types)) + self._wur_state = {} + + # @TODO: handle units + self._wur_state["timeout_millis"] = timeout.total_seconds() * 1000 + + self._wur_state["interval_millis"] = float(500) + self._wur_state["start"] = perf_counter() + + d = Deferred() + self._twisted_loop = task.LoopingCall(self._wait_until_ready, service_types_set, desired_state) + wur_d = self._twisted_loop.start(self._wur_state["interval_millis"] / 1000, now=True) + + def _on_okay(_): + d.callback(True) + + def _on_err(exc): + d.errback(exc) + + wur_d.addCallback(_on_okay) + wur_d.addErrback(_on_err) + return d + + def query( + self, + statement, # type: str + *options, # type: QueryOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[QueryResult]: + + query = N1QLQuery.create_query_object( + statement, *options, **kwargs) + request = N1QLRequest.generate_n1ql_request(self.connection, + self.loop, + query.params, + default_serializer=self.default_serializer) + d = Deferred() + + def _on_ok(_): + d.callback(QueryResult(request)) + + def _on_err(exc): + d.errback(exc) + + query_d = request.execute_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + + def analytics_query( + self, + statement, # type: str + *options, # type: AnalyticsOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[AnalyticsResult]: + + query = AnalyticsQuery.create_query_object( + statement, *options, **kwargs) + request = AnalyticsRequest.generate_analytics_request(self.connection, + self.loop, + query.params, + default_serializer=self.default_serializer) + d = Deferred() + + def _on_ok(_): + d.callback(AnalyticsResult(request)) + + def _on_err(exc): + d.errback(exc) + + query_d = request.execute_analytics_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + + def search_query(self, + index, # type: str + query, # type: SearchQuery + *options, # type: SearchOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[SearchResult]: + request_args = dict(default_serialize=self.default_serializer, + streaming_timeout=self.streaming_timeouts.get('search_timeout', None)) + query = SearchQueryBuilder.create_search_query_object(index, query, *options, **kwargs) + request = FullTextSearchRequest.generate_search_request(self.connection, + self.loop, + query.as_encodable(), + **request_args) + d = Deferred() + + def _on_ok(_): + d.callback(SearchResult(request)) + + def _on_err(exc): + d.errback(exc) + + query_d = request.execute_search_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + + def search(self, + index, # type: str + request, # type: SearchRequest + *options, # type: SearchOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[SearchResult]: + request_args = dict(default_serialize=self.default_serializer, + streaming_timeout=self.streaming_timeouts.get('search_timeout', None)) + query = SearchQueryBuilder.create_search_query_from_request(index, request, *options, **kwargs) + request = FullTextSearchRequest.generate_search_request(self.connection, + self.loop, + query.as_encodable(), + **request_args) + + d = Deferred() + + def _on_ok(_): + d.callback(SearchResult(request)) + + def _on_err(exc): + d.errback(exc) + + query_d = request.execute_search_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + + def buckets(self) -> BucketManager: + """ + Get the BucketManager. + + :return: A :class:`~.management.BucketManager` with which you can create or + modify buckets on the cluster. + """ + # TODO: AlreadyShutdownException? + return BucketManager(self.connection, self.loop) + + def users(self) -> UserManager: + """ + Get the UserManager. + + :return: A :class:`~.management.UserManager` with which you can create or update cluster users and roles. + """ + # TODO: AlreadyShutdownException? + return UserManager(self.connection, self.loop) + + def query_indexes(self) -> QueryIndexManager: + """ + Get the QueryIndexManager. + + :return: A :class:`~.management.queries.QueryIndexManager` with which you can + create or modify query indexes on the cluster. + """ + # TODO: AlreadyShutdownException? + return QueryIndexManager(self.connection, self.loop) + + def analytics_indexes(self) -> AnalyticsIndexManager: + """ + Get the AnalyticsIndexManager. + + :return: A :class:`~.management.AnalyticsIndexManager` with which you can create or modify analytics datasets, + dataverses, etc.. on the cluster. + """ + # TODO: AlreadyShutdownException? + return AnalyticsIndexManager(self.connection, self.loop) + + def search_indexes(self) -> SearchIndexManager: + """ + Get the SearchIndexManager. + + :return: A :class:`~.management.SearchIndexManager` with which you can create or modify analytics datasets, + dataverses, etc.. on the cluster. + """ + # TODO: AlreadyShutdownException? + return SearchIndexManager(self.connection, self.loop) + + +TxCluster = Cluster diff --git a/txcouchbase/collection.py b/txcouchbase/collection.py new file mode 100644 index 000000000..8344ff9f2 --- /dev/null +++ b/txcouchbase/collection.py @@ -0,0 +1,379 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# used to allow for unquoted (i.e. forward reference, Python >= 3.7, PEP563) +from __future__ import annotations + +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + Union) + +from twisted.internet.defer import Deferred + +from couchbase.logic.collection import CollectionLogic +from couchbase.options import forward_args +from couchbase.result import (CounterResult, + ExistsResult, + GetReplicaResult, + GetResult, + LookupInReplicaResult, + LookupInResult, + MutateInResult, + MutationResult) +from txcouchbase.binary_collection import BinaryCollection +from txcouchbase.logic import TxWrapper + +if TYPE_CHECKING: + from datetime import timedelta + + from couchbase._utils import JSONType + from couchbase.options import (AppendOptions, + DecrementOptions, + ExistsOptions, + GetAllReplicasOptions, + GetAndLockOptions, + GetAndTouchOptions, + GetAnyReplicaOptions, + GetOptions, + IncrementOptions, + InsertOptions, + LookupInAllReplicasOptions, + LookupInAnyReplicaOptions, + LookupInOptions, + MutateInOptions, + PrependOptions, + RemoveOptions, + ReplaceOptions, + TouchOptions, + UnlockOptions, + UpsertOptions) + from couchbase.subdocument import Spec + + +class Collection(CollectionLogic): + + def __init__(self, scope, name): + super().__init__(scope, name) + self._loop = scope.loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + def get(self, + key, # type: str + *opts, # type: GetOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[GetResult]: + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_internal(key, **final_args) + + @TxWrapper.inject_callbacks_and_decode(GetResult) + def _get_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Deferred[GetResult]: + super().get(key, **kwargs) + + def get_any_replica(self, + key, # type: str + *opts, # type: GetAnyReplicaOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[GetResult]: + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_any_replica_internal(key, **final_args) + + @TxWrapper.inject_callbacks_and_decode(GetReplicaResult) + def _get_any_replica_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Deferred[GetReplicaResult]: + super().get_any_replica(key, **kwargs) + + def get_all_replicas(self, + key, # type: str + *opts, # type: GetAllReplicasOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[Iterable[GetReplicaResult]]: + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_all_replicas_internal(key, **final_args) + + @TxWrapper.inject_callbacks_and_decode(GetReplicaResult) + def _get_all_replicas_internal( + self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> Deferred[Iterable[GetReplicaResult]]: + super().get_all_replicas(key, **kwargs) + + @TxWrapper.inject_callbacks(ExistsResult) + def exists( + self, + key, # type: str + *opts, # type: ExistsOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[ExistsResult]: + super().exists(key, *opts, **kwargs) + + @TxWrapper.inject_callbacks(MutationResult) + def insert( + self, # type: "Collection" + key, # type: str + value, # type: JSONType + *opts, # type: InsertOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[MutationResult]: + super().insert(key, value, *opts, **kwargs) + + @TxWrapper.inject_callbacks(MutationResult) + def upsert( + self, + key, # type: str + value, # type: JSONType + *opts, # type: UpsertOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[MutationResult]: + super().upsert(key, value, *opts, **kwargs) + + @TxWrapper.inject_callbacks(MutationResult) + def replace(self, + key, # type: str + value, # type: JSONType + *opts, # type: ReplaceOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + super().replace(key, value, *opts, **kwargs) + + @TxWrapper.inject_callbacks(MutationResult) + def remove(self, + key, # type: str + *opts, # type: RemoveOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + super().remove(key, *opts, **kwargs) + + @TxWrapper.inject_callbacks(MutationResult) + def touch(self, + key, # type: str + expiry, # type: timedelta + *opts, # type: TouchOptions + **kwargs, # type: Dict[str, Any] + ) -> MutationResult: + super().touch(key, expiry, *opts, **kwargs) + + def get_and_touch(self, + key, # type: str + expiry, # type: timedelta + *opts, # type: GetAndTouchOptions + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + # add to kwargs for conversion to int + kwargs["expiry"] = expiry + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_and_touch_internal(key, **final_args) + + @TxWrapper.inject_callbacks_and_decode(GetResult) + def _get_and_touch_internal(self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + super().get_and_touch(key, **kwargs) + + def get_and_lock( + self, + key, # type: str + lock_time, # type: timedelta + *opts, # type: GetAndLockOptions + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + # add to kwargs for conversion to int + kwargs["lock_time"] = lock_time + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + + return self._get_and_lock_internal(key, **final_args) + + @TxWrapper.inject_callbacks_and_decode(GetResult) + def _get_and_lock_internal(self, + key, # type: str + **kwargs, # type: Dict[str, Any] + ) -> GetResult: + super().get_and_lock(key, **kwargs) + + @TxWrapper.inject_callbacks(None) + def unlock(self, + key, # type: str + cas, # type: int + *opts, # type: UnlockOptions + **kwargs, # type: Dict[str, Any] + ) -> None: + super().unlock(key, cas, *opts, **kwargs) + + def lookup_in( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInOptions + **kwargs, # type: Dict[str, Any] + ) -> LookupInResult: + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_internal(key, spec, **final_args) + + @TxWrapper.inject_callbacks_and_decode(LookupInResult) + def _lookup_in_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Dict[str, Any] + ) -> LookupInResult: + super().lookup_in(key, spec, **kwargs) + + def lookup_in_any_replica( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInAnyReplicaOptions + **kwargs, # type: Dict[str, Any] + ) -> LookupInReplicaResult: + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_any_replica_internal(key, spec, **final_args) + + @TxWrapper.inject_callbacks_and_decode(LookupInReplicaResult) + def _lookup_in_any_replica_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Dict[str, Any] + ) -> LookupInReplicaResult: + super().lookup_in_any_replica(key, spec, **kwargs) + + def lookup_in_all_replicas( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: LookupInAllReplicasOptions + **kwargs, # type: Dict[str, Any] + ) -> Iterable[LookupInReplicaResult]: + final_args = forward_args(kwargs, *opts) + transcoder = final_args.get('transcoder', None) + if not transcoder: + transcoder = self.default_transcoder + final_args['transcoder'] = transcoder + return self._lookup_in_all_replicas_internal(key, spec, **final_args) + + @TxWrapper.inject_callbacks_and_decode(LookupInReplicaResult) + def _lookup_in_all_replicas_internal( + self, + key, # type: str + spec, # type: Iterable[Spec] + **kwargs, # type: Dict[str, Any] + ) -> Iterable[LookupInReplicaResult]: + super().lookup_in_all_replicas(key, spec, **kwargs) + + @TxWrapper.inject_callbacks(MutateInResult) + def mutate_in( + self, + key, # type: str + spec, # type: Iterable[Spec] + *opts, # type: MutateInOptions + **kwargs, # type: Dict[str, Any] + ) -> MutateInResult: + super().mutate_in(key, spec, *opts, **kwargs) + + def binary(self) -> BinaryCollection: + return BinaryCollection(self) + + @TxWrapper.inject_callbacks(MutationResult) + def _append( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: AppendOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[MutationResult]: + super().append(key, value, *opts, **kwargs) + + @TxWrapper.inject_callbacks(MutationResult) + def _prepend( + self, + key, # type: str + value, # type: Union[str,bytes,bytearray] + *opts, # type: PrependOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[MutationResult]: + super().prepend(key, value, *opts, **kwargs) + + @TxWrapper.inject_callbacks(CounterResult) + def _increment( + self, + key, # type: str + *opts, # type: IncrementOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[CounterResult]: + super().increment(key, *opts, **kwargs) + + @TxWrapper.inject_callbacks(CounterResult) + def _decrement( + self, + key, # type: str + *opts, # type: DecrementOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[CounterResult]: + super().decrement(key, *opts, **kwargs) + + @staticmethod + def default_name(): + return "_default" + + +TxCollection = Collection diff --git a/txcouchbase/connection.py b/txcouchbase/connection.py deleted file mode 100644 index 4805625ee..000000000 --- a/txcouchbase/connection.py +++ /dev/null @@ -1,18 +0,0 @@ -from txcouchbase.bucket import Bucket, RawBucket -from couchbase.bucket import _depr -from couchbase.connstr import convert_1x_args - -class TxAsyncConnection(RawBucket): - def __init__(self, bucket, **kwargs): - _depr('txcouchbase.connection.TxAsyncConnection', - 'txcouchbase.bucket.RawBucket') - kwargs = convert_1x_args(bucket, **kwargs) - super(TxAsyncConnection, self).__init__(**kwargs) - - -class Connection(Bucket): - def __init__(self, bucket, **kwargs): - _depr('txcouchbase.connection.Connection', - 'txcouchbase.bucket.Bucket') - kwargs = convert_1x_args(bucket, **kwargs) - super(Connection, self).__init__(**kwargs) diff --git a/txcouchbase/iops.py b/txcouchbase/iops.py deleted file mode 100644 index d91137d0b..000000000 --- a/txcouchbase/iops.py +++ /dev/null @@ -1,143 +0,0 @@ -from twisted.internet import error as TxErrors - -import couchbase._libcouchbase as LCB -from couchbase._libcouchbase import ( - Event, TimerEvent, IOEvent, - LCB_READ_EVENT, LCB_WRITE_EVENT, LCB_RW_EVENT, - PYCBC_EVSTATE_ACTIVE, - PYCBC_EVACTION_WATCH, - PYCBC_EVACTION_UNWATCH, - PYCBC_EVACTION_CLEANUP -) - -class TxIOEvent(IOEvent): - """ - IOEvent is a class implemented in C. It exposes - a 'fileno()' method, so we don't have to. - """ - __slots__ = [] - - def __init__(self): - super(TxIOEvent, self).__init__() - - def doRead(self): - self.ready_r() - - def doWrite(self): - self.ready_w() - - def connectionLost(self, reason): - if self.state == PYCBC_EVSTATE_ACTIVE: - self.ready_w() - - def logPrefix(self): - return "Couchbase IOEvent" - - -class TxTimer(TimerEvent): - __slots__ = ['_txev', 'lcb_active'] - - def __init__(self): - super(TxTimer, self).__init__() - self.lcb_active = False - self._txev = None - - - def _timer_wrap(self): - if not self.lcb_active: - return - - self.lcb_active = False - self.ready(0) - - - def schedule(self, usecs, reactor): - nsecs = usecs / 1000000.0 - if not self._txev or not self._txev.active(): - self._txev = reactor.callLater(nsecs, self._timer_wrap) - else: - self._txev.reset(nsecs) - - self.lcb_active = True - - def cancel(self): - self.lcb_active = False - - def cleanup(self): - if not self._txev: - return - - try: - self._txev.cancel() - except (TxErrors.AlreadyCalled, TxErrors.AlreadyCancelled): - pass - - self._txev = None - - -class v0Iops(object): - """ - IOPS Implementation to be used with Twisted's "FD" based reactors - """ - - __slots__ = [ 'reactor', 'is_sync', '_stop' ] - - def __init__(self, reactor, is_sync=False): - self.reactor = reactor - self.is_sync = is_sync - self._stop = False - - def update_event(self, event, action, flags): - """ - Called by libcouchbase to add/remove event watchers - """ - if action == PYCBC_EVACTION_UNWATCH: - if event.flags & LCB_READ_EVENT: - self.reactor.removeReader(event) - if event.flags & LCB_WRITE_EVENT: - self.reactor.removeWriter(event) - - elif action == PYCBC_EVACTION_WATCH: - if flags & LCB_READ_EVENT: - self.reactor.addReader(event) - if flags & LCB_WRITE_EVENT: - self.reactor.addWriter(event) - - if flags & LCB_READ_EVENT == 0: - self.reactor.removeReader(event) - if flags & LCB_WRITE_EVENT == 0: - self.reactor.removeWriter(event) - - def update_timer(self, timer, action, usecs): - """ - Called by libcouchbase to add/remove timers - """ - if action == PYCBC_EVACTION_WATCH: - timer.schedule(usecs, self.reactor) - - elif action == PYCBC_EVACTION_UNWATCH: - timer.cancel() - - elif action == PYCBC_EVACTION_CLEANUP: - timer.cleanup() - - def io_event_factory(self): - return TxIOEvent() - - def timer_event_factory(self): - return TxTimer() - - def start_watching(self): - """ - Start/Stop operations. This is a no-op in twisted because - it's a continuously running async loop - """ - if not self.is_sync: - return - - self._stop = False - while not self._stop: - self.reactor.doIteration(0) - - def stop_watching(self): - self._stop = True diff --git a/txcouchbase/logic/__init__.py b/txcouchbase/logic/__init__.py new file mode 100644 index 000000000..d63781166 --- /dev/null +++ b/txcouchbase/logic/__init__.py @@ -0,0 +1,16 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .wrappers import TxWrapper # noqa: F401 diff --git a/txcouchbase/logic/wrappers.py b/txcouchbase/logic/wrappers.py new file mode 100644 index 000000000..20f8eb428 --- /dev/null +++ b/txcouchbase/logic/wrappers.py @@ -0,0 +1,314 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from functools import partial, wraps + +from twisted.internet.defer import Deferred + +from acouchbase.logic import call_async_fn +from couchbase.exceptions import (PYCBC_ERROR_MAP, + CouchbaseException, + ErrorMapper, + ExceptionMap, + MissingConnectionException) +from couchbase.logic import decode_replicas, decode_value + + +class TxWrapper: + + @classmethod + def inject_connection_callbacks(cls): + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(res): + self._set_connection(res) + self.loop.call_soon_threadsafe(ft.set_result, True) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + call_async_fn(ft, self, fn, *args, **kwargs) + return Deferred.fromFuture(ft) + + return wrapped_fn + + return decorator + + @classmethod + def inject_close_callbacks(cls): + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(_): + self.loop.call_soon_threadsafe(ft.set_result, True) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + call_async_fn(ft, self, fn, *args, **kwargs) + return Deferred.fromFuture(ft) + + return wrapped_fn + + return decorator + + @classmethod + def chain_connect_futures(cls, ft, self, fn, cft, **kwargs): + """ + **INTERNAL** + """ + if cft.cancelled(): + ft.cancel() + return + + exc = cft.exception() + if exc is not None: + ft.set_exception(exc) + else: + self._connection = self._cluster.connection + args = kwargs.pop("args", None) + call_async_fn(ft, self, fn, *args, **kwargs) + + @classmethod + def inject_bucket_open_callbacks(cls): + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(_): + self._set_connected(True) + self.loop.call_soon_threadsafe(ft.set_result, True) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + cluster_conn_ft = Deferred.asFuture(self._cluster.on_connect(), self.loop) + kwargs["args"] = args + cluster_conn_ft.add_done_callback( + partial(cls.chain_connect_futures, ft, self, fn, **kwargs)) + else: + call_async_fn(ft, self, fn, *args, **kwargs) + + return Deferred.fromFuture(ft) + + return wrapped_fn + + return decorator + + @classmethod + def chain_futures(cls, ft, self, fn, cft, set_connection=False, **kwargs): + """ + **INTERNAL** + """ + if cft.cancelled(): + ft.cancel() + return + + exc = cft.exception() + if exc is not None: + ft.set_exception(exc) + else: + if set_connection is True: + # the bucket will set it's connection, need to make sure + # the connection is set w/ the scope and collection as well + self._scope._set_connection() + self._set_connection() + args = kwargs.pop("args", None) + call_async_fn(ft, self, fn, *args, **kwargs) + + @classmethod # noqa: C901 + def inject_cluster_callbacks(cls, return_cls, chain_connection=False, set_cluster_info=False): # noqa: C901 + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(res): + if return_cls is None: + retval = None + elif return_cls is True: + retval = res + else: + retval = return_cls(res) + + if set_cluster_info is True: + self._cluster_info = retval + + self.loop.call_soon_threadsafe(ft.set_result, retval) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + if chain_connection is True: + c_ft = Deferred.asFuture(self.on_connect(), self.loop) + # in order to keep arg passing simple, add operation args to kwargs + # this will keep the positional args passed to the callback only ones + # in the scope of handling logic w.r.t. to handling logic between the futures + # (the cluster connect future and the operation future) + kwargs["args"] = args + c_ft.add_done_callback( + partial(cls.chain_futures, ft, self, fn, **kwargs)) + else: + exc = MissingConnectionException('Not connected. Cannot perform operation.') + ft.set_exception(exc) + else: + call_async_fn(ft, self, fn, *args, **kwargs) + + return Deferred.fromFuture(ft) + + return wrapped_fn + + return decorator + + @classmethod # noqa: C901 + def inject_callbacks(cls, return_cls): # noqa: C901 + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(res): + if return_cls is None: + retval = None + elif return_cls is True: + retval = res + else: + retval = return_cls(res) + + self.loop.call_soon_threadsafe(ft.set_result, retval) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + bucket_conn_ft = Deferred.asFuture(self._scope._connect_bucket(), self.loop) + # in order to keep arg passing simple, add operation args to kwargs + # This allows the chain_futures callback to only worry about positional args + # outside the scope of the operation. + # Since, the kwargs passed to chain_futures only apply to the operation, chain_futures + # can easily determine what it needs to pass to the original wrapped fn + kwargs["args"] = args + bucket_conn_ft.add_done_callback( + partial(cls.chain_futures, ft, self, fn, set_connection=True, **kwargs)) + else: + call_async_fn(ft, self, fn, *args, **kwargs) + + return Deferred.fromFuture(ft) + + return wrapped_fn + + return decorator + + @classmethod # noqa: C901 + def inject_callbacks_and_decode(cls, return_cls): # noqa: C901 + + def decorator(fn): + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + transcoder = kwargs.pop('transcoder') + + def on_ok(res): + try: + is_subdoc = fn.__name__ in [ + '_lookup_in_internal', '_lookup_in_any_replica_internal', '_lookup_in_all_replicas_internal' + ] + + # special case for get_all_replicas and lookup_in_all_replicas + if fn.__name__ in ['_get_all_replicas_internal', '_lookup_in_all_replicas_internal']: + self.loop.call_soon_threadsafe( + ft.set_result, decode_replicas(transcoder, res, return_cls, is_subdoc=is_subdoc) + ) + return + + value = res.raw_result.get('value', None) + flags = res.raw_result.get('flags', None) + + res.raw_result['value'] = decode_value(transcoder, value, flags, is_subdoc=is_subdoc) + + if return_cls is None: + retval = None + elif return_cls is True: + retval = res + else: + retval = return_cls(res) + self.loop.call_soon_threadsafe(ft.set_result, retval) + except CouchbaseException as e: + self.loop.call_soon_threadsafe(ft.set_exception, e) + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(message=str(ex)) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + def on_err(exc): + excptn = ErrorMapper.build_exception(exc) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + bucket_conn_ft = Deferred.asFuture(self._scope._connect_bucket(), self.loop) + # in order to keep arg passing simple, add operation args to kwargs + # This allows the chain_futures callback to only worry about positional args + # outside the scope of the operation. + # Since, the kwargs passed to chain_futures only apply to the operation, chain_futures + # can easily determine what it needs to pass to the original wrapped fn + kwargs["args"] = args + bucket_conn_ft.add_done_callback( + partial(cls.chain_futures, ft, self, fn, set_connection=True, **kwargs)) + else: + call_async_fn(ft, self, fn, *args, **kwargs) + + return Deferred.fromFuture(ft) + + return wrapped_fn + + return decorator diff --git a/txcouchbase/management/__init__.py b/txcouchbase/management/__init__.py new file mode 100644 index 000000000..fc6346764 --- /dev/null +++ b/txcouchbase/management/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/txcouchbase/management/analytics.py b/txcouchbase/management/analytics.py new file mode 100644 index 000000000..1f65cb77a --- /dev/null +++ b/txcouchbase/management/analytics.py @@ -0,0 +1,220 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable, + Optional) + +from twisted.internet.defer import Deferred + +from couchbase.exceptions import InvalidArgumentException +from couchbase.management.logic.analytics_logic import (AnalyticsDataset, + AnalyticsDataType, + AnalyticsIndex, + AnalyticsLink, + AnalyticsManagerLogic) + +if TYPE_CHECKING: + from couchbase.management.options import (ConnectLinkOptions, + CreateAnalyticsIndexOptions, + CreateDatasetOptions, + CreateDataverseOptions, + CreateLinkAnalyticsOptions, + DisconnectLinkOptions, + DropAnalyticsIndexOptions, + DropDatasetOptions, + DropDataverseOptions, + DropLinkAnalyticsOptions, + GetAllAnalyticsIndexesOptions, + GetAllDatasetOptions, + GetLinksAnalyticsOptions, + GetPendingMutationsOptions, + ReplaceLinkAnalyticsOptions) + + +class AnalyticsIndexManager(AnalyticsManagerLogic): + + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + def create_dataverse(self, + dataverse_name, # type: str + options=None, # type: Optional[CreateDataverseOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + if not isinstance(dataverse_name, str): + raise ValueError("dataverse_name must be provided when creating an analytics dataverse.") + + return Deferred.fromFuture(super().create_dataverse(dataverse_name, options, **kwargs)) + + def drop_dataverse(self, + dataverse_name, # type: str + options=None, # type: Optional[DropDataverseOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + if not isinstance(dataverse_name, str): + raise ValueError("dataverse_name must be provided when dropping an analytics dataverse.") + + return Deferred.fromFuture(super().drop_dataverse(dataverse_name, options, **kwargs)) + + def create_dataset(self, + dataset_name, # type: str + bucket_name, # type: str + options=None, # type: Optional[CreateDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + if not isinstance(dataset_name, str): + raise ValueError("dataset_name must be provided when creating an analytics dataset.") + + if not isinstance(bucket_name, str): + raise ValueError("bucket_name must be provided when creating an analytics dataset.") + + return Deferred.fromFuture(super().create_dataset(dataset_name, bucket_name, options, **kwargs)) + + def drop_dataset(self, + dataset_name, # type: str + options=None, # type: Optional[DropDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + if not isinstance(dataset_name, str): + raise ValueError("dataset_name must be provided when dropping an analytics dataset.") + + return Deferred.fromFuture(super().drop_dataset(dataset_name, options, **kwargs)) + + def get_all_datasets(self, + options=None, # type: Optional[GetAllDatasetOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[Iterable[AnalyticsDataset]]: + + return Deferred.fromFuture(super().get_all_datasets(options, **kwargs)) + + def create_index(self, + index_name, # type: str + dataset_name, # type: str + fields, # type: Dict[str, AnalyticsDataType] + options=None, # type: Optional[CreateAnalyticsIndexOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + if not isinstance(index_name, str): + raise ValueError("index_name must be provided when creating an analytics index.") + + if not isinstance(dataset_name, str): + raise ValueError("dataset_name must be provided when creating an analytics index.") + + if fields is not None: + if not isinstance(fields, dict): + raise ValueError("fields must be provided when creating an analytics index.") + + if not all(map(lambda v: isinstance(v, AnalyticsDataType), fields.values())): + raise InvalidArgumentException("fields must all be an AnalyticsDataType.") + + return Deferred.fromFuture(super().create_index(index_name, dataset_name, fields, options, **kwargs)) + + def drop_index(self, + index_name, # type: str + dataset_name, # type: str + options=None, # type: Optional[DropAnalyticsIndexOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + if not isinstance(index_name, str): + raise ValueError("index_name must be provided when dropping an analytics index.") + + if not isinstance(dataset_name, str): + raise ValueError("dataset_name must be provided when dropping an analytics index.") + + return Deferred.fromFuture(super().drop_index(index_name, dataset_name, options, **kwargs)) + + def get_all_indexes(self, + options=None, # type: Optional[GetAllAnalyticsIndexesOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[Iterable[AnalyticsIndex]]: + + return Deferred.fromFuture(super().get_all_indexes(options, **kwargs)) + + def connect_link(self, + options=None, # type: Optional[ConnectLinkOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + return Deferred.fromFuture(super().connect_link(options, **kwargs)) + + def disconnect_link(self, + options=None, # type: Optional[DisconnectLinkOptions] + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + return Deferred.fromFuture(super().disconnect_link(options, **kwargs)) + + def get_pending_mutations(self, + options=None, # type: Optional[GetPendingMutationsOptions] + **kwargs # type: Dict[str, Any] + ) -> Dict[str, int]: + + return Deferred.fromFuture(super().get_pending_mutations(options, **kwargs)) + + def create_link( + self, + link, # type: AnalyticsLink + options=None, # type: Optional[CreateLinkAnalyticsOptions] + **kwargs + ) -> Deferred[None]: + return Deferred.fromFuture(super().create_link(link, options, **kwargs)) + + def replace_link( + self, + link, # type: AnalyticsLink + options=None, # type: Optional[ReplaceLinkAnalyticsOptions] + **kwargs + ) -> Deferred[None]: + return Deferred.fromFuture(super().replace_link(link, options, **kwargs)) + + def drop_link( + self, + link_name, # type: str + dataverse_name, # type: str + options=None, # type: Optional[DropLinkAnalyticsOptions] + **kwargs + ) -> Deferred[None]: + + if not isinstance(link_name, str): + raise ValueError("link_name must be provided when dropping an analytics link.") + + if not isinstance(dataverse_name, str): + raise ValueError("dataverse_name must be provided when dropping an analytics link.") + + return Deferred.fromFuture(super().drop_link(link_name, dataverse_name, options, **kwargs)) + + def get_links( + self, + options=None, # type: Optional[GetLinksAnalyticsOptions] + **kwargs + ) -> Deferred[Iterable[AnalyticsLink]]: + return Deferred.fromFuture(super().get_links(options, **kwargs)) diff --git a/txcouchbase/management/buckets.py b/txcouchbase/management/buckets.py new file mode 100644 index 000000000..2b7412830 --- /dev/null +++ b/txcouchbase/management/buckets.py @@ -0,0 +1,186 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict, + List) + +from twisted.internet.defer import Deferred + +from couchbase.management.logic import ManagementType +from couchbase.management.logic.buckets_logic import (BucketManagerLogic, + BucketSettings, + CreateBucketSettings) +from txcouchbase.management.logic.wrappers import TxMgmtWrapper + +if TYPE_CHECKING: + from couchbase.management.options import (CreateBucketOptions, + DropBucketOptions, + FlushBucketOptions, + GetAllBucketOptions, + GetBucketOptions, + UpdateBucketOptions) + + +class BucketManager(BucketManagerLogic): + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def create_bucket(self, + settings, # type: CreateBucketSettings + *options, # type: CreateBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + """Creates a new bucket. + + Args: + settings (:class:`.CreateBucketSettings`): The settings to use for the new bucket. + options (:class:`~couchbase.management.options.CreateBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + `Deferred`: An empty `Deferred` instance. + + Raises: + :class:`~couchbase.exceptions.BucketAlreadyExistsException`: If the bucket already exists. + :class:`~couchbase.exceptions.InvalidArgumentsException`: If an invalid type or value is provided for the + settings argument. + """ + super().create_bucket(settings, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def update_bucket(self, + settings, # type: BucketSettings + *options, # type: UpdateBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + """Update the settings for an existing bucket. + + Args: + settings (:class:`.BucketSettings`): The settings to use for the new bucket. + options (:class:`~couchbase.management.options.UpdateBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + `Deferred`: An empty `Deferred` instance. + + Raises: + :class:`~couchbase.exceptions.InvalidArgumentsException`: If an invalid type or value is provided for the + settings argument. + """ + super().update_bucket(settings, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def drop_bucket(self, + bucket_name, # type: str + *options, # type: DropBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + """Drops an existing bucket. + + Args: + bucket_name (str): The name of the bucket to drop. + options (:class:`~couchbase.management.options.DropBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + `Deferred`: An empty `Deferred` instance. + + Raises: + :class:`~couchbase.exceptions.BucketDoesNotExistException`: If the bucket does not exist. + """ + super().drop_bucket(bucket_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(BucketSettings, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def get_bucket(self, + bucket_name, # type: str + *options, # type: GetBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[BucketSettings]: + """Fetches the settings in use for a specified bucket. + + Args: + bucket_name (str): The name of the bucket to fetch. + options (:class:`~couchbase.management.options.GetBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Deferred[:class:`.BucketSettings`]: The settings of the specified bucket. + + Raises: + :class:`~couchbase.exceptions.BucketDoesNotExistException`: If the bucket does not exist. + """ + super().get_bucket(bucket_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(BucketSettings, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def get_all_buckets(self, + *options, # type: GetAllBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[List[BucketSettings]]: + """Returns a list of existing buckets in the cluster. + + Args: + options (:class:`~couchbase.management.options.GetAllBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Deferred[List[:class:`.BucketSettings`]]: A list of existing buckets in the cluster. + """ + super().get_all_buckets(*options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.BucketMgmt, BucketManagerLogic._ERROR_MAPPING) + def flush_bucket(self, + bucket_name, # type: str + *options, # type: FlushBucketOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + """Flushes the bucket, deleting all the existing data that is stored in it. + + Args: + bucket_name (str): The name of the bucket to flush. + options (:class:`~couchbase.management.options.FlushBucketOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + `Deferred`: An empty `Deferred` instance. + + Raises: + :class:`~couchbase.exceptions.BucketDoesNotExistException`: If the bucket does not exist. + :class:`~couchbase.exceptions.BucketNotFlushableException`: If the bucket's settings have + flushing disabled. + """ + super().flush_bucket(bucket_name, *options, **kwargs) diff --git a/txcouchbase/management/collections.py b/txcouchbase/management/collections.py new file mode 100644 index 000000000..651288dab --- /dev/null +++ b/txcouchbase/management/collections.py @@ -0,0 +1,254 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from inspect import Parameter, Signature +from typing import (Any, + Dict, + Iterable, + Optional) + +from twisted.internet.defer import Deferred + +from couchbase._utils import OverloadType +from couchbase.logic.supportability import Supportability +from couchbase.management.logic import ManagementType +from couchbase.management.logic.collections_logic import (CollectionManagerLogic, + CollectionSpec, + CreateCollectionSettings, + ScopeSpec, + UpdateCollectionSettings) +from couchbase.management.options import (CreateCollectionOptions, + CreateScopeOptions, + DropCollectionOptions, + DropScopeOptions, + GetAllScopesOptions, + UpdateCollectionOptions) +from txcouchbase.management.logic.wrappers import TxMgmtWrapper + + +class CollectionManager(CollectionManagerLogic): + + def __init__(self, connection, loop, bucket_name): + super().__init__(connection, bucket_name) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def create_scope(self, + scope_name, # type: str + *options, # type: CreateScopeOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + """Creates a new scope. + + Args: + scope_name (str): The name of the scope. + options (:class:`~couchbase.management.options.CreateScopeOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + `Deferred`: An empty `Deferred` instance. + + Raises: + :class:`~couchbase.exceptions.ScopeAlreadyExistsException`: If the scope already exists. + """ + super().create_scope(scope_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def drop_scope(self, + scope_name, # type: str + *options, # type: DropScopeOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + """Drops an existing scope. + + Args: + scope_name (str): The name of the scope. + options (:class:`~couchbase.management.options.DropScopeOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + `Deferred`: An empty `Deferred` instance. + + Raises: + :class:`~couchbase.exceptions.ScopeNotFoundException`: If the scope does not exist. + """ + super().drop_scope(scope_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks((ScopeSpec, CollectionSpec), + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING) + def get_all_scopes(self, + *options, # type: GetAllScopesOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Iterable[ScopeSpec]]: + """Returns all configured scopes along with their collections. + + Args: + scope_name (str): The name of the scope. + options (:class:`~couchbase.management.options.GetAllScopesOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + Deferred[Iterable[:class:`.ScopeSpec`]]: A list of all configured scopes. + """ + super().get_all_scopes(*options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.SECONDARY) + def create_collection(self, # noqa: F811 + collection: CollectionSpec, + *options: CreateCollectionOptions, + **kwargs: Dict[str, Any], + ) -> Deferred[None]: + Supportability.method_signature_deprecated( + 'create_collection', + Signature( + parameters=[ + Parameter('collection', Parameter.POSITIONAL_OR_KEYWORD, annotation=CollectionSpec), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=CreateCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=Deferred[None] + ), + Signature( + parameters=[ + Parameter('scope_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('collection_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('settings', Parameter.POSITIONAL_OR_KEYWORD, + annotation=Optional[CreateCollectionSettings]), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=CreateCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=Deferred[None] + ) + ) + settings = None + if collection.max_expiry is not None: + settings = CreateCollectionSettings(max_expiry=collection.max_expiry) + super().create_collection(collection.scope_name, collection.name, settings, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.DEFAULT) + def create_collection(self, # noqa: F811 + scope_name: str, + collection_name: str, + settings: Optional[CreateCollectionSettings] = None, + *options: CreateCollectionOptions, + **kwargs: Dict[str, Any] + ): + """Creates a new collection in a specified scope. + + Args: + collection (:class:`.CollectionSpec`): The collection details. + options (:class:`~couchbase.management.options.CreateCollectionOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + `Deferred`: An empty `Deferred` instance. + + Raises: + :class:`~couchbase.exceptions.CollectionAlreadyExistsException`: If the collection already exists. + :class:`~couchbase.exceptions.ScopeNotFoundException`: If the scope does not exist. + """ + super().create_collection(scope_name, collection_name, settings, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.CollectionMgmt, CollectionManagerLogic._ERROR_MAPPING) + def update_collection(self, + scope_name: str, + collection_name: str, + settings: UpdateCollectionSettings, + *options: UpdateCollectionOptions, + **kwargs: Dict[str, Any] + ) -> Deferred[None]: + super().update_collection(scope_name, collection_name, settings, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.SECONDARY) + def drop_collection(self, # noqa: F811 + collection: CollectionSpec, + *options: DropCollectionOptions, + **kwargs: Dict[str, Any] + ) -> Deferred[None]: + Supportability.method_signature_deprecated( + 'drop_collection', + Signature( + parameters=[ + Parameter('self', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('collection', Parameter.POSITIONAL_OR_KEYWORD, annotation=CollectionSpec), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=DropCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=Deferred[None], + ), + Signature( + parameters=[ + Parameter('self', Parameter.POSITIONAL_OR_KEYWORD), + Parameter('scope_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('collection_name', Parameter.POSITIONAL_OR_KEYWORD, annotation=str), + Parameter('options', Parameter.VAR_POSITIONAL, annotation=DropCollectionOptions), + Parameter('kwargs', Parameter.VAR_KEYWORD, annotation=Dict[str, Any]), + ], + return_annotation=Deferred[None] + ) + ) + super().drop_collection(collection.scope_name, collection.name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, + ManagementType.CollectionMgmt, + CollectionManagerLogic._ERROR_MAPPING, + OverloadType.DEFAULT) + def drop_collection(self, # noqa: F811 + scope_name: str, + collection_name: str, + *options: DropCollectionOptions, + **kwargs: Dict[str, Any] + ) -> Deferred[None]: + """Drops a collection from a scope. + + Args: + collection (:class:`.CollectionSpec`): The collection details. + options (:class:`~couchbase.management.options.DropCollectionOptions`): Optional parameters for this + operation. + **kwargs (Dict[str, Any]): keyword arguments that can be used as optional parameters + for this operation. + + Returns: + `Deferred`: An empty `Deferred` instance. + + Raises: + :class:`~couchbase.exceptions.CollectionNotFoundException`: If the collection does not exist. + """ + super().drop_collection(scope_name, collection_name, *options, **kwargs) diff --git a/txcouchbase/management/logic/__init__.py b/txcouchbase/management/logic/__init__.py new file mode 100644 index 000000000..fc6346764 --- /dev/null +++ b/txcouchbase/management/logic/__init__.py @@ -0,0 +1,14 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/txcouchbase/management/logic/wrappers.py b/txcouchbase/management/logic/wrappers.py new file mode 100644 index 000000000..a9530b0b3 --- /dev/null +++ b/txcouchbase/management/logic/wrappers.py @@ -0,0 +1,106 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + +from functools import wraps + +from twisted.internet.defer import Deferred + +from acouchbase.logic import call_async_fn +from couchbase._utils import Overload, OverloadType +from couchbase.exceptions import ErrorMapper, MissingConnectionException +from couchbase.management.logic import (ManagementType, + handle_analytics_index_mgmt_response, + handle_bucket_mgmt_response, + handle_collection_mgmt_response, + handle_eventing_function_mgmt_response, + handle_query_index_mgmt_response, + handle_search_index_mgmt_response, + handle_user_mgmt_response, + handle_view_index_mgmt_response) + + +def build_mgmt_exception(exc, mgmt_type, error_map): + return ErrorMapper.build_exception(exc, mapping=error_map) + + +mgmt_overload_registry = {} + + +class TxMgmtWrapper: + @classmethod # noqa: C901 + def inject_callbacks(cls, return_cls, mgmt_type, error_map, overload_type=None): # noqa: C901 + + def decorator(fn): + if overload_type is not None: + mgmt_overload = mgmt_overload_registry.get(fn.__qualname__) + if mgmt_overload is None: + mgmt_overload = mgmt_overload_registry[fn.__qualname__] = Overload(fn.__qualname__) + if overload_type is OverloadType.DEFAULT: + mgmt_overload.register_default(fn) + else: + mgmt_overload.register(fn) + + @wraps(fn) + def wrapped_fn(self, *args, **kwargs): + ft = self.loop.create_future() + + def on_ok(ret): + if return_cls is None: + retval = None + elif return_cls is True: + retval = ret + else: + if mgmt_type == ManagementType.BucketMgmt: + retval = handle_bucket_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.CollectionMgmt: + retval = handle_collection_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.UserMgmt: + retval = handle_user_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.QueryIndexMgmt: + retval = handle_query_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.AnalyticsIndexMgmt: + retval = handle_analytics_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.SearchIndexMgmt: + retval = handle_search_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.ViewIndexMgmt: + retval = handle_view_index_mgmt_response(ret, fn.__name__, return_cls) + elif mgmt_type == ManagementType.EventingFunctionMgmt: + retval = handle_eventing_function_mgmt_response(ret, fn.__name__, return_cls) + else: + retval = None + + self.loop.call_soon_threadsafe(ft.set_result, retval) + + def on_err(exc, exc_info=None, error_msg=None): + excptn = build_mgmt_exception(exc, mgmt_type, error_map) + self.loop.call_soon_threadsafe(ft.set_exception, excptn) + + kwargs["callback"] = on_ok + kwargs["errback"] = on_err + + if not self._connection: + exc = MissingConnectionException('Not connected. Cannot perform bucket management operation.') + ft.set_exception(exc) + else: + func = mgmt_overload_registry.get(fn.__qualname__, fn) + call_async_fn(ft, self, func, *args, **kwargs) + + return Deferred.fromFuture(ft) + + return wrapped_fn + + return decorator diff --git a/txcouchbase/management/queries.py b/txcouchbase/management/queries.py new file mode 100644 index 000000000..34e173d44 --- /dev/null +++ b/txcouchbase/management/queries.py @@ -0,0 +1,198 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# from datetime import timedelta +# from time import perf_counter, sleep +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable) + +from twisted.internet.defer import Deferred + +from couchbase.management.logic.query_index_logic import QueryIndex, QueryIndexManagerLogic + +# from couchbase.exceptions import QueryIndexNotFoundException, WatchQueryIndexTimeoutException +from couchbase.management.options import GetAllQueryIndexOptions + +# from couchbase.options import forward_args + +if TYPE_CHECKING: + from couchbase.management.options import (BuildDeferredQueryIndexOptions, + CreatePrimaryQueryIndexOptions, + CreateQueryIndexOptions, + DropPrimaryQueryIndexOptions, + DropQueryIndexOptions) + + +class QueryIndexManager(QueryIndexManagerLogic): + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + def create_index(self, + bucket_name, # type: str + index_name, # type: str + keys, # type: Iterable[str] + *options, # type: CreateQueryIndexOptions + **kwargs + ) -> Deferred[None]: + + if not isinstance(bucket_name, str): + raise ValueError("bucket_name must be provided when creating a secondary index.") + if not isinstance(index_name, str): + raise ValueError("index_name must be provided when creating a secondary index.") + if not isinstance(keys, (list, tuple)): + raise ValueError("keys must be provided when creating a secondary index.") + + return Deferred.fromFuture(super().create_index(bucket_name, index_name, keys, *options, **kwargs)) + + def create_primary_index(self, + bucket_name, # type: str + *options, # type: CreatePrimaryQueryIndexOptions + **kwargs + ) -> Deferred[None]: + + if not isinstance(bucket_name, str): + raise ValueError("bucket_name must be provided when creating a primary index.") + + return Deferred.fromFuture(super().create_primary_index(bucket_name, *options, **kwargs)) + + def drop_index(self, + bucket_name, # type: str + index_name, # type: str + *options, # type: DropQueryIndexOptions + **kwargs) -> Deferred[None]: + + if not isinstance(bucket_name, str): + raise ValueError("bucket_name must be provided when dropping a secondary index.") + if not isinstance(index_name, str): + raise ValueError("index_name must be provided when dropping a secondary index.") + + return Deferred.fromFuture(super().drop_index(bucket_name, index_name, *options, **kwargs)) + + def drop_primary_index(self, + bucket_name, # type: str + *options, # type: DropPrimaryQueryIndexOptions + **kwargs) -> Deferred[None]: + + if not isinstance(bucket_name, str): + raise ValueError("bucket_name must be provided when dropping a primary index.") + + return Deferred.fromFuture(super().drop_primary_index(bucket_name, *options, **kwargs)) + + def get_all_indexes(self, + bucket_name, # type: str + *options, # type: GetAllQueryIndexOptions + **kwargs # type: Dict[str,Any] + ) -> Deferred[Iterable[QueryIndex]]: + + if not isinstance(bucket_name, str): + raise ValueError("bucket_name must be provided when dropping a secondary index.") + + return Deferred.fromFuture(super().get_all_indexes(bucket_name, *options, **kwargs)) + + def build_deferred_indexes(self, + bucket_name, # type: str + *options, # type: BuildDeferredQueryIndexOptions + **kwargs + ) -> Deferred[None]: + """ + Build Deferred builds all indexes which are currently in deferred state. + + :param str bucket_name: name of the bucket. + :param BuildDeferredQueryIndexOptions options: Options for building deferred indexes. + :param Any kwargs: Override corresponding value in options. + :raise: InvalidArgumentsException + + """ + + if not isinstance(bucket_name, str): + raise ValueError("bucket_name must be provided when building deferred indexes.") + + return Deferred.fromFuture(super().build_deferred_indexes(bucket_name, *options, **kwargs)) + + # def watch_indexes(self, # noqa: C901 + # bucket_name, # type: str + # index_names, # type: Iterable[str] + # *options, # type: WatchQueryIndexOptions + # **kwargs # type: Dict[str,Any] + # ) -> Deferred[None]: + # """ + # Watch polls indexes until they are online. + + # :param str bucket_name: name of the bucket. + # :param Iterable[str] index_names: name(s) of the index(es). + # :param WatchQueryIndexOptions options: Options for request to watch indexes. + # :param Any kwargs: Override corresponding valud in options. + # :raises: QueryIndexNotFoundException + # :raises: WatchQueryIndexTimeoutException + # """ + # final_args = forward_args(kwargs, *options) + + # if final_args.get("watch_primary", False): + # index_names.append("#primary") + + # timeout = final_args.get("timeout", None) + # if not timeout: + # raise ValueError( + # 'Must specify a timeout condition for watch indexes') + + # def check_indexes(index_names, indexes): + # for idx_name in index_names: + # match = next((i for i in indexes if i.name == idx_name), None) + # if not match: + # raise QueryIndexNotFoundException( + # "Cannot find index with name: {}".format(idx_name)) + + # return all(map(lambda i: i.state == "online", indexes)) + + # # timeout is converted to microsecs via final_args() + # timeout_millis = timeout / 1000 + + # interval_millis = float(50) + # start = perf_counter() + # time_left = timeout_millis + # while True: + + # opts = GetAllQueryIndexOptions( + # timeout=timedelta(milliseconds=time_left)) + + # indexes = self.get_all_indexes(bucket_name, opts) + + # all_online = check_indexes(index_names, indexes) + # if all_online: + # break + + # interval_millis += 500 + # if interval_millis > 1000: + # interval_millis = 1000 + + # time_left = timeout_millis - ((perf_counter() - start) * 1000) + # if interval_millis > time_left: + # interval_millis = time_left + + # if time_left <= 0: + # raise WatchQueryIndexTimeoutException( + # "Failed to find all indexes online within the alloted time.") + + # sleep(interval_millis / 1000) diff --git a/txcouchbase/management/search.py b/txcouchbase/management/search.py new file mode 100644 index 000000000..a26828076 --- /dev/null +++ b/txcouchbase/management/search.py @@ -0,0 +1,298 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable) + +from twisted.internet.defer import Deferred + +from couchbase.management.logic import ManagementType +from couchbase.management.logic.search_index_logic import SearchIndex, SearchIndexManagerLogic +from txcouchbase.management.logic.wrappers import TxMgmtWrapper + +if TYPE_CHECKING: + from couchbase.management.options import (AllowQueryingSearchIndexOptions, + AnalyzeDocumentSearchIndexOptions, + DisallowQueryingSearchIndexOptions, + DropSearchIndexOptions, + FreezePlanSearchIndexOptions, + GetAllSearchIndexesOptions, + GetAllSearchIndexStatsOptions, + GetSearchIndexedDocumentsCountOptions, + GetSearchIndexOptions, + GetSearchIndexStatsOptions, + PauseIngestSearchIndexOptions, + ResumeIngestSearchIndexOptions, + UnfreezePlanSearchIndexOptions, + UpsertSearchIndexOptions) + + +class SearchIndexManager(SearchIndexManagerLogic): + def __init__(self, + connection, + loop + ): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def upsert_index(self, + index, # type: SearchIndex + *options, # type: UpsertSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().upsert_index(index, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + *options, # type: DropSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + super().drop_index(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(SearchIndex, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index(self, + index_name, # type: str + *options, # type: GetSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[SearchIndex]: + super().get_index(index_name, *options, **kwargs) + + def get_all_indexes(self, + *options, # type: GetAllSearchIndexesOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Iterable[SearchIndex]]: + super().get_all_indexes(*options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(int, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_indexed_documents_count(self, + index_name, # type: str + *options, # type: GetSearchIndexedDocumentsCountOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[int]: + super().get_indexed_documents_count(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def pause_ingest(self, + index_name, # type: str + *options, # type: PauseIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().pause_ingest(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def resume_ingest(self, + index_name, # type: str + *options, # type: ResumeIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().resume_ingest(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def allow_querying(self, + index_name, # type: str + *options, # type: AllowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().allow_querying(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def disallow_querying(self, + index_name, # type: str + *options, # type: DisallowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().disallow_querying(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def freeze_plan(self, + index_name, # type: str + *options, # type: FreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().freeze_plan(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def unfreeze_plan(self, + index_name, # type: str + *options, # type: UnfreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().unfreeze_plan(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def analyze_document(self, + index_name, # type: str + document, # type: Any + *options, # type: AnalyzeDocumentSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Dict[str, Any]]: + super().analyze_document(index_name, document, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index_stats(self, + index_name, # type: str + *options, # type: GetSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Dict[str, Any]]: + super().get_index_stats(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_index_stats(self, + *options, # type: GetAllSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Dict[str, Any]]: + super().get_all_index_stats(*options, **kwargs) + + +class ScopeSearchIndexManager(SearchIndexManagerLogic): + + def __init__(self, + connection, + loop, + bucket_name, # type: str + scope_name # type: str + ): + super().__init__(connection, bucket_name=bucket_name, scope_name=scope_name) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def upsert_index(self, + index, # type: SearchIndex + *options, # type: UpsertSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().upsert_index(index, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def drop_index(self, + index_name, # type: str + *options, # type: DropSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + super().drop_index(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(SearchIndex, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index(self, + index_name, # type: str + *options, # type: GetSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[SearchIndex]: + super().get_index(index_name, *options, **kwargs) + + def get_all_indexes(self, + *options, # type: GetAllSearchIndexesOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Iterable[SearchIndex]]: + super().get_all_indexes(*options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(int, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_indexed_documents_count(self, + index_name, # type: str + *options, # type: GetSearchIndexedDocumentsCountOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[int]: + super().get_indexed_documents_count(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def pause_ingest(self, + index_name, # type: str + *options, # type: PauseIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().pause_ingest(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def resume_ingest(self, + index_name, # type: str + *options, # type: ResumeIngestSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().resume_ingest(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def allow_querying(self, + index_name, # type: str + *options, # type: AllowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().allow_querying(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def disallow_querying(self, + index_name, # type: str + *options, # type: DisallowQueryingSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().disallow_querying(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def freeze_plan(self, + index_name, # type: str + *options, # type: FreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().freeze_plan(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(None, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def unfreeze_plan(self, + index_name, # type: str + *options, # type: UnfreezePlanSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + super().unfreeze_plan(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def analyze_document(self, + index_name, # type: str + document, # type: Any + *options, # type: AnalyzeDocumentSearchIndexOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Dict[str, Any]]: + super().analyze_document(index_name, document, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_index_stats(self, + index_name, # type: str + *options, # type: GetSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Dict[str, Any]]: + super().get_index_stats(index_name, *options, **kwargs) + + @TxMgmtWrapper.inject_callbacks(dict, ManagementType.SearchIndexMgmt, SearchIndexManagerLogic._ERROR_MAPPING) + def get_all_index_stats(self, + *options, # type: GetAllSearchIndexStatsOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Dict[str, Any]]: + super().get_all_index_stats(*options, **kwargs) diff --git a/txcouchbase/management/users.py b/txcouchbase/management/users.py new file mode 100644 index 000000000..d8bd2694c --- /dev/null +++ b/txcouchbase/management/users.py @@ -0,0 +1,112 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Iterable) + +from twisted.internet.defer import Deferred + +from couchbase.management.logic.users_logic import (Group, + RoleAndDescription, + User, + UserAndMetadata, + UserManagerLogic) + +if TYPE_CHECKING: + from couchbase.management.options import (DropGroupOptions, + DropUserOptions, + GetAllGroupsOptions, + GetAllUsersOptions, + GetGroupOptions, + GetRolesOptions, + GetUserOptions, + UpsertGroupOptions, + UpsertUserOptions) + + +class UserManager(UserManagerLogic): + + def __init__(self, connection, loop): + super().__init__(connection) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + def get_user(self, + username, # type: str + *options, # type: GetUserOptions + **kwargs # type: Any + ) -> Deferred[UserAndMetadata]: + return Deferred.fromFuture(super().get_user(username, *options, **kwargs)) + + def get_all_users(self, + *options, # type: GetAllUsersOptions + **kwargs # type: Any + ) -> Deferred[Iterable[UserAndMetadata]]: + return Deferred.fromFuture(super().get_all_users(*options, **kwargs)) + + def upsert_user(self, + user, # type: User + *options, # type: UpsertUserOptions + **kwargs # type: Any + ) -> Deferred[None]: + + return Deferred.fromFuture(super().upsert_user(user, *options, **kwargs)) + + def drop_user(self, + username, # type: str + *options, # type: DropUserOptions + **kwargs # type: Any + ) -> Deferred[None]: + return Deferred.fromFuture(super().drop_user(username, *options, **kwargs)) + + def get_roles(self, + *options, # type: GetRolesOptions + **kwargs # type: Any + ) -> Deferred[Iterable[RoleAndDescription]]: + return Deferred.fromFuture(super().get_roles(*options, **kwargs)) + + def get_group(self, + group_name, # type: str + *options, # type: GetGroupOptions + **kwargs # type: Any + ) -> Deferred[Group]: + return Deferred.fromFuture(super().get_group(group_name, *options, **kwargs)) + + def get_all_groups(self, + *options, # type: GetAllGroupsOptions + **kwargs # type: Any + ) -> Deferred[Iterable[Group]]: + return Deferred.fromFuture(super().get_all_groups(*options, **kwargs)) + + def upsert_group(self, + group, # type: Group + *options, # type: UpsertGroupOptions + **kwargs # type: Any + ) -> Deferred[None]: + return Deferred.fromFuture(super().upsert_group(group, *options, **kwargs)) + + def drop_group(self, + group_name, # type: str + *options, # type: DropGroupOptions + **kwargs # type: Any + ) -> Deferred[None]: + return Deferred.fromFuture(super().drop_group(group_name, *options, **kwargs)) diff --git a/txcouchbase/management/views.py b/txcouchbase/management/views.py new file mode 100644 index 000000000..f8ef4d6d0 --- /dev/null +++ b/txcouchbase/management/views.py @@ -0,0 +1,109 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict, + Iterable) + +from twisted.internet.defer import Deferred, inlineCallbacks + +from couchbase.management.logic.view_index_logic import (DesignDocument, + DesignDocumentNamespace, + ViewIndexManagerLogic) + +if TYPE_CHECKING: + from couchbase.management.options import (DropDesignDocumentOptions, + GetAllDesignDocumentsOptions, + GetDesignDocumentOptions, + PublishDesignDocumentOptions, + UpsertDesignDocumentOptions) + + +class ViewIndexManager(ViewIndexManagerLogic): + def __init__(self, connection, loop, bucket_name): + super().__init__(connection, bucket_name) + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + def get_design_document(self, + design_doc_name, # type: str + namespace, # type: DesignDocumentNamespace + *options, # type: GetDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[DesignDocument]: + return Deferred.fromFuture(super().get_design_document(design_doc_name, namespace, *options, **kwargs)) + + def get_all_design_documents(self, + namespace, # type: DesignDocumentNamespace + *options, # type: GetAllDesignDocumentsOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[Iterable[DesignDocument]]: + return Deferred.fromFuture(super().get_all_design_documents(namespace, *options, **kwargs)) + + def upsert_design_document(self, + design_doc_data, # type: DesignDocument + namespace, # type: DesignDocumentNamespace + *options, # type: UpsertDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + return Deferred.fromFuture(super().upsert_design_document(design_doc_data, namespace, *options, **kwargs)) + + def drop_design_document(self, + design_doc_name, # type: str + namespace, # type: DesignDocumentNamespace + *options, # type: DropDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + return Deferred.fromFuture(super().drop_design_document(design_doc_name, namespace, *options, **kwargs)) + + @inlineCallbacks + def _publish_design_document(self, + design_doc_name, # type: str + *options, # type: PublishDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + doc = yield self.get_design_document( + design_doc_name, DesignDocumentNamespace.DEVELOPMENT, *options, **kwargs) + + yield self.upsert_design_document( + doc, DesignDocumentNamespace.PRODUCTION, *options, **kwargs) + + def publish_design_document(self, + design_doc_name, # type: str + *options, # type: PublishDesignDocumentOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[None]: + + d = Deferred() + + def _on_ok(_): + d.callback(None) + + def _on_err(exc): + d.errback(exc) + + pub_ddod_d = self._publish_design_document(design_doc_name, *options, **kwargs) + pub_ddod_d.addCallback(_on_ok) + pub_ddod_d.addErrback(_on_err) + + return d diff --git a/txcouchbase/n1ql.py b/txcouchbase/n1ql.py new file mode 100644 index 000000000..3d65e6aea --- /dev/null +++ b/txcouchbase/n1ql.py @@ -0,0 +1,116 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet.defer import Deferred + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.n1ql import QueryRequestLogic + + +class N1QLRequest(QueryRequestLogic): + def __init__(self, + connection, + loop, + query_params, + row_factory=lambda x: x, + **kwargs + ): + super().__init__(connection, query_params, row_factory=row_factory, **kwargs) + self._query_request_ftr = None + self._query_d = None + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @classmethod + def generate_n1ql_request(cls, connection, loop, query_params, row_factory=lambda x: x, **kwargs): + return cls(connection, loop, query_params, row_factory=row_factory, **kwargs) + + def execute_query(self): + # if self._query_request_ftr is not None and self._query_request_ftr.done(): + if self.done_streaming: + raise AlreadyQueriedException() + + if self._query_request_ftr is None: + self._query_request_ftr = self.loop.create_future() + self._submit_query(callback=self._on_query_complete) + self._query_d = Deferred.fromFuture(self._query_request_ftr) + + return self._query_d + + def _on_query_complete(self, result): + self._loop.call_soon_threadsafe(self._query_request_ftr.set_result, result) + + def _get_metadata(self): + try: + query_response = next(self._streaming_result) + self._set_metadata(query_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + # def _get_metadata(self): + # if self._query_request_ftr.done(): + # if self._query_request_ftr.exception(): + # print('raising exception') + # raise self._query_request_ftr.exception() + # else: + # self._set_metadata() + # else: + # self._loop.run_until_complete(self._query_request_ftr) + # self._set_metadata() + + def __iter__(self): + return self + + def _get_next_row(self): + if self._done_streaming is True: + return + + row = next(self._streaming_result) + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopIteration + + return self.serializer.deserialize(row) + + def __next__(self): + try: + return self._get_next_row() + except StopIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/txcouchbase/scope.py b/txcouchbase/scope.py new file mode 100644 index 000000000..5234186ee --- /dev/null +++ b/txcouchbase/scope.py @@ -0,0 +1,250 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (TYPE_CHECKING, + Any, + Dict, + Optional) + +from twisted.internet.defer import Deferred + +from couchbase.logic.analytics import AnalyticsQuery +from couchbase.logic.n1ql import N1QLQuery +from couchbase.logic.search import SearchQueryBuilder +from couchbase.result import (AnalyticsResult, + QueryResult, + SearchResult) +from couchbase.transcoder import Transcoder +from txcouchbase.analytics import AnalyticsRequest +from txcouchbase.collection import Collection +from txcouchbase.management.search import ScopeSearchIndexManager +from txcouchbase.n1ql import N1QLRequest +from txcouchbase.search import FullTextSearchRequest + +if TYPE_CHECKING: + + from couchbase.options import (AnalyticsOptions, + QueryOptions, + SearchOptions) + from couchbase.search import SearchQuery, SearchRequest + + +class Scope: + def __init__(self, bucket, scope_name): + self._bucket = bucket + self._set_connection() + self._loop = bucket.loop + self._scope_name = scope_name + + @property + def connection(self): + """ + **INTERNAL** + """ + return self._connection + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @property + def default_transcoder(self) -> Optional[Transcoder]: + return self._bucket.default_transcoder + + @property + def name(self): + return self._scope_name + + @property + def bucket_name(self): + return self._bucket.name + + def collection(self, name # type: str + ) -> Collection: + + return Collection(self, name) + + def query( + self, + statement, # type: str + *options, # type: QueryOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[QueryResult]: + + opt = QueryOptions() + opts = list(options) + for o in opts: + if isinstance(o, QueryOptions): + opt = o + opts.remove(o) + + # set the query context as this bucket and scope if not provided + if not ('query_context' in opt or 'query_context' in kwargs): + kwargs['query_context'] = '`{}`.`{}`'.format(self.bucket_name, self.name) + + query = N1QLQuery.create_query_object( + statement, *options, **kwargs) + request = N1QLRequest.generate_n1ql_request(self.connection, + self.loop, + query.params, + default_serializer=self.default_serializer) + d = Deferred() + + def _on_ok(_): + d.callback(QueryResult(request)) + + def _on_err(exc): + d.errback(exc) + + query_d = request.execute_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + + def analytics_query( + self, + statement, # type: str + *options, # type: AnalyticsOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[AnalyticsResult]: + + opt = AnalyticsOptions() + opts = list(options) + for o in opts: + if isinstance(o, AnalyticsOptions): + opt = o + opts.remove(o) + + # set the query context as this bucket and scope if not provided + if not ('query_context' in opt or 'query_context' in kwargs): + kwargs['query_context'] = 'default:`{}`.`{}`'.format(self.bucket_name, self.name) + + query = AnalyticsQuery.execute_analytics_query( + statement, *options, **kwargs) + request = AnalyticsRequest.generate_analytics_request(self.connection, + self.loop, + query.params, + default_serializer=self.default_serializer) + d = Deferred() + + def _on_ok(_): + d.callback(AnalyticsResult(request)) + + def _on_err(exc): + d.errback(exc) + + query_d = request.execute_analytics_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + + def search_query( + self, + index, # type: str + query, # type: SearchQuery + *options, # type: SearchOptions + **kwargs # type: Dict[str, Any] + ) -> Deferred[SearchResult]: + + opt = SearchOptions() + opts = list(options) + for o in opts: + if isinstance(o, SearchOptions): + opt = o + opts.remove(o) + + # set the scope_name as this scope if not provided + if not ('scope_name' in opt or 'scope_name' in kwargs): + kwargs['scope_name'] = f'{self.name}' + + query = SearchQueryBuilder.create_search_query_object( + index, query, *options, **kwargs + ) + request = FullTextSearchRequest.generate_search_request(self.connection, + self.loop, + query.as_encodable(), + default_serializer=self.default_serializer) + d = Deferred() + + def _on_ok(_): + d.callback(SearchResult(request)) + + def _on_err(exc): + d.errback(exc) + + query_d = request.execute_search_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + + def search(self, + index, # type: str + request, # type: SearchRequest + *options, # type: SearchOptions + **kwargs, # type: Dict[str, Any] + ) -> Deferred[SearchResult]: + request_args = dict(default_serialize=self.default_serializer, + streaming_timeout=self.streaming_timeouts.get('search_timeout', None), + bucket_name=self.bucket_name, + scope_name=self.name) + query = SearchQueryBuilder.create_search_query_from_request(index, request, *options, **kwargs) + request = FullTextSearchRequest.generate_search_request(self.connection, + self.loop, + query.as_encodable(), + **request_args) + + d = Deferred() + + def _on_ok(_): + d.callback(SearchResult(request)) + + def _on_err(exc): + d.errback(exc) + + query_d = request.execute_search_query() + query_d.addCallback(_on_ok) + query_d.addErrback(_on_err) + return d + + def search_indexes(self) -> ScopeSearchIndexManager: + """ + Get a :class:`~txcouchbase.management.search.ScopeSearchIndexManager` which can be used to manage the search + indexes of this scope. + + Returns: + :class:`~txcouchbase.management.search.ScopeSearchIndexManager`: A :class:`~txcouchbase.management.search.ScopeSearchIndexManager` instance. + + """ # noqa: E501 + # TODO: AlreadyShutdownException? + return ScopeSearchIndexManager(self.connection, self.loop, self.bucket_name, self.name) + + def _connect_bucket(self): + """ + **INTERNAL** + """ + return self._bucket.on_connect() + + def _set_connection(self): + """ + **INTERNAL** + """ + self._connection = self._bucket.connection + + @staticmethod + def default_name(): + return "_default" diff --git a/txcouchbase/search.py b/txcouchbase/search.py new file mode 100644 index 000000000..a85c07723 --- /dev/null +++ b/txcouchbase/search.py @@ -0,0 +1,104 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet.defer import Deferred + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.search import FullTextSearchRequestLogic + + +class FullTextSearchRequest(FullTextSearchRequestLogic): + def __init__(self, + connection, + loop, + encoded_query, + **kwargs + ): + super().__init__(connection, encoded_query, **kwargs) + self._query_request_ftr = None + self._query_d = None + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @classmethod + def generate_search_request(cls, connection, loop, encoded_query, **kwargs): + return cls(connection, loop, encoded_query, **kwargs) + + def execute_search_query(self) -> Deferred: + # if self._query_request_ftr is not None and self._query_request_ftr.done(): + if self.done_streaming: + raise AlreadyQueriedException() + + if self._query_request_ftr is None: + self._query_request_ftr = self.loop.create_future() + self._submit_query(callback=self._on_query_complete) + self._query_d = Deferred.fromFuture(self._query_request_ftr) + + return self._query_d + + def _on_query_complete(self, result): + self._loop.call_soon_threadsafe(self._query_request_ftr.set_result, result) + + def _get_metadata(self): + try: + search_response = next(self._streaming_result) + self._set_metadata(search_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def __iter__(self): + return self + + def _get_next_row(self): + if self.done_streaming is True: + return + + row = next(self._streaming_result) + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopIteration + + return self._deserialize_row(row) + + def __next__(self): + try: + return self._get_next_row() + except StopIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn diff --git a/txcouchbase/tests/__init__.py b/txcouchbase/tests/__init__.py index e1d0a49b9..fc6346764 100644 --- a/txcouchbase/tests/__init__.py +++ b/txcouchbase/tests/__init__.py @@ -1,29 +1,14 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. # -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from couchbase.tests.base import SkipTest -import sys - -vi = sys.version_info -if vi[0] == 3: - from unittest.case import SkipTest - raise SkipTest("Twisted support unavailable on Python3") - -try: - from twisted.trial.unittest import TestCase -except: - raise SkipTest("Twisted not found") - +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/txcouchbase/tests/_test_utils.py b/txcouchbase/tests/_test_utils.py new file mode 100644 index 000000000..e22405820 --- /dev/null +++ b/txcouchbase/tests/_test_utils.py @@ -0,0 +1,477 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import (Any, + Callable, + Dict, + Optional, + Tuple, + Type, + Union) + +import pytest +from twisted.internet import defer, reactor + +from couchbase.auth import PasswordAuthenticator +from couchbase.exceptions import (AmbiguousTimeoutException, + BucketAlreadyExistsException, + BucketDoesNotExistException, + CollectionAlreadyExistsException, + CouchbaseException, + ScopeAlreadyExistsException, + ScopeNotFoundException, + UnAmbiguousTimeoutException) +from couchbase.management.buckets import (BucketType, + CreateBucketSettings, + StorageBackend) +from couchbase.management.collections import CollectionSpec +from couchbase.options import ClusterOptions +from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder +from tests.helpers import CollectionType # noqa: F401 +from tests.helpers import FakeTestObj # noqa: F401 +from tests.helpers import KVPair # noqa: F401 +from tests.helpers import CouchbaseTestEnvironment, CouchbaseTestEnvironmentException +from txcouchbase.bucket import Bucket +from txcouchbase.cluster import Cluster +from txcouchbase.management.buckets import BucketManager +from txcouchbase.management.collections import CollectionManager +from txcouchbase.scope import Scope + +from .conftest import run_in_reactor_thread + + +class TestEnvironment(CouchbaseTestEnvironment): + + TEST_SCOPE = "test-scope" + TEST_COLLECTION = "test-collection" + + def __init__(self, cluster, bucket, collection, cluster_config, **kwargs): + super().__init__(cluster, bucket, collection, cluster_config) + + if kwargs.get("manage_buckets", False) is True: + self.check_if_feature_supported('basic_bucket_mgmt') + self._bm = self.cluster.buckets() + + if kwargs.get("manage_collections", False) is True: + self.check_if_feature_supported('collections') + self._cm = self.bucket.collections() + + self._test_bucket = None + self._test_bucket_cm = None + self._collection_spec = None + self._scope = None + self._named_collection = None + + @property + def collection(self): + if self._named_collection is None: + return super().collection + return self._named_collection + + @property + def scope(self) -> Optional[Scope]: + return self._scope + + @property + def fqdn(self) -> Optional[str]: + return f'`{self.bucket.name}`.`{self.scope.name}`.`{self.collection.name}`' + + @property + def bm(self) -> Optional[BucketManager]: + """Returns the default bucket's BucketManager""" + return self._bm if hasattr(self, '_bm') else None + + @property + def cm(self) -> Optional[CollectionManager]: + """Returns the default CollectionManager""" + return self._cm if hasattr(self, '_cm') else None + + @property + def test_bucket(self) -> Optional[Bucket]: + """Returns the test bucket object""" + return self._test_bucket if hasattr(self, '_test_bucket') else None + + @property + def test_bucket_cm(self) -> Optional[CollectionManager]: + """Returns the test bucket's CollectionManager""" + return self._test_bucket_cm if hasattr(self, '_test_bucket_cm') else None + + @classmethod # noqa: C901 + def get_environment(cls, test_suite, couchbase_config, coll_type=CollectionType.DEFAULT, **kwargs): # noqa: C901 + + # this will only return False _if_ using the mock server + tmp_name = test_suite.split('.')[-1] + if 'collection_new' in tmp_name: + tmp_name = 'collection_t' + mock_supports = CouchbaseTestEnvironment.mock_supports_feature(tmp_name, + couchbase_config.is_mock_server) + if not mock_supports: + pytest.skip(f'Mock server does not support feature(s) required for test suite: {test_suite}') + + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + opts = ClusterOptions(PasswordAuthenticator(username, pw)) + transcoder = kwargs.pop('transcoder', None) + if transcoder: + opts['transcoder'] = transcoder + okay = False + for _ in range(3): + try: + cluster = Cluster(conn_string, opts) + run_in_reactor_thread(cluster.on_connect) + bucket = cluster.bucket(f"{couchbase_config.bucket_name}") + run_in_reactor_thread(bucket.on_connect) + run_in_reactor_thread(cluster.cluster_info) + okay = True + break + except (UnAmbiguousTimeoutException, AmbiguousTimeoutException): + continue + + if not okay and couchbase_config.is_mock_server: + pytest.skip(('CAVES does not seem to be happy. Skipping tests as failure is not' + ' an accurate representation of the state of the test, but rather' + ' there is an environment issue.')) + + coll = bucket.default_collection() + if coll_type == CollectionType.DEFAULT: + cb_env = cls(cluster, bucket, coll, couchbase_config, **kwargs) + elif coll_type == CollectionType.NAMED: + if 'manage_collections' not in kwargs: + kwargs['manage_collections'] = True + cb_env = cls(cluster, + bucket, + coll, + couchbase_config, + **kwargs) + + return cb_env + + def get_new_key_value(self, reset=True): + if reset is True: + try: + run_in_reactor_thread(self.collection.remove, self.NEW_KEY) + except BaseException: + pass + return self.NEW_KEY, self.NEW_CONTENT + + # standard data load/purge + + def load_data(self): + data_types, sample_json = self.load_data_from_file() + for dt in data_types: + data = sample_json.get(dt, None) + + if data and "results" in data: + stable = False + for _ in range(3): + try: + for idx, r in enumerate(data["results"]): + key = f"{r['type']}_{r['id']}" + run_in_reactor_thread(self.collection.upsert, key, r) + self._loaded_keys.append(key) + + stable = True + break + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + run_in_reactor_thread(TestEnvironment.deferred_sleep, 3) + continue + except Exception: + raise + + self.skip_if_mock_unstable(stable) + + def purge_data(self): + for key in self._loaded_keys: + try: + run_in_reactor_thread(self.collection.remove, key) + except CouchbaseException: + pass + + # binary data load/purge + + def load_utf8_binary_data(self, start_value=None): + utf8_key = self.get_binary_key('UTF8') + value = start_value or '' + tc = RawStringTranscoder() + run_in_reactor_thread(self.collection.upsert, utf8_key, value, transcoder=tc) + self.try_n_times(10, 1, self.collection.get, utf8_key, transcoder=tc) + return utf8_key, value + + def load_bytes_binary_data(self, start_value=None): + bytes_key = self.get_binary_key('BYTES') + value = start_value or b'' + tc = RawBinaryTranscoder() + run_in_reactor_thread(self.collection.upsert, bytes_key, value, transcoder=tc) + self.try_n_times(10, 1, self.collection.get, bytes_key, transcoder=tc) + return bytes_key, value + + def load_counter_binary_data(self, start_value=None): + counter_key = self.get_binary_key('COUNTER') + if start_value: + run_in_reactor_thread(self.collection.upsert, counter_key, start_value) + self.try_n_times(10, 1, self.collection.get, counter_key) + + return counter_key, start_value + + def purge_binary_data(self): + for k in self.get_binary_keys(): + try: + run_in_reactor_thread(self.collection.remove, k) + except CouchbaseException: + pass + + # Bucket MGMT + + def create_bucket(self, bucket_name, storage_backend=None): + try: + settings_kwargs = { + 'name': bucket_name, + 'bucket_type': BucketType.COUCHBASE, + 'ram_quota_mb': 100, + } + if storage_backend is not None: + settings_kwargs['storage_backend'] = storage_backend + if storage_backend is StorageBackend.MAGMA: + settings_kwargs['ram_quota_mb'] = 1024 + run_in_reactor_thread(self.bm.create_bucket, + CreateBucketSettings(**settings_kwargs)) + except BucketAlreadyExistsException: + pass + self.try_n_times(10, 1, self.bm.get_bucket, bucket_name) + + def purge_buckets(self, buckets): + for bucket in buckets: + try: + run_in_reactor_thread(self.bm.drop_bucket, bucket) + except BucketDoesNotExistException: + pass + except Exception: + raise + + # now be sure it is really gone + self.try_n_times_till_exception(10, + 3, + self.bm.get_bucket, + bucket, + expected_exceptions=(BucketDoesNotExistException)) + + # Collection MGMT + + def setup_collection_mgmt(self, bucket_name): + self.create_bucket(bucket_name) + self._test_bucket = self.try_n_times(3, 5, self.cluster.bucket, bucket_name, is_deferred=False) + self._test_bucket_cm = self._test_bucket.collections() + + def setup_named_collections(self): + try: + run_in_reactor_thread(self.cm.create_scope, self.TEST_SCOPE) + except ScopeAlreadyExistsException: + run_in_reactor_thread(self.cm.drop_scope, self.TEST_SCOPE) + run_in_reactor_thread(self.cm.create_scope, self.TEST_SCOPE) + + self._collection_spec = CollectionSpec(self.TEST_COLLECTION, self.TEST_SCOPE) + try: + run_in_reactor_thread(self.cm.create_collection, self._collection_spec) + except CollectionAlreadyExistsException: + run_in_reactor_thread(self.cm.drop_collection, self._collection_spec) + run_in_reactor_thread(self.cm.create_collection, self._collection_spec) + + c = self.get_collection(self.TEST_SCOPE, self.TEST_COLLECTION, bucket_name=self.bucket.name) + if c is None: + raise CouchbaseTestEnvironmentException("Unabled to create collection for name collection testing") + + self._scope = self.bucket.scope(self.TEST_SCOPE) + self._named_collection = self._scope.collection(self.TEST_COLLECTION) + + def teardown_named_collections(self): + run_in_reactor_thread(self.cm.drop_scope, self.TEST_SCOPE) + self.try_n_times_till_exception(10, + 1, + self.cm.drop_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeNotFoundException,) + ) + self._collection_spec = None + self._scope = None + self._named_collection = None + + def get_scope(self, scope_name, bucket_name=None): + if bucket_name is None and self._test_bucket is None: + raise CouchbaseTestEnvironmentException("Must provide a bucket name or have a valid test_bucket available") + + bucket_names = ['default'] + if self._test_bucket is not None: + bucket_names.append(self._test_bucket.name) + + bucket = bucket_name or self._test_bucket.name + if bucket not in bucket_names + [bucket_name]: + raise CouchbaseTestEnvironmentException( + f"{bucket} is an invalid bucket name.") + + scopes = [] + if bucket == self.bucket.name: + scopes = run_in_reactor_thread(self.cm.get_all_scopes) + else: + scopes = run_in_reactor_thread(self.test_bucket_cm.get_all_scopes) + return next((s for s in scopes if s.name == scope_name), None) + + def get_collection(self, scope_name, coll_name, bucket_name=None): + scope = self.get_scope(scope_name, bucket_name=bucket_name) + if scope: + return next( + (c for c in scope.collections if c.name == coll_name), None) + + return None + + # helper methods + + def sleep(self, sleep_seconds # type: float + ) -> None: + run_in_reactor_thread(TestEnvironment.deferred_sleep, sleep_seconds) + + @staticmethod + def deferred_sleep(sleep_seconds # type: float + ) -> None: + d = defer.Deferred() + reactor.callLater(sleep_seconds, d.callback, "") + return d + + def _try_n_times(self, + num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + is_deferred=True, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ) -> Any: + for _ in range(num_times): + try: + if is_deferred: + res = run_in_reactor_thread(func, *args, **kwargs) + else: + res = func(*args, **kwargs) + return res + except Exception: + print(f'trying {func} failed, sleeping for {seconds_between} seconds...') + run_in_reactor_thread(TestEnvironment.deferred_sleep, seconds_between) + + def try_n_times(self, + num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + is_deferred=True, # type: Optional[bool] + reset_on_timeout=False, # type: Optional[bool] + reset_num_times=None, # type: Optional[int] + **kwargs # type: Dict[str, Any] + ) -> Any: + if reset_on_timeout: + reset_times = reset_num_times or num_times + for _ in range(reset_times): + try: + return self._try_n_times(num_times, + seconds_between, + func, + *args, + is_deferred=is_deferred, + **kwargs) + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + continue + except Exception: + raise + else: + return self._try_n_times(num_times, + seconds_between, + func, + *args, + is_deferred=is_deferred, + **kwargs) + + raise CouchbaseTestEnvironmentException( + f"Unsuccessful execution of {func.__name__} after {num_times} times, " + f"waiting {seconds_between} seconds between calls.") + + def _try_n_times_until_exception(self, + num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + is_deferred=True, # type: Optional[bool] + expected_exceptions=(Exception,), # type: Tuple[Type[Exception],...] + raise_exception=False, # type: Optional[bool] + **kwargs # type: Dict[str, Any] + ) -> None: + for _ in range(num_times): + try: + if is_deferred: + run_in_reactor_thread(func, *args, **kwargs) + else: + func(*args, **kwargs) + run_in_reactor_thread(TestEnvironment.deferred_sleep, seconds_between) + except expected_exceptions: + if raise_exception: + raise + # helpful to have this print statement when tests fail + return + except Exception: + raise + + def try_n_times_till_exception(self, + num_times, # type: int + seconds_between, # type: Union[int, float] + func, # type: Callable + *args, # type: Any + is_deferred=True, # type: Optional[bool] + expected_exceptions=(Exception,), # type: Tuple[Type[Exception],...] + raise_exception=False, # type: Optional[bool] + raise_if_no_exception=True, # type: Optional[bool] + reset_on_timeout=False, # type: Optional[bool] + reset_num_times=None, # type: Optional[int] + **kwargs # type: Dict[str, Any] + ) -> None: + if reset_on_timeout: + reset_times = reset_num_times or num_times + for _ in range(reset_times): + try: + self._try_n_times_until_exception(num_times, + seconds_between, + func, + *args, + is_deferred=is_deferred, + expected_exceptions=expected_exceptions, + raise_exception=raise_exception, + **kwargs) + return + except (AmbiguousTimeoutException, UnAmbiguousTimeoutException): + continue + except Exception: + raise + else: + self._try_n_times_until_exception(num_times, + seconds_between, + func, + *args, + is_deferred=is_deferred, + expected_exceptions=expected_exceptions, + raise_exception=raise_exception, + **kwargs) + return # success -- return now + + # TODO: option to restart mock server? + if raise_if_no_exception is False: + return + + raise CouchbaseTestEnvironmentException((f"Exception not raised calling {func.__name__} {num_times} times " + f"waiting {seconds_between} seconds between calls.")) diff --git a/txcouchbase/tests/base.py b/txcouchbase/tests/base.py deleted file mode 100644 index aa03cfcfa..000000000 --- a/txcouchbase/tests/base.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from twisted.internet import defer -from twisted.trial.unittest import TestCase - -from txcouchbase.bucket import Bucket -from couchbase.tests.base import ConnectionTestCase - -def gen_base(basecls): - class _TxTestCase(basecls, TestCase): - def register_cleanup(self, obj): - d = defer.Deferred() - obj.registerDeferred('_dtor', d) - self.addCleanup(lambda x: d, None) - - # Add another callback (invoked _outside_ of C) to ensure - # the instance's destroy function is properly triggered - if hasattr(obj, '_async_shutdown'): - self.addCleanup(obj._async_shutdown) - - def make_connection(self, **kwargs): - ret = super(_TxTestCase, self).make_connection(**kwargs) - self.register_cleanup(ret) - return ret - - def checkCbRefcount(self): - pass - - @property - def factory(self): - return Bucket - - def setUp(self): - super(_TxTestCase, self).setUp() - self.cb = None - - def tearDown(self): - super(_TxTestCase, self).tearDown() - - return _TxTestCase diff --git a/txcouchbase/tests/binary_collection_t.py b/txcouchbase/tests/binary_collection_t.py new file mode 100644 index 000000000..21f526934 --- /dev/null +++ b/txcouchbase/tests/binary_collection_t.py @@ -0,0 +1,306 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from couchbase.exceptions import DocumentNotFoundException, InvalidArgumentException +from couchbase.options import (DecrementOptions, + DeltaValue, + IncrementOptions, + SignedInt64) +from couchbase.result import CounterResult, MutationResult +from couchbase.transcoder import RawBinaryTranscoder, RawStringTranscoder + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment, + run_in_reactor_thread) + + +class BinaryCollectionTests: + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment(__name__, couchbase_config, request.param) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + yield cb_env + + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + # key/value fixtures + + @pytest.fixture(name='utf8_empty_kvp') + def utf8_key_and_empty_value(self, cb_env) -> KVPair: + key, value = cb_env.try_n_times(5, 3, cb_env.load_utf8_binary_data, is_deferred=False) + yield KVPair(key, value) + run_in_reactor_thread(cb_env.collection.upsert, key, '', transcoder=RawStringTranscoder()) + + @pytest.fixture(name='utf8_kvp') + def utf8_key_and_value(self, cb_env) -> KVPair: + key, value = cb_env.try_n_times(5, 3, cb_env.load_utf8_binary_data, start_value='XXXX', is_deferred=False) + yield KVPair(key, value) + run_in_reactor_thread(cb_env.collection.upsert, key, '', transcoder=RawStringTranscoder()) + + @pytest.fixture(name='bytes_empty_kvp') + def bytes_key_and_empty_value(self, cb_env) -> KVPair: + key, value = cb_env.try_n_times(5, 3, cb_env.load_bytes_binary_data, is_deferred=False) + yield KVPair(key, value) + run_in_reactor_thread(cb_env.collection.upsert, key, b'', transcoder=RawBinaryTranscoder()) + + @pytest.fixture(name='bytes_kvp') + def bytes_key_and_value(self, cb_env) -> KVPair: + key, value = cb_env.try_n_times(5, 3, cb_env.load_bytes_binary_data, start_value=b'XXXX', is_deferred=False) + yield KVPair(key, value) + run_in_reactor_thread(cb_env.collection.upsert, key, b'', transcoder=RawBinaryTranscoder()) + + @pytest.fixture(name='counter_empty_kvp') + def counter_key_and_empty_value(self, cb_env) -> KVPair: + key, value = cb_env.try_n_times(5, 3, cb_env.load_counter_binary_data, is_deferred=False) + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,)) + + @pytest.fixture(name='counter_kvp') + def counter_key_and_value(self, cb_env) -> KVPair: + key, value = cb_env.try_n_times(5, 3, cb_env.load_counter_binary_data, start_value=100, is_deferred=False) + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,)) + + # tests + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_append_string(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + result = run_in_reactor_thread(cb.binary().append, key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = run_in_reactor_thread(cb.get, key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + def test_append_string_not_empty(self, cb_env, utf8_kvp): + cb = cb_env.collection + key = utf8_kvp.key + value = utf8_kvp.value + result = run_in_reactor_thread(cb.binary().append, key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = run_in_reactor_thread(cb.get, key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == value + 'foo' + + def test_append_string_nokey(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + run_in_reactor_thread(cb.remove, key) + cb_env.try_n_times_till_exception(10, + 1, + cb.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + # @TODO(jc): 3.2.x SDK tests for NotStoredException + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.binary().append, key, 'foo') + + def test_append_bytes(self, cb_env, bytes_empty_kvp): + cb = cb_env.collection + key = bytes_empty_kvp.key + result = run_in_reactor_thread(cb.binary().append, key, b'XXX') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = run_in_reactor_thread(cb.get, key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'XXX' + + def test_append_bytes_not_empty(self, cb_env, bytes_kvp): + cb = cb_env.collection + key = bytes_kvp.key + value = bytes_kvp.value + + result = run_in_reactor_thread(cb.binary().append, key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = run_in_reactor_thread(cb.get, key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == value + b'foo' + + def test_prepend_string(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + result = run_in_reactor_thread(cb.binary().prepend, key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = run_in_reactor_thread(cb.get, key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + + def test_prepend_string_not_empty(self, cb_env, utf8_kvp): + cb = cb_env.collection + key = utf8_kvp.key + value = utf8_kvp.value + + result = run_in_reactor_thread(cb.binary().prepend, key, 'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = run_in_reactor_thread(cb.get, key, transcoder=RawStringTranscoder()) + assert result.content_as[str] == 'foo' + value + + def test_prepend_string_nokey(self, cb_env, utf8_empty_kvp): + cb = cb_env.collection + key = utf8_empty_kvp.key + run_in_reactor_thread(cb.remove, key) + cb_env.try_n_times_till_exception(10, + 1, + cb.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + # @TODO(jc): 3.2.x SDK tests for NotStoredException + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.binary().prepend, key, 'foo') + + def test_prepend_bytes(self, cb_env, bytes_empty_kvp): + cb = cb_env.collection + key = bytes_empty_kvp.key + result = run_in_reactor_thread(cb.binary().prepend, key, b'XXX') + assert isinstance(result, MutationResult) + assert result.cas is not None + # make sure it really worked + result = run_in_reactor_thread(cb.get, key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'XXX' + + def test_prepend_bytes_not_empty(self, cb_env, bytes_kvp): + cb = cb_env.collection + key = bytes_kvp.key + value = bytes_kvp.value + + result = run_in_reactor_thread(cb.binary().prepend, key, b'foo') + assert isinstance(result, MutationResult) + assert result.cas is not None + result = run_in_reactor_thread(cb.get, key, transcoder=RawBinaryTranscoder()) + assert result.content_as[bytes] == b'foo' + value + + def test_counter_increment_initial_value(self, cb_env, counter_empty_kvp): + cb = cb_env.collection + key = counter_empty_kvp.key + + result = run_in_reactor_thread(cb.binary().increment, key, IncrementOptions(initial=SignedInt64(100))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == 100 + + def test_counter_decrement_initial_value(self, cb_env, counter_empty_kvp): + cb = cb_env.collection + key = counter_empty_kvp.key + + result = run_in_reactor_thread(cb.binary().decrement, key, DecrementOptions(initial=SignedInt64(100))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == 100 + + def test_counter_increment(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + result = run_in_reactor_thread(cb.binary().increment, key) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value + 1 + + def test_counter_decrement(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + result = run_in_reactor_thread(cb.binary().decrement, key) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value - 1 + + def test_counter_increment_non_default(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + result = run_in_reactor_thread(cb.binary().increment, key, IncrementOptions(delta=DeltaValue(3))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value + 3 + + def test_counter_decrement_non_default(self, cb_env, counter_kvp): + cb = cb_env.collection + key = counter_kvp.key + value = counter_kvp.value + + result = run_in_reactor_thread(cb.binary().decrement, key, DecrementOptions(delta=DeltaValue(3))) + assert isinstance(result, CounterResult) + assert result.cas is not None + assert result.content == value - 3 + + def test_counter_bad_initial_value(self, cb_env, counter_empty_kvp): + cb = cb_env.collection + key = counter_empty_kvp.key + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.binary().increment, key, initial=100) + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.binary().decrement, key, initial=100) + + def test_counter_bad_delta_value(self, cb_env, counter_empty_kvp): + cb = cb_env.collection + key = counter_empty_kvp.key + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.binary().increment, key, delta=5) + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.binary().decrement, key, delta=5) + + def test_unsigned_int(self): + with pytest.raises(InvalidArgumentException): + x = DeltaValue(-1) + with pytest.raises(InvalidArgumentException): + x = DeltaValue(0x7FFFFFFFFFFFFFFF + 1) + + x = DeltaValue(5) + assert 5 == x.value + + def test_signed_int_64(self): + with pytest.raises(InvalidArgumentException): + x = SignedInt64(-0x7FFFFFFFFFFFFFFF - 2) + + with pytest.raises(InvalidArgumentException): + x = SignedInt64(0x7FFFFFFFFFFFFFFF + 1) + + x = SignedInt64(0x7FFFFFFFFFFFFFFF) + assert 0x7FFFFFFFFFFFFFFF == x.value + x = SignedInt64(-0x7FFFFFFFFFFFFFFF - 1) + assert -0x7FFFFFFFFFFFFFFF - 1 == x.value diff --git a/txcouchbase/tests/bucketmgmt_t.py b/txcouchbase/tests/bucketmgmt_t.py new file mode 100644 index 000000000..8e631db74 --- /dev/null +++ b/txcouchbase/tests/bucketmgmt_t.py @@ -0,0 +1,282 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from random import choice + +import pytest + +from couchbase.durability import DurabilityLevel +from couchbase.exceptions import (BucketAlreadyExistsException, + BucketDoesNotExistException, + BucketNotFlushableException, + FeatureUnavailableException) +from couchbase.management.buckets import (BucketSettings, + BucketType, + ConflictResolutionType, + CreateBucketSettings, + StorageBackend) + +from ._test_utils import TestEnvironment, run_in_reactor_thread + + +@pytest.mark.flaky(reruns=5, reruns_delay=1) +class BucketManagementTests: + + @pytest.fixture(scope="class") + def test_buckets(self): + return [f"test-bucket-{i}" for i in range(5)] + + @pytest.fixture(scope="class", name="cb_env") + def couchbase_test_environment(self, couchbase_config, test_buckets): + cb_env = TestEnvironment.get_environment(__name__, couchbase_config, manage_buckets=True) + yield cb_env + cb_env.purge_buckets(test_buckets) + + @pytest.fixture(scope="class") + def check_bucket_mgmt_supported(self, cb_env): + cb_env.check_if_feature_supported('bucket_mgmt') + + @pytest.fixture(scope="class") + def check_bucket_min_durability_supported(self, cb_env): + cb_env.check_if_feature_supported('bucket_min_durability') + + @pytest.fixture(scope="class") + def check_bucket_storage_backend_supported(self, cb_env): + cb_env.check_if_feature_supported('bucket_storage_backend') + + @pytest.fixture(scope="class") + def check_custom_conflict_resolution_supported(self, cb_env): + cb_env.check_if_feature_supported('custom_conflict_resolution') + + @pytest.fixture() + def test_bucket(self, test_buckets): + return choice(test_buckets) + + @pytest.fixture() + def purge_buckets(self, cb_env, test_buckets): + yield + cb_env.purge_buckets(test_buckets) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_create(self, cb_env, test_bucket): + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100)) + bucket = cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + if cb_env.server_version_short >= 6.6: + assert bucket["minimum_durability_level"] == DurabilityLevel.NONE + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_create_replica_index_true(self, cb_env, test_bucket): + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + replica_index=True)) + bucket = cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + assert bucket.replica_index is True + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_create_replica_index_false(self, cb_env, test_bucket): + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + replica_index=False)) + bucket = cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + assert bucket.replica_index is False + + @pytest.mark.usefixtures("check_bucket_min_durability_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_create_durability(self, cb_env, test_bucket): + + min_durability = DurabilityLevel.MAJORITY_AND_PERSIST_TO_ACTIVE + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings(name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100, + minimum_durability_level=min_durability)) + bucket = cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + assert bucket["minimum_durability_level"] == min_durability + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_create_fail(self, cb_env, test_bucket): + settings = CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100) + run_in_reactor_thread(cb_env.bm.create_bucket, settings) + with pytest.raises(BucketAlreadyExistsException): + run_in_reactor_thread(cb_env.bm.create_bucket, settings) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + def test_bucket_drop_fail(self, cb_env, test_bucket): + settings = CreateBucketSettings( + name=test_bucket, + bucket_type=BucketType.COUCHBASE, + ram_quota_mb=100) + run_in_reactor_thread(cb_env.bm.create_bucket, settings) + cb_env.try_n_times(10, 1, cb_env.bm.drop_bucket, test_bucket) + with pytest.raises(BucketDoesNotExistException): + run_in_reactor_thread(cb_env.bm.drop_bucket, test_bucket) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_list(self, cb_env, test_buckets): + for bucket in test_buckets: + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=bucket, + ram_quota_mb=100)) + cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, bucket) + + buckets = run_in_reactor_thread(cb_env.bm.get_all_buckets) + assert set() == set(test_buckets).difference( + set(map(lambda b: b.name, buckets))) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_cluster_sees_bucket(self, cb_env, test_bucket): + # Create the bucket + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100)) + cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, test_bucket) + # cluster should be able to return it (though, not right away) + cb_env.try_n_times(10, 2, cb_env.cluster.bucket, test_bucket) + b = cb_env.cluster.bucket(test_bucket) + assert b is not None + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_change_expiry(self, cb_env, test_bucket): + # Create the bucket + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100)) + cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + + # change bucket TTL + cb_env.try_n_times(10, 3, cb_env.bm.update_bucket, BucketSettings( + name=test_bucket, max_expiry=timedelta(seconds=500))) + + def get_bucket_expiry_equal(name, expiry): + b = run_in_reactor_thread(cb_env.bm.get_bucket, name) + + if not expiry == b.max_expiry: + raise Exception("not equal") + + cb_env.try_n_times(10, 3, get_bucket_expiry_equal, test_bucket, timedelta(seconds=500), is_deferred=False) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_flush(self, cb_env, test_bucket): + # Create the bucket + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + flush_enabled=True)) + bucket = cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.flush_enabled is True + # flush the bucket + cb_env.try_n_times(10, 3, cb_env.bm.flush_bucket, bucket.name) + + @pytest.mark.usefixtures("check_bucket_mgmt_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_flush_fail(self, cb_env, test_bucket): + # Create the bucket + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + flush_enabled=False)) + bucket = cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.flush_enabled is False + + with pytest.raises(BucketNotFlushableException): + run_in_reactor_thread(cb_env.bm.flush_bucket, test_bucket) + + @pytest.mark.usefixtures("check_bucket_storage_backend_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_backend_default(self, cb_env, test_bucket): + # Create the bucket + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + flush_enabled=False)) + bucket = cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.storage_backend == StorageBackend.COUCHSTORE + + @pytest.mark.usefixtures("check_bucket_storage_backend_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_backend_magma(self, cb_env, test_bucket): + # Create the bucket + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=1024, + flush_enabled=False, + storage_backend=StorageBackend.MAGMA)) + bucket = cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.storage_backend == StorageBackend.MAGMA + + @pytest.mark.usefixtures("check_bucket_storage_backend_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_backend_ephemeral(self, cb_env, test_bucket): + # Create the bucket + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + bucket_type=BucketType.EPHEMERAL, + flush_enabled=False)) + bucket = cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.storage_backend == StorageBackend.UNDEFINED + + @pytest.mark.usefixtures("check_custom_conflict_resolution_supported") + @pytest.mark.usefixtures("purge_buckets") + def test_bucket_custom_conflict_resolution(self, cb_env, test_bucket): + + if cb_env.is_developer_preview: + # Create the bucket + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + conflict_resolution_type=ConflictResolutionType.CUSTOM, + flush_enabled=False)) + bucket = cb_env.try_n_times(10, 3, cb_env.bm.get_bucket, test_bucket) + assert bucket.conflict_resolution_type == ConflictResolutionType.CUSTOM + else: + with pytest.raises(FeatureUnavailableException): + run_in_reactor_thread(cb_env.bm.create_bucket, + CreateBucketSettings( + name=test_bucket, + ram_quota_mb=100, + conflict_resolution_type=ConflictResolutionType.CUSTOM, + flush_enabled=False)) diff --git a/txcouchbase/tests/chain_t.py b/txcouchbase/tests/chain_t.py new file mode 100644 index 000000000..749d0cbae --- /dev/null +++ b/txcouchbase/tests/chain_t.py @@ -0,0 +1,72 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from couchbase.auth import PasswordAuthenticator +from couchbase.options import ClusterOptions +from couchbase.result import GetResult +from txcouchbase.bucket import Bucket +from txcouchbase.cluster import Cluster, TxCluster + +from ._test_utils import TestEnvironment, run_in_reactor_thread + + +class ConnectionChainTests: + + def test_bucket_create_chain(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster = TxCluster(conn_string, ClusterOptions(auth)) + # validate the cluster, at this point, the connect deferred should be pending + assert isinstance(cluster, Cluster) + assert cluster._connect_d.called is False + + bucket = cluster.bucket(couchbase_config.bucket_name) + assert isinstance(bucket, Bucket) + run_in_reactor_thread(bucket.on_connect) + # after connecting the bucket, the cluster connection should now exist b/c the connection + # deferreds are chained (i.e. cluster.connect -> bucket.connect) + assert cluster._connect_d is not None + assert cluster._connect_d.called is True + assert cluster._connection is not None + assert bucket._connect_d.called is True + + def test_kv_op_chain(self, couchbase_config): + conn_string = couchbase_config.get_connection_string() + username, pw = couchbase_config.get_username_and_pw() + + auth = PasswordAuthenticator(username, pw) + cluster = Cluster(conn_string, ClusterOptions(auth)) + bucket = cluster.bucket(couchbase_config.bucket_name) + coll = bucket.default_collection() + key = 'async-test-conn-key' + doc = {'id': key, 'what': 'this is an asyncio test.'} + run_in_reactor_thread(coll.upsert, key, doc) + res = None + for _ in range(3): + try: + res = run_in_reactor_thread(coll.get, key) + except Exception: + run_in_reactor_thread(TestEnvironment.deferred_sleep, 1) + + # after executing the KV op, the cluster connection should now exist b/c the connection + # deferreds are chained (i.e. cluster.connect -> bucket.connect -> KV op) + assert cluster._connect_d.called is True + assert bucket._connect_d.called is True + assert isinstance(res, GetResult) + assert res.content_as[dict] == doc + + # @TODO(jc): PYCBC-1414 - once complete, validate cluster operations chaining diff --git a/txcouchbase/tests/collection_t.py b/txcouchbase/tests/collection_t.py new file mode 100644 index 000000000..15de3fa76 --- /dev/null +++ b/txcouchbase/tests/collection_t.py @@ -0,0 +1,793 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from datetime import datetime, timedelta +from time import time + +import pytest + +import couchbase.subdocument as SD +from couchbase.diagnostics import ServiceType +from couchbase.durability import DurabilityLevel, ServerDurability +from couchbase.exceptions import (AmbiguousTimeoutException, + CasMismatchException, + DocumentExistsException, + DocumentLockedException, + DocumentNotFoundException, + DocumentNotLockedException, + DocumentUnretrievableException, + DurabilityImpossibleException, + InvalidArgumentException, + PathNotFoundException, + TemporaryFailException) +from couchbase.options import (GetOptions, + InsertOptions, + RemoveOptions, + ReplaceOptions, + UpsertOptions) +from couchbase.result import (ExistsResult, + GetReplicaResult, + GetResult, + MutationResult) +from tests.mock_server import MockServerType + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment, + run_in_reactor_thread) + + +class CollectionTests: + NO_KEY = "not-a-key" + FIFTY_YEARS = 50 * 365 * 24 * 60 * 60 + THIRTY_DAYS = 30 * 24 * 60 * 60 + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment(__name__, couchbase_config, request.param, manage_buckets=True) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + cb_env.try_n_times(3, 5, cb_env.load_data, is_deferred=False) + yield cb_env + cb_env.try_n_times(3, 5, cb_env.purge_data, is_deferred=False) + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.fixture(scope="class") + def check_preserve_expiry_supported(self, cb_env): + cb_env.check_if_feature_supported('preserve_expiry') + + @pytest.fixture(scope="class") + def check_sync_durability_supported(self, cb_env): + cb_env.check_if_feature_supported('sync_durability') + + @pytest.fixture(scope="class") + def check_xattr_supported(self, cb_env): + cb_env.check_if_feature_supported('xattr') + + @pytest.fixture(scope="class") + def check_not_locked_supported(self, cb_env): + cb_env.check_if_feature_supported('kv_not_locked') + + @pytest.fixture(name="new_kvp") + def new_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="default_kvp") + def default_key_and_value(self, cb_env) -> KVPair: + key, value = cb_env.get_default_key_value() + yield KVPair(key, value) + + @pytest.fixture(name="default_kvp_and_reset") + def default_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_default_key_value() + yield KVPair(key, value) + cb_env.try_n_times(5, 3, cb_env.collection.upsert, key, value) + + @pytest.fixture(scope="class") + def check_replicas(self, cb_env): + bucket_settings = cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + ping_res = run_in_reactor_thread(cb_env.bucket.ping) + kv_endpoints = ping_res.endpoints.get(ServiceType.KeyValue, None) + if kv_endpoints is None or len(kv_endpoints) < (num_replicas + 1): + pytest.skip("Not all replicas are online") + + @pytest.fixture(scope="class") + def skip_if_go_caves(self, cb_env): + if cb_env.is_mock_server and cb_env.mock_server_type == MockServerType.GoCAVES: + pytest.skip("GoCAVES does not like this operation.") + + @pytest.fixture(scope="class") + def num_replicas(self, cb_env): + bucket_settings = cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + return num_replicas + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_exists(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + result = run_in_reactor_thread(cb.exists, key) + assert isinstance(result, ExistsResult) + assert result.exists is True + + def test_does_not_exists(self, cb_env): + cb = cb_env.collection + result = run_in_reactor_thread(cb.exists, self.NO_KEY) + assert isinstance(result, ExistsResult) + assert result.exists is False + + def test_get(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = run_in_reactor_thread(cb.get, key) + assert isinstance(result, GetResult) + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + assert result.content_as[dict] == value + + def test_get_options(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = run_in_reactor_thread(cb.get, + key, + GetOptions(timeout=timedelta(seconds=2), + with_expiry=False)) + assert isinstance(result, GetResult) + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + assert result.content_as[dict] == value + + def test_get_fails(self, cb_env): + cb = cb_env.collection + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, self.NO_KEY) + + @pytest.mark.usefixtures("check_xattr_supported") + def test_get_with_expiry(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + run_in_reactor_thread(cb.upsert, key, value, UpsertOptions(expiry=timedelta(seconds=1000))) + + expiry_path = "$document.exptime" + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry = res.content_as[int](0) + assert expiry is not None + assert expiry > 0 + expires_in = (datetime.fromtimestamp(expiry) - datetime.now()).total_seconds() + # when running local, this can be be up to 1050, so just make sure > 0 + assert expires_in > 0 + + def test_expiry_really_expires(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + result = run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(expiry=timedelta(seconds=2))) + assert result.cas != 0 + + cb_env.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + def test_project(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(expiry=timedelta(seconds=2))) + + def cas_matches(cb, new_cas): + r = run_in_reactor_thread(cb.get, key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + print('waiting for cas matches') + cb_env.try_n_times(10, 3, cas_matches, cb, result.cas, is_deferred=False) + result = run_in_reactor_thread(cb.get, key, GetOptions(project=["faa"])) + assert {"faa": "ORD"} == result.content_as[dict] + assert result.cas is not None + assert result.key == key + assert result.expiry_time is None + + def test_project_bad_path(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + with pytest.raises(PathNotFoundException): + run_in_reactor_thread(cb.get, + key, + GetOptions(project=["some", "qzx"])) + + def test_project_project_not_list(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.get, + key, + GetOptions(project="thiswontwork")) + + @pytest.mark.usefixtures('skip_if_go_caves') + def test_project_too_many_projections(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = {f'f{i}': i for i in range(1, 21)} + result = run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(expiry=timedelta(seconds=2))) + + def cas_matches(cb, new_cas): + r = run_in_reactor_thread(cb.get, key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + print('waiting for cas matches') + cb_env.try_n_times(10, 3, cas_matches, cb, result.cas, is_deferred=False) + + project = [f'f{i}' for i in range(1, 19)] + result = run_in_reactor_thread(cb.get, key, GetOptions(project=project)) + + assert result.cas is not None + res_dict = result.content_as[dict] + assert res_dict != {} + assert res_dict.get('f20') is None + for field in project: + assert res_dict.get(field) is not None + + def test_upsert(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = cb_env.try_n_times(10, 3, cb.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + def test_upsert_preserve_expiry_not_used(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value1 = new_kvp.value + run_in_reactor_thread(cb.upsert, key, value, UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = "$document.exptime" + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + run_in_reactor_thread(cb.upsert, key, value1) + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + cb_env.sleep(3.0) + result = run_in_reactor_thread(cb.get, key) + assert isinstance(result, GetResult) + assert result.content_as[dict] == value1 + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + def test_upsert_preserve_expiry(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value1 = new_kvp.value + + run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = "$document.exptime" + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + run_in_reactor_thread(cb.upsert, + key, + value1, + UpsertOptions(preserve_expiry=True)) + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + cb_env.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + def test_insert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + result = run_in_reactor_thread(cb.insert, + key, + value, + InsertOptions(timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = cb_env.try_n_times(10, 3, cb.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + def test_insert_document_exists(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + with pytest.raises(DocumentExistsException): + run_in_reactor_thread(cb.insert, key, value) + + def test_replace(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = run_in_reactor_thread(cb.replace, + key, + value, + ReplaceOptions(timeout=timedelta(seconds=3))) + assert result is not None + assert isinstance(result, MutationResult) + assert result.cas != 0 + g_result = cb_env.try_n_times(10, 3, cb.get, key) + assert g_result.key == key + assert value == g_result.content_as[dict] + + def test_replace_with_cas(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value1 = new_kvp.value + result = run_in_reactor_thread(cb.get, key) + old_cas = result.cas + result = run_in_reactor_thread(cb.replace, + key, + value1, + ReplaceOptions(cas=old_cas)) + assert isinstance(result, MutationResult) + assert result.cas != old_cas + + # try same cas again, must fail. + with pytest.raises(CasMismatchException): + run_in_reactor_thread(cb.replace, + key, + value1, + ReplaceOptions(cas=old_cas)) + + def test_replace_fail(self, cb_env): + cb = cb_env.collection + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.replace, self.NO_KEY, {"some": "content"}) + + def test_remove(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + result = run_in_reactor_thread(cb.remove, key) + assert isinstance(result, MutationResult) + + with pytest.raises(DocumentNotFoundException): + cb_env.try_n_times_till_exception(3, + 1, + cb.get, + key, + expected_exceptions=(DocumentNotFoundException,), + raise_exception=True) + + def test_remove_fail(self, cb_env): + cb = cb_env.collection + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.remove, self.NO_KEY) + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + def test_replace_preserve_expiry_not_used(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value1 = new_kvp.value + + run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = "$document.exptime" + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + run_in_reactor_thread(cb.replace, key, value1) + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + cb_env.sleep(3.0) + result = run_in_reactor_thread(cb.get, key) + assert isinstance(result, GetResult) + assert result.content_as[dict] == value1 + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + def test_replace_preserve_expiry(self, cb_env, default_kvp_and_reset, new_kvp): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + value1 = new_kvp.value + + run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(expiry=timedelta(seconds=2))) + expiry_path = "$document.exptime" + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + run_in_reactor_thread(cb.replace, + key, + value1, + ReplaceOptions(preserve_expiry=True)) + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + cb_env.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + @pytest.mark.usefixtures("check_preserve_expiry_supported") + def test_replace_preserve_expiry_fail(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + opts = ReplaceOptions( + expiry=timedelta( + seconds=5), + preserve_expiry=True) + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.replace, key, value, opts) + + def test_touch(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 1, cb.get, key) + result = run_in_reactor_thread(cb.touch, key, timedelta(seconds=2)) + assert isinstance(result, MutationResult) + cb_env.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + def test_touch_no_expire(self, cb_env, new_kvp): + # TODO: handle MOCK + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 1, cb.get, key) + run_in_reactor_thread(cb.touch, key, timedelta(seconds=15)) + g_result = run_in_reactor_thread(cb.get, key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is not None + run_in_reactor_thread(cb.touch, key, timedelta(seconds=0)) + g_result = run_in_reactor_thread(cb.get, key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is None + + def test_get_and_touch(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 1, cb.get, key) + result = run_in_reactor_thread(cb.get_and_touch, key, timedelta(seconds=2)) + assert isinstance(result, GetResult) + cb_env.sleep(3.0) + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + def test_get_and_touch_no_expire(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 1, cb.get, key) + run_in_reactor_thread(cb.get_and_touch, key, timedelta(seconds=15)) + g_result = run_in_reactor_thread(cb.get, key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is not None + run_in_reactor_thread(cb.get_and_touch, key, timedelta(seconds=0)) + g_result = run_in_reactor_thread(cb.get, key, GetOptions(with_expiry=True)) + assert g_result.expiry_time is None + + def test_get_and_lock(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = run_in_reactor_thread(cb.get_and_lock, key, timedelta(seconds=3)) + assert isinstance(result, GetResult) + with pytest.raises(DocumentLockedException): + run_in_reactor_thread(cb.upsert, key, value) + + cb_env.try_n_times(10, 1, cb.upsert, key, value) + + def test_get_after_lock(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + orig = run_in_reactor_thread(cb.get_and_lock, key, timedelta(seconds=5)) + assert isinstance(orig, GetResult) + result = run_in_reactor_thread(cb.get, key) + assert orig.content_as[dict] == result.content_as[dict] + assert orig.cas != result.cas + + # @TODO(jc): cxx client raises ambiguous timeout w/ retry reason: kv_temporary_failure + cb_env.try_n_times_till_exception(10, 1, + cb.unlock, key, orig.cas, + expected_exceptions=(TemporaryFailException, + DocumentNotLockedException)) + + def test_get_and_lock_replace_with_cas(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + result = run_in_reactor_thread(cb.get_and_lock, key, timedelta(seconds=5)) + assert isinstance(result, GetResult) + cas = result.cas + # TODO: handle retry reasons, looks to be where we can get the locked + # exception + with pytest.raises((AmbiguousTimeoutException, DocumentLockedException)): + run_in_reactor_thread(cb.upsert, key, value) + + run_in_reactor_thread(cb.replace, key, value, ReplaceOptions(cas=cas)) + # @TODO(jc): cxx client raises ambiguous timeout w/ retry reason: kv_temporary_failure + cb_env.try_n_times_till_exception(10, 1, + cb.unlock, key, cas, + expected_exceptions=(TemporaryFailException, + DocumentNotLockedException)) + + def test_unlock(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + result = run_in_reactor_thread(cb.get_and_lock, key, timedelta(seconds=5)) + assert isinstance(result, GetResult) + run_in_reactor_thread(cb.unlock, key, result.cas) + run_in_reactor_thread(cb.upsert, key, value) + + def test_unlock_wrong_cas(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + result = run_in_reactor_thread(cb.get_and_lock, key, timedelta(seconds=5)) + cas = result.cas + # @TODO(jc): MOCK - TemporaryFailException + with pytest.raises(CasMismatchException): + run_in_reactor_thread(cb.unlock, key, 100) + + cb_env.try_n_times_till_exception(10, 1, + cb.unlock, key, cas, + expected_exceptions=(TemporaryFailException, DocumentNotLockedException)) + + @pytest.mark.usefixtures("check_not_locked_supported") + def test_unlock_not_locked(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + result = run_in_reactor_thread(cb.get_and_lock, key, timedelta(seconds=5)) + cas = result.cas + run_in_reactor_thread(cb.unlock, key, cas) + with pytest.raises(DocumentNotLockedException): + run_in_reactor_thread(cb.unlock, key, cas) + + @pytest.mark.usefixtures("check_replicas") + def test_get_any_replica(self, cb_env, default_kvp): + result = cb_env.try_n_times(10, 3, cb_env.collection.get_any_replica, default_kvp.key) + assert isinstance(result, GetReplicaResult) + assert isinstance(result.is_replica, bool) + assert default_kvp.value == result.content_as[dict] + + @pytest.mark.usefixtures("check_replicas") + def test_get_any_replica_fail(self, cb_env): + with pytest.raises(DocumentUnretrievableException): + run_in_reactor_thread(cb_env.collection.get_any_replica, 'not-a-key') + + @pytest.mark.usefixtures("check_replicas") + def test_get_all_replicas(self, cb_env, default_kvp): + result = cb_env.try_n_times(10, 3, cb_env.collection.get_all_replicas, default_kvp.key) + # make sure we can iterate over results + while True: + try: + res = next(result) + assert isinstance(res, GetReplicaResult) + assert isinstance(res.is_replica, bool) + assert default_kvp.value == res.content_as[dict] + except StopIteration: + break + + @pytest.mark.usefixtures("check_replicas") + def test_get_all_replicas_fail(self, cb_env): + pytest.skip("Pending merge of CXXCBC-52") + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb_env.collection.get_all_replicas, 'not-a-key') + + @pytest.mark.usefixtures("check_replicas") + def test_get_all_replicas_results(self, cb_env, default_kvp): + result = cb_env.try_n_times(10, 3, cb_env.collection.get_all_replicas, default_kvp.key) + active_cnt = 0 + replica_cnt = 0 + for res in result: + assert isinstance(res, GetReplicaResult) + assert isinstance(res.is_replica, bool) + assert default_kvp.value == res.content_as[dict] + if res.is_replica: + replica_cnt += 1 + else: + active_cnt += 1 + + assert active_cnt == 1 + assert replica_cnt >= active_cnt + + @pytest.mark.usefixtures("check_sync_durability_supported") + def test_server_durable_upsert(self, cb_env, default_kvp_and_reset, num_replicas): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + + if num_replicas > 1: + run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(durability=durability)) + result = run_in_reactor_thread(cb.get, key) + assert value == result.content_as[dict] + else: + try: + run_in_reactor_thread(cb.upsert, + key, + value, + UpsertOptions(durability=durability)) + except DurabilityImpossibleException: + pass # this is okay -- server not setup correctly + + @pytest.mark.usefixtures("check_sync_durability_supported") + def test_server_durable_insert(self, cb_env, new_kvp, num_replicas): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + if num_replicas > 1: + run_in_reactor_thread(cb.insert, + key, + value, + InsertOptions(durability=durability)) + result = run_in_reactor_thread(cb.get, key) + assert value == result.content_as[dict] + else: + try: + run_in_reactor_thread(cb.insert, + key, + value, + InsertOptions(durability=durability)) + except DurabilityImpossibleException: + pass # this is okay -- server not setup correctly + + @pytest.mark.usefixtures("check_sync_durability_supported") + def test_server_durable_replace(self, cb_env, default_kvp_and_reset, num_replicas): + cb = cb_env.collection + key = default_kvp_and_reset.key + value = default_kvp_and_reset.value + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + + if num_replicas > 1: + run_in_reactor_thread(cb.replace, + key, + value, + ReplaceOptions(durability=durability)) + result = run_in_reactor_thread(cb.get, key) + assert value == result.content_as[dict] + else: + try: + run_in_reactor_thread(cb.replace, + key, + value, + ReplaceOptions(durability=durability)) + except DurabilityImpossibleException: + pass # this is okay -- server not setup correctly + + @pytest.mark.usefixtures("check_sync_durability_supported") + def test_server_durable_remove(self, cb_env, default_kvp_and_reset, num_replicas): + cb = cb_env.collection + key = default_kvp_and_reset.key + + durability = ServerDurability(level=DurabilityLevel.PERSIST_TO_MAJORITY) + if num_replicas > 1: + run_in_reactor_thread(cb.remove, key, RemoveOptions(durability=durability)) + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + else: + try: + run_in_reactor_thread(cb.remove, key, RemoveOptions(durability=durability)) + except DurabilityImpossibleException: + pass # this is okay -- server not setup correctly + + def test_client_durable_upsert(self): + pytest.skip("C++ client has not implemented replicate/persist durability.") + + def test_client_durable_insert(self): + pytest.skip("C++ client has not implemented replicate/persist durability.") + + def test_client_durable_replace(self): + pytest.skip("C++ client has not implemented replicate/persist durability.") + + def test_client_durable_remove(self): + pytest.skip("C++ client has not implemented replicate/persist durability.") + + # @TODO(jc): - should an expiry of -1 raise an InvalidArgumentException? + + @pytest.mark.usefixtures("check_xattr_supported") + @pytest.mark.parametrize("expiry", [FIFTY_YEARS + 1, + FIFTY_YEARS, + THIRTY_DAYS - 1, + THIRTY_DAYS, + int(time() - 1.0), 60, -1]) + def test_document_expiry_values(self, cb_env, new_kvp, expiry): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + before = int(time() - 1.0) + try: + result = run_in_reactor_thread(cb.upsert, key, value, expiry=timedelta(seconds=expiry)) + assert result.cas is not None + except InvalidArgumentException: + if expiry != -1: + raise + + expiry_path = "$document.exptime" + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + res_expiry = res.content_as[int](0) + + after = int(time() + 1.0) + before + expiry <= res_expiry <= after + expiry diff --git a/txcouchbase/tests/collectionmgmt_t.py b/txcouchbase/tests/collectionmgmt_t.py new file mode 100644 index 000000000..389477fba --- /dev/null +++ b/txcouchbase/tests/collectionmgmt_t.py @@ -0,0 +1,561 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import timedelta +from uuid import uuid4 + +import pytest + +from couchbase.exceptions import (BucketDoesNotExistException, + CollectionAlreadyExistsException, + CollectionNotFoundException, + DocumentNotFoundException, + FeatureUnavailableException, + InvalidArgumentException, + ScopeAlreadyExistsException, + ScopeNotFoundException) +from couchbase.management.buckets import StorageBackend +from couchbase.management.collections import (CollectionSpec, + CreateCollectionSettings, + UpdateCollectionSettings) +from tests.environments.test_environment import EnvironmentFeatures + +from ._test_utils import TestEnvironment, run_in_reactor_thread + + +class CollectionManagementTests: + + TEST_BUCKET = "test-bucket" + TEST_SCOPE = "test-scope" + TEST_COLLECTION = "test-collection" + + @pytest.fixture(scope="class", name="cb_env") + def couchbase_test_environment(self, couchbase_config): + cb_env = TestEnvironment.get_environment(__name__, + couchbase_config, + manage_buckets=True, + manage_collections=True) + # will create a new bucket w/ name test-bucket + cb_env.try_n_times(3, 5, cb_env.setup_collection_mgmt, self.TEST_BUCKET, is_deferred=False) + yield cb_env + if cb_env.is_feature_supported('bucket_mgmt'): + cb_env.purge_buckets([self.TEST_BUCKET]) + + @pytest.fixture(scope="class") + def check_non_deduped_history_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('non_deduped_history', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_update_collection_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('update_collection', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_update_collection_max_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('update_collection_max_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture(scope="class") + def check_negative_collection_max_expiry_supported(self, cb_env): + EnvironmentFeatures.check_if_feature_supported('negative_collection_max_expiry', + cb_env.server_version_short, + cb_env.mock_server_type) + + @pytest.fixture() + def cleanup_scope(self, cb_env): + cb_env.try_n_times_till_exception(5, 1, + cb_env.test_bucket_cm.drop_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeNotFoundException,)) + yield + cb_env.try_n_times_till_exception(5, 1, + cb_env.test_bucket_cm.drop_scope, + self.TEST_SCOPE, + expected_exceptions=(ScopeNotFoundException,)) + + @pytest.fixture() + def cleanup_collection(self, cb_env): + cb_env.try_n_times_till_exception(5, 1, + cb_env.test_bucket_cm.drop_collection, + CollectionSpec(self.TEST_COLLECTION), + expected_exceptions=(CollectionNotFoundException,)) + yield + cb_env.try_n_times_till_exception(5, 1, + cb_env.test_bucket_cm.drop_collection, + CollectionSpec(self.TEST_COLLECTION), + expected_exceptions=(CollectionNotFoundException,)) + + # temporary until we consoldate w/ new test env setup + def _get_collection(self, cm, scope_name, coll_name): + scopes = run_in_reactor_thread(cm.get_all_scopes) + scope = next((s for s in scopes if s.name == scope_name), None) + if scope: + return next( + (c for c in scope.collections if c.name == coll_name), None) + + return None + + @pytest.mark.usefixtures("cleanup_scope") + def test_create_scope(self, cb_env): + run_in_reactor_thread(cb_env.test_bucket_cm.create_scope, self.TEST_SCOPE) + assert cb_env.get_scope(self.TEST_SCOPE) is not None + + @pytest.mark.usefixtures("cleanup_scope") + def test_create_scope_already_exists(self, cb_env): + run_in_reactor_thread(cb_env.test_bucket_cm.create_scope, self.TEST_SCOPE) + assert cb_env.get_scope(self.TEST_SCOPE) is not None + with pytest.raises(ScopeAlreadyExistsException): + run_in_reactor_thread(cb_env.test_bucket_cm.create_scope, self.TEST_SCOPE) + + def test_get_all_scopes(self, cb_env): + scope_names = [str(uuid4())[:8] for _ in range(4)] + for name in scope_names: + run_in_reactor_thread(cb_env.test_bucket_cm.create_scope, name) + for _ in range(2): + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, name, str(uuid4())[:8]) + scopes = run_in_reactor_thread(cb_env.test_bucket_cm.get_all_scopes) + + assert (sum(s.name[0] != '_' for s in scopes) == len(scope_names)) + # should have a _default scope + assert any(map(lambda s: s.name == '_default', scopes)) + for scope_name in scope_names: + assert any(map(lambda s: s.name == scope_name, scopes)) + for s in scopes: + for c in s.collections: + assert isinstance(c, CollectionSpec) + assert isinstance(c.name, str) + assert isinstance(c.scope_name, str) + assert isinstance(c.max_expiry, timedelta) + assert c.history is None or isinstance(c.history, bool) + + def test_drop_scope(self, cb_env): + run_in_reactor_thread(cb_env.test_bucket_cm.create_scope, self.TEST_SCOPE) + assert cb_env.get_scope(self.TEST_SCOPE) is not None + run_in_reactor_thread(cb_env.test_bucket_cm.drop_scope, self.TEST_SCOPE) + with pytest.raises(ScopeNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.drop_scope, self.TEST_SCOPE) + + def test_drop_scope_not_found(self, cb_env): + with pytest.raises(ScopeNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.drop_scope, "some-random-scope") + + @pytest.mark.usefixtures("cleanup_collection") + def test_create_collection(self, cb_env): + # create a collection under default_ scope + collection_name = self.TEST_COLLECTION + scope_name = '_default' + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + assert cb_env.get_collection(scope_name, collection_name) is not None + + @pytest.mark.usefixtures("cleanup_scope") + def test_create_scope_and_collection(self, cb_env): + run_in_reactor_thread(cb_env.test_bucket_cm.create_scope, self.TEST_SCOPE) + assert cb_env.get_scope(self.TEST_SCOPE) is not None + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, self.TEST_SCOPE, self.TEST_COLLECTION) + assert cb_env.get_collection(self.TEST_SCOPE, self.TEST_COLLECTION) is not None + + @pytest.mark.usefixtures("cleanup_collection") + def test_create_collection_max_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection TTL.") + + scope_name = '_default' + collection_name = self.TEST_COLLECTION + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=2)) + + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name, settings) + assert cb_env.get_collection(scope_name, collection_name) is not None + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(collection_name) + key = "test-coll-key0" + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + cb_env.try_n_times(10, 1, coll.upsert, key, {"some": "thing"}) + cb_env.try_n_times(10, 1, coll.get, key) + cb_env.try_n_times_till_exception( + 4, 1, coll.get, key, expected_exceptions=( + DocumentNotFoundException,)) + + @pytest.mark.usefixtures("cleanup_collection") + def test_create_collection_max_expiry_default(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + # TODO: consistency + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + @pytest.mark.usefixtures("cleanup_collection") + def test_create_collection_max_expiry_invalid(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + collection_name = self.TEST_COLLECTION + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=-20)) + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name, settings) + + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures("cleanup_collection") + def test_create_collection_max_expiry_no_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=-1)) + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name, settings) + # TODO: consistency + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=-1) + + def test_create_collection_bad_scope(self, cb_env): + collection = CollectionSpec(self.TEST_COLLECTION, "im-a-fake-scope") + with pytest.raises(ScopeNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, collection) + + @pytest.mark.usefixtures("cleanup_collection") + def test_create_collection_already_exists(self, cb_env): + scope_name = '_default' + collection_name = self.TEST_COLLECTION + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + # verify the collection exists w/in other-bucket + assert cb_env.get_collection(scope_name, collection_name) is not None + # now, it will fail if we try to create it again... + with pytest.raises(CollectionAlreadyExistsException): + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + + @pytest.mark.usefixtures("cleanup_collection") + def test_collection_goes_in_correct_bucket(self, cb_env): + collection = CollectionSpec(self.TEST_COLLECTION) + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, collection) + # make sure it actually is in the other-bucket + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + # also be sure this isn't in the default bucket + assert cb_env.get_collection(collection.scope_name, + collection.name, + bucket_name=cb_env.bucket.name) is None + + @pytest.mark.usefixtures("cleanup_collection") + def test_deprecated_create_collection(self, cb_env): + # create a collection under default_ scope + collection = CollectionSpec(self.TEST_COLLECTION) + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, collection) + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + + @pytest.mark.usefixtures("cleanup_scope") + def test_deprecated_create_scope_and_collection(self, cb_env): + run_in_reactor_thread(cb_env.test_bucket_cm.create_scope, self.TEST_SCOPE) + assert cb_env.get_scope(self.TEST_SCOPE) is not None + collection = CollectionSpec(self.TEST_COLLECTION, self.TEST_SCOPE) + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, collection) + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + + @pytest.mark.usefixtures("cleanup_collection") + def test_deprecated_create_collection_max_ttl(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection TTL.") + + collection = CollectionSpec( + self.TEST_COLLECTION, + max_ttl=timedelta( + seconds=2)) + + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, collection) + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(collection.name) + key = "test-coll-key0" + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + cb_env.try_n_times(10, 1, coll.upsert, key, {"some": "thing"}) + cb_env.try_n_times(10, 1, coll.get, key) + cb_env.try_n_times_till_exception( + 4, 1, coll.get, key, expected_exceptions=( + DocumentNotFoundException,)) + + @pytest.mark.usefixtures("cleanup_collection") + def test_deprecated_create_collection_already_exists(self, cb_env): + collection = CollectionSpec(self.TEST_COLLECTION) + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, collection) + # verify the collection exists w/in other-bucket + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + # now, it will fail if we try to create it again... + with pytest.raises(CollectionAlreadyExistsException): + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, collection) + + @pytest.mark.usefixtures("cleanup_collection") + def test_deprecated_drop_collection(self, cb_env): + collection = CollectionSpec(self.TEST_COLLECTION) + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, collection) + # verify the collection exists w/in other-bucket + assert cb_env.get_collection(collection.scope_name, collection.name) is not None + # attempt to drop it again will raise CollectionNotFoundException + run_in_reactor_thread(cb_env.test_bucket_cm.drop_collection, collection) + with pytest.raises(CollectionNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.drop_collection, collection) + + def test_deprecated_drop_collection_not_found(self, cb_env): + collection = CollectionSpec("fake-collection") + with pytest.raises(CollectionNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.drop_collection, collection) + + def test_deprecated_drop_collection_scope_not_found(self, cb_env): + collection = CollectionSpec("fake-collection", "fake-scope") + with pytest.raises(ScopeNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.drop_collection, collection) + + @pytest.mark.usefixtures("cleanup_collection") + def test_drop_collection(self, cb_env): + collection_name = self.TEST_COLLECTION + scope_name = '_default' + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + # verify the collection exists w/in other-bucket + assert cb_env.get_collection(scope_name, collection_name) is not None + # attempt to drop it again will raise CollectionNotFoundException + run_in_reactor_thread(cb_env.test_bucket_cm.drop_collection, scope_name, collection_name) + with pytest.raises(CollectionNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.drop_collection, scope_name, collection_name) + + def test_drop_collection_not_found(self, cb_env): + collection_name = 'fake-collection' + scope_name = '_default' + with pytest.raises(CollectionNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.drop_collection, scope_name, collection_name) + + def test_drop_collection_scope_not_found(self, cb_env): + collection_name = 'fake-collection' + scope_name = 'fake-scope' + with pytest.raises(ScopeNotFoundException): + run_in_reactor_thread(cb_env.test_bucket_cm.drop_collection, scope_name, collection_name) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + def test_create_collection_history_retention(self, cb_env): + bucket_name = 'test-magma-bucket' + scope_name = '_default' + collection_name = self.TEST_COLLECTION + + cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA) + bucket = cb_env.cluster.bucket(bucket_name) + cb_env.try_n_times(10, 1, bucket.on_connect) + cm = bucket.collections() + + run_in_reactor_thread(cm.create_collection, + scope_name, + collection_name, + CreateCollectionSettings(history=True)) + collection_spec = None + retry = 0 + while retry < 5 and collection_spec is None: + collection_spec = self._get_collection(cm, scope_name, collection_name) + cb_env.sleep(1) + retry += 1 + assert collection_spec is not None + assert collection_spec.history + + cb_env.try_n_times_till_exception(10, + 3, + cb_env.bm.drop_bucket, + bucket_name, + expected_exceptions=(BucketDoesNotExistException,)) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + def test_create_collection_history_retention_unsupported(self, cb_env): + scope_name = '_default' + collection_name = self.TEST_COLLECTION + + # Couchstore does not support history retention + with pytest.raises(FeatureUnavailableException): + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, + scope_name, + collection_name, + CreateCollectionSettings(history=True)) + + with pytest.raises(FeatureUnavailableException): + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, + scope_name, + collection_name, + CreateCollectionSettings(history=False)) + + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures("cleanup_collection") + def test_update_collection_history_retention(self, cb_env): + bucket_name = 'test-magma-bucket' + scope_name = '_default' + collection_name = self.TEST_COLLECTION + + cb_env.create_bucket(bucket_name, storage_backend=StorageBackend.MAGMA) + bucket = cb_env.cluster.bucket(bucket_name) + cb_env.try_n_times(10, 1, bucket.on_connect) + cm = bucket.collections() + + run_in_reactor_thread(cm.create_collection, scope_name, collection_name, + CreateCollectionSettings(history=False)) + collection_spec = None + retry = 0 + while retry < 5 and collection_spec is None: + collection_spec = self._get_collection(cm, scope_name, collection_name) + cb_env.sleep(1) + retry += 1 + assert collection_spec is not None + assert not collection_spec.history + + run_in_reactor_thread(cm.update_collection, scope_name, collection_name, UpdateCollectionSettings(history=True)) + collection_spec = None + retry = 0 + while retry < 5 and collection_spec is None: + collection_spec = self._get_collection(cm, scope_name, collection_name) + cb_env.sleep(1) + retry += 1 + assert collection_spec is not None + assert collection_spec.history + + cb_env.try_n_times_till_exception(10, + 3, + cb_env.bm.drop_bucket, + bucket_name, + expected_exceptions=(BucketDoesNotExistException,)) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.usefixtures('check_non_deduped_history_supported') + @pytest.mark.usefixtures('check_update_collection_supported') + def test_update_collection_history_retention_unsupported(self, cb_env): + scope_name = '_default' + collection_name = self.TEST_COLLECTION + + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + collection_spec = cb_env.get_collection(scope_name, collection_name) + assert collection_spec is not None + assert collection_spec.history is False + + # Couchstore does not support history retention + with pytest.raises(FeatureUnavailableException): + run_in_reactor_thread(cb_env.test_bucket_cm.update_collection, + scope_name, + collection_name, + UpdateCollectionSettings(history=True)) + + # Collection history retention setting remains unchanged + collection_spec = cb_env.get_collection(scope_name, collection_name) + assert collection_spec is not None + assert collection_spec.history is False + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + def test_update_collection_max_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=2)) + run_in_reactor_thread(cb_env.test_bucket_cm.update_collection, scope_name, collection_name, settings) + + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=2) + + # pop a doc in with no ttl, verify it goes away... + coll = cb_env.test_bucket.collection(collection_name) + key = 'test-coll-key0' + # we _can_ get a temp fail here, as we just created the collection. So we + # retry the upsert. + cb_env.try_n_times(10, 1, coll.upsert, key, {'some': 'thing'}) + cb_env.try_n_times(10, 1, coll.get, key) + cb_env.try_n_times_till_exception(4, + 1, + coll.get, + key, + expected_exceptions=(DocumentNotFoundException,)) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_update_collection_max_expiry_bucket_default(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + settings = CreateCollectionSettings(max_expiry=timedelta(seconds=5)) + + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name, settings) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=5) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=0)) + run_in_reactor_thread(cb_env.test_bucket_cm.update_collection, scope_name, collection_name, settings) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_update_collection_max_expiry_invalid(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + collection_name = self.TEST_COLLECTION + scope_name = '_default' + + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=-20)) + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb_env.test_bucket_cm.update_collection, scope_name, collection_name, settings) + + @pytest.mark.usefixtures("cleanup_collection") + @pytest.mark.usefixtures('check_update_collection_supported') + @pytest.mark.usefixtures('check_update_collection_max_expiry_supported') + @pytest.mark.usefixtures('check_negative_collection_max_expiry_supported') + def test_update_collection_max_expiry_no_expiry(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES doesn't support collection expiry.") + + collection_name = self.TEST_COLLECTION + scope_name = '_default' + run_in_reactor_thread(cb_env.test_bucket_cm.create_collection, scope_name, collection_name) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=0) + + settings = UpdateCollectionSettings(max_expiry=timedelta(seconds=-1)) + run_in_reactor_thread(cb_env.test_bucket_cm.update_collection, scope_name, collection_name, settings) + coll_spec = cb_env.get_collection(scope_name, collection_name) + assert coll_spec is not None + assert coll_spec.max_expiry == timedelta(seconds=-1) diff --git a/txcouchbase/tests/conftest.py b/txcouchbase/tests/conftest.py new file mode 100644 index 000000000..6eead6539 --- /dev/null +++ b/txcouchbase/tests/conftest.py @@ -0,0 +1,76 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +from twisted.internet import threads +from twisted.internet.error import ReactorAlreadyInstalledError + + +class TwistedObjects: + _REACTOR = None + _GREENLET = None + _TWISTED_THREAD = None + + +def pytest_configure(config): + run_tx = config.getoption('--txcouchbase') + if config and config.option and config.option.markexpr: + run_tx = config.getoption('--txcouchbase') + if 'txcouchbase' in config.option.markexpr: + run_tx = True + + if run_tx is True: + init_reactor() + + +def run_in_reactor_thread(fn, *args, **kwargs): + result = threads.blockingCallFromThread(TwistedObjects._REACTOR, fn, *args, **kwargs) + return result + + +def run_reactor(reactor): + reactor.run() + + +def init_reactor(): + from twisted.internet import asyncioreactor + + from acouchbase import get_event_loop + try: + asyncioreactor.install(get_event_loop()) + except ReactorAlreadyInstalledError as ex: + print(f'Twisted setup: {ex}') + finally: + import twisted.internet.reactor + + TwistedObjects._REACTOR = twisted.internet.reactor + if not hasattr(TwistedObjects._REACTOR, '_asyncioEventloop'): + raise RuntimeError( + "Reactor installed is not the asyncioreactor.") + + TwistedObjects._TWISTED_THREAD = threading.Thread(target=lambda: run_reactor(TwistedObjects._REACTOR)) + TwistedObjects._REACTOR.suggestThreadPoolSize(10) + TwistedObjects._TWISTED_THREAD.start() + +# hook to catch prior to running tests +# def pytest_runtest_call(item): +# pass + + +def pytest_unconfigure(): + if TwistedObjects._TWISTED_THREAD: + threads.blockingCallFromThread(TwistedObjects._REACTOR, TwistedObjects._REACTOR.stop) + TwistedObjects._TWISTED_THREAD.join() diff --git a/txcouchbase/tests/mutation_tokens_t.py b/txcouchbase/tests/mutation_tokens_t.py new file mode 100644 index 000000000..0633c5341 --- /dev/null +++ b/txcouchbase/tests/mutation_tokens_t.py @@ -0,0 +1,122 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import couchbase.subdocument as SD +from couchbase.exceptions import DocumentNotFoundException + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment, + run_in_reactor_thread) + + +class MutationTokensEnabledTests: + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment(__name__, couchbase_config, request.param) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + yield cb_env + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.fixture(name="new_kvp") + def new_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + def verify_mutation_tokens(self, bucket_name, result): + mutation_token = result.mutation_token() + assert mutation_token is not None + partition_id, partition_uuid, sequence_number, mt_bucket_name = mutation_token.as_tuple() + assert isinstance(partition_id, int) + assert isinstance(partition_uuid, int) + assert isinstance(sequence_number, int) + assert bucket_name == mt_bucket_name + + def test_mutation_tokens_upsert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + result = cb_env.try_n_times(5, 3, cb.upsert, key, value) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + def test_mutation_tokens_insert(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + result = cb_env.try_n_times(5, 3, cb.insert, key, value) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + def test_mutation_tokens_replace(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + cb_env.try_n_times(5, 3, cb.upsert, key, value) + result = cb_env.try_n_times(5, 3, cb.replace, key, value) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + def test_mutation_tokens_remove(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + cb_env.try_n_times(5, 3, cb.upsert, key, value) + result = cb_env.try_n_times(5, 3, cb.remove, key) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + # @TODO: c++ client does not provide mutation token for touch + # def test_mutation_tokens_touch(self, cb_env, new_kvp): + # cb = cb_env.collection + # key = new_kvp.key + # value = new_kvp.value + # cb.upsert(key, value) + # result = cb.touch(key, timedelta(seconds=3)) + # self.verify_mutation_tokens(cb_env.bucket.name, result) + + def test_mutation_tokens_mutate_in(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + + def cas_matches(key, cas): + result = run_in_reactor_thread(cb.get, key) + if result.cas == cas: + return result + raise Exception("nope") + + res = cb_env.try_n_times(5, 3, cb.upsert, key, {"a": "aaa", "b": {"c": {"d": "yo!"}}}) + cas = res.cas + cb_env.try_n_times(10, 3, cas_matches, key, cas, is_deferred=False) + result = cb_env.try_n_times(5, 3, cb.mutate_in, key, (SD.upsert("c", "ccc"), SD.replace("b", "XXX"),)) + self.verify_mutation_tokens(cb_env.bucket.name, result) + + +# @TODO: need to update client settings first +class MutationTokensDisabledTests: + pass diff --git a/txcouchbase/tests/subdoc_t.py b/txcouchbase/tests/subdoc_t.py new file mode 100644 index 000000000..326e4f756 --- /dev/null +++ b/txcouchbase/tests/subdoc_t.py @@ -0,0 +1,1017 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from datetime import datetime, timedelta + +import pytest + +import couchbase.subdocument as SD +from couchbase.durability import DurabilityLevel, ServerDurability +from couchbase.exceptions import (DocumentExistsException, + DocumentNotFoundException, + DocumentUnretrievableException, + DurabilityImpossibleException, + InvalidArgumentException, + InvalidValueException, + PathExistsException, + PathMismatchException, + PathNotFoundException) +from couchbase.options import (GetOptions, + LookupInAllReplicasOptions, + LookupInAnyReplicaOptions, + LookupInOptions, + MutateInOptions) +from couchbase.result import (GetResult, + LookupInReplicaResult, + LookupInResult, + MutateInResult) + +from ._test_utils import (CollectionType, + KVPair, + TestEnvironment, + run_in_reactor_thread) + + +class SubDocumentTests: + NO_KEY = "not-a-key" + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment(__name__, couchbase_config, request.param) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + cb_env.try_n_times(3, 5, cb_env.load_data, is_deferred=False) + yield cb_env + cb_env.try_n_times(3, 5, cb_env.purge_data, is_deferred=False) + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.fixture(scope="class") + def skip_if_less_than_cheshire_cat(self, cb_env): + if cb_env.cluster.server_version_short < 7.0: + pytest.skip("Feature only available on CBS >= 7.0") + + @pytest.fixture(scope="class") + def skip_if_less_than_alice(self, cb_env): + if cb_env.cluster.server_version_short < 6.5: + pytest.skip("Feature only available on CBS >= 6.5") + + @pytest.fixture(scope="class") + def check_xattr_supported(self, cb_env): + cb_env.check_if_feature_supported('xattr') + + @pytest.fixture(scope="class") + def check_replica_read_supported(self, cb_env): + cb_env.check_if_feature_supported('subdoc_replica_read') + + @pytest.fixture(scope="class") + def num_replicas(self, cb_env): + pytest.skip('num_replicas not supported, pending txcouchbase mgmt updates.') + bucket_settings = cb_env.try_n_times(10, 1, cb_env.bm.get_bucket, cb_env.bucket.name) + num_replicas = bucket_settings.get("num_replicas") + return num_replicas + + @pytest.fixture(name="new_kvp") + def new_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="default_kvp") + def default_key_and_value(self, cb_env) -> KVPair: + key, value = cb_env.get_default_key_value() + yield KVPair(key, value) + + @pytest.fixture(name="default_kvp_and_reset") + def default_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_default_key_value() + yield KVPair(key, value) + cb_env.try_n_times(5, 3, cb_env.collection.upsert, key, value) + + @pytest.fixture(scope="class") + def skip_mock_mutate_in(self, cb_env): + if cb_env.is_mock_server: + pytest.skip("CAVES + couchbase++ not playing nice...") + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_lookup_in_simple_get(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in, key, (SD.get("geo"),)) + assert isinstance(result, LookupInResult) + assert result.content_as[dict](0) == value["geo"] + + def test_lookup_in_simple_get_spec_as_list(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = run_in_reactor_thread(cb.lookup_in, key, [SD.get("geo")]) + assert isinstance(result, LookupInResult) + assert result.content_as[dict](0) == value["geo"] + + def test_lookup_in_simple_exists(self, cb_env): + cb = cb_env.collection + key, _ = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in, key, (SD.exists("geo"),)) + assert isinstance(result, LookupInResult) + assert result.exists(0) + assert result.content_as[bool](0) + + def test_lookup_in_simple_exists_bad_path(self, cb_env): + cb = cb_env.collection + key, _ = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in, key, (SD.exists("qzzxy"),)) + assert isinstance(result, LookupInResult) + assert result.exists(0) is False + assert not result.content_as[bool](0) + + def test_lookup_in_one_path_not_found(self, cb_env): + cb = cb_env.collection + key, _ = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in, + key, + (SD.exists("geo"), SD.exists("qzzxy"),)) + assert isinstance(result, LookupInResult) + assert result.exists(0) + assert result.exists(1) is False + assert result.content_as[bool](0) + assert not result.content_as[bool](1) + + def test_lookup_in_simple_long_path(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + # add longer path to doc + value["long_path"] = {"a": {"b": {"c": "yo!"}}} + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.lookup_in, + key, + (SD.get("long_path.a.b.c"),)) + assert isinstance(result, LookupInResult) + assert result.content_as[str](0) == value["long_path"]["a"]["b"]["c"] + # reset to norm + run_in_reactor_thread(cb.remove, key) + + def test_lookup_in_simple_with_timeout(self, cb_env, default_kvp): + cb = cb_env.collection + key = default_kvp.key + value = default_kvp.value + result = run_in_reactor_thread(cb.lookup_in, + key, + (SD.get('geo'),), + LookupInOptions(timeout=timedelta(milliseconds=5000))) + assert isinstance(result, LookupInResult) + assert result.content_as[dict](0) == value['geo'] + + @pytest.mark.usefixtures("check_xattr_supported") + def test_lookup_in_multiple_specs(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in, + key, + (SD.get("$document.exptime", xattr=True), + SD.exists("geo"), + SD.get("geo"), SD.get("geo.alt"),)) + assert isinstance(result, LookupInResult) + assert result.content_as[int](0) == 0 + assert result.exists(1) is True + assert result.content_as[dict](2) == value["geo"] + assert result.content_as[int](3) == value["geo"]["alt"] + + def test_count(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["count"] = [1, 2, 3, 4, 5] + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.lookup_in, key, (SD.count("count"),)) + assert isinstance(result, LookupInResult) + assert result.content_as[int](0) == 5 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_get(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in_any_replica, key, [SD.get('city')]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['city'] + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_get_full(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in_any_replica, key, [SD.get_full()]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[dict](0) == value + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_get_bad_path(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in_any_replica, key, [SD.get('qzzxy')]) + assert isinstance(result, LookupInReplicaResult) + with pytest.raises(PathNotFoundException): + result.content_as[str](0) + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_exists(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in_any_replica, key, [SD.exists('country')]) + assert isinstance(result, LookupInReplicaResult) + assert result.exists(0) + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_exists_bad_path(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in_any_replica, key, [SD.exists('qzzxy')]) + assert isinstance(result, LookupInReplicaResult) + assert not result.exists(0) + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_bad_key(self, cb_env): + cb = cb_env.collection + with pytest.raises(DocumentUnretrievableException): + run_in_reactor_thread(cb.lookup_in_any_replica, 'asdfgh', [SD.exists('country')]) + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_multiple_specs(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + result = run_in_reactor_thread(cb.lookup_in_any_replica, key, [SD.get('country'), SD.exists('geo.lat')]) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['country'] + assert result.exists(1) + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_any_replica_with_timeout(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + opts = LookupInAnyReplicaOptions(timeout=timedelta(milliseconds=5000)) + result = run_in_reactor_thread(cb.lookup_in_any_replica, key, [SD.get('country')], opts) + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['country'] + assert result.is_replica is not None + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_get(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + results = run_in_reactor_thread(cb.lookup_in_all_replicas, key, [SD.get('city')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['city'] + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_get_full(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + results = run_in_reactor_thread(cb.lookup_in_all_replicas, key, [SD.get_full()]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[dict](0) == value + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_get_bad_path(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + results = run_in_reactor_thread(cb.lookup_in_all_replicas, key, [SD.get('qzzxy')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + with pytest.raises(PathNotFoundException): + result.content_as[str](0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_exists(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + results = run_in_reactor_thread(cb.lookup_in_all_replicas, key, [SD.exists('country')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.exists(0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_exists_bad_path(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + results = run_in_reactor_thread(cb.lookup_in_all_replicas, key, [SD.exists('qzzxy')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert not result.exists(0) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_bad_key(self, cb_env): + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb_env.collection.lookup_in_all_replicas, 'asdfgh', [SD.exists('country')]) + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_multiple_specs(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + results = run_in_reactor_thread(cb.lookup_in_all_replicas, key, [SD.get('country'), SD.exists('geo.lat')]) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['country'] + assert result.exists(1) + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('check_replica_read_supported') + def test_lookup_in_all_replicas_with_timeout(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_default_key_value() + opts = LookupInAllReplicasOptions(timeout=timedelta(milliseconds=5000)) + results = run_in_reactor_thread(cb.lookup_in_all_replicas, key, [SD.get('country')], opts) + active_count = 0 + for result in results: + assert isinstance(result, LookupInReplicaResult) + assert result.content_as[str](0) == value['country'] + assert result.is_replica is not None + active_count += not result.is_replica + assert active_count == 1 + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_mutate_in_simple(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert("city", "New City"), + SD.replace("faa", "CTY")), + MutateInOptions(expiry=timedelta(seconds=1000))) + + value["city"] = "New City" + value["faa"] = "CTY" + + def cas_matches(cb, new_cas): + r = run_in_reactor_thread(cb.get, key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + cb_env.try_n_times(10, 3, cas_matches, cb, result.cas, is_deferred=False) + + result = run_in_reactor_thread(cb.get, key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_mutate_in_simple_spec_as_list(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + + result = run_in_reactor_thread(cb.mutate_in, + key, + [SD.upsert("city", "New City"), + SD.replace("faa", "CTY")], + MutateInOptions(expiry=timedelta(seconds=1000))) + + value["city"] = "New City" + value["faa"] = "CTY" + + def cas_matches(cb, new_cas): + r = run_in_reactor_thread(cb.get, key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + cb_env.try_n_times(10, 3, cas_matches, cb, result.cas, is_deferred=False) + + result = run_in_reactor_thread(cb.get, key) + assert value == result.content_as[dict] + + @pytest.mark.usefixtures("check_xattr_supported") + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_mutate_in_expiry(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert("city", "New City"), + SD.replace("faa", "CTY")), + MutateInOptions(expiry=timedelta(seconds=1000))) + + def cas_matches(cb, new_cas): + r = run_in_reactor_thread(cb.get, key) + if new_cas != r.cas: + raise Exception(f"{new_cas} != {r.cas}") + + cb_env.try_n_times(10, 3, cas_matches, cb, result.cas, is_deferred=False) + + result = run_in_reactor_thread(cb.get, key, GetOptions(with_expiry=True)) + expires_in = (result.expiry_time - datetime.now()).total_seconds() + assert expires_in > 0 and expires_in < 1021 + + # reset to norm + run_in_reactor_thread(cb.remove, key) + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_mutate_in_remove(self, cb_env, new_kvp): + + cb = cb_env.collection + key = new_kvp.key + value = new_kvp.value + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + + run_in_reactor_thread(cb.mutate_in, key, [SD.remove('geo.alt')]) + result = run_in_reactor_thread(cb.get, key) + assert 'alt' not in result.content_as[dict]['geo'] + + @pytest.mark.usefixtures("skip_if_less_than_cheshire_cat") + def test_mutate_in_preserve_expiry_not_used(self, cb_env, default_kvp_and_reset): + if cb_env.is_mock_server: + pytest.skip("Mock will not return expiry in the xaddrs.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert("city", "New City"), + SD.replace("faa", "CTY")), + MutateInOptions(expiry=timedelta(seconds=5))) + + expiry_path = "$document.exptime" + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + run_in_reactor_thread(cb.mutate_in, key, (SD.upsert("city", "Updated City"),)) + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 != expiry2 + # if expiry was set, should be expired by now + cb_env.sleep(3) + result = run_in_reactor_thread(cb.get, key) + assert isinstance(result, GetResult) + assert result.content_as[dict]['city'] == 'Updated City' + + @pytest.mark.usefixtures("skip_if_less_than_cheshire_cat") + def test_mutate_in_preserve_expiry(self, cb_env, default_kvp_and_reset): + if cb_env.is_mock_server: + pytest.skip("Mock will not return expiry in the xaddrs.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert("city", "New City"), + SD.replace("faa", "CTY")), + MutateInOptions(expiry=timedelta(seconds=2))) + + expiry_path = "$document.exptime" + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry1 = res.content_as[int](0) + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert("city", "Updated City"),), + MutateInOptions(preserve_expiry=True)) + res = cb_env.try_n_times(10, 3, cb.lookup_in, key, (SD.get(expiry_path, xattr=True),)) + expiry2 = res.content_as[int](0) + + assert expiry1 is not None + assert expiry2 is not None + assert expiry1 == expiry2 + # if expiry was set, should be expired by now + cb_env.sleep(3) + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + @pytest.mark.usefixtures("skip_if_less_than_cheshire_cat") + def test_mutate_in_preserve_expiry_fails(self, cb_env, default_kvp_and_reset): + if cb_env.is_mock_server: + pytest.skip("Mock will not return expiry in the xaddrs.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.mutate_in, + key, + (SD.insert("c", "ccc"),), + MutateInOptions(preserve_expiry=True), + ) + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.mutate_in, + key, + (SD.replace("c", "ccc"),), + MutateInOptions( + expiry=timedelta( + seconds=5), + preserve_expiry=True), + ) + + @pytest.mark.usefixtures("skip_if_less_than_alice") + def test_mutate_in_server_durability(self, cb_env, default_kvp_and_reset, num_replicas): + if cb_env.is_mock_server: + pytest.skip("Mock will not return expiry in the xaddrs.") + + cb = cb_env.collection + key = default_kvp_and_reset.key + if num_replicas > 1: + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert("city", "New City"), + SD.replace("faa", "CTY")), + MutateInOptions(durability=ServerDurability( + level=DurabilityLevel.PERSIST_TO_MAJORITY))) + else: + try: + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert("city", "New City"), + SD.replace("faa", "CTY")), + MutateInOptions(durability=ServerDurability( + level=DurabilityLevel.PERSIST_TO_MAJORITY))) + except DurabilityImpossibleException: + pass # this is okay -- server not setup correctly + + @pytest.mark.usefixtures("skip_if_less_than_alice") + def test_mutate_in_client_durability(self, cb_env, default_kvp_and_reset, num_replicas): + pytest.skip("C++ client has not implemented replicate/persist durability.") + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_mutate_in_upsert_semantics(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.UPSERT)) + + res = cb_env.try_n_times(10, 3, cb.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_mutate_in_upsert_semantics_kwargs(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert('new_path', 'im new'),), + upsert_doc=True) + + res = cb_env.try_n_times(10, 3, cb.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_mutate_in_insert_semantics(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.insert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.INSERT)) + + res = cb_env.try_n_times(10, 3, cb.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_mutate_in_insert_semantics_kwargs(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.get, key) + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.insert('new_path', 'im new'),), + insert_doc=True) + + res = cb_env.try_n_times(10, 3, cb.get, key) + assert res.content_as[dict] == {'new_path': 'im new'} + + def test_mutate_in_insert_semantics_fail(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + + with pytest.raises(DocumentExistsException): + run_in_reactor_thread(cb.mutate_in, + key, + (SD.insert('new_path', 'im new'),), + insert_doc=True) + + def test_mutate_in_replace_semantics(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert('new_path', 'im new'),), + MutateInOptions(store_semantics=SD.StoreSemantics.REPLACE)) + + res = cb_env.try_n_times(10, 3, cb.get, key) + assert res.content_as[dict]['new_path'] == 'im new' + + def test_mutate_in_replace_semantics_kwargs(self, cb_env, default_kvp_and_reset): + cb = cb_env.collection + key = default_kvp_and_reset.key + + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert('new_path', 'im new'),), + replace_doc=True) + + res = cb_env.try_n_times(10, 3, cb.get, key) + assert res.content_as[dict]['new_path'] == 'im new' + + def test_mutate_in_replace_semantics_fail(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + + with pytest.raises(DocumentNotFoundException): + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert('new_path', 'im new'),), + replace_doc=True) + + def test_mutate_in_store_semantics_fail(self, cb_env, new_kvp): + cb = cb_env.collection + key = new_kvp.key + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert('new_path', 'im new'),), + insert_doc=True, upsert_doc=True) + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert('new_path', 'im new'),), + insert_doc=True, replace_doc=True) + + with pytest.raises(InvalidArgumentException): + run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert('new_path', 'im new'),), + upsert_doc=True, replace_doc=True) + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_array_append(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["array"] = [1, 2, 3, 4] + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_append("array", 5),)) + assert isinstance(result, MutateInResult) + result = cb_env.try_n_times(10, 3, cb.get, key) + val = result.content_as[dict] + assert isinstance(val["array"], list) + assert len(val["array"]) == 5 + assert val["array"][4] == 5 + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_array_prepend(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["array"] = [1, 2, 3, 4] + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_prepend("array", 0),)) + assert isinstance(result, MutateInResult) + result = cb_env.try_n_times(10, 3, cb.get, key) + val = result.content_as[dict] + assert isinstance(val["array"], list) + assert len(val["array"]) == 5 + assert val["array"][0] == 0 + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_array_insert(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["array"] = [1, 2, 4, 5] + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_insert("array.[2]", 3),)) + assert isinstance(result, MutateInResult) + result = cb_env.try_n_times(10, 3, cb.get, key) + val = result.content_as[dict] + assert isinstance(val["array"], list) + assert len(val["array"]) == 5 + assert val["array"][2] == 3 + + def test_array_add_unique(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["array"] = [0, 1, 2, 3] + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_addunique("array", 4),)) + assert isinstance(result, MutateInResult) + result = cb_env.try_n_times(10, 3, cb.get, key) + val = result.content_as[dict] + assert isinstance(val["array"], list) + assert len(val["array"]) == 5 + assert 4 in val["array"] + + def test_array_as_document(self, cb_env): + cb = cb_env.collection + key = "simple-key" + run_in_reactor_thread(cb.upsert, key, []) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_append("", 2), + SD.array_prepend("", 0), + SD.array_insert("[1]", 1),)) + assert isinstance(result, MutateInResult) + result = cb_env.try_n_times(10, 3, cb.get, key) + val = result.content_as[list] + assert isinstance(val, list) + assert len(val) == 3 + assert val[0] == 0 + assert val[1] == 1 + assert val[2] == 2 + + # clean-up + run_in_reactor_thread(cb.remove, key) + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_array_append_multi_insert(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["array"] = [1, 2, 3, 4, 5, 6, 7] + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_append("array", 8, 9, 10),)) + assert isinstance(result, MutateInResult) + result = cb_env.try_n_times(10, 3, cb.get, key) + val = result.content_as[dict] + assert isinstance(val["array"], list) + app_res = val["array"][7:] + assert len(app_res) == 3 + assert app_res == [8, 9, 10] + + def test_array_prepend_multi_insert(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["array"] = [4, 5, 6, 7, 8, 9, 10] + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_prepend("array", 1, 2, 3),)) + assert isinstance(result, MutateInResult) + result = cb_env.try_n_times(10, 3, cb.get, key) + val = result.content_as[dict] + assert isinstance(val["array"], list) + pre_res = val["array"][:3] + assert len(pre_res) == 3 + assert pre_res == [1, 2, 3] + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_array_insert_multi_insert(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["array"] = [1, 2, 3, 4, 8, 9, 10] + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_insert("array.[4]", 5, 6, 7),)) + assert isinstance(result, MutateInResult) + result = cb_env.try_n_times(10, 3, cb.get, key) + val = result.content_as[dict] + assert isinstance(val["array"], list) + ins_res = val["array"][4:7] + assert len(ins_res) == 3 + assert ins_res == [5, 6, 7] + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_array_add_unique_fail(self, cb_env): + cb = cb_env.collection + key = "simple-key" + value = { + "a": "aaa", + "b": [0, 1, 2, 3], + "c": [1.25, 1.5, {"nested": ["str", "array"]}], + } + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + + with pytest.raises(PathExistsException): + run_in_reactor_thread(cb.mutate_in, key, (SD.array_addunique("b", 3),)) + + with pytest.raises(InvalidValueException): + run_in_reactor_thread(cb.mutate_in, key, (SD.array_addunique("b", [4, 5, 6]),)) + + with pytest.raises(InvalidValueException): + run_in_reactor_thread(cb.mutate_in, key, (SD.array_addunique("b", {"c": "d"}),)) + + with pytest.raises(PathMismatchException): + run_in_reactor_thread(cb.mutate_in, key, (SD.array_addunique("c", 2),)) + + def test_increment(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["count"] = 100 + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.increment("count", 50),)) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + assert result.content_as[dict]["count"] == 150 + + def test_decrement(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + value["count"] = 100 + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.decrement("count", 50),)) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + assert result.content_as[dict]["count"] == 50 + + def test_insert_create_parents(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.insert("new.path", + "parents created", + create_parents=True),)) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + assert result.content_as[dict]["new"]["path"] == "parents created" + + def test_upsert_create_parents(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.upsert("new.path", + "parents created", + create_parents=True),)) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + assert result.content_as[dict]["new"]["path"] == "parents created" + + def test_array_append_create_parents(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_append("new.array", "Hello,", create_parents=True), + SD.array_append("new.array", "World!"),)) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + assert result.content_as[dict]["new"]["array"] == [ + "Hello,", "World!"] + + def test_array_prepend_create_parents(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_prepend("new.array", "World!", create_parents=True), + SD.array_prepend("new.array", "Hello,"),)) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + assert result.content_as[dict]["new"]["array"] == [ + "Hello,", "World!"] + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_array_add_unique_create_parents(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.array_addunique("new.set", "new", create_parents=True), + SD.array_addunique("new.set", "unique"), + SD.array_addunique("new.set", "set"))) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + new_set = result.content_as[dict]["new"]["set"] + assert isinstance(new_set, list) + assert "new" in new_set + assert "unique" in new_set + assert "set" in new_set + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_increment_create_parents(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.increment("new.counter", 100, create_parents=True),)) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + assert result.content_as[dict]["new"]["counter"] == 100 + + @pytest.mark.usefixtures('skip_mock_mutate_in') + def test_decrement_create_parents(self, cb_env): + cb = cb_env.collection + key, value = cb_env.get_new_key_value() + run_in_reactor_thread(cb.upsert, key, value) + cb_env.try_n_times(10, 3, cb.get, key) + result = run_in_reactor_thread(cb.mutate_in, + key, + (SD.decrement("new.counter", 100, create_parents=True),)) + assert isinstance(result, MutateInResult) + result = run_in_reactor_thread(cb.get, key) + assert result.content_as[dict]["new"]["counter"] == -100 diff --git a/txcouchbase/tests/test_n1ql.py b/txcouchbase/tests/test_n1ql.py deleted file mode 100644 index 56ac620da..000000000 --- a/txcouchbase/tests/test_n1ql.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2015, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -from twisted.internet import defer - -from txcouchbase.bucket import BatchedN1QLRequest -from couchbase.exceptions import HTTPError -from couchbase.async.n1ql import AsyncN1QLRequest - -from couchbase.tests.base import MockTestCase -from txcouchbase.tests.base import gen_base - -class RowsHandler(AsyncN1QLRequest): - def __init__(self, *args, **kwargs): - super(RowsHandler, self).__init__(*args, **kwargs) - self.rows = [] - self.done_called = False - self.deferred = None - self.cached_error = None - - def on_rows(self, rowiter): - self.rows = list(rowiter) - - def on_done(self): - assert not self.done_called - self.done_called = True - self.deferred.callback(None) - - def on_error(self, ex): - self.cached_error = ex - self.deferred.errback(ex) - - -class TxN1QLTests(gen_base(MockTestCase)): - def testIncremental(self): - cb = self.make_connection() - d = defer.Deferred() - o = cb.n1qlQueryEx(RowsHandler, 'SELECT mockrow') - self.assertIsInstance(o, RowsHandler) - - def verify(*args): - self.assertEqual(len(o.rows), 1) - self.assertTrue(o.done_called) - - o.deferred = d - d.addCallback(verify) - return d - - def testBatched(self): - cb = self.make_connection() - d = cb.n1qlQueryAll('SELECT mockrow') - - def verify(o): - self.assertIsInstance(o, BatchedN1QLRequest) - rows = [r for r in o] - self.assertEqual(1, len(rows)) - - return d.addCallback(verify) \ No newline at end of file diff --git a/txcouchbase/tests/test_ops.py b/txcouchbase/tests/test_ops.py deleted file mode 100644 index 3984aebdd..000000000 --- a/txcouchbase/tests/test_ops.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from couchbase.tests.base import ConnectionTestCase - -from txcouchbase.tests.base import gen_base -from couchbase.exceptions import NotFoundError -from couchbase.result import ( - Result, OperationResult, ValueResult, MultiResult) - - -class OperationTestCase(gen_base(ConnectionTestCase)): - def testSimpleSet(self): - cb = self.make_connection() - key = self.gen_key("test_simple_set") - d = cb.set(key, "simple_Value") - def t(ret): - self.assertIsInstance(ret, OperationResult) - self.assertEqual(ret.key, key) - del ret - - d.addCallback(t) - del cb - return d - - def testSimpleGet(self): - cb = self.make_connection() - key = self.gen_key("test_simple_get") - value = "simple_value" - - cb.set(key, value) - d_get = cb.get(key) - def t(ret): - self.assertIsInstance(ret, ValueResult) - self.assertEqual(ret.key, key) - self.assertEqual(ret.value, value) - - d_get.addCallback(t) - return d_get - - def testMultiSet(self): - cb = self.make_connection() - kvs = self.gen_kv_dict(prefix="test_multi_set") - d_set = cb.setMulti(kvs) - - def t(ret): - self.assertEqual(len(ret), len(kvs)) - self.assertEqual(ret.keys(), kvs.keys()) - self.assertTrue(ret.all_ok) - for k in kvs: - self.assertEqual(ret[k].key, k) - self.assertTrue(ret[k].success) - - del ret - - d_set.addCallback(t) - return d_set - - def testSingleError(self): - cb = self.make_connection() - key = self.gen_key("test_single_error") - - d_del = cb.delete(key, quiet=True) - - d = cb.get(key, quiet=False) - def t(err): - self.assertIsInstance(err.value, NotFoundError) - return True - - d.addCallback(lambda x: self.assertTrue(False)) - d.addErrback(t) - return d - - def testMultiErrors(self): - cb = self.make_connection() - kv = self.gen_kv_dict(prefix = "test_multi_errors") - cb.setMulti(kv) - - rmkey = kv.keys()[0] - cb.delete(rmkey) - - d = cb.getMulti(kv.keys()) - - def t(err): - self.assertIsInstance(err.value, NotFoundError) - all_results = err.value.all_results - for k, v in kv.items(): - self.assertTrue(k in all_results) - res = all_results[k] - self.assertEqual(res.key, k) - if k != rmkey: - self.assertTrue(res.success) - self.assertEqual(res.value, v) - - res_fail = err.value.result - self.assertFalse(res_fail.success) - self.assertTrue(NotFoundError._can_derive(res_fail.rc)) - - d.addErrback(t) - return d diff --git a/txcouchbase/tests/test_txconn.py b/txcouchbase/tests/test_txconn.py deleted file mode 100644 index 39cf40dca..000000000 --- a/txcouchbase/tests/test_txconn.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from twisted.internet import reactor, defer -from couchbase.exceptions import ( - BucketNotFoundError, - ObjectDestroyedError) - -from couchbase.tests.base import ConnectionTestCase -from couchbase.connstr import ConnectionString -from txcouchbase.tests.base import gen_base -from txcouchbase.bucket import Bucket - -class BasicConnectionTest(gen_base(ConnectionTestCase)): - def testConnectionSuccess(self): - cb = self.make_connection() - d = cb.connect() - d.addCallback(lambda x: self.assertTrue(cb.connected)) - return d - - def testConnectionFailure(self): - cb = self.make_connection(bucket='blahblah') - d = cb.connect() - d.addCallback(lambda x: x, cb) - return self.assertFailure(d, BucketNotFoundError) - - def testBadEvent(self): - cb = self.make_connection() - self.assertRaises(ValueError, cb.registerDeferred, - 'blah', - defer.Deferred()) - - d = defer.Deferred() - cb.registerDeferred('connect', d) - d.addBoth(lambda x: None) - return d - - def testMultiHost(self): - info = self.cluster_info - cs = ConnectionString.parse(self.make_connargs()['connection_string']) - cs.hosts = [ info.host + ':' + '10', info.host + ':' + str(info.port) ] - cb = self.make_connection(connection_string=cs.encode()) - d = cb.connect() - d.addCallback(lambda x: self.assertTrue(cb.connected)) - return d - - def testConnstrFirstArg(self): - info = self.cluster_info - s = self.make_connargs()['connection_string'] - cb = Bucket(s) - d = cb.connect().addCallback(lambda x: self.assertTrue(cb.connected)) - self.register_cleanup(cb) - return d - - def testConnectionDestroyed(self): - cb = self.make_connection() - d = cb.connect() - self.assertFailure(d, ObjectDestroyedError) - return d diff --git a/txcouchbase/tests/test_views.py b/txcouchbase/tests/test_views.py deleted file mode 100644 index 08f8592d8..000000000 --- a/txcouchbase/tests/test_views.py +++ /dev/null @@ -1,88 +0,0 @@ -# Copyright 2013, Couchbase, Inc. -# All Rights Reserved -# -# Licensed under the Apache License, Version 2.0 (the "License") -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -from twisted.internet import defer - -from txcouchbase.bucket import BatchedView -from couchbase.exceptions import HTTPError, ArgumentError -from couchbase.async.view import AsyncViewBase - -from couchbase.tests.base import ViewTestCase -from txcouchbase.tests.base import gen_base - -class RowsHandler(AsyncViewBase): - def __init__(self, *args, **kwargs): - super(RowsHandler, self).__init__(*args, **kwargs) - self._rows_received = 0 - self._done_called = False - self._call_count = 0 - self._cached_ex = None - - def on_rows(self, rows): - l = list(rows) - self._rows_received += len(l) - self._call_count += 1 - - def on_done(self): - self._done_called = True - self._d.callback(None) - - def on_error(self, ex): - self._cached_ex = ex - self._d.errback(ex) - - -class TxViewsTests(gen_base(ViewTestCase)): - def make_connection(self, **kwargs): - return super(TxViewsTests, self).make_connection(bucket='beer-sample') - - def testEmptyView(self): - cb = self.make_connection() - return cb.queryAll('beer', 'brewery_beers', limit=0) - - def testLimitView(self): - cb = self.make_connection() - d = cb.queryAll('beer', 'brewery_beers', limit=10) - - def _verify(o): - self.assertIsInstance(o, BatchedView) - rows = list(o) - self.assertEqual(len(rows), 10) - - return d.addCallback(_verify) - - def testBadView(self): - cb = self.make_connection() - d = cb.queryAll('blah', 'blah_blah') - self.assertFailure(d, HTTPError) - return d - - def testIncrementalRows(self): - d = defer.Deferred() - cb = self.make_connection() - o = cb.queryEx(RowsHandler, 'beer', 'brewery_beers') - self.assertIsInstance(o, RowsHandler) - - def verify(unused): - self.assertTrue(o.indexed_rows > 7000) - self.assertEqual(o._rows_received, o.indexed_rows) - - ## Commented because we can't really verify this now, - ## can we? - #self.assertTrue(o._call_count > 1) - - d.addCallback(verify) - o._d = d - return d diff --git a/txcouchbase/tests/transcoder_t.py b/txcouchbase/tests/transcoder_t.py new file mode 100644 index 000000000..732e2f064 --- /dev/null +++ b/txcouchbase/tests/transcoder_t.py @@ -0,0 +1,854 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +from datetime import timedelta + +import pytest + +from couchbase.exceptions import (DocumentLockedException, + DocumentNotFoundException, + ValueFormatException) +from couchbase.options import (GetAndLockOptions, + GetAndTouchOptions, + GetOptions, + InsertOptions, + ReplaceOptions, + UpsertOptions) +from couchbase.transcoder import (LegacyTranscoder, + RawBinaryTranscoder, + RawJSONTranscoder, + RawStringTranscoder) + +from ._test_utils import (CollectionType, + FakeTestObj, + KVPair, + TestEnvironment, + run_in_reactor_thread) + + +class DefaultTranscoderTests: + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment(__name__, couchbase_config, request.param) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + yield cb_env + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.fixture(name="new_kvp") + def new_key_and_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="str_kvp") + def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="bytes_kvp") + def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="json_kvp") + def json_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_default_tc_json_upsert(self, cb_env, json_kvp): + key, value = json_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + def test_default_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + + res = run_in_reactor_thread(cb_env.collection.get, key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + def test_default_tc_json_replace(self, cb_env, json_kvp): + key, value = json_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + value['new_content'] = 'new content!' + run_in_reactor_thread(cb_env.collection.replace, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + result = res.content_as[dict] + assert result is not None + assert isinstance(result, dict) + assert result == value + + # default TC: no transcoder set in ClusterOptions or KV options + + def test_default_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == value + + def test_default_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == value + + def test_default_tc_string_replace(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + new_content = "new string content" + run_in_reactor_thread(cb_env.collection.replace, key, new_content) + res = run_in_reactor_thread(cb_env.collection.get, key) + result = res.content_as[str] + assert result is not None + assert isinstance(result, str) + assert result == new_content + + def test_default_tc_binary_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.upsert, key, value) + + def test_default_tc_bytearray_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.upsert, key, bytearray(value)) + + def test_default_tc_binary_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.insert, key, value) + + def test_default_tc_binary_replace(self, cb_env, str_kvp, bytes_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.replace, key, bytes_kvp.value) + + +class RawJsonTranscoderTests: + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment( + __name__, couchbase_config, request.param, transcoder=RawJSONTranscoder()) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + yield cb_env + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.fixture(name="str_kvp") + def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="bytes_kvp") + def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="json_kvp") + def json_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_raw_json_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes].decode('utf-8') + + def test_raw_json_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes].decode('utf-8') + + def test_raw_json_tc_string_replace(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + new_content = "new string content" + run_in_reactor_thread(cb_env.collection.replace, key, new_content) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes].decode('utf-8') + + def test_raw_json_tc_bytes_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_json_tc_bytes_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_json_tc_bytes_replace(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + new_content = 'new string content'.encode('utf-8') + run_in_reactor_thread(cb_env.collection.replace, key, new_content) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + def test_pass_through(self, cb_env, json_kvp): + key, value = json_kvp + json_str = json.dumps(value) + run_in_reactor_thread(cb_env.collection.upsert, key, json_str) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] != value + + decoded = json.loads(res.content_as[bytes].decode('utf-8')) + assert decoded == value + + def test_raw_json_tc_json_upsert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.upsert, key, value) + + def test_raw_json_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.insert, key, value) + + def test_raw_json_tc_json_replace(self, cb_env, str_kvp, json_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.replace, key, json_kvp.value) + + +class RawStringTranscoderTests: + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment( + __name__, couchbase_config, request.param, transcoder=RawStringTranscoder()) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + yield cb_env + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.fixture(name="str_kvp") + def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="bytes_kvp") + def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="json_kvp") + def json_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_raw_string_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + def test_raw_string_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + def test_raw_string_tc_string_replace(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + new_content = "new string content" + run_in_reactor_thread(cb_env.collection.replace, key, new_content) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, str) + assert new_content == res.content_as[str] + + def test_raw_string_tc_bytes_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.upsert, key, value) + + def test_raw_string_tc_bytes_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.insert, key, value) + + def test_raw_string_tc_bytes_replace(self, cb_env, str_kvp, bytes_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.replace, key, bytes_kvp.value) + + def test_raw_string_tc_json_upsert(self, cb_env, json_kvp): + key = json_kvp.key + value = json_kvp.value + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.upsert, key, value) + + def test_raw_string_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.insert, key, value) + + def test_raw_string_tc_json_replace(self, cb_env, str_kvp, json_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.replace, key, json_kvp.value) + + +class RawBinaryTranscoderTests: + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment( + __name__, couchbase_config, request.param, transcoder=RawBinaryTranscoder()) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + yield cb_env + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.fixture(name="str_kvp") + def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="bytes_kvp") + def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="hex_kvp") + def hex_value_with_reset(self, cb_env) -> KVPair: + key = 'key_hex_bytes' + hex_arr = ['ff0102030405060708090a0b0c0d0e0f', + '101112131415161718191a1b1c1d1e1f', + '202122232425262728292a2b2c2d2e2f', + '303132333435363738393a3b3c3d3e3f'] + value = bytes.fromhex(''.join(hex_arr)) + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="json_kvp") + def json_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_raw_binary_tc_bytes_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_binary_tc_bytes_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_binary_tc_bytes_replace(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + new_content = 'new string content'.encode('utf-8') + run_in_reactor_thread(cb_env.collection.replace, key, new_content) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + def test_raw_binary_tc_hex_upsert(self, cb_env, hex_kvp): + key, value = hex_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_binary_tc_hex_insert(self, cb_env, hex_kvp): + key, value = hex_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_raw_binary_tc_hex_replace(self, cb_env, hex_kvp): + key, value = hex_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + new_content = b'\xFF' + run_in_reactor_thread(cb_env.collection.replace, key, new_content) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + def test_raw_binary_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.upsert, key, value) + + def test_raw_binary_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.insert, key, value) + + def test_raw_binary_tc_string_replace(self, cb_env, bytes_kvp, str_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.replace, key, str_kvp.value) + + def test_raw_binary_tc_json_upsert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.upsert, key, value) + + def test_raw_binary_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.insert, key, value) + + def test_raw_binary_tc_json_replace(self, cb_env, bytes_kvp, json_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.replace, key, json_kvp.value) + + +class LegacyTranscoderTests: + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment( + __name__, couchbase_config, request.param, transcoder=LegacyTranscoder()) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + yield cb_env + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.fixture(name="str_kvp") + def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="bytes_kvp") + def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="json_kvp") + def json_value_with_reset(self, cb_env) -> KVPair: + key, value = cb_env.get_new_key_value() + yield KVPair(key, value) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="obj_kvp") + def obj_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_obj' + yield KVPair(key, FakeTestObj()) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + def test_legacy_tc_bytes_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_legacy_tc_bytes_insert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert value == res.content_as[bytes] + + def test_legacy_tc_bytes_replace(self, cb_env, bytes_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + new_content = 'new string content'.encode('utf-8') + run_in_reactor_thread(cb_env.collection.replace, key, new_content) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, bytes) + assert new_content == res.content_as[bytes] + + def test_legacy_tc_obj_upsert(self, cb_env, obj_kvp): + key, value = obj_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, FakeTestObj) + assert value.PROP == res.value.PROP + assert value.PROP1 == res.value.PROP1 + + def test_legacy_tc_obj_insert(self, cb_env, obj_kvp): + key, value = obj_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, FakeTestObj) + assert value.PROP == res.value.PROP + assert value.PROP1 == res.value.PROP1 + + def test_legacy_tc_obj_replace(self, cb_env, bytes_kvp, obj_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + run_in_reactor_thread(cb_env.collection.replace, key, obj_kvp.value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, FakeTestObj) + assert obj_kvp.value.PROP == res.value.PROP + assert obj_kvp.value.PROP1 == res.value.PROP1 + + def test_legacy_tc_string_upsert(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + def test_legacy_tc_string_insert(self, cb_env, str_kvp): + key, value = str_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, str) + assert value == res.content_as[str] + + def test_legacy_tc_string_replace(self, cb_env, bytes_kvp, str_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + run_in_reactor_thread(cb_env.collection.replace, key, str_kvp.value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, str) + assert str_kvp.value == res.content_as[str] + + def test_legacy_tc_json_upsert(self, cb_env, json_kvp): + key, value = json_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + def test_legacy_tc_json_insert(self, cb_env, json_kvp): + key, value = json_kvp + run_in_reactor_thread(cb_env.collection.insert, key, value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, dict) + assert value == res.content_as[dict] + + def test_legacy_tc_json_replace(self, cb_env, bytes_kvp, json_kvp): + key, value = bytes_kvp + run_in_reactor_thread(cb_env.collection.upsert, key, value) + run_in_reactor_thread(cb_env.collection.replace, key, json_kvp.value) + res = run_in_reactor_thread(cb_env.collection.get, key) + assert isinstance(res.value, dict) + assert json_kvp.value == res.content_as[dict] + + +class KeyValueOpTranscoderTests: + + @pytest.fixture(scope="class", name="cb_env", params=[CollectionType.DEFAULT, CollectionType.NAMED]) + def couchbase_test_environment(self, couchbase_config, request): + cb_env = TestEnvironment.get_environment(__name__, couchbase_config, request.param) + + if request.param == CollectionType.NAMED: + cb_env.try_n_times(5, 3, cb_env.setup_named_collections, is_deferred=False) + + yield cb_env + if request.param == CollectionType.NAMED: + cb_env.try_n_times_till_exception(5, 3, + cb_env.teardown_named_collections, + raise_if_no_exception=False, + is_deferred=False) + + @pytest.mark.flaky(reruns=5, reruns_delay=1) + @pytest.fixture(name="str_kvp") + def str_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_str' + yield KVPair(key, 'some string content') + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + @pytest.fixture(name="bytes_kvp") + def bytes_value_with_reset(self, cb_env) -> KVPair: + key = 'key_tc_bytes' + yield KVPair(key, 'some bytes content'.encode('utf-8'),) + cb_env.try_n_times_till_exception(10, + 1, + cb_env.collection.remove, + key, + expected_exceptions=(DocumentNotFoundException,), + reset_on_timeout=True, + reset_num_times=3) + + def test_upsert(self, cb_env, bytes_kvp): + key, value = bytes_kvp + # use RawBinaryTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + run_in_reactor_thread(cb_env.collection.upsert, + key, + value, + UpsertOptions(transcoder=RawBinaryTranscoder())) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.get, key) + + def test_insert(self, cb_env, str_kvp): + key, value = str_kvp + # use RawStringTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + run_in_reactor_thread(cb_env.collection.upsert, key, value, InsertOptions(transcoder=RawStringTranscoder())) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.get, key) + + def test_replace(self, cb_env, bytes_kvp): + key, value = bytes_kvp + # use RawBinaryTranscoder() so that get() fails as expected + # since get() w/o passing in transcoder uses the default JSONTranscoder() + tc = RawBinaryTranscoder() + run_in_reactor_thread(cb_env.collection.upsert, key, value, UpsertOptions(transcoder=tc)) + new_content = 'some new bytes content'.encode('utf-8') + run_in_reactor_thread(cb_env.collection.replace, key, new_content, ReplaceOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.get, key) + + def test_get(self, cb_env, bytes_kvp): + key, value = bytes_kvp + tc = RawBinaryTranscoder() + run_in_reactor_thread(cb_env.collection.upsert, key, value, UpsertOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.get, key) + res = run_in_reactor_thread(cb_env.collection.get, key, GetOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + + def test_get_and_touch(self, cb_env, bytes_kvp): + key, value = bytes_kvp + tc = RawBinaryTranscoder() + run_in_reactor_thread(cb_env.collection.upsert, key, value, UpsertOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.get_and_touch, key, timedelta(seconds=30)) + + res = run_in_reactor_thread(cb_env.collection.get_and_touch, + key, + timedelta(seconds=3), + GetAndTouchOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + cb_env.try_n_times_till_exception( + 10, 3, cb_env.collection.get, key, GetOptions(transcoder=tc), DocumentNotFoundException) + + def test_get_and_lock(self, cb_env, bytes_kvp): + key, value = bytes_kvp + tc = RawBinaryTranscoder() + run_in_reactor_thread(cb_env.collection.upsert, key, value, UpsertOptions(transcoder=tc)) + with pytest.raises(ValueFormatException): + run_in_reactor_thread(cb_env.collection.get_and_lock, key, timedelta(seconds=1)) + + cb_env.try_n_times(10, 1, cb_env.collection.upsert, key, + value, UpsertOptions(transcoder=tc)) + res = run_in_reactor_thread(cb_env.collection.get_and_lock, + key, + timedelta(seconds=3), + GetAndLockOptions(transcoder=tc)) + assert isinstance(res.value, bytes) + assert res.content_as[bytes] == value + # upsert should definitely fail + with pytest.raises(DocumentLockedException): + run_in_reactor_thread(cb_env.collection.upsert, key, value, transcoder=tc) + # but succeed eventually + cb_env.try_n_times(10, 1, cb_env.collection.upsert, key, value, transcoder=tc) diff --git a/txcouchbase/twisted_bucket.py b/txcouchbase/twisted_bucket.py deleted file mode 120000 index e7b4468f8..000000000 --- a/txcouchbase/twisted_bucket.py +++ /dev/null @@ -1 +0,0 @@ -bucket.py \ No newline at end of file diff --git a/txcouchbase/views.py b/txcouchbase/views.py new file mode 100644 index 000000000..18d180a17 --- /dev/null +++ b/txcouchbase/views.py @@ -0,0 +1,112 @@ +# Copyright 2016-2022. Couchbase, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License") +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from twisted.internet.defer import Deferred + +from couchbase.exceptions import (PYCBC_ERROR_MAP, + AlreadyQueriedException, + CouchbaseException, + ErrorMapper, + ExceptionMap) +from couchbase.exceptions import exception as CouchbaseBaseException +from couchbase.logic.views import ViewRequestLogic, ViewRow + + +class ViewRequest(ViewRequestLogic): + def __init__(self, + connection, + loop, + encoded_query, + **kwargs + ): + super().__init__(connection, encoded_query, **kwargs) + self._query_request_ftr = None + self._query_d = None + self._loop = loop + + @property + def loop(self): + """ + **INTERNAL** + """ + return self._loop + + @classmethod + def generate_view_request(cls, connection, loop, encoded_query, **kwargs): + return cls(connection, loop, encoded_query, **kwargs) + + def execute_view_query(self): + # if self._query_request_ftr is not None and self._query_request_ftr.done(): + if self.done_streaming: + raise AlreadyQueriedException() + + if self._query_request_ftr is None: + self._query_request_ftr = self.loop.create_future() + self._submit_query(callback=self._on_query_complete) + self._query_d = Deferred.fromFuture(self._query_request_ftr) + + return self._query_d + + def _on_query_complete(self, result): + self._loop.call_soon_threadsafe(self._query_request_ftr.set_result, result) + + def _get_metadata(self): + try: + view_response = next(self._streaming_result) + self._set_metadata(view_response) + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn + + def __iter__(self): + return self + + def _get_next_row(self): + if self._done_streaming is True: + return + + row = next(self._streaming_result) + if isinstance(row, CouchbaseBaseException): + raise ErrorMapper.build_exception(row) + # should only be None one query request is complete and _no_ errors found + if row is None: + raise StopIteration + + # TODO: until streaming, a dict is returned, no deserializing... + # deserialized_row = self.serializer.deserialize(row) + deserialized_row = row + if issubclass(self.row_factory, ViewRow): + if hasattr(self.row_factory, 'from_json'): + return self.row_factory.from_json(deserialized_row) + return self.row_factory(**deserialized_row) + else: + return deserialized_row + + def __next__(self): + try: + return self._get_next_row() + except StopIteration: + self._done_streaming = True + self._get_metadata() + raise + except CouchbaseException as ex: + raise ex + except Exception as ex: + exc_cls = PYCBC_ERROR_MAP.get(ExceptionMap.InternalSDKException.value, CouchbaseException) + excptn = exc_cls(str(ex)) + raise excptn